Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve pairing planning and statistics visibility

Browse Source
This commit is contained in:
Harmon
2026-07-07 01:04:47 +08:00
parent 2da2121829
commit 36a406ae49
22 changed files with 2085 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/ops_maintenance_service.py
+82 -240
View File
@@ -4,9 +4,9 @@ import os
import shutil
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
from typing import Any, Dict, List, Optional
from sqlalchemy import delete, func, or_, select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
@@ -14,25 +14,18 @@ from ..models import (
DinsarProductionExecutionORM,
DinsarProductionRunItemORM,
DinsarProductionRunORM,
DinsarResultORM,
ResultProductORM,
SystemJobORM,
SystemTaskORM,
TaskLogORM,
WorkflowArtifactORM,
WorkflowRunORM,
WorkflowStepORM,
)
from .dinsar_production_service import dinsar_production_service
TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "PARTIAL_SUCCESS", "CANCELLED"}
MAINTENANCE_LIST_STATUSES = {"FAILED", "PARTIAL_SUCCESS", "CANCELLED", "PENDING", "RUNNING"}
MAINTENANCE_LIST_STATUSES = {"COMPLETED", "FAILED", "PARTIAL_SUCCESS", "CANCELLED", "PENDING", "RUNNING"}
ACTIVE_TASK_STATUSES = {"PENDING", "RUNNING"}
ACTIVE_JOB_STATUSES = {"READY", "RETRY", "RUNNING"}
DINSAR_TASK_TYPES = {"LANDSAR_RUN", "LANDSAR_CLUSTER_RUN", "PYINT_RUN", "IDL_RUN_DINSAR"}
SUPPORTED_CLEANUP_TASK_TYPES = DINSAR_TASK_TYPES | {"COPY_DATA"}
DEFAULT_TASK_TYPES = SUPPORTED_CLEANUP_TASK_TYPES | {"PAIRING_CACHE_REBUILD"}
SUPPORTED_CLEANUP_TASK_TYPES = {"LANDSAR_RUN", "LANDSAR_CLUSTER_RUN"}
DEFAULT_TASK_TYPES = DINSAR_TASK_TYPES | {"COPY_DATA", "PAIRING_CACHE_REBUILD"}
LANDSAR_WORK_TERMINAL_EXECUTION_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
def _utcnow_naive() -> datetime:
@@ -68,8 +61,15 @@ def _path_exists(path: str) -> bool:
return bool(path) and os.path.exists(path)
def _safe_count(rows: Iterable[Any]) -> int:
return len(list(rows))
def _is_within_path(path: str, parent: str) -> bool:
path_text = _normalize_path_text(path)
parent_text = _normalize_path_text(parent)
if not path_text or not parent_text:
return False
try:
return os.path.commonpath([os.path.abspath(path_text), os.path.abspath(parent_text)]) == os.path.abspath(parent_text)
except ValueError:
return False
class OpsMaintenanceService:
@@ -108,11 +108,12 @@ class OpsMaintenanceService:
if item.get("issue_level") in {"warning", "danger"}
or _norm_status(item.get("status")) in {"FAILED", "PARTIAL_SUCCESS", "CANCELLED"}
]
visible_items = items if status_filter else abnormal_items
return {
"items": abnormal_items,
"items": visible_items,
"limit": safe_limit,
"offset": safe_offset,
"returned": len(abnormal_items),
"returned": len(visible_items),
}
async def diagnose_task(self, db: AsyncSession, task_id: str) -> Optional[Dict[str, Any]]:
@@ -124,19 +125,11 @@ class OpsMaintenanceService:
item_counts: Dict[str, int] = {}
execution_counts: Dict[str, int] = {}
disk_paths: List[Dict[str, Any]] = []
products: List[Dict[str, Any]] = []
if run is not None:
item_counts = await self._status_counts(db, DinsarProductionRunItemORM, run.run_id)
execution_counts = await self._status_counts(db, DinsarProductionExecutionORM, run.run_id)
disk_paths.extend(await self._collect_run_disk_paths(db, run))
products = await self._collect_result_products(db, run)
related_tasks = await self._related_copy_tasks_for_run(db, run) if run is not None else []
copy_dest = self._copy_task_dest_dir(task)
if copy_dest:
disk_paths.append(self._path_payload(copy_dest, "task_pool"))
recent_logs = await self._recent_logs(db, task.task_id)
findings, cleanup_supported, cleanup_blockers = self._diagnose_findings(
task=task,
@@ -152,8 +145,6 @@ class OpsMaintenanceService:
"production_run": self._run_payload(run) if run else None,
"production_item_counts": item_counts,
"production_execution_counts": execution_counts,
"result_products": products,
"related_tasks": [self._task_payload(item) for item in related_tasks],
"recent_logs": recent_logs,
"disk_paths": disk_paths,
"diagnosis": {
@@ -179,15 +170,17 @@ class OpsMaintenanceService:
if task_status in ACTIVE_TASK_STATUSES:
blockers.append("任务仍处于活动状态，不能清理。")
db_counts = await self._cleanup_db_counts(db, diagnosis)
disk_deletes = self._cleanup_disk_targets(diagnosis)
has_existing_target = any(item.get("exists") and item.get("allowed") for item in disk_deletes)
if not has_existing_target:
blockers.append("未发现可清理的 LandSAR_WORK_ROOT/run_* 目录。")
blocked = bool(blockers) or not cleanup_supported
return {
"task_id": task_id,
"blocked": blocked,
"blockers": blockers,
"cleanup_supported": cleanup_supported and not blocked,
"database_deletes": db_counts,
"database_deletes": {},
"disk_deletes": disk_deletes,
}
@@ -204,82 +197,17 @@ class OpsMaintenanceService:
if preview.get("blocked"):
raise ValueError("; ".join(preview.get("blockers") or ["清理被阻止。"]))
deleted_db: Dict[str, int] = {}
task = await self._get_task(db, task_id)
if task is None:
return None
jobs = await self._get_jobs(db, task_id)
run = await self._get_production_run_for_task(db, task, jobs)
delete_logs = bool(options.get("delete_logs", True))
delete_task_records = bool(options.get("delete_task_records", True))
delete_production_records = bool(options.get("delete_production_records", True))
delete_result_products = bool(options.get("delete_result_products", True))
delete_task_pool_dir = bool(options.get("delete_task_pool_dir", True))
related_copy_tasks = await self._related_copy_tasks_for_run(db, run) if run is not None else []
if delete_result_products and run is not None:
products = await self._result_product_orms_for_run(db, run)
product_ids = [item.product_id for item in products]
compat_ids = await self._compat_ids_for_products(db, product_ids)
if product_ids:
result = await db.execute(delete(ResultProductORM).where(ResultProductORM.product_id.in_(product_ids)))
deleted_db["result_products"] = int(result.rowcount or 0)
if compat_ids:
result = await db.execute(delete(DinsarResultORM).where(DinsarResultORM.id.in_(compat_ids)))
deleted_db["dinsar_results"] = int(result.rowcount or 0)
if delete_production_records and run is not None:
deleted_run = await dinsar_production_service.delete_run_record(run.run_id, db=db)
deleted_db["dinsar_production_runs"] = 1 if deleted_run else 0
deleted_db["dinsar_production_run_items"] = int(preview["database_deletes"].get("dinsar_production_run_items", 0))
deleted_db["dinsar_production_executions"] = int(preview["database_deletes"].get("dinsar_production_executions", 0))
deleted_db["system_jobs"] = int(preview["database_deletes"].get("system_jobs", 0))
deleted_db["system_tasks"] = 1
deleted_db["task_logs"] = int(preview["database_deletes"].get("task_logs", 0))
if delete_task_pool_dir:
related_deleted = await self._delete_related_tasks(db, related_copy_tasks)
for key, value in related_deleted.items():
deleted_db[key] = int(deleted_db.get(key, 0)) + int(value or 0)
else:
if delete_logs:
result = await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id == task_id))
deleted_db["task_logs"] = int(result.rowcount or 0)
if delete_task_records:
result = await db.execute(delete(SystemJobORM).where(SystemJobORM.task_id == task_id))
deleted_db["system_jobs"] = int(result.rowcount or 0)
result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id == task_id))
deleted_db["system_tasks"] = int(result.rowcount or 0)
await db.commit()
disk_result = await self._delete_disk_targets(preview, options)
return {
"task_id": task_id,
"deleted_database": deleted_db,
"deleted_database": {},
"deleted_disk": disk_result,
}
async def _delete_related_tasks(
self,
db: AsyncSession,
tasks: List[SystemTaskORM],
) -> Dict[str, int]:
task_ids = [task.task_id for task in tasks if task.task_id]
if not task_ids:
return {}
result = await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids)))
logs = int(result.rowcount or 0)
result = await db.execute(delete(SystemJobORM).where(SystemJobORM.task_id.in_(task_ids)))
jobs = int(result.rowcount or 0)
result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids)))
task_count = int(result.rowcount or 0)
await db.commit()
return {
"related_task_logs": logs,
"related_system_jobs": jobs,
"related_system_tasks": task_count,
}
async def _get_task(self, db: AsyncSession, task_id: str) -> Optional[SystemTaskORM]:
result = await db.execute(select(SystemTaskORM).where(SystemTaskORM.task_id == str(task_id or "").strip()))
return result.scalar_one_or_none()
@@ -383,87 +311,33 @@ class OpsMaintenanceService:
async def _collect_run_disk_paths(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[Dict[str, Any]]:
paths: Dict[str, Dict[str, Any]] = {}
if run.source_root:
paths[_normalize_path_text(run.source_root)] = self._path_payload(run.source_root, "task_pool")
result = await db.execute(select(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run.run_id))
for execution in result.scalars().all():
if execution.output_dir:
publish_dir = self._publish_package_dir(execution.output_dir)
paths[publish_dir] = self._path_payload(publish_dir, "production_result")
log_path = dinsar_production_service.read_run_log(run.run_id, max_bytes=1).get("path")
if log_path:
paths[_normalize_path_text(log_path)] = self._path_payload(log_path, "run_log")
landsar_work_dir = self._landsar_work_dir_for_execution(run, execution)
if landsar_work_dir:
payload = self._path_payload(landsar_work_dir, "landsar_work")
payload["run_key"] = execution.run_key
payload["execution_status"] = execution.status
paths[_normalize_path_text(landsar_work_dir).lower()] = payload
return list(paths.values())
async def _collect_result_products(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[Dict[str, Any]]:
products = await self._result_product_orms_for_run(db, run)
return [
{
"product_id": item.product_id,
"display_name": item.display_name,
"status": item.status,
"health_status": item.health_status,
"publish_dir": item.publish_dir,
"manifest_path": item.manifest_path,
}
for item in products
]
async def _result_product_orms_for_run(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[ResultProductORM]:
result = await db.execute(select(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run.run_id))
dirs = [self._publish_package_dir(item.output_dir) for item in result.scalars().all() if item.output_dir]
clauses = []
for path in dirs:
clauses.append(ResultProductORM.publish_dir == path)
clauses.append(ResultProductORM.native_output_dir.like(path + "%"))
clauses.append(ResultProductORM.manifest_path.like(path + "%"))
clauses.append(ResultProductORM.primary_asset_path.like(path + "%"))
if not clauses:
return []
products = await db.execute(select(ResultProductORM).where(or_(*clauses)))
by_id: Dict[str, ResultProductORM] = {}
for product in products.scalars().all():
by_id[product.product_id] = product
return list(by_id.values())
async def _compat_ids_for_products(self, db: AsyncSession, product_ids: List[str]) -> List[int]:
if not product_ids:
return []
result = await db.execute(select(DinsarResultORM).where(DinsarResultORM.compat_product_id.in_(product_ids)))
return [int(item.id) for item in result.scalars().all()]
def _publish_package_dir(self, output_dir: str) -> str:
normalized = _normalize_path_text(output_dir)
marker = os.sep + "runs" + os.sep
if marker.lower() in normalized.lower():
lower = normalized.lower()
index = lower.index(marker.lower())
return normalized[:index]
return normalized
def _copy_task_dest_dir(self, task: SystemTaskORM) -> str:
params = task.params if isinstance(task.params, dict) else {}
return _normalize_path_text(params.get("dest_dir")) if params.get("dest_dir") else ""
async def _related_copy_tasks_for_run(
def _landsar_work_dir_for_execution(
self,
db: AsyncSession,
run: Optional[DinsarProductionRunORM],
) -> List[SystemTaskORM]:
source_root = _normalize_path_text(run.source_root if run is not None else "")
if not source_root:
return []
result = await db.execute(
select(SystemTaskORM)
.where(SystemTaskORM.task_type == "COPY_DATA")
.order_by(SystemTaskORM.updated_at.desc(), SystemTaskORM.id.desc())
.limit(1000)
)
tasks = []
for task in result.scalars().all():
if _normalize_path_text(self._copy_task_dest_dir(task)).lower() == source_root.lower():
tasks.append(task)
return tasks
run: DinsarProductionRunORM,
execution: DinsarProductionExecutionORM,
) -> str:
if _norm_status(run.engine_code) != "LANDSAR":
return ""
if _norm_status(run.status) in ACTIVE_TASK_STATUSES:
return ""
if _norm_status(execution.status) not in LANDSAR_WORK_TERMINAL_EXECUTION_STATUSES:
return ""
work_root = _normalize_path_text(settings.LANDSAR_WORK_ROOT)
run_key = str(execution.run_key or "").strip()
if not work_root or not run_key or Path(run_key).name != run_key or not run_key.startswith("run_"):
return ""
candidate = _normalize_path_text(os.path.join(work_root, run_key))
return candidate if self._is_landsar_work_delete_path(candidate) else ""
def _diagnose_findings(
self,
@@ -481,9 +355,9 @@ class OpsMaintenanceService:
if status == "FAILED":
findings.append("任务已失败，需要人工确认后清理。")
elif status == "PARTIAL_SUCCESS":
findings.append("任务部分成功，清理前请确认保留策略。")
findings.append("任务部分成功，可按需清理 LandSAR 工作目录。")
elif status == "CANCELLED":
findings.append("任务已取消，可按需清理残留记录和目录。")
findings.append("任务已取消，可按需清理 LandSAR 工作目录。")
elif status in ACTIVE_TASK_STATUSES:
blockers.append("任务仍处于活动状态。")
@@ -507,58 +381,29 @@ class OpsMaintenanceService:
missing_paths = [item for item in disk_paths if item.get("path") and not item.get("exists")]
if missing_paths:
findings.append(f"有 {len(missing_paths)} 个登记路径已不存在。")
landsar_work_paths = [
item
for item in disk_paths
if item.get("kind") == "landsar_work" and item.get("exists")
]
if landsar_work_paths:
findings.append(f"发现 {len(landsar_work_paths)} 个可清理的 LandSAR 工作目录。")
cleanup_supported = _norm_status(task.task_type) in SUPPORTED_CLEANUP_TASK_TYPES and not blockers
cleanup_supported = (
_norm_status(task.task_type) in SUPPORTED_CLEANUP_TASK_TYPES
and not blockers
and bool(landsar_work_paths)
)
return findings, cleanup_supported, blockers
async def _cleanup_db_counts(self, db: AsyncSession, diagnosis: Dict[str, Any]) -> Dict[str, int]:
task_id = diagnosis["task"]["task_id"]
run = diagnosis.get("production_run") or {}
run_id = run.get("run_id")
workflow_run_id = run.get("workflow_run_id")
counts = {
"system_tasks": await self._count(db, select(func.count()).select_from(SystemTaskORM).where(SystemTaskORM.task_id == task_id)),
"system_jobs": await self._count(db, select(func.count()).select_from(SystemJobORM).where(SystemJobORM.task_id == task_id)),
"task_logs": await self._count(db, select(func.count()).select_from(TaskLogORM).where(TaskLogORM.task_id == task_id)),
"related_system_tasks": 0,
"related_system_jobs": 0,
"related_task_logs": 0,
"dinsar_production_runs": 0,
"dinsar_production_run_items": 0,
"dinsar_production_executions": 0,
"result_products": len(diagnosis.get("result_products") or []),
"dinsar_results": 0,
"workflow_runs": 0,
"workflow_steps": 0,
"workflow_artifacts": 0,
}
if run_id:
counts["dinsar_production_runs"] = await self._count(db, select(func.count()).select_from(DinsarProductionRunORM).where(DinsarProductionRunORM.run_id == run_id))
counts["dinsar_production_run_items"] = await self._count(db, select(func.count()).select_from(DinsarProductionRunItemORM).where(DinsarProductionRunItemORM.run_id == run_id))
counts["dinsar_production_executions"] = await self._count(db, select(func.count()).select_from(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run_id))
run_obj = await self._get_run_by_id(db, run_id)
related_tasks = await self._related_copy_tasks_for_run(db, run_obj)
related_task_ids = [item.task_id for item in related_tasks if item.task_id]
counts["related_system_tasks"] = len(related_task_ids)
if related_task_ids:
counts["related_system_jobs"] = await self._count(db, select(func.count()).select_from(SystemJobORM).where(SystemJobORM.task_id.in_(related_task_ids)))
counts["related_task_logs"] = await self._count(db, select(func.count()).select_from(TaskLogORM).where(TaskLogORM.task_id.in_(related_task_ids)))
if workflow_run_id:
counts["workflow_runs"] = await self._count(db, select(func.count()).select_from(WorkflowRunORM).where(WorkflowRunORM.run_id == workflow_run_id))
counts["workflow_steps"] = await self._count(db, select(func.count()).select_from(WorkflowStepORM).where(WorkflowStepORM.run_id == workflow_run_id))
counts["workflow_artifacts"] = await self._count(db, select(func.count()).select_from(WorkflowArtifactORM).where(WorkflowArtifactORM.run_id == workflow_run_id))
return counts
async def _get_run_by_id(self, db: AsyncSession, run_id: str) -> Optional[DinsarProductionRunORM]:
result = await db.execute(select(DinsarProductionRunORM).where(DinsarProductionRunORM.run_id == run_id))
return result.scalar_one_or_none()
async def _count(self, db: AsyncSession, stmt: Any) -> int:
return int((await db.execute(stmt)).scalar_one() or 0)
def _cleanup_disk_targets(self, diagnosis: Dict[str, Any]) -> List[Dict[str, Any]]:
targets: Dict[str, Dict[str, Any]] = {}
for item in diagnosis.get("disk_paths") or []:
if item.get("kind") != "landsar_work":
continue
path = _normalize_path_text(item.get("path"))
if not path:
continue
@@ -581,29 +426,26 @@ class OpsMaintenanceService:
normalized = _normalize_path_text(path)
if not normalized:
return False
roots = [
settings.DINSAR_TASK_POOL_ROOT,
settings.DINSAR_PRODUCT_DIR,
os.path.join(settings.PROJECT_ROOT, "backend", "runtime", "dinsar_production"),
]
return self._is_landsar_work_delete_path(normalized)
def _is_landsar_work_delete_path(self, path: str) -> bool:
work_root = _normalize_path_text(settings.LANDSAR_WORK_ROOT)
normalized = _normalize_path_text(path)
if not work_root or not normalized:
return False
root_full = os.path.abspath(work_root)
full = os.path.abspath(normalized)
for root in roots:
root_text = _normalize_path_text(root)
if not root_text:
continue
root_full = os.path.abspath(root_text)
if full == root_full:
return False
try:
if os.path.commonpath([full, root_full]) == root_full:
return True
except ValueError:
continue
return False
if full == root_full or not _is_within_path(full, root_full):
return False
try:
relative = os.path.relpath(full, root_full)
except ValueError:
return False
parts = [part for part in relative.split(os.sep) if part]
return len(parts) == 1 and parts[0].startswith("run_")
async def _delete_disk_targets(self, preview: Dict[str, Any], options: Dict[str, bool]) -> Dict[str, Any]:
delete_production_dirs = bool(options.get("delete_production_dirs", True))
delete_task_pool_dir = bool(options.get("delete_task_pool_dir", True))
delete_landsar_work_dir = bool(options.get("delete_landsar_work_dir", True))
deleted: List[str] = []
missing: List[str] = []
skipped: List[str] = []
@@ -611,13 +453,13 @@ class OpsMaintenanceService:
for item in preview.get("disk_deletes") or []:
kind = item.get("kind")
path = _normalize_path_text(item.get("path"))
if kind != "landsar_work":
skipped.append(path)
continue
if not item.get("allowed"):
skipped.append(path)
continue
if kind == "task_pool" and not delete_task_pool_dir:
skipped.append(path)
continue
if kind == "production_result" and not delete_production_dirs:
if not delete_landsar_work_dir:
skipped.append(path)
continue
if not _path_exists(path):