Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Distinguish failed D-InSAR engine attempts

Browse Source
This commit is contained in:
Harmon
2026-07-01 16:46:25 +08:00
parent ce052b909e
commit 23958dbafb
3 changed files with 167 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/dinsar_engine_matrix.py
+89 -3
View File
@@ -5,8 +5,9 @@ from typing import Any, Dict, Iterable, List, Optional
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..models import DinsarTaskItemORM, ResultProductORM
from ..models import DinsarProductionRunItemORM, DinsarTaskItemORM, ResultProductORM
from ..utils import normalize_satellite_family
@@ -102,11 +103,32 @@ def _product_status(product: ResultProductORM) -> str:
return "ready"
def _status_timestamp(value: Optional[datetime]) -> float:
return _timestamp(value)
def _latest_failed_attempt(attempts: Iterable[DinsarProductionRunItemORM]) -> Optional[DinsarProductionRunItemORM]:
latest: Optional[DinsarProductionRunItemORM] = None
for attempt in attempts:
if str(attempt.status or "").strip().upper() != "FAILED":
continue
if latest is None or (
_status_timestamp(attempt.ended_at or attempt.updated_at or attempt.created_at),
attempt.id or 0,
) > (
_status_timestamp(latest.ended_at or latest.updated_at or latest.created_at),
latest.id or 0,
):
latest = attempt
return latest
def serialize_engine_result(
*,
engine_code: str,
data_family: str,
product: Optional[ResultProductORM],
failed_attempt: Optional[DinsarProductionRunItemORM] = None,
allowed: bool,
legacy: bool = False,
) -> Dict[str, Any]:
@@ -119,6 +141,10 @@ def serialize_engine_result(
status = "blocked"
can_dispatch = False
skip_reason = "unsupported_data_family"
elif product is None and failed_attempt is not None:
status = "failed"
can_dispatch = True
skip_reason = "production_failed"
elif product is None:
status = "missing"
can_dispatch = True
@@ -140,11 +166,18 @@ def serialize_engine_result(
),
"latest_product_id": product.id if product is not None else None,
"product_id": product.product_id if product is not None else None,
"run_key": product.run_key if product is not None else None,
"run_key": product.run_key if product is not None else failed_attempt.latest_run_key if failed_attempt is not None else None,
"published_at": product.published_at if product is not None else None,
"health_status": product.health_status if product is not None else None,
"primary_asset_path": product.primary_asset_path if product is not None else None,
"preview_path": product.preview_path if product is not None else None,
"production_run_id": failed_attempt.run_id if failed_attempt is not None else None,
"production_item_id": failed_attempt.id if failed_attempt is not None else None,
"production_status": failed_attempt.status if failed_attempt is not None else None,
"production_error": failed_attempt.last_error if failed_attempt is not None else None,
"latest_log_path": failed_attempt.latest_log_path if failed_attempt is not None else None,
"latest_output_dir": failed_attempt.latest_output_dir if failed_attempt is not None else None,
"attempt_ended_at": failed_attempt.ended_at if failed_attempt is not None else None,
"can_dispatch": can_dispatch,
"skip_reason": skip_reason,
}
@@ -153,6 +186,7 @@ def serialize_engine_result(
def build_engine_results(
*,
products: Iterable[ResultProductORM],
failed_attempts: Iterable[DinsarProductionRunItemORM] = (),
data_family: str = "unknown",
include_legacy: bool = False,
) -> Dict[str, Dict[str, Any]]:
@@ -168,6 +202,17 @@ def build_engine_results(
):
target[engine] = product
failed_by_engine: Dict[str, DinsarProductionRunItemORM] = {}
attempts_by_engine: Dict[str, List[DinsarProductionRunItemORM]] = {}
for attempt in failed_attempts:
engine = normalize_dinsar_engine_code(getattr(getattr(attempt, "run", None), "engine_code", ""))
if engine:
attempts_by_engine.setdefault(engine, []).append(attempt)
for engine, attempts in attempts_by_engine.items():
latest = _latest_failed_attempt(attempts)
if latest is not None:
failed_by_engine[engine] = latest
allowed = allowed_engines_for_data_family(data_family)
matrix: Dict[str, Dict[str, Any]] = {}
for engine in CURRENT_DINSAR_ENGINE_ORDER:
@@ -175,6 +220,7 @@ def build_engine_results(
engine_code=engine,
data_family=data_family,
product=latest_by_engine.get(engine),
failed_attempt=failed_by_engine.get(engine) if latest_by_engine.get(engine) is None else None,
allowed=engine in allowed,
)
@@ -184,6 +230,7 @@ def build_engine_results(
engine_code=engine,
data_family=data_family,
product=product,
failed_attempt=None,
allowed=False,
legacy=True,
)
@@ -229,6 +276,27 @@ async def build_engine_results_for_task_items(
)
products = result.scalars().all()
attempt_conditions = []
if pair_keys:
attempt_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys))
if aliases:
attempt_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases))
attempt_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases))
failed_attempts: List[DinsarProductionRunItemORM] = []
if attempt_conditions:
attempt_result = await db.execute(
select(DinsarProductionRunItemORM)
.options(selectinload(DinsarProductionRunItemORM.run))
.where(DinsarProductionRunItemORM.status == "FAILED")
.where(or_(*attempt_conditions))
.order_by(
DinsarProductionRunItemORM.ended_at.desc().nullslast(),
DinsarProductionRunItemORM.updated_at.desc().nullslast(),
DinsarProductionRunItemORM.id.desc(),
)
)
failed_attempts = attempt_result.scalars().all()
by_pair_key: Dict[str, List[ResultProductORM]] = {}
by_alias: Dict[str, List[ResultProductORM]] = {}
for product in products:
@@ -239,15 +307,33 @@ async def build_engine_results_for_task_items(
if alias:
by_alias.setdefault(alias, []).append(product)
failed_by_pair_key: Dict[str, List[DinsarProductionRunItemORM]] = {}
failed_by_alias: Dict[str, List[DinsarProductionRunItemORM]] = {}
for attempt in failed_attempts:
pair_key = str(attempt.pair_key or "").strip()
if pair_key:
failed_by_pair_key.setdefault(pair_key, []).append(attempt)
for alias in {str(attempt.task_alias or "").strip(), str(attempt.task_name or "").strip()}:
if alias:
failed_by_alias.setdefault(alias, []).append(attempt)
output: Dict[int, Dict[str, Dict[str, Any]]] = {}
for item in items:
item_products: List[ResultProductORM] = []
item_failed_attempts: List[DinsarProductionRunItemORM] = []
pair_key = str(item.pair_key or "").strip()
alias = str(item.task_alias or item.task_name or "").strip()
if pair_key:
item_products.extend(by_pair_key.get(pair_key, []))
item_failed_attempts.extend(failed_by_pair_key.get(pair_key, []))
if not item_products and alias:
item_products.extend(by_alias.get(alias, []))
if not item_failed_attempts and alias:
item_failed_attempts.extend(failed_by_alias.get(alias, []))
data_family = infer_dinsar_data_family(item.master_satellite, item.slave_satellite)
output[int(item.id)] = build_engine_results(products=item_products, data_family=data_family)
output[int(item.id)] = build_engine_results(
products=item_products,
failed_attempts=item_failed_attempts,
data_family=data_family,
)
return output