Represent partial D-InSAR success separately
This commit is contained in:
@@ -39,6 +39,7 @@ TASK_TYPE_LANDSAR_CLUSTER_PRODUCTION = "LANDSAR_CLUSTER_RUN"
|
|||||||
RUN_STATUS_PENDING = "PENDING"
|
RUN_STATUS_PENDING = "PENDING"
|
||||||
RUN_STATUS_RUNNING = "RUNNING"
|
RUN_STATUS_RUNNING = "RUNNING"
|
||||||
RUN_STATUS_COMPLETED = "COMPLETED"
|
RUN_STATUS_COMPLETED = "COMPLETED"
|
||||||
|
RUN_STATUS_PARTIAL_SUCCESS = "PARTIAL_SUCCESS"
|
||||||
RUN_STATUS_FAILED = "FAILED"
|
RUN_STATUS_FAILED = "FAILED"
|
||||||
RUN_STATUS_CANCELLED = "CANCELLED"
|
RUN_STATUS_CANCELLED = "CANCELLED"
|
||||||
RUN_ITEM_STATUS_PENDING = "PENDING"
|
RUN_ITEM_STATUS_PENDING = "PENDING"
|
||||||
@@ -66,6 +67,7 @@ RUNS_STEP_ID = "execute_items"
|
|||||||
RUNS_STEP_NAME = "Execute ENVI D-InSAR items"
|
RUNS_STEP_NAME = "Execute ENVI D-InSAR items"
|
||||||
TERMINAL_RUN_STATUSES = {
|
TERMINAL_RUN_STATUSES = {
|
||||||
RUN_STATUS_COMPLETED,
|
RUN_STATUS_COMPLETED,
|
||||||
|
RUN_STATUS_PARTIAL_SUCCESS,
|
||||||
RUN_STATUS_FAILED,
|
RUN_STATUS_FAILED,
|
||||||
RUN_STATUS_CANCELLED,
|
RUN_STATUS_CANCELLED,
|
||||||
}
|
}
|
||||||
@@ -549,6 +551,8 @@ def _public_run_status(value: str) -> str:
|
|||||||
normalized = str(value or "").strip().upper()
|
normalized = str(value or "").strip().upper()
|
||||||
if normalized == RUN_STATUS_COMPLETED:
|
if normalized == RUN_STATUS_COMPLETED:
|
||||||
return "success"
|
return "success"
|
||||||
|
if normalized == RUN_STATUS_PARTIAL_SUCCESS:
|
||||||
|
return "partial_success"
|
||||||
if normalized == RUN_STATUS_FAILED:
|
if normalized == RUN_STATUS_FAILED:
|
||||||
return "failed"
|
return "failed"
|
||||||
if normalized == RUN_STATUS_CANCELLED:
|
if normalized == RUN_STATUS_CANCELLED:
|
||||||
@@ -573,12 +577,14 @@ class DinsarProductionService:
|
|||||||
task_status = str(task.status or "").strip().upper()
|
task_status = str(task.status or "").strip().upper()
|
||||||
if run_status in TERMINAL_RUN_STATUSES:
|
if run_status in TERMINAL_RUN_STATUSES:
|
||||||
return False
|
return False
|
||||||
if task_status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if task_status not in {"COMPLETED", "PARTIAL_SUCCESS", "FAILED", "CANCELLED", "CANCELED"}:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if task_status == "COMPLETED":
|
if task_status == "COMPLETED":
|
||||||
next_status = RUN_STATUS_COMPLETED
|
next_status = RUN_STATUS_COMPLETED
|
||||||
elif task_status == "CANCELLED":
|
elif task_status == "PARTIAL_SUCCESS":
|
||||||
|
next_status = RUN_STATUS_PARTIAL_SUCCESS
|
||||||
|
elif task_status in {"CANCELLED", "CANCELED"}:
|
||||||
next_status = RUN_STATUS_CANCELLED
|
next_status = RUN_STATUS_CANCELLED
|
||||||
run.cancel_requested = True
|
run.cancel_requested = True
|
||||||
else:
|
else:
|
||||||
@@ -1132,6 +1138,12 @@ class DinsarProductionService:
|
|||||||
f"LandSAR cluster cancelled. completed={completed} failed={failed} "
|
f"LandSAR cluster cancelled. completed={completed} failed={failed} "
|
||||||
f"cancelled={cancelled} total={total_items}"
|
f"cancelled={cancelled} total={total_items}"
|
||||||
)
|
)
|
||||||
|
elif failed > 0 and (completed > 0 or skipped > 0):
|
||||||
|
final_status = RUN_STATUS_PARTIAL_SUCCESS
|
||||||
|
latest_message = (
|
||||||
|
f"LandSAR cluster partially completed. completed={completed} "
|
||||||
|
f"failed={failed} skipped={skipped} total={total_items}"
|
||||||
|
)
|
||||||
elif failed > 0:
|
elif failed > 0:
|
||||||
final_status = RUN_STATUS_FAILED
|
final_status = RUN_STATUS_FAILED
|
||||||
latest_message = (
|
latest_message = (
|
||||||
|
|||||||
@@ -32,7 +32,14 @@ from .cluster_transport import (
|
|||||||
)
|
)
|
||||||
from .dinsar_compat_service import dinsar_compat_service
|
from .dinsar_compat_service import dinsar_compat_service
|
||||||
from .dinsar_naming import build_run_key
|
from .dinsar_naming import build_run_key
|
||||||
from .dinsar_production_service import dinsar_production_service
|
from .dinsar_production_service import (
|
||||||
|
RUN_STATUS_CANCELLED,
|
||||||
|
RUN_STATUS_COMPLETED,
|
||||||
|
RUN_STATUS_FAILED,
|
||||||
|
RUN_STATUS_PARTIAL_SUCCESS,
|
||||||
|
TERMINAL_RUN_STATUSES,
|
||||||
|
dinsar_production_service,
|
||||||
|
)
|
||||||
from .dinsar_read_service import dinsar_read_service
|
from .dinsar_read_service import dinsar_read_service
|
||||||
from .dinsar_result_layout_service import (
|
from .dinsar_result_layout_service import (
|
||||||
get_run_disp_asset_paths,
|
get_run_disp_asset_paths,
|
||||||
@@ -73,6 +80,16 @@ from ..ai_service import (
|
|||||||
JobHandler = Callable[[SystemJobORM], Awaitable[None]]
|
JobHandler = Callable[[SystemJobORM], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
|
FINAL_TASK_STATUSES = {
|
||||||
|
RUN_STATUS_COMPLETED,
|
||||||
|
RUN_STATUS_PARTIAL_SUCCESS,
|
||||||
|
RUN_STATUS_FAILED,
|
||||||
|
RUN_STATUS_CANCELLED,
|
||||||
|
"CANCELED",
|
||||||
|
"CANCELLED",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
JOB_TYPE_SCAN_DATA = "SCAN_DATA"
|
JOB_TYPE_SCAN_DATA = "SCAN_DATA"
|
||||||
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
|
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
|
||||||
JOB_TYPE_COPY_DATA = "COPY_DATA"
|
JOB_TYPE_COPY_DATA = "COPY_DATA"
|
||||||
@@ -246,6 +263,28 @@ def _classify_dinsar_failure(error_message: Any) -> str:
|
|||||||
return compact if compact != "Unknown error" else "Unclassified D-InSAR failure"
|
return compact if compact != "Unknown error" else "Unclassified D-InSAR failure"
|
||||||
|
|
||||||
|
|
||||||
|
def _dinsar_final_status(completed: Any, failed: Any, skipped: Any = 0) -> str:
|
||||||
|
completed_count = int(completed or 0)
|
||||||
|
failed_count = int(failed or 0)
|
||||||
|
skipped_count = int(skipped or 0)
|
||||||
|
if failed_count > 0 and (completed_count > 0 or skipped_count > 0):
|
||||||
|
return RUN_STATUS_PARTIAL_SUCCESS
|
||||||
|
if failed_count > 0:
|
||||||
|
return RUN_STATUS_FAILED
|
||||||
|
return RUN_STATUS_COMPLETED
|
||||||
|
|
||||||
|
|
||||||
|
def _task_status_for_dinsar_run(final_status: str) -> str:
|
||||||
|
normalized = str(final_status or "").strip().upper()
|
||||||
|
if normalized == RUN_STATUS_COMPLETED:
|
||||||
|
return "COMPLETED"
|
||||||
|
if normalized == RUN_STATUS_PARTIAL_SUCCESS:
|
||||||
|
return "PARTIAL_SUCCESS"
|
||||||
|
if normalized in {RUN_STATUS_CANCELLED, "CANCELLED"}:
|
||||||
|
return "CANCELLED"
|
||||||
|
return "FAILED"
|
||||||
|
|
||||||
|
|
||||||
async def _build_dinsar_failure_summary(db, run) -> Dict[str, Any]:
|
async def _build_dinsar_failure_summary(db, run) -> Dict[str, Any]:
|
||||||
result = await db.execute(
|
result = await db.execute(
|
||||||
select(DinsarProductionRunItemORM)
|
select(DinsarProductionRunItemORM)
|
||||||
@@ -2204,22 +2243,28 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
|||||||
cancelled = await _refresh_cancel_state()
|
cancelled = await _refresh_cancel_state()
|
||||||
await db.refresh(run)
|
await db.refresh(run)
|
||||||
latest_message = ""
|
latest_message = ""
|
||||||
final_status = "COMPLETED"
|
final_status = RUN_STATUS_COMPLETED
|
||||||
if publish_error:
|
if publish_error:
|
||||||
final_status = "FAILED"
|
final_status = RUN_STATUS_FAILED
|
||||||
latest_message = f"Result catalog publish failed: {publish_error}"
|
latest_message = f"Result catalog publish failed: {publish_error}"
|
||||||
elif cancelled:
|
elif cancelled:
|
||||||
final_status = "CANCELLED"
|
final_status = RUN_STATUS_CANCELLED
|
||||||
latest_message = (
|
latest_message = (
|
||||||
f"D-InSAR production cancelled. completed={run.completed_items} "
|
f"D-InSAR production cancelled. completed={run.completed_items} "
|
||||||
f"failed={run.failed_items} total={run.total_items}"
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
)
|
)
|
||||||
elif int(run.failed_items or 0) > 0:
|
elif int(run.failed_items or 0) > 0:
|
||||||
final_status = "FAILED"
|
final_status = _dinsar_final_status(run.completed_items, run.failed_items, run.skipped_items)
|
||||||
latest_message = (
|
if final_status == RUN_STATUS_PARTIAL_SUCCESS:
|
||||||
f"D-InSAR production finished with failures. completed={run.completed_items} "
|
latest_message = (
|
||||||
f"failed={run.failed_items} total={run.total_items}"
|
f"D-InSAR production partially completed. completed={run.completed_items} "
|
||||||
)
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
latest_message = (
|
||||||
|
f"D-InSAR production failed. completed={run.completed_items} "
|
||||||
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
latest_message = (
|
latest_message = (
|
||||||
f"D-InSAR production completed. completed={run.completed_items} "
|
f"D-InSAR production completed. completed={run.completed_items} "
|
||||||
@@ -2259,23 +2304,23 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
|||||||
)
|
)
|
||||||
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
|
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
|
||||||
|
|
||||||
if final_status == "COMPLETED":
|
if final_status in {RUN_STATUS_COMPLETED, RUN_STATUS_PARTIAL_SUCCESS}:
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
status="COMPLETED",
|
status=_task_status_for_dinsar_run(final_status),
|
||||||
progress=100,
|
progress=100,
|
||||||
message=latest_message,
|
message=latest_message,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
task_status = "CANCELLED" if final_status == "CANCELLED" else "FAILED"
|
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
status=task_status,
|
status=_task_status_for_dinsar_run(final_status),
|
||||||
progress=100,
|
progress=100,
|
||||||
message=latest_message,
|
message=latest_message,
|
||||||
)
|
)
|
||||||
raise RuntimeError(latest_message)
|
if final_status == RUN_STATUS_FAILED:
|
||||||
|
raise RuntimeError(latest_message)
|
||||||
|
|
||||||
|
|
||||||
async def _handle_idl_run_import(job: SystemJobORM) -> None:
|
async def _handle_idl_run_import(job: SystemJobORM) -> None:
|
||||||
@@ -2298,7 +2343,7 @@ async def _handle_idl_run_dinsar(job: SystemJobORM) -> None:
|
|||||||
try:
|
try:
|
||||||
async with AsyncSessionLocal() as db:
|
async with AsyncSessionLocal() as db:
|
||||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||||
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if run is not None and str(run.status or "").strip().upper() not in TERMINAL_RUN_STATUSES:
|
||||||
summary_payload = dict(run.summary_json or {})
|
summary_payload = dict(run.summary_json or {})
|
||||||
summary_payload["controller_error"] = str(exc)
|
summary_payload["controller_error"] = str(exc)
|
||||||
await dinsar_production_service.finalize_run(
|
await dinsar_production_service.finalize_run(
|
||||||
@@ -2317,7 +2362,7 @@ async def _handle_idl_run_dinsar(job: SystemJobORM) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
current_task = await task_service.get_task(job.task_id)
|
current_task = await task_service.get_task(job.task_id)
|
||||||
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if current_task and str(current_task.status or "").strip().upper() not in FINAL_TASK_STATUSES:
|
||||||
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
@@ -3191,22 +3236,28 @@ async def _run_wsl_dinsar_production_controller(
|
|||||||
cancelled = await _refresh_cancel_state()
|
cancelled = await _refresh_cancel_state()
|
||||||
await dinsar_production_service.refresh_run_counters(run, db=db)
|
await dinsar_production_service.refresh_run_counters(run, db=db)
|
||||||
if publish_error:
|
if publish_error:
|
||||||
final_status = "FAILED"
|
final_status = RUN_STATUS_FAILED
|
||||||
latest_message = f"Result catalog publish failed: {publish_error}"
|
latest_message = f"Result catalog publish failed: {publish_error}"
|
||||||
elif cancelled:
|
elif cancelled:
|
||||||
final_status = "CANCELLED"
|
final_status = RUN_STATUS_CANCELLED
|
||||||
latest_message = (
|
latest_message = (
|
||||||
f"{engine_title} D-InSAR production cancelled. completed={run.completed_items} "
|
f"{engine_title} D-InSAR production cancelled. completed={run.completed_items} "
|
||||||
f"failed={run.failed_items} total={run.total_items}"
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
)
|
)
|
||||||
elif int(run.failed_items or 0) > 0:
|
elif int(run.failed_items or 0) > 0:
|
||||||
final_status = "FAILED"
|
final_status = _dinsar_final_status(run.completed_items, run.failed_items, run.skipped_items)
|
||||||
latest_message = (
|
if final_status == RUN_STATUS_PARTIAL_SUCCESS:
|
||||||
f"{engine_title} D-InSAR production finished with failures. completed={run.completed_items} "
|
latest_message = (
|
||||||
f"failed={run.failed_items} total={run.total_items}"
|
f"{engine_title} D-InSAR production partially completed. completed={run.completed_items} "
|
||||||
)
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
latest_message = (
|
||||||
|
f"{engine_title} D-InSAR production failed. completed={run.completed_items} "
|
||||||
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
final_status = "COMPLETED"
|
final_status = RUN_STATUS_COMPLETED
|
||||||
latest_message = (
|
latest_message = (
|
||||||
f"{engine_title} D-InSAR production completed. completed={run.completed_items} "
|
f"{engine_title} D-InSAR production completed. completed={run.completed_items} "
|
||||||
f"failed={run.failed_items} total={run.total_items}"
|
f"failed={run.failed_items} total={run.total_items}"
|
||||||
@@ -3246,23 +3297,23 @@ async def _run_wsl_dinsar_production_controller(
|
|||||||
)
|
)
|
||||||
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
|
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
|
||||||
|
|
||||||
if final_status == "COMPLETED":
|
if final_status in {RUN_STATUS_COMPLETED, RUN_STATUS_PARTIAL_SUCCESS}:
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
status="COMPLETED",
|
status=_task_status_for_dinsar_run(final_status),
|
||||||
progress=100,
|
progress=100,
|
||||||
message=latest_message,
|
message=latest_message,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
task_status = "CANCELLED" if final_status == "CANCELLED" else "FAILED"
|
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
status=task_status,
|
status=_task_status_for_dinsar_run(final_status),
|
||||||
progress=100,
|
progress=100,
|
||||||
message=latest_message,
|
message=latest_message,
|
||||||
)
|
)
|
||||||
raise RuntimeError(latest_message)
|
if final_status == RUN_STATUS_FAILED:
|
||||||
|
raise RuntimeError(latest_message)
|
||||||
|
|
||||||
|
|
||||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||||
@@ -3281,7 +3332,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
|||||||
try:
|
try:
|
||||||
async with AsyncSessionLocal() as db:
|
async with AsyncSessionLocal() as db:
|
||||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||||
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if run is not None and str(run.status or "").strip().upper() not in TERMINAL_RUN_STATUSES:
|
||||||
summary_payload = dict(run.summary_json or {})
|
summary_payload = dict(run.summary_json or {})
|
||||||
summary_payload["controller_error"] = str(exc)
|
summary_payload["controller_error"] = str(exc)
|
||||||
await dinsar_production_service.finalize_run(
|
await dinsar_production_service.finalize_run(
|
||||||
@@ -3300,7 +3351,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
current_task = await task_service.get_task(job.task_id)
|
current_task = await task_service.get_task(job.task_id)
|
||||||
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if current_task and str(current_task.status or "").strip().upper() not in FINAL_TASK_STATUSES:
|
||||||
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
@@ -3335,7 +3386,7 @@ async def _handle_pyint_run(job: SystemJobORM) -> None:
|
|||||||
try:
|
try:
|
||||||
async with AsyncSessionLocal() as db:
|
async with AsyncSessionLocal() as db:
|
||||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||||
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if run is not None and str(run.status or "").strip().upper() not in TERMINAL_RUN_STATUSES:
|
||||||
summary_payload = dict(run.summary_json or {})
|
summary_payload = dict(run.summary_json or {})
|
||||||
summary_payload["controller_error"] = str(exc)
|
summary_payload["controller_error"] = str(exc)
|
||||||
await dinsar_production_service.finalize_run(
|
await dinsar_production_service.finalize_run(
|
||||||
@@ -3354,7 +3405,7 @@ async def _handle_pyint_run(job: SystemJobORM) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
current_task = await task_service.get_task(job.task_id)
|
current_task = await task_service.get_task(job.task_id)
|
||||||
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if current_task and str(current_task.status or "").strip().upper() not in FINAL_TASK_STATUSES:
|
||||||
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
@@ -3389,7 +3440,7 @@ async def _handle_landsar_run(job: SystemJobORM) -> None:
|
|||||||
try:
|
try:
|
||||||
async with AsyncSessionLocal() as db:
|
async with AsyncSessionLocal() as db:
|
||||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||||
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if run is not None and str(run.status or "").strip().upper() not in TERMINAL_RUN_STATUSES:
|
||||||
summary_payload = dict(run.summary_json or {})
|
summary_payload = dict(run.summary_json or {})
|
||||||
summary_payload["controller_error"] = str(exc)
|
summary_payload["controller_error"] = str(exc)
|
||||||
await dinsar_production_service.finalize_run(
|
await dinsar_production_service.finalize_run(
|
||||||
@@ -3408,7 +3459,7 @@ async def _handle_landsar_run(job: SystemJobORM) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
current_task = await task_service.get_task(job.task_id)
|
current_task = await task_service.get_task(job.task_id)
|
||||||
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
if current_task and str(current_task.status or "").strip().upper() not in FINAL_TASK_STATUSES:
|
||||||
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
@@ -4002,10 +4053,9 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if final_status:
|
if final_status:
|
||||||
task_status = "COMPLETED" if final_status == "COMPLETED" else ("CANCELLED" if final_status == "CANCELLED" else "FAILED")
|
|
||||||
await task_service.update_task(
|
await task_service.update_task(
|
||||||
job.task_id,
|
job.task_id,
|
||||||
status=task_status,
|
status=_task_status_for_dinsar_run(final_status),
|
||||||
progress=100,
|
progress=100,
|
||||||
message=run.latest_message,
|
message=run.latest_message,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -48,6 +48,13 @@ TASK_QUERY_MAX_OFFSET = read_int_env(
|
|||||||
minimum=0,
|
minimum=0,
|
||||||
maximum=20000000,
|
maximum=20000000,
|
||||||
)
|
)
|
||||||
|
TERMINAL_TASK_STATUSES = {
|
||||||
|
"COMPLETED",
|
||||||
|
"PARTIAL_SUCCESS",
|
||||||
|
"FAILED",
|
||||||
|
"CANCELLED",
|
||||||
|
"CANCELED",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _clamp_pagination(limit: int, offset: int, *, default_limit: int, max_limit: int) -> tuple[int, int]:
|
def _clamp_pagination(limit: int, offset: int, *, default_limit: int, max_limit: int) -> tuple[int, int]:
|
||||||
@@ -270,7 +277,7 @@ class TaskService:
|
|||||||
await self._add_log(task_id, "INFO", message, db=db)
|
await self._add_log(task_id, "INFO", message, db=db)
|
||||||
|
|
||||||
# 如果任务结束,更新结束时间
|
# 如果任务结束,更新结束时间
|
||||||
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
|
if status and str(status).strip().upper() in TERMINAL_TASK_STATUSES:
|
||||||
task.ended_at = datetime.now()
|
task.ended_at = datetime.now()
|
||||||
await self._add_log(task_id, "INFO", f"任务已结束: {status}", db=db)
|
await self._add_log(task_id, "INFO", f"任务已结束: {status}", db=db)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -5857,6 +5857,12 @@ input[type="checkbox"] {
|
|||||||
border-color: rgba(245, 158, 11, 0.25);
|
border-color: rgba(245, 158, 11, 0.25);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dinsar-status-pill.tone-partial {
|
||||||
|
background: rgba(217, 119, 6, 0.13);
|
||||||
|
color: #92400e;
|
||||||
|
border-color: rgba(217, 119, 6, 0.28);
|
||||||
|
}
|
||||||
|
|
||||||
.dinsar-status-pill.tone-error {
|
.dinsar-status-pill.tone-error {
|
||||||
background: rgba(220, 38, 38, 0.12);
|
background: rgba(220, 38, 38, 0.12);
|
||||||
color: #b91c1c;
|
color: #b91c1c;
|
||||||
|
|||||||
@@ -18,7 +18,18 @@ const RUN_HISTORY_PAGE_SIZE = 200;
|
|||||||
const TASK_HISTORY_PAGE_SIZE = 200;
|
const TASK_HISTORY_PAGE_SIZE = 200;
|
||||||
const TASK_LOG_PAGE_SIZE = 1000;
|
const TASK_LOG_PAGE_SIZE = 1000;
|
||||||
const INLINE_TASK_LOG_LIMIT = 200;
|
const INLINE_TASK_LOG_LIMIT = 200;
|
||||||
const TERMINAL_STATUS_VALUES = new Set(['COMPLETED', 'FAILED', 'CANCELLED', 'CANCELED', 'success', 'failed', 'cancelled', 'canceled']);
|
const TERMINAL_STATUS_VALUES = new Set([
|
||||||
|
'COMPLETED',
|
||||||
|
'PARTIAL_SUCCESS',
|
||||||
|
'FAILED',
|
||||||
|
'CANCELLED',
|
||||||
|
'CANCELED',
|
||||||
|
'success',
|
||||||
|
'partial_success',
|
||||||
|
'failed',
|
||||||
|
'cancelled',
|
||||||
|
'canceled',
|
||||||
|
]);
|
||||||
|
|
||||||
const ENGINE_STATUS_COLOR = {
|
const ENGINE_STATUS_COLOR = {
|
||||||
ok: '#22c55e',
|
ok: '#22c55e',
|
||||||
@@ -54,10 +65,12 @@ const STATUS_LABEL = {
|
|||||||
PENDING: '等待中',
|
PENDING: '等待中',
|
||||||
RUNNING: '运行中',
|
RUNNING: '运行中',
|
||||||
COMPLETED: '已完成',
|
COMPLETED: '已完成',
|
||||||
|
PARTIAL_SUCCESS: '部分成功',
|
||||||
FAILED: '失败',
|
FAILED: '失败',
|
||||||
CANCELLED: '已取消',
|
CANCELLED: '已取消',
|
||||||
CANCELED: '已取消',
|
CANCELED: '已取消',
|
||||||
success: '成功',
|
success: '成功',
|
||||||
|
partial_success: '部分成功',
|
||||||
failed: '失败',
|
failed: '失败',
|
||||||
running: '运行中',
|
running: '运行中',
|
||||||
pending: '等待中',
|
pending: '等待中',
|
||||||
@@ -133,6 +146,7 @@ function taskTypeToEngine(taskType) {
|
|||||||
function taskStatusToRunStatus(status) {
|
function taskStatusToRunStatus(status) {
|
||||||
const normalized = String(status || '').toUpperCase();
|
const normalized = String(status || '').toUpperCase();
|
||||||
if (normalized === 'COMPLETED') return 'success';
|
if (normalized === 'COMPLETED') return 'success';
|
||||||
|
if (normalized === 'PARTIAL_SUCCESS') return 'partial_success';
|
||||||
if (normalized === 'FAILED') return 'failed';
|
if (normalized === 'FAILED') return 'failed';
|
||||||
if (normalized === 'CANCELLED' || normalized === 'CANCELED') return 'cancelled';
|
if (normalized === 'CANCELLED' || normalized === 'CANCELED') return 'cancelled';
|
||||||
if (normalized === 'RUNNING') return 'running';
|
if (normalized === 'RUNNING') return 'running';
|
||||||
@@ -309,6 +323,7 @@ function formatRunCounts(run) {
|
|||||||
function statusToneClass(status) {
|
function statusToneClass(status) {
|
||||||
const normalized = String(status || '').toLowerCase();
|
const normalized = String(status || '').toLowerCase();
|
||||||
if (normalized === 'success' || normalized === 'completed') return 'tone-ready';
|
if (normalized === 'success' || normalized === 'completed') return 'tone-ready';
|
||||||
|
if (normalized === 'partial_success' || normalized === 'partial' || normalized === 'completed_with_errors') return 'tone-partial';
|
||||||
if (normalized === 'failed') return 'tone-error';
|
if (normalized === 'failed') return 'tone-error';
|
||||||
if (normalized === 'running') return 'tone-info';
|
if (normalized === 'running') return 'tone-info';
|
||||||
if (normalized === 'cancelled' || normalized === 'canceled') return 'tone-neutral';
|
if (normalized === 'cancelled' || normalized === 'canceled') return 'tone-neutral';
|
||||||
@@ -828,7 +843,10 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
|||||||
? (latestRunWithTask.mode === 'cluster' ? 'LANDSAR_CLUSTER_RUN' : 'LANDSAR_RUN')
|
? (latestRunWithTask.mode === 'cluster' ? 'LANDSAR_CLUSTER_RUN' : 'LANDSAR_RUN')
|
||||||
: 'IDL_RUN_DINSAR',
|
: 'IDL_RUN_DINSAR',
|
||||||
status: latestRunWithTask.raw_status || latestRunWithTask.status,
|
status: latestRunWithTask.raw_status || latestRunWithTask.status,
|
||||||
progress: latestRunWithTask.raw_status === 'COMPLETED' || latestRunWithTask.status === 'success' ? 100 : null,
|
progress: ['COMPLETED', 'PARTIAL_SUCCESS'].includes(String(latestRunWithTask.raw_status || '').toUpperCase())
|
||||||
|
|| ['success', 'partial_success'].includes(String(latestRunWithTask.status || '').toLowerCase())
|
||||||
|
? 100
|
||||||
|
: null,
|
||||||
message: latestRunWithTask.message || '最近一次任务',
|
message: latestRunWithTask.message || '最近一次任务',
|
||||||
}
|
}
|
||||||
: null
|
: null
|
||||||
@@ -836,7 +854,12 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
|||||||
const logTaskId = monitoredTask?.task_id || '';
|
const logTaskId = monitoredTask?.task_id || '';
|
||||||
const showingRecentTask = !taskMonitor.isBusy && !!monitoredTask;
|
const showingRecentTask = !taskMonitor.isBusy && !!monitoredTask;
|
||||||
const activeRunCount = runs.filter(run => ['running', 'pending'].includes(String(run.status || '').toLowerCase())).length;
|
const activeRunCount = runs.filter(run => ['running', 'pending'].includes(String(run.status || '').toLowerCase())).length;
|
||||||
const failedRunCount = runs.filter(run => ['failed', 'FAILED'].includes(String(run.status || ''))).length;
|
const issueRunCount = runs.filter(run => {
|
||||||
|
const status = String(run.status || '').toLowerCase();
|
||||||
|
const rawStatus = String(run.raw_status || '').toUpperCase();
|
||||||
|
return ['failed', 'partial_success'].includes(status)
|
||||||
|
|| ['FAILED', 'PARTIAL_SUCCESS'].includes(rawStatus);
|
||||||
|
}).length;
|
||||||
const productionReady = !!currentEngineObj?.available && !!rootDir.trim() && !pyintPreviewBlocksSubmit && !readOnly;
|
const productionReady = !!currentEngineObj?.available && !!rootDir.trim() && !pyintPreviewBlocksSubmit && !readOnly;
|
||||||
|
|
||||||
const loadEngines = useCallback(async () => {
|
const loadEngines = useCallback(async () => {
|
||||||
@@ -1471,9 +1494,9 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
|||||||
<span>运行中</span>
|
<span>运行中</span>
|
||||||
<strong>{activeRunCount}</strong>
|
<strong>{activeRunCount}</strong>
|
||||||
</div>
|
</div>
|
||||||
<div className={`dinsar-production-signal ${failedRunCount > 0 ? 'warn' : ''}`}>
|
<div className={`dinsar-production-signal ${issueRunCount > 0 ? 'warn' : ''}`}>
|
||||||
<span>失败记录</span>
|
<span>异常记录</span>
|
||||||
<strong>{failedRunCount}</strong>
|
<strong>{issueRunCount}</strong>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -25,6 +25,22 @@ const toneColor = {
|
|||||||
text: '#b91c1c',
|
text: '#b91c1c',
|
||||||
fill: '#dc2626',
|
fill: '#dc2626',
|
||||||
},
|
},
|
||||||
|
partial: {
|
||||||
|
border: '#fcd34d',
|
||||||
|
bg: '#fffbeb',
|
||||||
|
text: '#92400e',
|
||||||
|
fill: '#d97706',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const TASK_STATUS_LABEL = {
|
||||||
|
PENDING: '等待中',
|
||||||
|
RUNNING: '运行中',
|
||||||
|
COMPLETED: '已完成',
|
||||||
|
PARTIAL_SUCCESS: '部分成功',
|
||||||
|
FAILED: '失败',
|
||||||
|
CANCELLED: '已取消',
|
||||||
|
CANCELED: '已取消',
|
||||||
};
|
};
|
||||||
|
|
||||||
const normalizeProgress = (value) => {
|
const normalizeProgress = (value) => {
|
||||||
@@ -34,6 +50,11 @@ const normalizeProgress = (value) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const isFailed = (task) => String(task?.status || '').toUpperCase() === 'FAILED';
|
const isFailed = (task) => String(task?.status || '').toUpperCase() === 'FAILED';
|
||||||
|
const isPartialSuccess = (task) => String(task?.status || '').toUpperCase() === 'PARTIAL_SUCCESS';
|
||||||
|
const formatTaskStatus = (status) => {
|
||||||
|
const normalized = String(status || '-').toUpperCase();
|
||||||
|
return TASK_STATUS_LABEL[normalized] || normalized;
|
||||||
|
};
|
||||||
|
|
||||||
export default function TaskStatusPanel({
|
export default function TaskStatusPanel({
|
||||||
title = '任务状态',
|
title = '任务状态',
|
||||||
@@ -48,7 +69,9 @@ export default function TaskStatusPanel({
|
|||||||
}) {
|
}) {
|
||||||
const task = latestTask || activeTasks[0] || recentTasks[0] || null;
|
const task = latestTask || activeTasks[0] || recentTasks[0] || null;
|
||||||
const showingRecent = !isBusy && !!task;
|
const showingRecent = !isBusy && !!task;
|
||||||
const tone = task ? (isFailed(task) ? 'error' : (showingRecent ? 'recent' : 'active')) : 'idle';
|
const tone = task
|
||||||
|
? (isFailed(task) ? 'error' : (isPartialSuccess(task) ? 'partial' : (showingRecent ? 'recent' : 'active')))
|
||||||
|
: 'idle';
|
||||||
const colors = toneColor[tone];
|
const colors = toneColor[tone];
|
||||||
const progress = normalizeProgress(task?.progress);
|
const progress = normalizeProgress(task?.progress);
|
||||||
|
|
||||||
@@ -82,7 +105,7 @@ export default function TaskStatusPanel({
|
|||||||
<div style={{ marginTop: 8, color: colors.text, fontSize: 12, wordBreak: 'break-all' }}>
|
<div style={{ marginTop: 8, color: colors.text, fontSize: 12, wordBreak: 'break-all' }}>
|
||||||
<span style={{ fontWeight: 700 }}>{getTaskTypeLabel(task.task_type)}</span>
|
<span style={{ fontWeight: 700 }}>{getTaskTypeLabel(task.task_type)}</span>
|
||||||
<span style={{ margin: '0 6px' }}>·</span>
|
<span style={{ margin: '0 6px' }}>·</span>
|
||||||
<span>{String(task.status || '-').toUpperCase()}</span>
|
<span>{formatTaskStatus(task.status)}</span>
|
||||||
{task.task_id && (
|
{task.task_id && (
|
||||||
<>
|
<>
|
||||||
<span style={{ margin: '0 6px' }}>·</span>
|
<span style={{ margin: '0 6px' }}>·</span>
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import apiClient from '../api/client';
|
|||||||
import { normalizeTaskStatus } from '../utils/appUiHelpers';
|
import { normalizeTaskStatus } from '../utils/appUiHelpers';
|
||||||
|
|
||||||
const ACTIVE_TASK_FALLBACK_POLL_MS = 10000;
|
const ACTIVE_TASK_FALLBACK_POLL_MS = 10000;
|
||||||
|
const TERMINAL_TASK_STATUSES = new Set(['COMPLETED', 'PARTIAL_SUCCESS', 'FAILED', 'CANCELLED', 'CANCELED']);
|
||||||
|
|
||||||
export default function useGlobalTaskControl({
|
export default function useGlobalTaskControl({
|
||||||
currentUser,
|
currentUser,
|
||||||
@@ -56,7 +57,7 @@ export default function useGlobalTaskControl({
|
|||||||
const taskStatus = normalizeTaskStatus(taskInfo?.status);
|
const taskStatus = normalizeTaskStatus(taskInfo?.status);
|
||||||
|
|
||||||
// 检查任务是否真正完成:解析 message 中的进度信息
|
// 检查任务是否真正完成:解析 message 中的进度信息
|
||||||
let isReallyFinished = taskStatus === 'COMPLETED' || taskStatus === 'FAILED';
|
let isReallyFinished = TERMINAL_TASK_STATUSES.has(taskStatus);
|
||||||
|
|
||||||
// 如果任务状态是 PENDING,检查进度信息
|
// 如果任务状态是 PENDING,检查进度信息
|
||||||
if (taskStatus === 'PENDING' && taskInfo?.message) {
|
if (taskStatus === 'PENDING' && taskInfo?.message) {
|
||||||
@@ -77,7 +78,7 @@ export default function useGlobalTaskControl({
|
|||||||
|
|
||||||
if (isReallyFinished) {
|
if (isReallyFinished) {
|
||||||
reallyFinishedIds.push(taskId);
|
reallyFinishedIds.push(taskId);
|
||||||
if (taskStatus === 'COMPLETED' || taskStatus === 'FAILED') {
|
if (TERMINAL_TASK_STATUSES.has(taskStatus)) {
|
||||||
handleTaskCompletionRef.current?.(taskInfo);
|
handleTaskCompletionRef.current?.(taskInfo);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user