Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden LandSAR cluster handoff

Browse Source
This commit is contained in:
Harmon
2026-06-26 12:28:47 +08:00
parent fced6f4a7f
commit 40178ad1a4
10 changed files with 418 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/cluster_transport.py
+66 -24
View File
@@ -67,6 +67,16 @@ def safe_extract_zip(zf: zipfile.ZipFile, target_dir: str) -> None:
zf.extractall(target_root)
def _zip_directory_contents(source_dir: str, zip_path: str) -> None:
source_root = os.path.abspath(source_dir)
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
for current, _, files in os.walk(source_root):
for name in files:
path = os.path.join(current, name)
arcname = os.path.relpath(path, source_root)
zf.write(path, arcname)
def _resolve_cluster_server_url() -> str:
"""Return the main-server HTTP base URL for cluster data transport.
@@ -99,23 +109,52 @@ def _is_remote_worker() -> bool:
return host not in {"", "127.0.0.1", "localhost"}
def resolve_cluster_local_task_dir(item: DinsarProductionRunItemORM) -> str:
"""Return the worker-local Task_* directory for a cluster item."""
source_task_dir = os.path.normpath(str(item.source_task_dir or ""))
worker_root = _read_cluster_env("CLUSTER_WORKER_TASK_ROOT")
if not worker_root:
return source_task_dir
task_name = os.path.basename(source_task_dir) or f"Task_item_{item.id}"
return os.path.normpath(
os.path.join(worker_root, f"item_{item.id}", task_name)
)
def resolve_cluster_local_run_dir(
item: DinsarProductionRunItemORM,
run_key: str,
) -> str:
"""Return the worker-local managed result run directory."""
worker_root = _read_cluster_env("CLUSTER_WORKER_RESULT_ROOT")
if not worker_root:
return os.path.join(str(item.results_root_dir or ""), "runs", run_key)
pair_fragment = str(item.pair_key or f"item_{item.id}").strip() or f"item_{item.id}"
safe_pair = "".join(
ch if ch.isalnum() or ch in "._-" else "_"
for ch in pair_fragment
).strip("._") or f"item_{item.id}"
return os.path.normpath(os.path.join(worker_root, safe_pair, "runs", run_key))
async def materialize_cluster_input(
item: DinsarProductionRunItemORM,
source_task_dir: str,
local_task_dir: str,
task_id: str,
) -> None:
"""Download and extract the input data for a cluster item.
Calls ``GET /api/cluster/input-package/{item_id}`` on the main
server, retrieves a zip containing the Task_Pool directory tree,
and extracts it so that *source_task_dir* exists locally.
and extracts it so that *local_task_dir* exists locally.
"""
from .task_service import task_service
server_url = _resolve_cluster_server_url()
download_url = f"{server_url}/api/cluster/input-package/{item.id}"
parent_dir = os.path.dirname(source_task_dir)
task_name = os.path.basename(source_task_dir)
parent_dir = os.path.dirname(local_task_dir)
task_name = os.path.basename(local_task_dir)
package_task_name = os.path.basename(str(item.source_task_dir or "")) or task_name
await task_service.add_log(
task_id,
@@ -124,8 +163,8 @@ async def materialize_cluster_input(
)
tmp_zip = os.path.join(
tempfile.gettempdir(),
f"cluster_input_{item.id}_{task_name}.zip",
tempfile.mkdtemp(prefix=f"cluster_input_{item.id}_"),
f"{package_task_name}.zip",
)
try:
req = urllib.request.Request(
@@ -138,18 +177,28 @@ async def materialize_cluster_input(
shutil.copyfileobj(resp, fh, 8 * 1024 * 1024)
os.makedirs(parent_dir, exist_ok=True)
if os.path.isdir(local_task_dir):
shutil.rmtree(local_task_dir)
with zipfile.ZipFile(tmp_zip, "r") as zf:
safe_extract_zip(zf, parent_dir)
if not os.path.isdir(source_task_dir):
extracted_task_dir = os.path.join(parent_dir, package_task_name)
if (
os.path.isdir(extracted_task_dir)
and os.path.normcase(os.path.abspath(extracted_task_dir))
!= os.path.normcase(os.path.abspath(local_task_dir))
):
os.replace(extracted_task_dir, local_task_dir)
if not os.path.isdir(local_task_dir):
raise RuntimeError(
f"Extraction did not create expected directory: {source_task_dir}"
f"Extraction did not create expected directory: {local_task_dir}"
)
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Input data ready: {source_task_dir}",
f"[cluster] Input data ready: {local_task_dir}",
)
except urllib.error.HTTPError as exc:
body_text = ""
@@ -167,7 +216,9 @@ async def materialize_cluster_input(
) from exc
finally:
try:
os.unlink(tmp_zip)
tmp_root = os.path.dirname(tmp_zip)
if os.path.isdir(tmp_root):
shutil.rmtree(tmp_root)
except Exception:
pass
@@ -194,19 +245,10 @@ async def upload_cluster_result(
"[cluster] Packaging results for upload ...",
)
run_dir_name = os.path.basename(os.path.normpath(managed_run_dir))
tmp_zip = os.path.join(
tempfile.gettempdir(),
f"cluster_result_{item.id}.zip",
)
tmp_root = tempfile.mkdtemp(prefix=f"cluster_result_{item.id}_")
tmp_zip = os.path.join(tmp_root, "result.zip")
try:
parent = os.path.dirname(managed_run_dir)
shutil.make_archive(
tmp_zip.replace(".zip", ""),
"zip",
root_dir=parent,
base_dir=run_dir_name,
)
_zip_directory_contents(managed_run_dir, tmp_zip)
await task_service.add_log(
task_id,
@@ -284,7 +326,7 @@ async def upload_cluster_result(
) from exc
finally:
try:
if os.path.isfile(tmp_zip):
os.unlink(tmp_zip)
if os.path.isdir(tmp_root):
shutil.rmtree(tmp_root)
except Exception:
pass
backend/app/services/dinsar_completion_files.py
+12 -2
View File
@@ -71,9 +71,11 @@ def _current_pointer_path(results_root_dir: str, *, engine_code: str, profile_co
def _runtime_id_for_engine(engine_code: str) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized == "isce2":
return settings.ISCE2_RUNTIME_ID or None
return getattr(settings, "ISCE2_RUNTIME_ID", "") or None
if normalized == "landsar":
return getattr(settings, "LANDSAR_RUNTIME_ID", "") or None
if normalized in {"pyint", "gamma"}:
return settings.PYINT_RUNTIME_ID or None
return getattr(settings, "PYINT_RUNTIME_ID", "") or None
return None
@@ -137,9 +139,17 @@ def repair_managed_completion_files(
task_name = _first_text(payload.get("task_name"), payload.get("task_alias"), run_key)
task_alias = _first_text(payload.get("task_alias"), payload.get("task_name"), task_name)
output_dir = _normalize_path(payload.get("output_dir") or normalized_run_dir)
if normalized_primary.startswith(normalized_run_dir + os.sep):
output_dir = normalized_run_dir
native_output_dir = _normalize_path(
payload.get("native_output_dir") or os.path.join(normalized_run_dir, "native")
)
if not native_output_dir.startswith(normalized_run_dir + os.sep):
local_native_dir = os.path.join(normalized_run_dir, "native")
if os.path.isdir(local_native_dir):
native_output_dir = _normalize_path(local_native_dir)
else:
native_output_dir = output_dir
results_root_dir = _infer_results_root_dir(normalized_run_dir, pair_key)
publish_root_dir = _normalize_path(os.path.dirname(results_root_dir))
backend/app/services/dinsar_production_service.py
+4 -1
View File
@@ -1184,9 +1184,12 @@ class DinsarProductionService:
run: DinsarProductionRunORM,
item: DinsarProductionRunItemORM,
run_key: str,
output_dir_override: Optional[str] = None,
db: AsyncSession,
) -> DinsarProductionExecutionORM:
output_dir = _execution_dir(item, run_key)
output_dir = os.path.normpath(
os.path.abspath(output_dir_override)
) if output_dir_override else _execution_dir(item, run_key)
_ensure_dir(output_dir)
execution = DinsarProductionExecutionORM(
execution_id=run_key,
backend/app/services/job_handlers.py
+91 -52
View File
@@ -26,6 +26,8 @@ from .asset_inventory_service import asset_inventory_service
from .cluster_transport import (
_is_remote_worker,
materialize_cluster_input,
resolve_cluster_local_run_dir,
resolve_cluster_local_task_dir,
upload_cluster_result,
)
from .dinsar_compat_service import dinsar_compat_service
@@ -164,15 +166,30 @@ def _normalize_positive_int(value: Any) -> Optional[int]:
def _cluster_source_task_dir_ready(path: str) -> bool:
if not path or not os.path.isdir(path):
return False
def _has_landsar_source_file(directory: str) -> bool:
try:
with os.scandir(directory) as entries:
for entry in entries:
if not entry.is_file():
continue
name = entry.name.lower()
if name.startswith("lt1") and name.endswith((".xml", ".tif", ".tiff")):
return True
except OSError:
return False
return False
input_data_dir = os.path.join(path, "Input_Data")
if os.path.isdir(input_data_dir) and _has_landsar_source_file(input_data_dir):
return True
if not os.path.isfile(os.path.join(path, ".dinsar_pair.json")):
return False
for child_name in ("master", "slave"):
child_dir = os.path.join(path, child_name)
if not os.path.isdir(child_dir):
return False
try:
with os.scandir(child_dir) as entries:
if not any(entries):
return False
except OSError:
if not os.path.isdir(child_dir) or not _has_landsar_source_file(child_dir):
return False
return True
@@ -3275,14 +3292,14 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
raise ValueError(f"LandSAR cluster item not found: {item_id}")
await db.refresh(item)
source_task_dir = os.path.normpath(str(item.source_task_dir or ""))
if source_task_dir and not _cluster_source_task_dir_ready(source_task_dir):
await materialize_cluster_input(item, source_task_dir, job.task_id)
item_status = str(item.status or "").strip().upper()
if item_status in {"COMPLETED", "FAILED", "SKIPPED", "CANCELLED"}:
await dinsar_production_service.finalize_cluster_run_if_complete(run, db=db)
return
source_task_dir = os.path.normpath(str(item.source_task_dir or ""))
local_task_dir = resolve_cluster_local_task_dir(item)
if local_task_dir and not _cluster_source_task_dir_ready(local_task_dir):
await materialize_cluster_input(item, local_task_dir, job.task_id)
current_task = await task_service.get_task(job.task_id)
task_cancelled = bool(current_task and current_task.status == "CANCELLED")
@@ -3323,10 +3340,12 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
item_index = max(1, int(item.order_index or 1))
item_label = item.task_alias or item.task_name
run_key = f"{build_run_key('landsar', run.profile_code, started_at=datetime.utcnow())}_{item.id}_{uuid.uuid4().hex[:6]}"
local_run_dir = os.path.normpath(resolve_cluster_local_run_dir(item, run_key))
execution = await dinsar_production_service.begin_item_execution(
run=run,
item=item,
run_key=run_key,
output_dir_override=local_run_dir,
db=db,
)
@@ -3425,7 +3444,7 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
request = RunRequest(
engine_code="landsar",
profile=run.profile_code,
root_dir=str(item.source_task_dir),
root_dir=local_task_dir,
job_id=job.job_id,
num_to_process=1,
timeout_seconds=per_task_timeout or None,
@@ -3438,6 +3457,7 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
"__managed_orbit_output_dir": managed_orbit_output_dir,
"__managed_run_key": run_key,
"__source_root_override": run.source_root,
"__source_task_dir_override": source_task_dir,
"__rerun_mode": "rerun_all",
"__cluster_item": True,
},
@@ -3497,24 +3517,17 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
native_output_dir=native_output_dir,
metrics=metrics,
)
await asyncio.to_thread(
dinsar_production_service.write_current_pointer,
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
primary_file=primary_file,
source_files=source_files,
native_output_dir=native_output_dir,
)
await dinsar_production_service.mark_item_completed(
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
metrics=metrics,
db=db,
)
if not _is_remote_worker():
await asyncio.to_thread(
dinsar_production_service.write_current_pointer,
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
primary_file=primary_file,
source_files=source_files,
native_output_dir=native_output_dir,
)
# ---- Post-flight: upload or local publish ----
if _is_remote_worker():
@@ -3544,6 +3557,14 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
"WARNING",
f"[cluster {item_index}/{total_items}] Result catalog publish failed for {item_label}: {publish_error}",
)
await dinsar_production_service.mark_item_completed(
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
metrics=metrics,
db=db,
)
await task_service.add_log(
job.task_id,
@@ -3554,28 +3575,46 @@ async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
except Exception as exc:
run_exception_text = str(exc)
item_error = run_exception_text
await dinsar_production_service.mark_item_failed(
run=run,
item=item,
execution=execution,
error_message=item_error,
db=db,
)
await task_service.add_log(
job.task_id,
"WARNING",
f"[cluster {item_index}/{total_items}] Failed {item_label}: {item_error}",
)
dinsar_production_service.append_run_log(
run.run_id,
f"[cluster-item-failed] {item_index}/{total_items} {item_label}: {item_error}",
)
if task_result.get("command"):
await task_service.add_log(job.task_id, "INFO", f"LandSAR command [{item_label}]: {task_result.get('command')}")
if task_result.get("stdout_tail"):
await task_service.add_log(job.task_id, "INFO", f"LandSAR stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}")
if task_result.get("stderr_tail"):
await task_service.add_log(job.task_id, "WARNING", f"LandSAR stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}")
remote_completed = False
if _is_remote_worker():
try:
await db.refresh(item)
remote_completed = str(item.status or "").strip().upper() == "COMPLETED"
except Exception:
remote_completed = False
if remote_completed:
await task_service.add_log(
job.task_id,
"INFO",
f"[cluster {item_index}/{total_items}] Main server already completed {item_label}; keeping completed state after local error: {item_error}",
)
dinsar_production_service.append_run_log(
run.run_id,
f"[cluster-item-ok-after-upload] {item_index}/{total_items} {item_label}: {item_error}",
)
else:
await dinsar_production_service.mark_item_failed(
run=run,
item=item,
execution=execution,
error_message=item_error,
db=db,
)
await task_service.add_log(
job.task_id,
"WARNING",
f"[cluster {item_index}/{total_items}] Failed {item_label}: {item_error}",
)
dinsar_production_service.append_run_log(
run.run_id,
f"[cluster-item-failed] {item_index}/{total_items} {item_label}: {item_error}",
)
if task_result.get("command"):
await task_service.add_log(job.task_id, "INFO", f"LandSAR command [{item_label}]: {task_result.get('command')}")
if task_result.get("stdout_tail"):
await task_service.add_log(job.task_id, "INFO", f"LandSAR stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}")
if task_result.get("stderr_tail"):
await task_service.add_log(job.task_id, "WARNING", f"LandSAR stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}")
finally:
keepalive_task.cancel()
try:
backend/app/services/result_catalog_service.py
+12 -3
View File
@@ -142,9 +142,11 @@ def _coerce_optional_int(value: Any) -> Optional[int]:
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized == "isce2":
return settings.ISCE2_RUNTIME_ID or None
return getattr(settings, "ISCE2_RUNTIME_ID", "") or None
if normalized == "landsar":
return getattr(settings, "LANDSAR_RUNTIME_ID", "") or None
if normalized in {"pyint", "gamma"}:
return settings.PYINT_RUNTIME_ID or None
return getattr(settings, "PYINT_RUNTIME_ID", "") or None
return None
@@ -822,6 +824,13 @@ class ResultCatalogService:
package_dir = _ensure_directory(os.path.join(target_root, pair_key, "runs", run_key))
source_dir = _normalize_path(candidate["source_dir"])
in_place_source = _is_path_within(package_dir, source_dir)
if candidate_meta["engine_code"] in {"isce2", "landsar"} and in_place_source:
candidate_meta = dict(candidate_meta)
candidate_meta["output_dir"] = package_dir
native_dir = os.path.join(package_dir, RUN_NATIVE_DIRNAME)
candidate_meta["native_output_dir"] = (
native_dir if os.path.isdir(native_dir) else package_dir
)
task_item = await self._lookup_task_item(
db,
pair_key=pair_key,
@@ -994,7 +1003,7 @@ class ResultCatalogService:
manifest_path = os.path.join(package_dir, "manifest.json")
with open(manifest_path, "w", encoding="utf-8") as fp:
json.dump(manifest, fp, ensure_ascii=False, indent=2)
if candidate["engine_code"] == "isce2" and in_place_source:
if candidate["engine_code"] in {"isce2", "landsar"} and in_place_source:
try:
completion_files_result = repair_managed_completion_files(
package_dir,