Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync production runtime and docs

Browse Source
This commit is contained in:
Harmon
2026-04-25 15:02:40 +08:00
parent cb518b431f
commit 1e44101eb2
50 changed files with 8575 additions and 2177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/config.py
+140 -7
View File
@@ -30,6 +30,17 @@ def _clean_path_text(value: str | None) -> str:
return str(value or "").strip().strip('"').strip("'")
def _infer_conda_env_name_from_python(path: str | None) -> str:
text = _clean_path_text(path)
marker = "/envs/"
if not text or marker not in text:
return ""
tail = text.split(marker, 1)[1].strip("/")
if not tail:
return ""
return tail.split("/", 1)[0].strip()
def _default_idl_runtime_dir() -> str:
return os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
@@ -58,6 +69,14 @@ def _resolve_idl_runtime_dir(value: str | None) -> str:
return os.path.normpath(os.path.abspath(normalized))
def _default_result_publish_root(project_root: str) -> str:
normalized_root = os.path.normpath(project_root)
drive, _tail = os.path.splitdrive(normalized_root)
if drive:
return os.path.join(drive + os.sep, "production_results")
return os.path.join(normalized_root, "production_results")
def _read_env_pairs(env_path: str) -> dict[str, str]:
env_map: dict[str, str] = {}
if not os.path.isfile(env_path):
@@ -208,10 +227,18 @@ class Settings(BaseSettings):
RESULT_PUBLISH_ROOT: str = ""
DINSAR_PRODUCT_DIR: str = ""
TIMESERIES_PRODUCT_DIR: str = ""
PSINSAR_PRODUCT_DIR: str = ""
RESULT_QUARANTINE_ROOT: str = ""
RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP: bool = True
WSL_DISTRO: str = ""
WSL_SHARED_CONDA_ENV: str = ""
WSL_SHARED_PYTHON: str = ""
WSL_BROKER_JOB_ROOT: str = ""
ISCE2_RUNTIME_ID: str = ""
PYINT_RUNTIME_ID: str = ""
ISCE2_ENABLED: bool = False
ISCE2_WSL_DISTRO: str = "Ubuntu-24.04"
ISCE2_PYTHON: str = "/home/administrator/miniconda3/envs/isce2/bin/python"
@@ -325,7 +352,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"RESULT_PUBLISH_ROOT",
os.path.join(backend_dir, "result_products"),
_default_result_publish_root(project_root),
)
if not self.DINSAR_PRODUCT_DIR:
object.__setattr__(
@@ -333,11 +360,21 @@ class Settings(BaseSettings):
"DINSAR_PRODUCT_DIR",
os.path.join(self.RESULT_PUBLISH_ROOT, "dinsar"),
)
if not self.PSINSAR_PRODUCT_DIR:
timeseries_product_dir = (
_clean_path_text(self.TIMESERIES_PRODUCT_DIR)
or _clean_path_text(self.PSINSAR_PRODUCT_DIR)
or os.path.join(self.RESULT_PUBLISH_ROOT, "timeseries")
)
object.__setattr__(
self,
"TIMESERIES_PRODUCT_DIR",
os.path.normpath(timeseries_product_dir),
)
if not self.PSINSAR_PRODUCT_DIR or _clean_path_text(self.PSINSAR_PRODUCT_DIR) != self.TIMESERIES_PRODUCT_DIR:
object.__setattr__(
self,
"PSINSAR_PRODUCT_DIR",
os.path.join(self.RESULT_PUBLISH_ROOT, "psinsar"),
self.TIMESERIES_PRODUCT_DIR,
)
if not self.RESULT_QUARANTINE_ROOT:
object.__setattr__(
@@ -345,6 +382,18 @@ class Settings(BaseSettings):
"RESULT_QUARANTINE_ROOT",
os.path.join(self.RESULT_PUBLISH_ROOT, "_quarantine"),
)
if not self.ISCE2_WORK_ROOT:
object.__setattr__(
self,
"ISCE2_WORK_ROOT",
os.path.join(backend_dir, "runtime", "isce2_work"),
)
if not self.ISCE2_OUTPUT_ROOT:
object.__setattr__(
self,
"ISCE2_OUTPUT_ROOT",
self.DINSAR_PRODUCT_DIR,
)
if not self.ISCE2_PIPELINE_SCRIPT:
local_pipeline = os.path.join(
project_root,
@@ -358,10 +407,45 @@ class Settings(BaseSettings):
"ISCE2_PIPELINE_SCRIPT",
_windows_path_to_wsl_mount(local_pipeline),
)
if not self.WSL_DISTRO:
fallback_distro = str(
self.ISCE2_WSL_DISTRO
or self.PYINT_WSL_DISTRO
or "Ubuntu-24.04"
).strip()
object.__setattr__(self, "WSL_DISTRO", fallback_distro)
if not self.WSL_SHARED_CONDA_ENV:
shared_conda_env = (
_infer_conda_env_name_from_python(self.WSL_SHARED_PYTHON)
or _infer_conda_env_name_from_python(self.ISCE2_PYTHON)
or _infer_conda_env_name_from_python(self.PYINT_WSL_PYTHON)
or "insar_wsl_v1"
)
object.__setattr__(self, "WSL_SHARED_CONDA_ENV", shared_conda_env)
if not self.WSL_SHARED_PYTHON:
shared_python = (
_clean_path_text(self.ISCE2_PYTHON)
or _clean_path_text(self.PYINT_WSL_PYTHON)
or (
f"/home/administrator/miniconda3/envs/"
f"{self.WSL_SHARED_CONDA_ENV}/bin/python"
)
)
object.__setattr__(self, "WSL_SHARED_PYTHON", shared_python)
if not self.WSL_BROKER_JOB_ROOT:
object.__setattr__(
self,
"WSL_BROKER_JOB_ROOT",
os.path.join(backend_dir, "runtime", "wsl_jobs"),
)
if not self.ISCE2_RUNTIME_ID:
object.__setattr__(self, "ISCE2_RUNTIME_ID", "isce2_runtime_v1")
if not self.PYINT_RUNTIME_ID:
object.__setattr__(self, "PYINT_RUNTIME_ID", "gamma_pyint_runtime_v1")
if not self.PYINT_WSL_DISTRO:
object.__setattr__(self, "PYINT_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
object.__setattr__(self, "PYINT_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO)
if not self.PYINT_WSL_PYTHON:
object.__setattr__(self, "PYINT_WSL_PYTHON", self.ISCE2_PYTHON)
object.__setattr__(self, "PYINT_WSL_PYTHON", self.WSL_SHARED_PYTHON or self.ISCE2_PYTHON)
if not self.PYINT_HOME:
object.__setattr__(
self,
@@ -390,7 +474,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"PYINT_OUTPUT_ROOT",
os.path.join(backend_dir, "runtime", "pyint_output"),
self.DINSAR_PRODUCT_DIR,
)
if not self.PYINT_DEM_ROOT:
object.__setattr__(
@@ -420,7 +504,7 @@ class Settings(BaseSettings):
if not self.PYINT_ORBIT_POOL_TXT:
object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI)
if not self.TIMESERIES_WSL_DISTRO:
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO)
if not self.TIMESERIES_ENV_NAME:
object.__setattr__(self, "TIMESERIES_ENV_NAME", "isce2_mintpy_v1")
if not self.TIMESERIES_PYTHON:
@@ -518,8 +602,10 @@ class Settings(BaseSettings):
os.makedirs(settings.IDL_WORKER_RUNTIME_DIR, exist_ok=True)
os.makedirs(settings.RESULT_PUBLISH_ROOT, exist_ok=True)
os.makedirs(settings.DINSAR_PRODUCT_DIR, exist_ok=True)
os.makedirs(settings.TIMESERIES_PRODUCT_DIR, exist_ok=True)
os.makedirs(settings.PSINSAR_PRODUCT_DIR, exist_ok=True)
os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True)
os.makedirs(settings.WSL_BROKER_JOB_ROOT, exist_ok=True)
os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True)
os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True)
os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True)
@@ -668,6 +754,16 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="DINSAR_PRODUCT_DIR", value=settings.DINSAR_PRODUCT_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(
label="TIMESERIES_PRODUCT_DIR",
value=settings.TIMESERIES_PRODUCT_DIR,
errors=errors,
warnings=warnings,
expect_file=False,
)
_check_path(label="RESULT_QUARANTINE_ROOT", value=settings.RESULT_QUARANTINE_ROOT, errors=errors, warnings=warnings, expect_file=False)
for label, raw_value in (
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
@@ -780,6 +876,43 @@ def validate_runtime_config() -> dict[str, Any]:
expect_file=True,
)
if settings.ISCE2_ENABLED or settings.PYINT_ENABLED:
info.append(
"WSL shared runtime: "
f"distro={settings.WSL_DISTRO or '<empty>'}, "
f"conda_env={settings.WSL_SHARED_CONDA_ENV or '<empty>'}, "
f"isce2_runtime={settings.ISCE2_RUNTIME_ID}, "
f"pyint_runtime={settings.PYINT_RUNTIME_ID}"
)
_check_path(
label="WSL_BROKER_JOB_ROOT",
value=settings.WSL_BROKER_JOB_ROOT,
errors=errors,
warnings=warnings,
expect_file=False,
)
if settings.ISCE2_PYTHON and settings.WSL_SHARED_PYTHON:
isce_python = _clean_path_text(settings.ISCE2_PYTHON)
shared_python = _clean_path_text(settings.WSL_SHARED_PYTHON)
if isce_python != shared_python:
warnings.append(
"ISCE2_PYTHON differs from WSL_SHARED_PYTHON. "
"Legacy execution path and new shared runtime are not aligned."
)
if settings.PYINT_WSL_PYTHON and settings.WSL_SHARED_PYTHON:
pyint_python = _clean_path_text(settings.PYINT_WSL_PYTHON)
shared_python = _clean_path_text(settings.WSL_SHARED_PYTHON)
if pyint_python != shared_python:
warnings.append(
"PYINT_WSL_PYTHON differs from WSL_SHARED_PYTHON. "
"Gamma/PyINT still depends on a legacy Python path override."
)
if settings.PYINT_GAMMA_ENV_SCRIPT:
warnings.append(
"PYINT_GAMMA_ENV_SCRIPT is still configured. "
"Gamma runtime has not been fully migrated to the fixed profile model."
)
if settings.TIMESERIES_ENABLED:
_check_path(
label="TIMESERIES_PYTHON",
backend/app/dinsar_engines/isce2_engine.py
+402 -109
View File
@@ -1,7 +1,9 @@
"""ISCE2 D-InSAR engine backed by a WSL pipeline script."""
from __future__ import annotations
import json
import os
import shlex
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List
@@ -15,6 +17,10 @@ from ..services.dinsar_naming import (
find_json_sidecar,
write_run_metadata,
)
from ..services.dinsar_result_layout_service import (
normalize_isce2_run_layout,
)
from ..services.isce2_result_validator import validate_isce2_result_files
LT1_FIXED_WAVELENGTH = 0.23793052222222222
@@ -25,6 +31,7 @@ ORBIT_MARGIN_MIN_SEC = 60.0
ORBIT_MARGIN_MAX_SEC = 120.0
TARGET_GRID_SIZE_MIN_M = 5
TARGET_GRID_SIZE_MAX_M = 100
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
def _read_env(name: str, default: str = "") -> str:
@@ -47,6 +54,60 @@ def _windows_path_to_wsl_mount(path: str) -> str:
return f"/mnt/{drive_letter}/{normalized_tail}"
def _join_argv_for_log(argv: List[str]) -> str:
return " ".join(shlex.quote(str(item)) for item in argv if str(item))
def _normalize_rerun_mode(value: Any) -> str:
normalized = str(value or "").strip().lower()
return normalized if normalized == RERUN_MODE_UNFINISHED_ONLY else "rerun_all"
def _normalize_optional_path(value: Any) -> str:
text = str(value or "").strip()
if not text:
return ""
return os.path.normpath(os.path.abspath(text))
def _load_json_file(path: str) -> Dict[str, Any]:
try:
with open(path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
return payload if isinstance(payload, dict) else {}
except Exception:
return {}
def _find_isce2_export_outputs(export_dir: str) -> Dict[str, str]:
normalized_export_dir = _normalize_optional_path(export_dir)
if not normalized_export_dir or not os.path.isdir(normalized_export_dir):
return {}
outputs: Dict[str, str] = {}
with os.scandir(normalized_export_dir) as entries:
files = sorted(
[entry for entry in entries if entry.is_file()],
key=lambda entry: entry.name.lower(),
)
for entry in files:
lower_name = entry.name.lower()
if lower_name.endswith(("_disp_full.tif", "_disp_full.tiff")):
continue
if "disp" not in outputs and (
lower_name in {"disp.tif", "disp.tiff"}
or lower_name.endswith(("_disp.tif", "_disp.tiff"))
):
outputs["disp"] = entry.path
continue
if "coh" not in outputs and (
lower_name in {"coh.tif", "coh.tiff"}
or lower_name.endswith(("_coh.tif", "_coh.tiff"))
):
outputs["coh"] = entry.path
return outputs
class Isce2Engine(DinsarEngine):
@property
def engine_code(self) -> str:
@@ -79,6 +140,10 @@ class Isce2Engine(DinsarEngine):
"/home/administrator/miniconda3/envs/isce2/bin/python",
)
@property
def _runtime_id(self) -> str:
return _read_env("ISCE2_RUNTIME_ID", settings.ISCE2_RUNTIME_ID or "isce2_runtime_v1")
@property
def _dem_path(self) -> str:
explicit = _read_env("ISCE2_DEM_PATH", "")
@@ -315,7 +380,45 @@ class Isce2Engine(DinsarEngine):
# Task discovery
# ------------------------------------------------------------------
def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
def _has_completed_task_result(self, task_dir: str, profile_code: str) -> bool:
task_name = os.path.basename(os.path.normpath(task_dir))
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
pointer_path = os.path.join(
settings.DINSAR_PRODUCT_DIR,
pair_key,
"current",
f"{self.engine_code}__{str(profile_code or '').strip()}.json",
)
if not os.path.isfile(pointer_path):
return False
payload = _load_json_file(pointer_path)
if str(payload.get("status") or "").strip().upper() != "COMPLETED":
return False
primary_file = _normalize_optional_path(payload.get("primary_file"))
source_files = payload.get("source_files") if isinstance(payload.get("source_files"), list) else []
validation = validate_isce2_result_files(primary_file, source_files)
if not bool(validation.get("accepted")):
return False
manifest_path = _normalize_optional_path(payload.get("manifest_path"))
if manifest_path and os.path.isfile(manifest_path):
return True
output_dir = _normalize_optional_path(payload.get("output_dir"))
if output_dir and os.path.isfile(os.path.join(output_dir, "execution_manifest.json")):
return True
return False
def validate_root_dir(
self,
root_dir: str,
num_to_process: int = 0,
rerun_mode: str = "rerun_all",
) -> Dict[str, Any]:
normalized_root = os.path.normpath(os.path.abspath(str(root_dir or "").strip()))
if not root_dir or not os.path.isdir(normalized_root):
raise ValueError(f"ISCE2 root_dir does not exist or is not a directory: {root_dir}")
@@ -353,18 +456,95 @@ class Isce2Engine(DinsarEngine):
f"{detail}"
)
discovered_task_count = len(task_dirs)
skipped_completed_count = 0
selected_task_dirs = task_dirs
if _normalize_rerun_mode(rerun_mode) == RERUN_MODE_UNFINISHED_ONLY:
selected_task_dirs = []
for task_dir in task_dirs:
if self._has_completed_task_result(task_dir, "lt1_stripmap"):
skipped_completed_count += 1
continue
selected_task_dirs.append(task_dir)
selected_count = int(num_to_process or 0)
if selected_count > 0:
task_dirs = task_dirs[:selected_count]
selected_task_dirs = selected_task_dirs[:selected_count]
return {
"root_dir": normalized_root,
"mode": mode,
"task_dirs": task_dirs,
"task_count": len(task_dirs),
"task_dirs": selected_task_dirs,
"task_count": len(selected_task_dirs),
"selected_task_count": len(selected_task_dirs),
"discovered_task_count": discovered_task_count,
"skipped_completed_count": skipped_completed_count,
"invalid_candidates": invalid_candidates,
}
def _build_lt1_manifest_payload(
self,
*,
request: RunRequest,
task_dir: str,
task_name: str,
task_alias: str,
pair_key: str,
run_key: str,
work_dir: str,
output_dir: str,
orbit_output_dir: str,
wsl_task_dir: str,
wsl_work_dir: str,
wsl_output_dir: str,
wsl_orbit_output_dir: str,
wsl_orbit_root: str,
wsl_dem: str,
force: bool,
target_grid_size_m: int,
bbox: str,
coh_threshold: Any,
bbox_margin: Any,
wavelength: Any,
orbit_margin_sec: Any,
pair_meta: Dict[str, Any],
) -> Dict[str, Any]:
return {
"job_id": request.job_id,
"profile": request.profile,
"task_name": task_name,
"task_alias": task_alias,
"pair_key": pair_key,
"run_key": run_key,
"paths": {
"source_root_windows": os.path.normpath(
str((request.extra or {}).get("__source_root_override") or request.root_dir)
),
"task_dir_windows": os.path.normpath(task_dir),
"task_dir_wsl": wsl_task_dir,
"work_dir_windows": work_dir,
"work_dir_wsl": wsl_work_dir,
"output_dir_windows": output_dir,
"output_dir_wsl": wsl_output_dir,
"orbit_output_dir_windows": orbit_output_dir,
"orbit_output_dir_wsl": wsl_orbit_output_dir,
"orbit_root_windows": self._orbit_pool_isce2,
"orbit_root_wsl": wsl_orbit_root,
"dem_path_windows": self._dem_path,
"dem_path_wsl": wsl_dem,
},
"params": {
"force": bool(force),
"target_grid_size_m": int(target_grid_size_m),
"bbox": str(bbox or "").strip(),
"coh_threshold": coh_threshold,
"bbox_margin": bbox_margin,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
},
"pair_meta": dict(pair_meta or {}),
}
@staticmethod
def _iter_child_dirs(root_dir: str):
with os.scandir(root_dir) as entries:
@@ -409,16 +589,50 @@ class Isce2Engine(DinsarEngine):
return self._run_lt1_stripmap(request)
def _run_lt1_stripmap(self, request: RunRequest) -> RunResult:
from ..services.wsl_service import run_wsl_command, windows_path_to_wsl
from ..services.wsl_broker import wsl_broker
from ..services.wsl_runtime_registry import get_wsl_runtime
from ..services.wsl_service import windows_path_to_wsl
extra = self.normalize_extra(request.extra)
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
validation = self.validate_root_dir(
request.root_dir,
request.num_to_process,
str((request.extra or {}).get("__rerun_mode") or "rerun_all"),
)
task_dirs: List[str] = validation["task_dirs"]
total_tasks = len(task_dirs)
run_started_at = datetime.utcnow()
run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z"
run_key = build_run_key(self.engine_code, request.profile, started_at=run_started_at)
run_key = str(extra.get("__managed_run_key") or "").strip() or build_run_key(
self.engine_code,
request.profile,
started_at=run_started_at,
)
progress_callback = request.progress_callback
runtime = get_wsl_runtime(self._runtime_id)
managed_run_dir_override = _normalize_optional_path(extra.get("__managed_run_dir"))
managed_native_output_dir_override = _normalize_optional_path(extra.get("__managed_native_output_dir"))
managed_work_dir_override = _normalize_optional_path(extra.get("__managed_work_dir"))
managed_export_dir_override = _normalize_optional_path(extra.get("__managed_export_dir"))
managed_orbit_output_dir_override = _normalize_optional_path(extra.get("__managed_orbit_output_dir"))
source_root_override = _normalize_optional_path(extra.get("__source_root_override")) or os.path.normpath(request.root_dir)
has_managed_override = any(
[
managed_run_dir_override,
managed_native_output_dir_override,
managed_work_dir_override,
managed_export_dir_override,
managed_orbit_output_dir_override,
]
)
if has_managed_override and total_tasks > 1:
return RunResult(
success=False,
engine_code=self.engine_code,
profile=request.profile,
job_id=request.job_id,
error="Managed ISCE2 directory overrides require a single task request.",
)
def emit_progress(event_type: str, **payload: Any) -> None:
if not callable(progress_callback):
@@ -439,11 +653,11 @@ class Isce2Engine(DinsarEngine):
wsl_isce2_pool = ""
if self._orbit_pool_isce2:
wsl_isce2_pool = windows_path_to_wsl(self._orbit_pool_isce2, distro=self._distro)
wsl_isce2_pool = windows_path_to_wsl(self._orbit_pool_isce2, distro=runtime.distro)
wsl_dem = ""
if self._dem_path:
wsl_dem = windows_path_to_wsl(self._dem_path, distro=self._distro)
wsl_dem = windows_path_to_wsl(self._dem_path, distro=runtime.distro)
task_results: List[Dict[str, Any]] = []
output_dirs: List[str] = []
@@ -455,15 +669,18 @@ class Isce2Engine(DinsarEngine):
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
work_root = self._work_root or os.path.join(task_dir, "isce2_work")
output_root = self._output_root or os.path.join(task_dir, "isce2_output")
work_dir = os.path.normpath(os.path.join(work_root, pair_key, run_key))
output_dir = os.path.normpath(os.path.join(output_root, pair_key, run_key, "native"))
orbit_output_dir = os.path.join(work_dir, "orbits")
wsl_task_dir = windows_path_to_wsl(task_dir, distro=self._distro)
wsl_work_dir = windows_path_to_wsl(work_dir, distro=self._distro)
wsl_output_dir = windows_path_to_wsl(output_dir, distro=self._distro)
wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=self._distro)
run_dir = managed_run_dir_override or os.path.normpath(
os.path.join(output_root, pair_key, "runs", run_key)
)
native_output_dir = managed_native_output_dir_override or os.path.join(run_dir, "native")
work_dir = managed_work_dir_override or os.path.join(native_output_dir, "workflow")
export_dir = managed_export_dir_override or os.path.join(native_output_dir, "export")
orbit_output_dir = managed_orbit_output_dir_override or os.path.join(work_dir, "orbits")
wsl_task_dir = windows_path_to_wsl(task_dir, distro=runtime.distro)
wsl_work_dir = windows_path_to_wsl(work_dir, distro=runtime.distro)
wsl_output_dir = windows_path_to_wsl(export_dir, distro=runtime.distro)
wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=runtime.distro)
emit_progress(
"pair_started",
pair_index=pair_index,
@@ -473,7 +690,7 @@ class Isce2Engine(DinsarEngine):
pair_key=pair_key,
task_dir=task_dir,
work_dir=work_dir,
output_dir=output_dir,
output_dir=run_dir,
)
if not wsl_task_dir:
pairs_failed += 1
@@ -496,8 +713,11 @@ class Isce2Engine(DinsarEngine):
"pair_key": pair_key,
"run_key": run_key,
"task_dir": task_dir,
"run_dir": run_dir,
"native_output_dir": native_output_dir,
"work_dir": work_dir,
"output_dir": output_dir,
"output_dir": run_dir,
"export_dir": export_dir,
"success": False,
"returncode": -2,
"error": error_text,
@@ -531,8 +751,11 @@ class Isce2Engine(DinsarEngine):
"pair_key": pair_key,
"run_key": run_key,
"task_dir": task_dir,
"run_dir": run_dir,
"native_output_dir": native_output_dir,
"work_dir": work_dir,
"output_dir": output_dir,
"output_dir": run_dir,
"export_dir": export_dir,
"success": False,
"returncode": -2,
"error": error_text,
@@ -546,92 +769,137 @@ class Isce2Engine(DinsarEngine):
)
continue
cmd_parts = [
"export PROJ_DATA=/home/administrator/miniconda3/envs/isce2/share/proj",
f"&& {self._python} '{self._pipeline_script}' '{wsl_task_dir}'",
f"--task-name '{task_alias}'",
f"--output-prefix '{task_alias}'",
f"--work-dir '{wsl_work_dir}'",
f"--output-dir '{wsl_output_dir}'",
f"--orbit-output-dir '{wsl_orbit_output_dir}'",
]
if wsl_isce2_pool:
cmd_parts.append(f"--orbit-root '{wsl_isce2_pool}'")
if wsl_dem:
cmd_parts.append(f"--dem '{wsl_dem}'")
if force:
cmd_parts.append("--force")
if target_grid_size_m:
cmd_parts.append(f"--target-grid-size-m {target_grid_size_m}")
if bbox:
cmd_parts.append(f"--bbox '{bbox}'")
if coh_threshold is not None:
cmd_parts.append(f"--coh-threshold {coh_threshold}")
if bbox_margin is not None:
cmd_parts.append(f"--bbox-margin {bbox_margin}")
if wavelength is not None:
cmd_parts.append(f"--wavelength {wavelength}")
if orbit_margin_sec is not None:
cmd_parts.append(f"--orbit-margin-sec {orbit_margin_sec}")
cmd = " ".join(cmd_parts)
rc, stdout, stderr = run_wsl_command(
cmd,
distro=self._distro,
timeout=timeout,
manifest_payload = self._build_lt1_manifest_payload(
request=request,
task_dir=task_dir,
task_name=task_name,
task_alias=task_alias,
pair_key=pair_key,
run_key=run_key,
work_dir=work_dir,
output_dir=export_dir,
orbit_output_dir=orbit_output_dir,
wsl_task_dir=wsl_task_dir,
wsl_work_dir=wsl_work_dir,
wsl_output_dir=wsl_output_dir,
wsl_orbit_output_dir=wsl_orbit_output_dir,
wsl_orbit_root=wsl_isce2_pool,
wsl_dem=wsl_dem,
force=force,
target_grid_size_m=target_grid_size_m,
bbox=bbox,
coh_threshold=coh_threshold,
bbox_margin=bbox_margin,
wavelength=wavelength,
orbit_margin_sec=orbit_margin_sec,
pair_meta=pair_meta,
)
broker_result = wsl_broker.run_manifest(
runtime_id=runtime.runtime_id,
operation="lt1_stripmap",
payload=manifest_payload,
job_id=f"{request.job_id}_{pair_key}",
timeout_seconds=timeout,
)
rc = broker_result.returncode
stdout = broker_result.stdout
stderr = broker_result.stderr
command = _join_argv_for_log(list(broker_result.argv))
success = rc == 0
if success:
pairs_processed += 1
os.makedirs(output_dir, exist_ok=True)
write_run_metadata(
output_dir,
{
"run_key": run_key,
"pair_key": pair_key,
"task_name": task_name,
"task_alias": task_alias,
"engine_code": self.engine_code,
"profile_code": request.profile,
"source_root": os.path.normpath(request.root_dir),
"task_dir": os.path.normpath(task_dir),
"work_dir": work_dir,
"output_dir": output_dir,
"orbit_output_dir": orbit_output_dir,
"started_at": run_started_at_text,
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"params": {
"force": force,
"target_grid_size_m": target_grid_size_m,
"bbox": bbox,
"coh_threshold": coh_threshold,
"bbox_margin": bbox_margin,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
success = False
layout_result: Dict[str, Any] = {}
validation_result: Dict[str, Any] = {}
error_text = stderr.strip() if stderr else ""
if rc == 0:
try:
os.makedirs(run_dir, exist_ok=True)
export_outputs = _find_isce2_export_outputs(export_dir)
disp_path = export_outputs.get("disp", "")
coh_path = export_outputs.get("coh", "")
if not disp_path:
raise FileNotFoundError(
f"No ISCE2 displacement GeoTIFF found under export dir: {export_dir}"
)
source_files = [disp_path]
if coh_path:
source_files.append(coh_path)
validation_result = validate_isce2_result_files(disp_path, source_files)
if not bool(validation_result.get("accepted")):
issues = validation_result.get("issues") or []
issue_text = "; ".join(str(item) for item in issues[:3]) or "unknown validation error"
raise RuntimeError(f"ISCE2 output validation failed: {issue_text}")
layout_result = normalize_isce2_run_layout(
run_dir,
primary_file=str(validation_result.get("primary_file") or disp_path),
source_files=list(validation_result.get("source_files") or source_files),
rewrite_metadata=False,
)
write_run_metadata(
run_dir,
{
"run_key": run_key,
"pair_key": pair_key,
"task_name": task_name,
"task_alias": task_alias,
"engine_code": self.engine_code,
"profile_code": request.profile,
"source_root": source_root_override,
"task_dir": os.path.normpath(task_dir),
"work_dir": work_dir,
"export_dir": export_dir,
"output_dir": run_dir,
"native_output_dir": layout_result["native_output_dir"],
"orbit_output_dir": orbit_output_dir,
"runtime_id": runtime.runtime_id,
"manifest_path_windows": broker_result.manifest.manifest_path_windows,
"manifest_path_wsl": broker_result.manifest.manifest_path_wsl,
"started_at": run_started_at_text,
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"primary_file": layout_result["primary_file"],
"source_files": layout_result["source_files"],
"acceptance": validation_result,
"params": {
"force": force,
"target_grid_size_m": target_grid_size_m,
"bbox": bbox,
"coh_threshold": coh_threshold,
"bbox_margin": bbox_margin,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
},
"master_path": pair_meta.get("master_path"),
"slave_path": pair_meta.get("slave_path"),
"master_satellite": pair_meta.get("master_satellite"),
"slave_satellite": pair_meta.get("slave_satellite"),
"master_imaging_date": pair_meta.get("master_imaging_date"),
"slave_imaging_date": pair_meta.get("slave_imaging_date"),
"master_imaging_mode": pair_meta.get("master_imaging_mode"),
"slave_imaging_mode": pair_meta.get("slave_imaging_mode"),
"master_polarization": pair_meta.get("master_polarization"),
"slave_polarization": pair_meta.get("slave_polarization"),
"time_baseline_days": pair_meta.get("time_baseline_days"),
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
"network_run_id": pair_meta.get("network_run_id"),
"network_edge_id": pair_meta.get("network_edge_id"),
"policy_version": pair_meta.get("policy_version"),
"selection_strategy": pair_meta.get("selection_strategy"),
},
"master_path": pair_meta.get("master_path"),
"slave_path": pair_meta.get("slave_path"),
"master_satellite": pair_meta.get("master_satellite"),
"slave_satellite": pair_meta.get("slave_satellite"),
"master_imaging_date": pair_meta.get("master_imaging_date"),
"slave_imaging_date": pair_meta.get("slave_imaging_date"),
"master_imaging_mode": pair_meta.get("master_imaging_mode"),
"slave_imaging_mode": pair_meta.get("slave_imaging_mode"),
"master_polarization": pair_meta.get("master_polarization"),
"slave_polarization": pair_meta.get("slave_polarization"),
"time_baseline_days": pair_meta.get("time_baseline_days"),
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
"network_run_id": pair_meta.get("network_run_id"),
"network_edge_id": pair_meta.get("network_edge_id"),
"policy_version": pair_meta.get("policy_version"),
"selection_strategy": pair_meta.get("selection_strategy"),
},
)
output_dirs.append(output_dir)
else:
)
output_dirs.append(run_dir)
pairs_processed += 1
success = True
except Exception as exc:
error_text = str(exc)
if stderr:
stderr = stderr.rstrip() + "\n" + error_text
else:
stderr = error_text
if not success:
pairs_failed += 1
emit_progress(
@@ -643,7 +911,7 @@ class Isce2Engine(DinsarEngine):
pair_key=pair_key,
success=success,
returncode=rc,
error=stderr.strip() if stderr else "",
error=error_text,
)
task_results.append(
{
@@ -652,17 +920,30 @@ class Isce2Engine(DinsarEngine):
"pair_key": pair_key,
"run_key": run_key,
"task_dir": task_dir,
"run_dir": run_dir,
"native_output_dir": (
layout_result.get("native_output_dir")
or native_output_dir
),
"work_dir": work_dir,
"output_dir": output_dir,
"output_dir": run_dir,
"export_dir": export_dir,
"primary_file": layout_result.get("primary_file", ""),
"source_files": layout_result.get("source_files", []),
"validation": validation_result,
"wsl_task_dir": wsl_task_dir,
"wsl_work_dir": wsl_work_dir,
"wsl_output_dir": wsl_output_dir,
"command": cmd,
"runtime_id": runtime.runtime_id,
"command": command,
"runner_argv": list(broker_result.argv),
"manifest_path_windows": broker_result.manifest.manifest_path_windows,
"manifest_path_wsl": broker_result.manifest.manifest_path_wsl,
"success": success,
"returncode": rc,
"stdout_tail": stdout[-3000:] if stdout else "",
"stderr_tail": stderr[-3000:] if stderr else "",
"error": stderr.strip() if stderr else "",
"error": error_text,
}
)
@@ -706,7 +987,11 @@ class Isce2Engine(DinsarEngine):
"target_grid_size_m": target_grid_size_m,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
"runtime_id": runtime.runtime_id,
"command": last_task_result.get("command", ""),
"runner_argv": last_task_result.get("runner_argv", []),
"manifest_path_windows": last_task_result.get("manifest_path_windows", ""),
"manifest_path_wsl": last_task_result.get("manifest_path_wsl", ""),
"stdout_tail": last_task_result.get("stdout_tail", ""),
"stderr_tail": last_task_result.get("stderr_tail", ""),
"wsl_task_dir": last_task_result.get("wsl_task_dir", ""),
@@ -714,7 +999,15 @@ class Isce2Engine(DinsarEngine):
"wsl_output_dir": last_task_result.get("wsl_output_dir", ""),
"wsl_orbit_pool": wsl_isce2_pool,
"wsl_dem": wsl_dem,
"wsl_work_root": windows_path_to_wsl(self._work_root, distro=self._distro) if self._work_root else "",
"wsl_output_root": windows_path_to_wsl(self._output_root, distro=self._distro) if self._output_root else "",
"wsl_work_root": (
os.path.dirname(str(last_task_result.get("wsl_work_dir") or "").strip())
if str(last_task_result.get("wsl_work_dir") or "").strip()
else (windows_path_to_wsl(self._work_root, distro=runtime.distro) if self._work_root else "")
),
"wsl_output_root": (
os.path.dirname(str(last_task_result.get("wsl_output_dir") or "").strip())
if str(last_task_result.get("wsl_output_dir") or "").strip()
else (windows_path_to_wsl(self._output_root, distro=runtime.distro) if self._output_root else "")
),
},
)
backend/app/dinsar_engines/pyint_engine.py
+80 -5
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import os
import json
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List
@@ -25,6 +26,7 @@ from ..services.pyint_service import (
infer_scene_date_from_archives,
infer_task_identity,
quote_shell,
resolve_gamma_env_script,
resolve_time_baseline_days,
to_wsl_path,
validate_pyint_root_dir,
@@ -32,6 +34,8 @@ from ..services.pyint_service import (
from ..services.wsl_service import run_wsl_command
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
def _read_env(name: str, default: str = "") -> str:
return get_env_text(name, default) or default
@@ -53,6 +57,11 @@ def _windows_path_to_wsl_mount(path: str) -> str:
return f"/mnt/{drive_letter}/{normalized_tail}"
def _normalize_rerun_mode(value: Any) -> str:
normalized = str(value or "").strip().lower()
return normalized if normalized == RERUN_MODE_UNFINISHED_ONLY else "rerun_all"
class PyintEngine(DinsarEngine):
@property
def engine_code(self) -> str:
@@ -138,7 +147,7 @@ class PyintEngine(DinsarEngine):
@property
def _gamma_env_script(self) -> str:
return _read_env("PYINT_GAMMA_ENV_SCRIPT", "")
return resolve_gamma_env_script()
@property
def _lt1_precise_orbit_enabled(self) -> bool:
@@ -269,8 +278,70 @@ class PyintEngine(DinsarEngine):
return normalized
def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
return validate_pyint_root_dir(root_dir, num_to_process)
def _has_completed_task_result(self, task_dir: str, profile_code: str) -> bool:
task_identity = infer_task_identity(task_dir)
pair_key = task_identity["pair_key"]
output_root = self._output_root or os.path.join(task_dir, "pyint_output")
runs_root = os.path.join(output_root, pair_key, "runs")
if not os.path.isdir(runs_root):
return False
with os.scandir(runs_root) as entries:
run_dirs = [entry.path for entry in entries if entry.is_dir()]
run_dirs.sort(key=lambda path: os.path.basename(path).lower(), reverse=True)
for run_dir in run_dirs:
metadata_path = os.path.join(run_dir, "native", ".dinsar_run.json")
if not os.path.isfile(metadata_path):
metadata_path = os.path.join(run_dir, ".dinsar_run.json")
if not os.path.isfile(metadata_path):
continue
try:
with open(metadata_path, "r", encoding="utf-8") as fp:
metadata = json.load(fp) or {}
except Exception:
continue
if str(metadata.get("engine_code") or "").strip().lower() != self.engine_code:
continue
if str(metadata.get("profile_code") or "").strip() != str(profile_code or "").strip():
continue
output_dir = str(metadata.get("output_dir") or os.path.join(run_dir, "native")).strip()
if output_dir and os.path.isdir(output_dir):
return True
return False
def validate_root_dir(
self,
root_dir: str,
num_to_process: int = 0,
rerun_mode: str = "rerun_all",
) -> Dict[str, Any]:
validation = validate_pyint_root_dir(root_dir, 0)
task_dirs: List[str] = list(validation.get("task_dirs") or [])
discovered_task_count = len(task_dirs)
skipped_completed_count = 0
if _normalize_rerun_mode(rerun_mode) == RERUN_MODE_UNFINISHED_ONLY:
filtered_task_dirs: List[str] = []
for task_dir in task_dirs:
if self._has_completed_task_result(task_dir, "lt1_gamma_dinsar"):
skipped_completed_count += 1
continue
filtered_task_dirs.append(task_dir)
task_dirs = filtered_task_dirs
selected_count = int(num_to_process or 0)
if selected_count > 0:
task_dirs = task_dirs[:selected_count]
return {
**validation,
"task_dirs": task_dirs,
"task_count": len(task_dirs),
"selected_task_count": len(task_dirs),
"discovered_task_count": discovered_task_count,
"skipped_completed_count": skipped_completed_count,
}
def check_available(self) -> EngineAvailability:
report = check_pyint_environment(
@@ -333,7 +404,11 @@ class PyintEngine(DinsarEngine):
def _run_lt1_gamma_dinsar(self, request: RunRequest) -> RunResult:
extra = self.normalize_extra(request.extra)
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
validation = self.validate_root_dir(
request.root_dir,
request.num_to_process,
str((request.extra or {}).get("__rerun_mode") or "rerun_all"),
)
task_dirs: List[str] = validation["task_dirs"]
total_tasks = len(task_dirs)
run_started_at = datetime.utcnow()
@@ -383,7 +458,7 @@ class PyintEngine(DinsarEngine):
slave_date = task_identity["slave_date"]
work_run_root = os.path.normpath(os.path.join(self._work_root, pair_key, run_key))
output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, run_key, "native"))
output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, "runs", run_key, "native"))
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
project_name = build_project_name(pair_key, run_key)
project_dir = os.path.join(work_run_root, project_name)
backend/app/dinsar_engines/sarscape_engine.py
+21 -2
View File
@@ -50,6 +50,7 @@ class SarscapeEngine(DinsarEngine):
status_raw = envi_service.get_status()
idl_ok: bool = status_raw.get("idl_installed", False)
dem_ok: bool = status_raw.get("dem_exists", False)
runner_ok: bool = status_raw.get("runner_ready", False)
checks = [
{
@@ -62,12 +63,30 @@ class SarscapeEngine(DinsarEngine):
"ok": dem_ok,
"detail": status_raw.get("dem_base_file", ""),
},
{
"name": "Python Runner",
"ok": runner_ok,
"detail": (
status_raw.get("runner_message")
or status_raw.get("runner_python")
or ""
),
},
]
if idl_ok and dem_ok:
if idl_ok and dem_ok and runner_ok:
status = "ok"
available = True
message = "ENVI/SARscape 可用"
elif not idl_ok:
status = "unavailable"
available = False
message = "IDL/ENVI 未安装或路径错误"
elif not runner_ok:
status = "unavailable"
available = False
runner_message = str(status_raw.get("runner_message") or "").strip()
message = runner_message or "ENVI Python runner unavailable."
elif idl_ok:
status = "degraded"
available = True
@@ -75,7 +94,7 @@ class SarscapeEngine(DinsarEngine):
else:
status = "unavailable"
available = False
message = "IDL/ENVI 未安装或路径错误"
message = "ENVI/SARscape unavailable."
return EngineAvailability(
engine_code=self.engine_code,
backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py
+103 -7
View File
@@ -29,6 +29,7 @@ class Scene:
tiff_path: Path
meta_path: Path
date_yyyymmdd: str
satellite: str
orbit_xml_path: Path
@@ -157,6 +158,16 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Delete an existing work directory before rerunning",
)
parser.add_argument(
"--reference-satellite",
default=None,
help="Optional LT-1 satellite for the reference/master scene (LT1A or LT1B)",
)
parser.add_argument(
"--secondary-satellite",
default=None,
help="Optional LT-1 satellite for the secondary/slave scene (LT1A or LT1B)",
)
parser.add_argument(
"--dry-run",
action="store_true",
@@ -209,10 +220,17 @@ def choose_scene_tiff(scene_dir: Path, scene_glob: str, prefer_scene_keyword: st
def scene_meta_from_tiff(tiff_path: Path) -> Path:
meta_path = Path(str(tiff_path).replace(".tiff", ".meta.xml"))
if not meta_path.exists():
raise FileNotFoundError(f"Missing meta XML for {tiff_path}: {meta_path}")
return meta_path
candidates = [tiff_path.with_suffix(".meta.xml")]
legacy_path = Path(str(tiff_path).replace(".tiff", ".meta.xml"))
if legacy_path not in candidates:
candidates.append(legacy_path)
for meta_path in candidates:
if meta_path.exists():
return meta_path
searched = ", ".join(str(path) for path in candidates)
raise FileNotFoundError(f"Missing meta XML for {tiff_path}. Searched: {searched}")
def extract_scene_date(name: str) -> str:
@@ -224,8 +242,71 @@ def extract_scene_date(name: str) -> str:
return match.group(1)
def normalize_lt1_satellite(value: str | None) -> str:
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
if "LT1A" in text or text in {"A", "LTA"}:
return "LT1A"
if "LT1B" in text or text in {"B", "LTB"}:
return "LT1B"
return ""
def extract_scene_satellite_from_name(name: str) -> str:
match = re.search(r"(LT1[AB])", str(name or ""), re.IGNORECASE)
return normalize_lt1_satellite(match.group(1) if match else "")
def extract_scene_satellite_from_meta(meta_path: Path) -> str:
try:
root = ET.parse(meta_path).getroot()
except Exception:
return ""
for element in root.iter():
tag = str(element.tag or "").rsplit("}", 1)[-1].strip().lower()
if tag not in {"mission", "satellite", "platform", "platformid", "missionid"}:
continue
satellite = normalize_lt1_satellite(element.text)
if satellite:
return satellite
return ""
def resolve_scene_satellite(
tiff_path: Path,
meta_path: Path,
explicit_satellite: str | None = None,
) -> str:
explicit = normalize_lt1_satellite(explicit_satellite)
name_satellite = extract_scene_satellite_from_name(tiff_path.name)
meta_satellite = extract_scene_satellite_from_meta(meta_path)
if explicit:
if name_satellite and name_satellite != explicit:
raise ValueError(
f"Explicit satellite {explicit} does not match filename for {tiff_path.name}: {name_satellite}"
)
if meta_satellite and meta_satellite != explicit:
raise ValueError(
f"Explicit satellite {explicit} does not match metadata for {meta_path.name}: {meta_satellite}"
)
return explicit
if name_satellite and meta_satellite and name_satellite != meta_satellite:
raise ValueError(
f"Satellite mismatch between filename and metadata for {tiff_path.name}: "
f"{name_satellite} vs {meta_satellite}"
)
if name_satellite:
return name_satellite
if meta_satellite:
return meta_satellite
raise ValueError(f"Unable to resolve LT-1 satellite from {tiff_path} / {meta_path}")
def ensure_orbit_xml(
date_yyyymmdd: str,
satellite: str,
annotation_xml: Path,
orbit_root: Path,
orbit_out_dir: Path,
@@ -233,7 +314,7 @@ def ensure_orbit_xml(
) -> Path:
resolution = ensure_lt1_orbit_xml(
date_yyyymmdd=date_yyyymmdd,
satellite="LT1A",
satellite=satellite,
annotation_xml=annotation_xml,
orbit_root=orbit_root,
orbit_output_dir=orbit_out_dir,
@@ -268,8 +349,14 @@ def resolve_task(
slave_dir_name: str,
scene_glob: str,
prefer_scene_keyword: str,
reference_satellite: str | None = None,
secondary_satellite: str | None = None,
) -> tuple[Scene, Scene]:
scenes: list[Scene] = []
satellite_hints = {
"master": reference_satellite,
"slave": secondary_satellite,
}
for role, subdir in (("master", master_dir_name), ("slave", slave_dir_name)):
scene_dir = task_dir / subdir
if not scene_dir.exists():
@@ -278,8 +365,14 @@ def resolve_task(
tiff_path = choose_scene_tiff(scene_dir, scene_glob, prefer_scene_keyword)
meta_path = scene_meta_from_tiff(tiff_path)
date_yyyymmdd = extract_scene_date(tiff_path.name)
satellite = resolve_scene_satellite(
tiff_path=tiff_path,
meta_path=meta_path,
explicit_satellite=satellite_hints.get(role),
)
orbit_xml_path = ensure_orbit_xml(
date_yyyymmdd=date_yyyymmdd,
satellite=satellite,
annotation_xml=meta_path,
orbit_root=orbit_root,
orbit_out_dir=orbit_out_dir,
@@ -291,6 +384,7 @@ def resolve_task(
tiff_path=tiff_path,
meta_path=meta_path,
date_yyyymmdd=date_yyyymmdd,
satellite=satellite,
orbit_xml_path=orbit_xml_path,
)
)
@@ -467,8 +561,8 @@ def print_summary(
print(f"Work dir: {work_dir}")
print(f"Output dir: {output_dir}")
print(f"DEM: {config.dem_path}")
print(f"Reference: {config.reference.tiff_path}")
print(f"Secondary: {config.secondary.tiff_path}")
print(f"Reference: {config.reference.tiff_path} [{config.reference.satellite}]")
print(f"Secondary: {config.secondary.tiff_path} [{config.secondary.satellite}]")
print(f"Ref orbit: {config.reference.orbit_xml_path}")
print(f"Sec orbit: {config.secondary.orbit_xml_path}")
print(f"BBox: {config.bbox if config.bbox is not None else 'auto'}")
@@ -510,6 +604,8 @@ def main() -> int:
slave_dir_name=args.slave_dir_name,
scene_glob=args.scene_glob,
prefer_scene_keyword=args.prefer_scene_keyword,
reference_satellite=args.reference_satellite,
secondary_satellite=args.secondary_satellite,
)
dem_path = resolve_dem(args.dem)
bbox = parse_bbox_arg(args.bbox)
backend/app/main.py
+12 -5
View File
@@ -76,7 +76,7 @@ async def lifespan(app: FastAPI):
ps_catalog_bootstrap = await psinsar_catalog_service.bootstrap_catalog_on_startup_clean()
except Exception as exc:
ps_catalog_bootstrap = {
"storage_root": settings.PSINSAR_PRODUCT_DIR,
"storage_root": settings.TIMESERIES_PRODUCT_DIR,
"manifest_count": 0,
"db_count": 0,
"needs_rebuild": False,
@@ -163,7 +163,7 @@ async def lifespan(app: FastAPI):
if catalog_bootstrap.get("compat_error"):
print(f">>> [Catalog Compat] Startup sync failed: {catalog_bootstrap['compat_error']}")
print(
">>> [PS Catalog] root={0} manifests={1} db={2} rebuild={3} queued={4}".format(
">>> [Timeseries Catalog] root={0} manifests={1} db={2} rebuild={3} queued={4}".format(
ps_catalog_bootstrap.get("storage_root") or "?",
ps_catalog_bootstrap.get("manifest_count", 0),
ps_catalog_bootstrap.get("db_count", 0),
@@ -172,7 +172,7 @@ async def lifespan(app: FastAPI):
)
)
if ps_catalog_bootstrap.get("error"):
print(f">>> [PS Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}")
print(f">>> [Timeseries Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}")
print(
">>> [Pairing] status={0} scenes={1} pairs={2} dirty={3} metric={4} rebuild={5}".format(
pairing_bootstrap.get("status") or "?",
@@ -195,16 +195,23 @@ async def lifespan(app: FastAPI):
schema_ok = health.get("database", {}).get("schema_ok")
worker_ok = health.get("worker", {}).get("ok")
dinsar_catalog_ok = health.get("dinsar_result_catalog", {}).get("ok")
psinsar_catalog_ok = health.get("psinsar_result_catalog", {}).get("ok")
psinsar_catalog_ok = (
health.get("timeseries_result_catalog", {})
or health.get("psinsar_result_catalog", {})
).get("ok")
pairing_ok = health.get("pairing_system", {}).get("ok")
idl_ok = health.get("idl", {}).get("ok")
product_packages_ok = health.get("product_packages", {}).get("ok")
wsl_runtime_ok = health.get("wsl_runtime", {}).get("ok")
print(
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} PSInSAR-Catalog:{4} Pairing:{5} IDL:{6}".format(
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} Timeseries-Catalog:{4} Packages:{5} WSL:{6} Pairing:{7} IDL:{8}".format(
"OK" if db_ok else "FAIL",
"OK" if schema_ok else "FAIL",
"OK" if worker_ok else "FAIL",
"OK" if dinsar_catalog_ok else "FAIL",
"OK" if psinsar_catalog_ok else "FAIL",
"OK" if product_packages_ok else "FAIL",
"OK" if wsl_runtime_ok else "FAIL",
"OK" if pairing_ok else "FAIL",
"OK" if idl_ok else "FAIL",
)
backend/app/models/orm.py
+13
View File
@@ -83,11 +83,13 @@ class ResultProductORM(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
product_id = Column(String(64), unique=True, index=True, nullable=False)
catalog_name = Column(String(32), index=True, nullable=False, default="dinsar")
product_family = Column(String(32), index=True, nullable=True)
product_type = Column(String(32), index=True, nullable=False, default="dinsar")
display_name = Column(String(255), nullable=False)
task_name = Column(String(255), index=True, nullable=True)
task_alias = Column(String(255), index=True, nullable=True)
pair_key = Column(String(128), index=True, nullable=True)
stack_key = Column(String(128), index=True, nullable=True)
pair_uid = Column(String(64), index=True, nullable=True)
run_key = Column(String(128), index=True, nullable=True)
network_run_id = Column(String(64), index=True, nullable=True)
@@ -97,12 +99,17 @@ class ResultProductORM(Base):
profile_code = Column(String(64), index=True, nullable=True)
engine_code = Column(String(32), index=True, nullable=False)
engine_version = Column(String(64), nullable=True)
package_schema = Column(String(64), nullable=True)
package_layout = Column(String(64), nullable=True)
processor_code = Column(String(64), nullable=True)
runtime_id = Column(String(64), nullable=True)
status = Column(String(32), index=True, nullable=False, default="READY")
health_status = Column(String(16), index=True, nullable=False, default="OK")
publish_dir = Column(String, unique=True, nullable=False)
manifest_path = Column(String, unique=True, nullable=False)
source_primary_path = Column(String, nullable=True)
native_output_dir = Column(String, nullable=True)
preview_path = Column(String, nullable=True)
primary_asset_path = Column(String, nullable=True)
@@ -260,6 +267,7 @@ class ResultCatalogStateORM(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
catalog_name = Column(String(32), unique=True, index=True, nullable=False)
product_family = Column(String(32), index=True, nullable=True)
storage_root = Column(String, nullable=False)
status = Column(String(16), index=True, nullable=False, default="READY")
needs_rebuild = Column(Boolean, nullable=False, default=False)
@@ -789,10 +797,12 @@ class DinsarProductionRunORM(Base):
task_id = Column(String, index=True, nullable=True)
workflow_run_id = Column(String, index=True, nullable=True)
product_family = Column(String(32), index=True, nullable=True)
engine_code = Column(String(32), index=True, nullable=False, default="sarscape")
profile_code = Column(String(64), index=True, nullable=False, default="custom6")
mode = Column(String(32), index=True, nullable=False, default="custom")
source_root = Column(String, nullable=False)
publish_root_dir = Column(String, nullable=True)
status = Column(String(32), index=True, nullable=False, default="PENDING")
cancel_requested = Column(Boolean, nullable=False, default=False)
@@ -928,11 +938,14 @@ class PsTimeseriesRunORM(Base):
run_id = Column(String(64), unique=True, index=True, nullable=False)
batch_id = Column(String, ForeignKey("ps_task_batches.batch_id"), index=True, nullable=False)
product_family = Column(String(32), index=True, nullable=True)
run_name = Column(String(255), nullable=False)
catalog_name = Column(String(32), index=True, nullable=False, default="psinsar")
stack_key = Column(String(128), index=True, nullable=True)
mode = Column(String(32), nullable=False, default="sbas")
engine_code = Column(String(32), index=True, nullable=False, default="isce2")
processor_code = Column(String(64), nullable=False, default="isce2_stack_mintpy")
runtime_id = Column(String(64), nullable=True)
env_name = Column(String(128), nullable=True)
wsl_distro = Column(String(128), nullable=True)
backend/app/models/schemas.py
+5
View File
@@ -66,10 +66,12 @@ class DinsarResult(BaseModel):
id: int
product_id: Optional[str] = None
compat_result_id: Optional[int] = None
product_family: Optional[str] = None
name: str
task_name: Optional[str] = None
task_alias: Optional[str] = None
pair_key: Optional[str] = None
stack_key: Optional[str] = None
pair_uid: Optional[str] = None
run_key: Optional[str] = None
network_run_id: Optional[str] = None
@@ -472,11 +474,14 @@ class PsTaskItem(BaseModel):
class PsTimeseriesRun(BaseModel):
run_id: str
batch_id: str
product_family: Optional[str] = None
run_name: str
catalog_name: str
stack_key: Optional[str] = None
mode: str
engine_code: str
processor_code: str
runtime_id: Optional[str] = None
env_name: Optional[str] = None
wsl_distro: Optional[str] = None
status: str
backend/app/pyint_pipeline/pyint_gamma_env.sh
+7 -72
View File
@@ -1,78 +1,13 @@
#!/usr/bin/env bash
set -euo pipefail
_pyint_gamma_die() {
echo "$1" >&2
_pyint_legacy_profile="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)/deploy/wsl/profiles/gamma_env.sh"
if [ ! -f "${_pyint_legacy_profile}" ]; then
echo "Gamma profile not found: ${_pyint_legacy_profile}" >&2
return 1 2>/dev/null || exit 1
}
_pyint_gamma_home=""
if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then
_pyint_gamma_home="${PYINT_GAMMA_HOME}"
elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then
_pyint_gamma_home="${GAMMA_HOME}"
else
for _candidate in \
/usr/local/GAMMA_SOFTWARE-20240627 \
/usr/local/GAMMA_SOFTWARE-* \
/opt/GAMMA_SOFTWARE-*; do
[ -d "${_candidate}" ] || continue
_pyint_gamma_home="${_candidate}"
break
done
fi
[ -n "${_pyint_gamma_home}" ] || _pyint_gamma_die "Gamma home not found."
. "${_pyint_legacy_profile}"
unset _pyint_legacy_profile
export GAMMA_HOME="${_pyint_gamma_home}"
export MSP_HOME="${GAMMA_HOME}/MSP"
export ISP_HOME="${GAMMA_HOME}/ISP"
export DIFF_HOME="${GAMMA_HOME}/DIFF"
export DISP_HOME="${GAMMA_HOME}/DISP"
export LAT_HOME="${GAMMA_HOME}/LAT"
export IPTA_HOME="${GAMMA_HOME}/IPTA"
export GEO_HOME="${GAMMA_HOME}/GEO"
_pyint_gamma_prepend_path() {
local _dir="$1"
[ -d "${_dir}" ] || return 0
case ":${PATH}:" in
*":${_dir}:"*) ;;
*) PATH="${_dir}:${PATH}" ;;
esac
}
for _gamma_dir in \
"${MSP_HOME}/bin" \
"${ISP_HOME}/bin" \
"${DIFF_HOME}/bin" \
"${DISP_HOME}/bin" \
"${LAT_HOME}/bin" \
"${IPTA_HOME}/bin" \
"${GEO_HOME}/bin" \
"${MSP_HOME}/scripts" \
"${ISP_HOME}/scripts" \
"${DIFF_HOME}/scripts" \
"${DISP_HOME}/scripts" \
"${LAT_HOME}/scripts" \
"${IPTA_HOME}/scripts" \
"${GEO_HOME}/scripts"; do
_pyint_gamma_prepend_path "${_gamma_dir}"
done
export PATH
export OS="linux64"
export HDF5_DISABLE_VERSION_CHECK="1"
export GNUTERM="${GNUTERM:-qt}"
export GAMMA_RASTER="${GAMMA_RASTER:-BMP}"
export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}"
_pyint_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
_pyint_script_dir="${_pyint_repo_root}/third_party/PyINT/pyint"
_pyint_gamma_prepend_path "${_pyint_script_dir}"
unset _pyint_gamma_home
unset _gamma_dir
unset _pyint_repo_root
unset _pyint_script_dir
unset -f _pyint_gamma_prepend_path
unset -f _pyint_gamma_die
return 0 2>/dev/null || exit 0
backend/app/routers/dinsar_production.py
+48 -6
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
import asyncio
from typing import Any, Dict, Optional
from typing import Any, Dict, Literal, Optional
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field
@@ -43,6 +43,10 @@ class RunJobRequest(BaseModel):
profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap / lt1_gamma_dinsar")
root_dir: str = Field(..., description="Windows root directory")
num_to_process: int = Field(default=0, ge=0, description="How many tasks to process; 0 means all")
rerun_mode: Literal["unfinished_only", "rerun_all"] = Field(
default="unfinished_only",
description="unfinished_only = only run unfinished tasks; rerun_all = rerun everything",
)
timeout_seconds: Optional[int] = Field(default=None, ge=60)
extra: Dict[str, Any] = Field(default_factory=dict, description="Engine-specific parameters")
@@ -178,9 +182,23 @@ async def submit_run(
engine.validate_root_dir,
req.root_dir,
req.num_to_process,
req.rerun_mode,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
if int(validation_summary.get("task_count", 0) or 0) <= 0:
if (
req.rerun_mode == "unfinished_only"
and int(validation_summary.get("skipped_completed_count", 0) or 0) > 0
):
raise HTTPException(
status_code=400,
detail=(
f"All discovered Task_* directories already have completed "
f"{req.engine_code}/{req.profile} results under: {req.root_dir}"
),
)
raise HTTPException(status_code=400, detail="No valid task directories selected.")
effective_timeout_seconds = req.timeout_seconds
if effective_timeout_seconds is None:
@@ -189,7 +207,13 @@ async def submit_run(
effective_timeout_seconds = int(engine_default_timeout)
pyint_preview = None
if req.engine_code == "pyint":
skip_pyint_submit_preview = (
req.engine_code == "pyint"
and req.rerun_mode == "unfinished_only"
and validation_summary is not None
and int(validation_summary.get("skipped_completed_count", 0) or 0) > 0
)
if req.engine_code == "pyint" and not skip_pyint_submit_preview:
try:
pyint_preview = await asyncio.to_thread(
build_pyint_input_preview,
@@ -216,36 +240,43 @@ async def submit_run(
"profile": req.profile,
"root_dir": req.root_dir,
"num_to_process": req.num_to_process,
"rerun_mode": req.rerun_mode,
"timeout_seconds": effective_timeout_seconds,
"extra": dict(req.extra or {}),
}
from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN, JOB_TYPE_PYINT_RUN
create_managed_run = False
normalized_extra = dict(payload["extra"])
if req.engine_code == "sarscape":
payload["mode"] = "custom" if req.profile == "custom6" else "metatask"
job_type = JOB_TYPE_IDL_RUN_DINSAR
max_attempts = DINSAR_PRODUCTION_JOB_MAX_ATTEMPTS
create_managed_run = True
elif req.engine_code in {"isce2", "pyint"}:
if hasattr(engine, "normalize_extra"):
try:
payload["extra"] = engine.normalize_extra(payload["extra"])
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
normalized_extra = dict(payload["extra"])
if req.engine_code == "isce2":
job_type = JOB_TYPE_ISCE2_RUN
max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS
create_managed_run = True
else:
job_type = JOB_TYPE_PYINT_RUN
max_attempts = PYINT_PRODUCTION_JOB_MAX_ATTEMPTS
if validation_summary is not None:
validated_task_count = validation_summary.get("task_count", 0)
if pyint_preview is not None:
validated_task_count = int(pyint_preview.get("selected_task_count", validated_task_count) or 0)
payload["extra"].update(
{
"__validated_task_count": validated_task_count,
"__validated_mode": validation_summary.get("mode", ""),
"__rerun_mode": req.rerun_mode,
"__discovered_task_count": int(validation_summary.get("discovered_task_count", validated_task_count) or 0),
"__skipped_completed_count": int(validation_summary.get("skipped_completed_count", 0) or 0),
}
)
else:
@@ -255,15 +286,16 @@ async def submit_run(
)
try:
if req.engine_code == "sarscape":
if create_managed_run:
async with _new_session() as db:
result = await dinsar_production_service.create_run(
engine_code=req.engine_code,
profile_code=req.profile,
root_dir=req.root_dir,
num_to_process=req.num_to_process,
rerun_mode=req.rerun_mode,
timeout_seconds=req.timeout_seconds,
extra=req.extra,
extra=normalized_extra,
created_by=getattr(current_user, "username", None),
db=db,
)
@@ -276,6 +308,9 @@ async def submit_run(
"engine_code": req.engine_code,
"profile": req.profile,
"selected_task_count": result.get("selected_task_count", 0),
"discovered_task_count": result.get("discovered_task_count", result.get("selected_task_count", 0)),
"skipped_completed_count": result.get("skipped_completed_count", 0),
"rerun_mode": result.get("rerun_mode", req.rerun_mode),
"message": "Task queued.",
}
@@ -303,6 +338,13 @@ async def submit_run(
"engine_code": req.engine_code,
"profile": req.profile,
"selected_task_count": validation_summary.get("task_count", 1) if validation_summary else 1,
"discovered_task_count": (
validation_summary.get("discovered_task_count", validation_summary.get("task_count", 1))
if validation_summary
else 1
),
"skipped_completed_count": validation_summary.get("skipped_completed_count", 0) if validation_summary else 0,
"rerun_mode": req.rerun_mode,
"message": "Task queued.",
}
backend/app/services/dinsar_layout_migration_service.py
+517
View File
@@ -0,0 +1,517 @@
from __future__ import annotations
import json
import os
import re
from typing import Any, Dict, Iterable, List, Optional
from ..config import settings
from .dinsar_naming import RUN_META_FILENAME
from .dinsar_result_layout_service import (
EXECUTION_MANIFEST_FILENAME,
PACKAGE_MANIFEST_FILENAME,
get_run_disp_asset_paths,
get_run_native_output_dir,
normalize_envi_run_layout,
)
_ENVI_ENGINE_CODES = {"envi", "sarscape"}
_LEGACY_DISP_RE = re.compile(r"^.+_rsp_disp$", re.IGNORECASE)
def _normalize_path(path: str) -> str:
return os.path.normpath(os.path.abspath(str(path or "").strip()))
def _load_json(path: str) -> Optional[Dict[str, Any]]:
try:
with open(path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
return payload if isinstance(payload, dict) else None
except Exception:
return None
def _iter_managed_run_dirs(root_dir: str) -> Iterable[str]:
normalized_root = _normalize_path(root_dir)
if not os.path.isdir(normalized_root):
return
for pair_name in sorted(os.listdir(normalized_root)):
pair_dir = os.path.join(normalized_root, pair_name)
if not os.path.isdir(pair_dir):
continue
runs_dir = os.path.join(pair_dir, "runs")
if not os.path.isdir(runs_dir):
continue
for run_name in sorted(os.listdir(runs_dir)):
run_dir = os.path.join(runs_dir, run_name)
if os.path.isdir(run_dir):
yield run_dir
def _read_run_payloads(run_dir: str) -> Dict[str, Dict[str, Any]]:
normalized_run_dir = _normalize_path(run_dir)
payloads: Dict[str, Dict[str, Any]] = {}
execution_manifest = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME))
if execution_manifest:
payloads["execution"] = execution_manifest
run_meta = _load_json(os.path.join(normalized_run_dir, RUN_META_FILENAME))
if run_meta:
payloads["run_meta"] = run_meta
package_manifest = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME))
if package_manifest:
payloads["package"] = package_manifest
return payloads
def _first_text(*values: Any) -> Optional[str]:
for value in values:
text = str(value or "").strip()
if text:
return text
return None
def _resolve_engine_code(payloads: Dict[str, Dict[str, Any]]) -> str:
package_payload = payloads.get("package") or {}
package_engine = package_payload.get("engine") if isinstance(package_payload.get("engine"), dict) else {}
return (
_first_text(
(payloads.get("execution") or {}).get("engine_code"),
(payloads.get("run_meta") or {}).get("engine_code"),
package_engine.get("code"),
package_payload.get("engine_code"),
)
or ""
).strip().lower()
def _resolve_pair_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]:
package_payload = payloads.get("package") or {}
package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {}
return _first_text(
(payloads.get("execution") or {}).get("pair_key"),
(payloads.get("run_meta") or {}).get("pair_key"),
package_identity.get("pair_key"),
package_payload.get("pair_key"),
os.path.basename(os.path.dirname(os.path.dirname(_normalize_path(run_dir)))),
)
def _resolve_run_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]:
package_payload = payloads.get("package") or {}
package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {}
return _first_text(
(payloads.get("execution") or {}).get("run_key"),
(payloads.get("run_meta") or {}).get("run_key"),
package_identity.get("run_key"),
package_payload.get("run_key"),
os.path.basename(_normalize_path(run_dir)),
)
def _relpath_lower(base_dir: str, path: str) -> str:
return os.path.relpath(_normalize_path(path), _normalize_path(base_dir)).replace("/", os.sep).lower()
def _is_standard_disp_path(run_dir: str, path: str) -> bool:
disp_paths = get_run_disp_asset_paths(run_dir)
normalized_path = _normalize_path(path)
return normalized_path in {
_normalize_path(disp_paths["primary"]),
_normalize_path(disp_paths["hdr"]),
_normalize_path(disp_paths["sml"]),
}
def _looks_like_legacy_primary(path: str) -> bool:
return bool(_LEGACY_DISP_RE.match(os.path.basename(str(path or "").strip())))
def _looks_like_envi_primary(run_dir: str, path: str) -> bool:
normalized_path = _normalize_path(path)
if not os.path.isfile(normalized_path):
return False
if _is_standard_disp_path(run_dir, normalized_path):
return True
return _looks_like_legacy_primary(normalized_path)
def _build_source_files(primary_file: str, source_files: Optional[List[str]] = None) -> List[str]:
normalized_primary = _normalize_path(primary_file)
candidates: List[str] = [normalized_primary]
for raw_path in source_files or []:
normalized = _normalize_path(raw_path)
if normalized and normalized not in candidates and os.path.isfile(normalized):
candidates.append(normalized)
for ext in (".hdr", ".sml"):
sidecar = normalized_primary + ext
if os.path.isfile(sidecar) and sidecar not in candidates:
candidates.append(sidecar)
return candidates
def _find_standardized_run_files(run_dir: str) -> Optional[Dict[str, Any]]:
disp_paths = get_run_disp_asset_paths(run_dir)
primary_file = disp_paths["primary"]
if not os.path.isfile(primary_file):
return None
return {
"primary_file": _normalize_path(primary_file),
"source_files": _build_source_files(primary_file),
"discovery": "standard_asset",
}
def _iter_metadata_candidates(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
execution_payload = payloads.get("execution") or {}
if execution_payload:
primary_file = execution_payload.get("primary_file")
source_files = execution_payload.get("source_files") if isinstance(execution_payload.get("source_files"), list) else None
if primary_file:
yield {
"primary_file": primary_file,
"source_files": source_files,
"discovery": "execution_manifest",
}
package_payload = payloads.get("package") or {}
source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {}
if source_payload:
primary_file = source_payload.get("primary_path")
if primary_file:
yield {
"primary_file": primary_file,
"source_files": None,
"discovery": "package_manifest",
}
def _find_latest_legacy_primary(search_dir: str) -> Optional[str]:
normalized_search_dir = _normalize_path(search_dir)
if not os.path.isdir(normalized_search_dir):
return None
candidates: List[str] = []
try:
with os.scandir(normalized_search_dir) as entries:
for entry in entries:
try:
if entry.is_file(follow_symlinks=False) and _looks_like_legacy_primary(entry.name):
candidates.append(entry.path)
except OSError:
continue
except OSError:
return None
if not candidates:
return None
candidates.sort(key=lambda item: os.path.getmtime(item), reverse=True)
return _normalize_path(candidates[0])
def _discover_envi_run_files(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
standardized = _find_standardized_run_files(run_dir)
if standardized:
return standardized
for candidate in _iter_metadata_candidates(run_dir, payloads):
primary_file = _normalize_path(str(candidate.get("primary_file") or "").strip())
if not _looks_like_envi_primary(run_dir, primary_file):
continue
return {
"primary_file": primary_file,
"source_files": _build_source_files(primary_file, candidate.get("source_files")),
"discovery": candidate["discovery"],
}
search_dirs = [
_normalize_path(run_dir),
get_run_native_output_dir(run_dir),
]
for search_dir in search_dirs:
primary_file = _find_latest_legacy_primary(search_dir)
if primary_file:
return {
"primary_file": primary_file,
"source_files": _build_source_files(primary_file),
"discovery": "filesystem_scan",
}
return None
def validate_envi_run_layout(run_dir: str) -> Dict[str, Any]:
normalized_run_dir = _normalize_path(run_dir)
native_output_dir = get_run_native_output_dir(normalized_run_dir)
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
issues: List[str] = []
if not os.path.isdir(normalized_run_dir):
issues.append("run_dir_missing")
if not os.path.isdir(native_output_dir):
issues.append("native_dir_missing")
if not os.path.isfile(disp_paths["primary"]):
issues.append("standard_disp_missing")
execution_payload = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)) or {}
if execution_payload:
if _normalize_path(str(execution_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]):
issues.append("execution_manifest_primary_mismatch")
if _normalize_path(str(execution_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
issues.append("execution_manifest_native_mismatch")
package_payload = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME)) or {}
if package_payload:
source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {}
if _normalize_path(str(source_payload.get("primary_path") or "")) != _normalize_path(disp_paths["primary"]):
issues.append("package_manifest_primary_mismatch")
if _normalize_path(str(source_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
issues.append("package_manifest_native_mismatch")
assets_payload = package_payload.get("assets") if isinstance(package_payload.get("assets"), list) else []
disp_asset = next(
(item for item in assets_payload if str((item or {}).get("role") or "").strip() == "disp"),
None,
)
expected_relative = _relpath_lower(normalized_run_dir, disp_paths["primary"])
actual_relative = _relpath_lower(
normalized_run_dir,
os.path.join(normalized_run_dir, str((disp_asset or {}).get("relative_path") or "")),
) if disp_asset else ""
if not disp_asset or actual_relative != expected_relative:
issues.append("package_manifest_disp_asset_mismatch")
run_key = _resolve_run_key(normalized_run_dir, _read_run_payloads(normalized_run_dir))
current_dir = os.path.join(os.path.dirname(os.path.dirname(normalized_run_dir)), "current")
matched_pointer_count = 0
if run_key and os.path.isdir(current_dir):
for name in os.listdir(current_dir):
if not name.lower().endswith(".json"):
continue
pointer_payload = _load_json(os.path.join(current_dir, name))
if not pointer_payload:
continue
if str(pointer_payload.get("run_key") or "").strip() != run_key:
continue
matched_pointer_count += 1
if _normalize_path(str(pointer_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]):
issues.append(f"current_pointer_primary_mismatch:{name}")
if _normalize_path(str(pointer_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
issues.append(f"current_pointer_native_mismatch:{name}")
return {
"ok": len(issues) == 0,
"issues": issues,
"matched_current_pointer_count": matched_pointer_count,
}
def inspect_envi_run_layout(run_dir: str) -> Dict[str, Any]:
normalized_run_dir = _normalize_path(run_dir)
payloads = _read_run_payloads(normalized_run_dir)
engine_code = _resolve_engine_code(payloads)
files = _discover_envi_run_files(normalized_run_dir, payloads)
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
standardized = bool(os.path.isfile(disp_paths["primary"]))
native_exists = bool(os.path.isdir(get_run_native_output_dir(normalized_run_dir)))
is_envi_run = bool(files) or engine_code in _ENVI_ENGINE_CODES
layout_state = "unknown"
if not is_envi_run:
layout_state = "not_envi"
elif standardized and native_exists:
layout_state = "normalized"
elif files and _looks_like_legacy_primary(files["primary_file"]):
layout_state = "legacy"
elif standardized:
layout_state = "partial_normalized"
validation = validate_envi_run_layout(normalized_run_dir) if is_envi_run else {"ok": True, "issues": [], "matched_current_pointer_count": 0}
needs_migration = bool(
is_envi_run
and (
layout_state in {"legacy", "partial_normalized"}
or (layout_state == "normalized" and not validation["ok"])
)
)
return {
"run_dir": normalized_run_dir,
"pair_key": _resolve_pair_key(normalized_run_dir, payloads),
"run_key": _resolve_run_key(normalized_run_dir, payloads),
"engine_code": engine_code,
"is_envi_run": is_envi_run,
"layout_state": layout_state,
"needs_migration": needs_migration,
"primary_file": (files or {}).get("primary_file"),
"source_files": (files or {}).get("source_files") or [],
"discovery": (files or {}).get("discovery"),
"standard_disp_exists": standardized,
"native_dir_exists": native_exists,
"validation": validation,
}
class DinsarLayoutMigrationService:
def inspect_managed_envi_runs(
self,
*,
root_dir: Optional[str] = None,
pair_keys: Optional[List[str]] = None,
limit: Optional[int] = None,
) -> Dict[str, Any]:
normalized_root = _normalize_path(root_dir or settings.DINSAR_PRODUCT_DIR)
filter_pair_keys = {
str(item or "").strip()
for item in pair_keys or []
if str(item or "").strip()
}
details: List[Dict[str, Any]] = []
inspected = 0
envi_run_count = 0
legacy_count = 0
normalized_count = 0
pending_count = 0
for run_dir in _iter_managed_run_dirs(normalized_root):
inspection = inspect_envi_run_layout(run_dir)
if filter_pair_keys and inspection.get("pair_key") not in filter_pair_keys:
continue
inspected += 1
if inspection["is_envi_run"]:
envi_run_count += 1
if inspection["layout_state"] == "legacy":
legacy_count += 1
if inspection["layout_state"] == "normalized":
normalized_count += 1
if inspection["needs_migration"]:
pending_count += 1
details.append(inspection)
if limit is not None and len(details) >= int(limit):
break
return {
"root_dir": normalized_root,
"inspected_run_count": inspected,
"envi_run_count": envi_run_count,
"legacy_run_count": legacy_count,
"normalized_run_count": normalized_count,
"pending_migration_count": pending_count,
"details": details,
}
def migrate_managed_envi_runs(
self,
*,
root_dir: Optional[str] = None,
pair_keys: Optional[List[str]] = None,
limit: Optional[int] = None,
dry_run: bool = False,
) -> Dict[str, Any]:
inspection = self.inspect_managed_envi_runs(
root_dir=root_dir,
pair_keys=pair_keys,
limit=limit,
)
details: List[Dict[str, Any]] = []
migrated_count = 0
rewritten_count = 0
skipped_count = 0
failed_count = 0
for item in inspection["details"]:
if not item["is_envi_run"]:
skipped_count += 1
details.append(
{
**item,
"status": "skipped_not_envi",
}
)
continue
if not item["primary_file"]:
failed_count += 1
details.append(
{
**item,
"status": "failed_no_primary",
"error": "Unable to resolve ENVI primary displacement file.",
}
)
continue
if not item["needs_migration"]:
skipped_count += 1
details.append(
{
**item,
"status": "skipped_already_normalized",
}
)
continue
if dry_run:
details.append(
{
**item,
"status": "would_migrate",
}
)
continue
try:
result = normalize_envi_run_layout(
item["run_dir"],
primary_file=item["primary_file"],
source_files=item["source_files"],
rewrite_metadata=True,
)
validation = validate_envi_run_layout(item["run_dir"])
status = "migrated"
if not result["promoted_files"] and not result["moved_entries"]:
status = "rewritten"
if validation["ok"]:
if status == "rewritten":
rewritten_count += 1
else:
migrated_count += 1
else:
status = "failed_validation"
failed_count += 1
details.append(
{
**item,
**result,
"status": status,
"post_validation": validation,
}
)
except Exception as exc:
failed_count += 1
details.append(
{
**item,
"status": "failed",
"error": str(exc),
}
)
return {
"root_dir": inspection["root_dir"],
"dry_run": bool(dry_run),
"inspected_run_count": inspection["inspected_run_count"],
"envi_run_count": inspection["envi_run_count"],
"pending_migration_count": inspection["pending_migration_count"],
"migrated_count": migrated_count,
"rewritten_count": rewritten_count,
"skipped_count": skipped_count,
"failed_count": failed_count,
"details": details,
}
dinsar_layout_migration_service = DinsarLayoutMigrationService()
backend/app/services/dinsar_production_service.py
+244 -18
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
import os
import re
import subprocess
import uuid
from datetime import datetime
@@ -12,6 +13,7 @@ from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..config import settings
from ..models import (
DinsarProductionExecutionORM,
DinsarProductionRunItemORM,
@@ -24,6 +26,7 @@ from .workflow_service import workflow_service
TASK_TYPE_DINSAR_PRODUCTION = "IDL_RUN_DINSAR"
TASK_TYPE_ISCE2_DINSAR_PRODUCTION = "ISCE2_RUN"
RUN_STATUS_PENDING = "PENDING"
RUN_STATUS_RUNNING = "RUNNING"
RUN_STATUS_COMPLETED = "COMPLETED"
@@ -40,8 +43,15 @@ EXECUTION_STATUS_RUNNING = "RUNNING"
EXECUTION_STATUS_COMPLETED = "COMPLETED"
EXECUTION_STATUS_FAILED = "FAILED"
EXECUTION_STATUS_CANCELLED = "CANCELLED"
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
RERUN_MODE_RERUN_ALL = "rerun_all"
VALID_RERUN_MODES = {
RERUN_MODE_UNFINISHED_ONLY,
RERUN_MODE_RERUN_ALL,
}
CURRENT_POINTER_FILENAME = "current.json"
CURRENT_POINTER_DIRNAME = "current"
EXECUTION_MANIFEST_FILENAME = "execution_manifest.json"
RUNS_STEP_ID = "execute_items"
RUNS_STEP_NAME = "Execute ENVI D-InSAR items"
@@ -56,6 +66,34 @@ TERMINAL_ITEM_STATUSES = {
RUN_ITEM_STATUS_SKIPPED,
RUN_ITEM_STATUS_CANCELLED,
}
_SAFE_POINTER_RE = re.compile(r"[^0-9A-Za-z._-]+")
def _task_type_for_engine(engine_code: str) -> str:
normalized = str(engine_code or "").strip().lower()
if normalized == "sarscape":
return TASK_TYPE_DINSAR_PRODUCTION
if normalized == "isce2":
return TASK_TYPE_ISCE2_DINSAR_PRODUCTION
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
def _workflow_name_for_engine(engine_code: str) -> str:
normalized = str(engine_code or "").strip().lower()
if normalized == "sarscape":
return "dinsar_sarscape_production"
if normalized == "isce2":
return "dinsar_isce2_production"
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
def _workflow_step_name_for_engine(engine_code: str) -> str:
normalized = str(engine_code or "").strip().lower()
if normalized == "sarscape":
return RUNS_STEP_NAME
if normalized == "isce2":
return "Execute ISCE2 D-InSAR items"
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
def _new_session() -> AsyncSession:
@@ -100,10 +138,15 @@ def _looks_like_task_dir(path: str) -> bool:
return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave"))
def _discover_run_items(root_dir: str, num_to_process: int) -> List[Dict[str, Any]]:
def _normalize_rerun_mode(value: Optional[str]) -> str:
normalized = str(value or "").strip().lower()
if normalized in VALID_RERUN_MODES:
return normalized
return RERUN_MODE_UNFINISHED_ONLY
def _discover_run_items(root_dir: str) -> List[Dict[str, Any]]:
task_folders = [root_dir] if _looks_like_task_dir(root_dir) else _collect_task_folders(root_dir)
if num_to_process > 0:
task_folders = task_folders[:num_to_process]
items: List[Dict[str, Any]] = []
for order_index, folder in enumerate(task_folders, start=1):
@@ -121,12 +164,119 @@ def _discover_run_items(root_dir: str, num_to_process: int) -> List[Dict[str, An
"policy_version": pair_meta.get("policy_version"),
"selection_strategy": pair_meta.get("selection_strategy"),
"source_task_dir": folder,
"results_root_dir": os.path.join(folder, "dinsar_results"),
"results_root_dir": os.path.join(settings.DINSAR_PRODUCT_DIR, pair_key),
}
)
return items
def _current_pointer_path_for_root(
results_root_dir: str,
*,
engine_code: Optional[str] = None,
profile_code: Optional[str] = None,
) -> str:
pointer_dir = os.path.join(results_root_dir, CURRENT_POINTER_DIRNAME)
if engine_code or profile_code:
pointer_name = (
f"{_sanitize_pointer_fragment(engine_code or 'engine', 'engine')}__"
f"{_sanitize_pointer_fragment(profile_code or 'profile', 'profile')}.json"
)
return os.path.join(pointer_dir, pointer_name)
return os.path.join(pointer_dir, CURRENT_POINTER_FILENAME)
def _normalize_existing_file(path: Any) -> str:
text = str(path or "").strip()
if not text:
return ""
normalized = os.path.normpath(os.path.abspath(_to_local_path(text)))
return normalized if os.path.isfile(normalized) else ""
def _normalize_existing_dir(path: Any) -> str:
text = str(path or "").strip()
if not text:
return ""
normalized = os.path.normpath(os.path.abspath(_to_local_path(text)))
return normalized if os.path.isdir(normalized) else ""
def _has_completed_current_result(
item_payload: Dict[str, Any],
*,
engine_code: str,
profile_code: str,
) -> bool:
pointer_path = _current_pointer_path_for_root(
str(item_payload.get("results_root_dir") or ""),
engine_code=engine_code,
profile_code=profile_code,
)
if not os.path.isfile(pointer_path):
return False
try:
with open(pointer_path, "r", encoding="utf-8") as fp:
payload = json.load(fp) or {}
except Exception:
return False
if str(payload.get("status") or "").strip().upper() != EXECUTION_STATUS_COMPLETED:
return False
manifest_path = _normalize_existing_file(payload.get("manifest_path"))
if manifest_path:
return True
output_dir = _normalize_existing_dir(payload.get("output_dir"))
if output_dir and os.path.isfile(_execution_manifest_path(output_dir)):
return True
return False
def _select_run_items(
root_dir: str,
*,
engine_code: str,
profile_code: str,
num_to_process: int,
rerun_mode: Optional[str],
) -> Dict[str, Any]:
discovered_items = _discover_run_items(root_dir)
normalized_mode = _normalize_rerun_mode(rerun_mode)
skipped_completed_count = 0
selected_items: List[Dict[str, Any]] = []
for item_payload in discovered_items:
if (
normalized_mode == RERUN_MODE_UNFINISHED_ONLY
and _has_completed_current_result(
item_payload,
engine_code=engine_code,
profile_code=profile_code,
)
):
skipped_completed_count += 1
continue
selected_items.append(dict(item_payload))
if num_to_process > 0:
selected_items = selected_items[:num_to_process]
for order_index, item_payload in enumerate(selected_items, start=1):
item_payload["order_index"] = order_index
return {
"items": selected_items,
"rerun_mode": normalized_mode,
"discovered_task_count": len(discovered_items),
"skipped_completed_count": skipped_completed_count,
"selected_task_count": len(selected_items),
}
def _run_log_path(run_id: str) -> str:
return os.path.join(RUNTIME_DIR, f"{run_id}.log")
@@ -179,8 +329,22 @@ def _execution_dir(item: DinsarProductionRunItemORM, run_key: str) -> str:
return os.path.join(item.results_root_dir, "runs", run_key)
def _current_pointer_path(item: DinsarProductionRunItemORM) -> str:
return os.path.join(item.results_root_dir, CURRENT_POINTER_FILENAME)
def _sanitize_pointer_fragment(value: str, default: str) -> str:
text = _SAFE_POINTER_RE.sub("_", str(value or "").strip()).strip("._")
return text or default
def _current_pointer_path(
item: DinsarProductionRunItemORM,
*,
engine_code: Optional[str] = None,
profile_code: Optional[str] = None,
) -> str:
return _current_pointer_path_for_root(
item.results_root_dir,
engine_code=engine_code,
profile_code=profile_code,
)
def _execution_manifest_path(execution_dir: str) -> str:
@@ -193,6 +357,15 @@ def _safe_epoch(value: Optional[datetime]) -> Optional[int]:
return int(value.timestamp())
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized == "isce2":
return settings.ISCE2_RUNTIME_ID or None
if normalized in {"pyint", "gamma"}:
return settings.PYINT_RUNTIME_ID or None
return None
def _public_run_status(value: str) -> str:
normalized = str(value or "").strip().upper()
if normalized == RUN_STATUS_COMPLETED:
@@ -249,6 +422,7 @@ class DinsarProductionService:
profile_code: str,
root_dir: str,
num_to_process: int,
rerun_mode: Optional[str],
timeout_seconds: Optional[int],
extra: Optional[Dict[str, Any]],
created_by: Optional[str],
@@ -256,26 +430,44 @@ class DinsarProductionService:
) -> Dict[str, Any]:
normalized_engine = str(engine_code or "").strip().lower()
normalized_profile = str(profile_code or "").strip()
if normalized_engine != "sarscape":
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
task_type = _task_type_for_engine(normalized_engine)
workflow_name = _workflow_name_for_engine(normalized_engine)
workflow_step_name = _workflow_step_name_for_engine(normalized_engine)
normalized_root = _normalize_dir(root_dir, "root_dir")
item_payloads = await asyncio.to_thread(
_discover_run_items,
selection = await asyncio.to_thread(
_select_run_items,
normalized_root,
max(0, int(num_to_process or 0)),
engine_code=normalized_engine,
profile_code=normalized_profile,
num_to_process=max(0, int(num_to_process or 0)),
rerun_mode=rerun_mode,
)
item_payloads = selection["items"]
if not item_payloads:
if (
selection["discovered_task_count"] > 0
and selection["skipped_completed_count"] > 0
and selection["rerun_mode"] == RERUN_MODE_UNFINISHED_ONLY
):
raise ValueError(
f"All discovered Task_* directories already have completed "
f"{normalized_engine}/{normalized_profile} results under: {normalized_root}"
)
raise ValueError(f"No Task_* directories found under: {normalized_root}")
run_id = str(uuid.uuid4())
mode = "custom" if normalized_profile == "custom6" else "metatask"
if normalized_engine == "sarscape":
mode = "custom" if normalized_profile == "custom6" else "metatask"
else:
mode = "managed"
task_name = f"D-InSAR production: {normalized_engine}/{normalized_profile}"
task_params = {
"engine_code": normalized_engine,
"profile": normalized_profile,
"root_dir": normalized_root,
"num_to_process": int(num_to_process or 0),
"rerun_mode": selection["rerun_mode"],
"timeout_seconds": timeout_seconds,
"extra": dict(extra or {}),
"mode": mode,
@@ -285,7 +477,7 @@ class DinsarProductionService:
task_id: Optional[str] = None
try:
task_id = await task_service.create_task(
task_type=TASK_TYPE_DINSAR_PRODUCTION,
task_type=task_type,
task_name=task_name,
params=task_params,
db=db,
@@ -294,10 +486,12 @@ class DinsarProductionService:
run = DinsarProductionRunORM(
run_id=run_id,
task_id=task_id,
product_family="dinsar",
engine_code=normalized_engine,
profile_code=normalized_profile,
mode=mode,
source_root=normalized_root,
publish_root_dir=settings.DINSAR_PRODUCT_DIR,
status=RUN_STATUS_PENDING,
cancel_requested=False,
total_items=len(item_payloads),
@@ -309,6 +503,11 @@ class DinsarProductionService:
summary_json={
"phase": "queued",
"selected_task_count": len(item_payloads),
"discovered_task_count": selection["discovered_task_count"],
"skipped_completed_count": selection["skipped_completed_count"],
"rerun_mode": selection["rerun_mode"],
"product_family": "dinsar",
"publish_root_dir": settings.DINSAR_PRODUCT_DIR,
},
created_by=created_by,
)
@@ -337,12 +536,12 @@ class DinsarProductionService:
await db.flush()
workflow_run_id = await workflow_service.create_run(
workflow_name="dinsar_sarscape_production",
workflow_name=workflow_name,
steps=[
{
"step_id": RUNS_STEP_ID,
"step_name": RUNS_STEP_NAME,
"job_type": TASK_TYPE_DINSAR_PRODUCTION,
"step_name": workflow_step_name,
"job_type": task_type,
"payload": {"production_run_id": run_id},
"task_id": task_id,
"max_attempts": 1,
@@ -381,7 +580,11 @@ class DinsarProductionService:
await asyncio.to_thread(
_append_run_log_sync,
run_id,
f"[queued] run_id={run_id} profile={normalized_profile} root={normalized_root} items={len(item_payloads)}",
(
f"[queued] run_id={run_id} profile={normalized_profile} root={normalized_root} "
f"items={len(item_payloads)} rerun_mode={selection['rerun_mode']} "
f"skipped_completed={selection['skipped_completed_count']}"
),
)
return {
"run_id": run_id,
@@ -389,6 +592,9 @@ class DinsarProductionService:
"workflow_run_id": run.workflow_run_id,
"status": run.status,
"selected_task_count": len(item_payloads),
"discovered_task_count": selection["discovered_task_count"],
"skipped_completed_count": selection["skipped_completed_count"],
"rerun_mode": selection["rerun_mode"],
}
async def list_runs(
@@ -434,6 +640,7 @@ class DinsarProductionService:
"runs": [
{
"run_id": run.run_id,
"product_family": run.product_family,
"engine": run.engine_code,
"profile_code": run.profile_code,
"status": _public_run_status(run.status),
@@ -443,6 +650,7 @@ class DinsarProductionService:
"task_id": run.task_id,
"workflow_run_id": run.workflow_run_id,
"root_dir": run.source_root,
"publish_root_dir": run.publish_root_dir,
"message": run.latest_message,
"total_items": run.total_items,
"completed_items": run.completed_items,
@@ -684,15 +892,18 @@ class DinsarProductionService:
execution: DinsarProductionExecutionORM,
primary_file: str,
source_files: List[str],
native_output_dir: Optional[str],
metrics: Optional[Dict[str, Any]],
) -> str:
manifest_payload = {
"format_version": 1,
"run_id": run.run_id,
"product_family": run.product_family or "dinsar",
"run_key": execution.run_key,
"task_id": run.task_id,
"engine_code": run.engine_code,
"profile_code": run.profile_code,
"runtime_id": _runtime_id_for_engine(run.engine_code),
"mode": run.mode,
"task_name": item.task_name,
"task_alias": item.task_alias,
@@ -704,7 +915,10 @@ class DinsarProductionService:
"selection_strategy": item.selection_strategy,
"source_root": run.source_root,
"source_task_dir": item.source_task_dir,
"results_root_dir": item.results_root_dir,
"publish_root_dir": run.publish_root_dir,
"output_dir": execution.output_dir,
"native_output_dir": str(native_output_dir or execution.output_dir),
"primary_file": primary_file,
"source_files": source_files,
"status": EXECUTION_STATUS_COMPLETED,
@@ -718,24 +932,36 @@ class DinsarProductionService:
def write_current_pointer(
self,
*,
run: DinsarProductionRunORM,
item: DinsarProductionRunItemORM,
execution: DinsarProductionExecutionORM,
manifest_path: str,
primary_file: str,
source_files: List[str],
native_output_dir: Optional[str],
) -> str:
pointer_payload = {
"format_version": 1,
"product_family": run.product_family or "dinsar",
"engine_code": run.engine_code,
"profile_code": run.profile_code,
"runtime_id": _runtime_id_for_engine(run.engine_code),
"run_key": execution.run_key,
"execution_id": execution.execution_id,
"status": EXECUTION_STATUS_COMPLETED,
"output_dir": execution.output_dir,
"native_output_dir": str(native_output_dir or execution.output_dir),
"manifest_path": manifest_path,
"primary_file": primary_file,
"source_files": source_files,
"updated_at": _utc_text(),
}
return _write_json(_current_pointer_path(item), pointer_payload)
pointer_path = _current_pointer_path(
item,
engine_code=run.engine_code,
profile_code=run.profile_code,
)
return _write_json(pointer_path, pointer_payload)
async def get_active_execution_by_task_id(
self,
backend/app/services/dinsar_result_layout_service.py
+562
View File
@@ -0,0 +1,562 @@
from __future__ import annotations
import json
import os
import shutil
from typing import Any, Dict, List, Optional
from .dinsar_naming import RUN_META_FILENAME
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
RUN_NATIVE_DIRNAME = "native"
RUN_ASSETS_DIRNAME = "assets"
RUN_PREVIEW_DIRNAME = "preview"
RUN_CURRENT_DIRNAME = "current"
RUN_DISP_DIRNAME = "disp"
RUN_COH_DIRNAME = "coh"
EXECUTION_MANIFEST_FILENAME = "execution_manifest.json"
PACKAGE_MANIFEST_FILENAME = "manifest.json"
STANDARD_ENVI_DISP_BASENAME = "disp"
STANDARD_ISCE2_DISP_NAME = "disp.tif"
STANDARD_ISCE2_COH_NAME = "coh.tif"
_KEEP_RUN_ROOT_NAMES = {
RUN_NATIVE_DIRNAME,
RUN_ASSETS_DIRNAME,
RUN_PREVIEW_DIRNAME,
RUN_CURRENT_DIRNAME,
RUN_META_FILENAME,
EXECUTION_MANIFEST_FILENAME,
PACKAGE_MANIFEST_FILENAME,
}
_DISP_ASSET_ROLES = {"disp", "disp_header", "disp_sidecar"}
_ISCE2_ASSET_ROLES = {"disp", "coh"}
def _normalize_path(path: str) -> str:
return os.path.normpath(os.path.abspath(str(path or "").strip()))
def _load_json(path: str) -> Optional[Dict[str, Any]]:
try:
with open(path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
return payload if isinstance(payload, dict) else None
except Exception:
return None
def _write_json(path: str, payload: Dict[str, Any]) -> str:
target = _normalize_path(path)
os.makedirs(os.path.dirname(target), exist_ok=True)
with open(target, "w", encoding="utf-8") as fp:
json.dump(payload, fp, ensure_ascii=False, indent=2)
return target
def get_run_native_output_dir(run_dir: str) -> str:
return os.path.join(_normalize_path(run_dir), RUN_NATIVE_DIRNAME)
def get_run_disp_asset_base(run_dir: str) -> str:
return os.path.join(
_normalize_path(run_dir),
RUN_ASSETS_DIRNAME,
RUN_DISP_DIRNAME,
STANDARD_ENVI_DISP_BASENAME,
)
def get_run_disp_asset_paths(run_dir: str) -> Dict[str, str]:
base = get_run_disp_asset_base(run_dir)
return {
"primary": base,
"hdr": base + ".hdr",
"sml": base + ".sml",
}
def get_run_isce2_disp_asset_path(run_dir: str) -> str:
return os.path.join(
_normalize_path(run_dir),
RUN_ASSETS_DIRNAME,
RUN_DISP_DIRNAME,
STANDARD_ISCE2_DISP_NAME,
)
def get_run_isce2_coh_asset_path(run_dir: str) -> str:
return os.path.join(
_normalize_path(run_dir),
RUN_ASSETS_DIRNAME,
RUN_COH_DIRNAME,
STANDARD_ISCE2_COH_NAME,
)
def is_standard_envi_disp_file(run_root: str, file_path: str) -> bool:
run_root = _normalize_path(run_root)
file_path = _normalize_path(file_path)
rel_parts = [
part.lower()
for part in os.path.relpath(file_path, run_root).replace("/", os.sep).split(os.sep)
if part
]
if len(rel_parts) < 3:
return False
tail = rel_parts[-3:]
return tuple(tail) in {
("assets", "disp", "disp"),
("assets", "disp", "disp.hdr"),
("assets", "disp", "disp.sml"),
}
def is_standard_isce2_disp_file(run_root: str, file_path: str) -> bool:
run_root = _normalize_path(run_root)
file_path = _normalize_path(file_path)
rel_parts = [
part.lower()
for part in os.path.relpath(file_path, run_root).replace("/", os.sep).split(os.sep)
if part
]
if len(rel_parts) < 3:
return False
tail = rel_parts[-3:]
return tuple(tail) in {
("assets", "disp", "disp.tif"),
("assets", "disp", "disp.tiff"),
}
def is_path_within_native_dir(run_root: str, path: str) -> bool:
run_root = _normalize_path(run_root)
path = _normalize_path(path)
try:
rel_path = os.path.relpath(path, run_root)
except ValueError:
return False
parts = [part.lower() for part in rel_path.split(os.sep) if part]
return RUN_NATIVE_DIRNAME in parts
def _move_file(src: str, dst: str) -> bool:
src_path = _normalize_path(src)
dst_path = _normalize_path(dst)
if src_path == dst_path or not os.path.isfile(src_path):
return False
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
if os.path.exists(dst_path):
raise FileExistsError(f"Target already exists: {dst_path}")
os.replace(src_path, dst_path)
return True
def _move_entry(src: str, dst: str) -> bool:
src_path = _normalize_path(src)
dst_path = _normalize_path(dst)
if src_path == dst_path or not os.path.exists(src_path):
return False
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
if os.path.exists(dst_path):
raise FileExistsError(f"Target already exists: {dst_path}")
os.replace(src_path, dst_path)
return True
def _copy_file(src: str, dst: str) -> bool:
src_path = _normalize_path(src)
dst_path = _normalize_path(dst)
if src_path == dst_path or not os.path.isfile(src_path):
return False
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
shutil.copy2(src_path, dst_path)
return True
def _rewrite_run_metadata(run_dir: str, native_output_dir: str) -> None:
meta_path = os.path.join(_normalize_path(run_dir), RUN_META_FILENAME)
payload = _load_json(meta_path)
if not payload:
return
payload["output_dir"] = _normalize_path(run_dir)
payload["native_output_dir"] = _normalize_path(native_output_dir)
_write_json(meta_path, payload)
def _rewrite_execution_manifest(
run_dir: str,
*,
native_output_dir: str,
primary_file: str,
source_files: List[str],
) -> None:
manifest_path = os.path.join(_normalize_path(run_dir), EXECUTION_MANIFEST_FILENAME)
payload = _load_json(manifest_path)
if not payload:
return
payload["output_dir"] = _normalize_path(run_dir)
payload["native_output_dir"] = _normalize_path(native_output_dir)
payload["primary_file"] = _normalize_path(primary_file)
payload["source_files"] = [_normalize_path(path) for path in source_files]
_write_json(manifest_path, payload)
def _rewrite_current_pointers(
run_dir: str,
*,
native_output_dir: str,
primary_file: str,
source_files: List[str],
) -> None:
normalized_run_dir = _normalize_path(run_dir)
runs_dir = os.path.dirname(normalized_run_dir)
if os.path.basename(runs_dir).lower() != "runs":
return
pair_root = os.path.dirname(runs_dir)
current_dir = os.path.join(pair_root, RUN_CURRENT_DIRNAME)
if not os.path.isdir(current_dir):
return
execution_manifest_path = os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)
execution_manifest = _load_json(execution_manifest_path) or {}
run_key = str(execution_manifest.get("run_key") or "").strip()
if not run_key:
return
for name in os.listdir(current_dir):
if not name.lower().endswith(".json"):
continue
pointer_path = os.path.join(current_dir, name)
payload = _load_json(pointer_path)
if not payload:
continue
if str(payload.get("run_key") or "").strip() != run_key:
continue
payload["output_dir"] = normalized_run_dir
payload["native_output_dir"] = _normalize_path(native_output_dir)
payload["manifest_path"] = execution_manifest_path
payload["primary_file"] = _normalize_path(primary_file)
payload["source_files"] = [_normalize_path(path) for path in source_files]
_write_json(pointer_path, payload)
def _rewrite_package_manifest(
run_dir: str,
*,
native_output_dir: str,
primary_file: str,
source_files: List[str],
) -> None:
manifest_path = os.path.join(_normalize_path(run_dir), PACKAGE_MANIFEST_FILENAME)
payload = _load_json(manifest_path)
if not payload:
return
normalized_run_dir = _normalize_path(run_dir)
normalized_primary = _normalize_path(primary_file)
normalized_sources = [_normalize_path(path) for path in source_files]
primary_relative = os.path.relpath(normalized_primary, normalized_run_dir)
source_payload = payload.get("source") if isinstance(payload.get("source"), dict) else {}
source_payload["output_dir"] = normalized_run_dir
source_payload["native_output_dir"] = _normalize_path(native_output_dir)
source_payload["primary_path"] = normalized_primary
source_payload["source_dir"] = normalized_run_dir
source_payload["publish_dir"] = normalized_run_dir
payload["source"] = source_payload
summary_payload = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
summary_payload["primary_asset_relative"] = primary_relative
payload["summary"] = summary_payload
preview_relative = str(summary_payload.get("preview_relative") or "").strip()
existing_assets = payload.get("assets") if isinstance(payload.get("assets"), list) else []
preserved_assets = [
item
for item in existing_assets
if str((item or {}).get("role") or "").strip() not in _DISP_ASSET_ROLES
]
disp_assets: List[Dict[str, Any]] = [
{
"role": "disp",
"asset_name": os.path.basename(normalized_primary) or STANDARD_ENVI_DISP_BASENAME,
"relative_path": primary_relative,
"format": "envi",
"media_type": "application/octet-stream",
"is_required": True,
"is_primary": True,
}
]
if len(normalized_sources) > 1 and os.path.isfile(normalized_sources[1]):
disp_assets.append(
{
"role": "disp_header",
"asset_name": os.path.basename(normalized_sources[1]),
"relative_path": os.path.relpath(normalized_sources[1], normalized_run_dir),
"format": "hdr",
"media_type": "text/plain",
"is_required": True,
"is_primary": False,
}
)
if len(normalized_sources) > 2 and os.path.isfile(normalized_sources[2]):
disp_assets.append(
{
"role": "disp_sidecar",
"asset_name": os.path.basename(normalized_sources[2]),
"relative_path": os.path.relpath(normalized_sources[2], normalized_run_dir),
"format": "sml",
"media_type": "text/plain",
"is_required": False,
"is_primary": False,
}
)
payload["assets"] = disp_assets + preserved_assets
payload["native_output_dir"] = _normalize_path(native_output_dir)
canonical = build_canonical_descriptor(
payload["assets"],
product_family=str(payload.get("product_family") or "dinsar"),
)
if preview_relative:
canonical["preview_asset_relative"] = preview_relative
payload["canonical"] = canonical
normalized_payload = normalize_package_manifest(payload)
_write_json(manifest_path, normalized_payload)
def _rewrite_isce2_package_manifest(
run_dir: str,
*,
native_output_dir: str,
primary_file: str,
source_files: List[str],
) -> None:
manifest_path = os.path.join(_normalize_path(run_dir), PACKAGE_MANIFEST_FILENAME)
payload = _load_json(manifest_path)
if not payload:
return
normalized_run_dir = _normalize_path(run_dir)
normalized_primary = _normalize_path(primary_file)
normalized_sources = [_normalize_path(path) for path in source_files]
primary_relative = os.path.relpath(normalized_primary, normalized_run_dir)
source_payload = payload.get("source") if isinstance(payload.get("source"), dict) else {}
source_payload["output_dir"] = normalized_run_dir
source_payload["native_output_dir"] = _normalize_path(native_output_dir)
source_payload["primary_path"] = normalized_primary
source_payload["source_dir"] = normalized_run_dir
source_payload["publish_dir"] = normalized_run_dir
payload["source"] = source_payload
summary_payload = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
summary_payload["primary_asset_relative"] = primary_relative
payload["summary"] = summary_payload
preview_relative = str(summary_payload.get("preview_relative") or "").strip()
existing_assets = payload.get("assets") if isinstance(payload.get("assets"), list) else []
preserved_assets = [
item
for item in existing_assets
if str((item or {}).get("role") or "").strip() not in _ISCE2_ASSET_ROLES
]
isce2_assets: List[Dict[str, Any]] = [
{
"role": "disp",
"asset_name": os.path.basename(normalized_primary) or STANDARD_ISCE2_DISP_NAME,
"relative_path": primary_relative,
"format": "geotiff",
"media_type": "image/tiff",
"is_required": True,
"is_primary": True,
}
]
if len(normalized_sources) > 1 and os.path.isfile(normalized_sources[1]):
isce2_assets.append(
{
"role": "coh",
"asset_name": os.path.basename(normalized_sources[1]) or STANDARD_ISCE2_COH_NAME,
"relative_path": os.path.relpath(normalized_sources[1], normalized_run_dir),
"format": "geotiff",
"media_type": "image/tiff",
"is_required": False,
"is_primary": False,
}
)
payload["assets"] = isce2_assets + preserved_assets
payload["native_output_dir"] = _normalize_path(native_output_dir)
canonical = build_canonical_descriptor(
payload["assets"],
product_family=str(payload.get("product_family") or "dinsar"),
)
if preview_relative:
canonical["preview_asset_relative"] = preview_relative
payload["canonical"] = canonical
normalized_payload = normalize_package_manifest(payload)
_write_json(manifest_path, normalized_payload)
def normalize_envi_run_layout(
run_dir: str,
*,
primary_file: str,
source_files: List[str],
rewrite_metadata: bool = True,
) -> Dict[str, Any]:
normalized_run_dir = _normalize_path(run_dir)
if not os.path.isdir(normalized_run_dir):
raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}")
native_output_dir = get_run_native_output_dir(normalized_run_dir)
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
normalized_primary = _normalize_path(primary_file)
normalized_sources = [_normalize_path(path) for path in source_files if str(path or "").strip()]
if normalized_primary and normalized_primary not in normalized_sources:
normalized_sources.insert(0, normalized_primary)
promoted_files: List[str] = []
if os.path.isfile(normalized_primary) and normalized_primary != disp_paths["primary"]:
if _move_file(normalized_primary, disp_paths["primary"]):
promoted_files.append(disp_paths["primary"])
for path in normalized_sources:
lower = path.lower()
if lower.endswith(".hdr"):
if _move_file(path, disp_paths["hdr"]):
promoted_files.append(disp_paths["hdr"])
elif lower.endswith(".sml"):
if _move_file(path, disp_paths["sml"]):
promoted_files.append(disp_paths["sml"])
os.makedirs(native_output_dir, exist_ok=True)
moved_entries: List[str] = []
for name in os.listdir(normalized_run_dir):
if name in _KEEP_RUN_ROOT_NAMES:
continue
src_path = os.path.join(normalized_run_dir, name)
dst_path = os.path.join(native_output_dir, name)
if _move_entry(src_path, dst_path):
moved_entries.append(dst_path)
final_sources = [disp_paths["primary"]]
if os.path.isfile(disp_paths["hdr"]):
final_sources.append(disp_paths["hdr"])
if os.path.isfile(disp_paths["sml"]):
final_sources.append(disp_paths["sml"])
if rewrite_metadata:
_rewrite_run_metadata(normalized_run_dir, native_output_dir)
_rewrite_execution_manifest(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_paths["primary"],
source_files=final_sources,
)
_rewrite_current_pointers(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_paths["primary"],
source_files=final_sources,
)
_rewrite_package_manifest(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_paths["primary"],
source_files=final_sources,
)
return {
"run_dir": normalized_run_dir,
"native_output_dir": native_output_dir,
"primary_file": disp_paths["primary"],
"source_files": final_sources,
"promoted_files": promoted_files,
"moved_entries": moved_entries,
}
def normalize_isce2_run_layout(
run_dir: str,
*,
primary_file: str,
source_files: List[str],
rewrite_metadata: bool = True,
) -> Dict[str, Any]:
normalized_run_dir = _normalize_path(run_dir)
if not os.path.isdir(normalized_run_dir):
raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}")
native_output_dir = get_run_native_output_dir(normalized_run_dir)
disp_asset_path = get_run_isce2_disp_asset_path(normalized_run_dir)
coh_asset_path = get_run_isce2_coh_asset_path(normalized_run_dir)
normalized_primary = _normalize_path(primary_file)
normalized_sources = [_normalize_path(path) for path in source_files if str(path or "").strip()]
if normalized_primary and normalized_primary not in normalized_sources:
normalized_sources.insert(0, normalized_primary)
copied_files: List[str] = []
if os.path.isfile(normalized_primary) and _copy_file(normalized_primary, disp_asset_path):
copied_files.append(disp_asset_path)
coh_source = ""
for path in normalized_sources[1:]:
if os.path.isfile(path):
coh_source = path
break
if coh_source and _copy_file(coh_source, coh_asset_path):
copied_files.append(coh_asset_path)
if not os.path.isfile(disp_asset_path):
raise FileNotFoundError(f"ISCE2 displacement asset not found: {disp_asset_path}")
os.makedirs(native_output_dir, exist_ok=True)
moved_entries: List[str] = []
for name in os.listdir(normalized_run_dir):
if name in _KEEP_RUN_ROOT_NAMES:
continue
src_path = os.path.join(normalized_run_dir, name)
dst_path = os.path.join(native_output_dir, name)
if _move_entry(src_path, dst_path):
moved_entries.append(dst_path)
final_sources = [disp_asset_path]
if os.path.isfile(coh_asset_path):
final_sources.append(coh_asset_path)
if rewrite_metadata:
_rewrite_run_metadata(normalized_run_dir, native_output_dir)
_rewrite_execution_manifest(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_asset_path,
source_files=final_sources,
)
_rewrite_current_pointers(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_asset_path,
source_files=final_sources,
)
_rewrite_isce2_package_manifest(
normalized_run_dir,
native_output_dir=native_output_dir,
primary_file=disp_asset_path,
source_files=final_sources,
)
return {
"run_dir": normalized_run_dir,
"native_output_dir": native_output_dir,
"primary_file": disp_asset_path,
"source_files": final_sources,
"copied_files": copied_files,
"moved_entries": moved_entries,
}
backend/app/services/envi_service.py
+105 -2
View File
@@ -9,6 +9,8 @@ from __future__ import annotations
import json
import math
import os
import subprocess
import sys
import time
import defusedxml.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
@@ -25,6 +27,7 @@ from .dinsar_naming import (
find_json_sidecar,
write_run_metadata,
)
from .dinsar_result_layout_service import get_run_native_output_dir
_BACKEND_DIR = type(settings).BACKEND_DIR
@@ -57,6 +60,93 @@ def _to_local_path(value: Any) -> str:
return os.path.normpath(raw.replace("/", os.sep))
def get_envi_runner_python() -> str:
configured = _to_local_path(getattr(settings, "PYTHON_PATH", "") or "")
if configured:
return configured
return os.path.normpath(sys.executable)
def get_envi_runner_cwd() -> str:
return os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
def get_envi_runner_env() -> Dict[str, str]:
env = os.environ.copy()
project_root = get_envi_runner_cwd()
existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()]
ordered = [project_root, *existing]
deduped: List[str] = []
seen = set()
for raw_path in ordered:
try:
key = os.path.normcase(os.path.normpath(os.path.abspath(str(raw_path))))
except Exception:
key = str(raw_path)
if key in seen:
continue
seen.add(key)
deduped.append(str(raw_path))
env["PYTHONPATH"] = os.pathsep.join(deduped)
return env
def build_envi_runner_command(*args: Any) -> List[str]:
command = [
get_envi_runner_python(),
"-m",
"backend.app.services.envi_runner_cli",
]
command.extend(str(arg) for arg in args if arg is not None)
return command
def probe_envi_runner() -> Dict[str, Any]:
python_path = get_envi_runner_python()
project_root = get_envi_runner_cwd()
result: Dict[str, Any] = {
"python_path": python_path,
"cwd": project_root,
"ready": False,
"returncode": None,
"message": "",
}
if not python_path:
result["message"] = "PYTHON_PATH is empty."
return result
if not os.path.isfile(python_path):
result["message"] = f"Python executable not found: {python_path}"
return result
if not os.path.isdir(project_root):
result["message"] = f"Project root not found: {project_root}"
return result
try:
completed = subprocess.run(
build_envi_runner_command("--help"),
cwd=project_root,
env=get_envi_runner_env(),
capture_output=True,
text=True,
timeout=20,
check=False,
)
except Exception as exc:
result["message"] = str(exc)
return result
result["returncode"] = int(completed.returncode)
if completed.returncode == 0:
result["ready"] = True
result["message"] = "Runner entrypoint is available."
return result
stderr_text = str(completed.stderr or "").strip()
stdout_text = str(completed.stdout or "").strip()
result["message"] = (stderr_text or stdout_text or f"returncode={completed.returncode}")[:1000]
return result
# ---------------------------------------------------------------------------
# Configuration (read once at import time)
# ---------------------------------------------------------------------------
@@ -245,6 +335,7 @@ def _write_envi_run_sidecar(
started_at: str,
params: Dict[str, Any],
metrics: Dict[str, Any],
native_output_dir: Optional[str] = None,
) -> None:
write_run_metadata(
output_dir,
@@ -258,6 +349,7 @@ def _write_envi_run_sidecar(
"source_root": os.path.normpath(root_dir),
"task_dir": os.path.normpath(task_dir),
"output_dir": os.path.normpath(output_dir),
"native_output_dir": os.path.normpath(native_output_dir or output_dir),
"started_at": started_at,
"finished_at": _utc_now_text(),
"params": params,
@@ -1632,17 +1724,20 @@ def run_single_task_workflow(
if not DEM_BASE_FILE:
raise ValueError("DEM path not configured. Set IDL_DINSAR_DEM_BASE_FILE in .env")
native_output_dir = get_run_native_output_dir(output_dir)
master_dir = os.path.join(task_dir, "master")
slave_dir = os.path.join(task_dir, "slave")
if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir):
raise RuntimeError(f"{task_name}: master/slave dir missing")
os.makedirs(output_dir, exist_ok=True)
os.makedirs(native_output_dir, exist_ok=True)
log_lines: List[str] = [
f"[envi] single task workflow={normalized_workflow}",
f"[envi] source_root={source_root}",
f"[envi] task_dir={task_dir}",
f"[envi] output_dir={output_dir}",
f"[envi] native_output_dir={native_output_dir}",
f"[envi] dem={DEM_BASE_FILE}",
]
auto_imported = 0
@@ -1689,7 +1784,7 @@ def run_single_task_workflow(
"REFERENCE_SARSCAPEDATA": _build_sarscapedata(master_base),
"SECONDARY_SARSCAPEDATA": _build_sarscapedata(slave_base),
"DEM_SARSCAPEDATA": _build_sarscapedata(DEM_BASE_FILE),
"OUTPUT_FOLDER": _normalize_path(output_dir),
"OUTPUT_FOLDER": _normalize_path(native_output_dir),
},
)
_write_progress(job_id, 1, 1, "Completed", output_dir, 1, 1, task_name)
@@ -1698,7 +1793,7 @@ def run_single_task_workflow(
master_base,
slave_base,
DEM_BASE_FILE,
os.path.join(output_dir, "workflow"),
os.path.join(native_output_dir, "workflow"),
log_lines,
job_id=job_id,
pair_index=1,
@@ -1746,6 +1841,7 @@ def run_single_task_workflow(
metrics={
"elapsed_seconds": elapsed,
},
native_output_dir=native_output_dir,
)
return {
"summary": {
@@ -1764,6 +1860,7 @@ def run_single_task_workflow(
"run_key": resolved_run_key,
"task_dir": task_dir,
"output_dir": output_dir,
"native_output_dir": native_output_dir,
"success": True,
"status": "ok",
"elapsed_seconds": elapsed,
@@ -1917,6 +2014,7 @@ def get_status() -> Dict[str, Any]:
"""Return ENVI/IDL system status and DEM configuration."""
idl_installed = bool(IDL_EXECUTABLE and os.path.isfile(IDL_EXECUTABLE))
is_running = is_any_process_running(["idl.exe", "idlde.exe", "taskengine.exe"])
runner_status = probe_envi_runner()
dem_ok = bool(
DEM_BASE_FILE
@@ -1930,6 +2028,11 @@ def get_status() -> Dict[str, Any]:
"idl_running": is_running,
"dem_base_file": DEM_BASE_FILE or "(not configured)",
"dem_exists": dem_ok,
"runner_python": runner_status.get("python_path", ""),
"runner_cwd": runner_status.get("cwd", ""),
"runner_ready": bool(runner_status.get("ready")),
"runner_returncode": runner_status.get("returncode"),
"runner_message": runner_status.get("message", ""),
}
backend/app/services/health_service.py
+192 -6
View File
@@ -17,7 +17,9 @@ from ..models import (
SystemWorkerHeartbeatORM,
)
from ..idl_service import get_idl_status
from .product_package_schema import CANONICAL_PACKAGE_SCHEMA
from .pairing_state_service import pairing_state_service
from .wsl_runtime_registry import wsl_runtime_registry
DEFAULT_WORKER_TIMEOUT_SECONDS = 60
@@ -282,14 +284,18 @@ async def _check_result_catalog() -> Dict[str, Any]:
)
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
async def _check_timeseries_result_catalog() -> Dict[str, Any]:
return await _check_catalog(
catalog_name="psinsar",
storage_root=settings.PSINSAR_PRODUCT_DIR,
storage_root=settings.TIMESERIES_PRODUCT_DIR,
enabled=bool(settings.TIMESERIES_ENABLED),
)
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
return await _check_timeseries_result_catalog()
async def _check_nginx() -> Dict[str, Any]:
status = {"ok": False, "error": None, "status_code": None}
nginx_health_url = settings.NGINX_HEALTH_URL
@@ -351,6 +357,30 @@ def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]:
}
def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
"total_count": int(payload.get("total_count") or 0),
"canonical_count": int(payload.get("canonical_count") or 0),
"missing_manifest_count": int(payload.get("missing_manifest_count") or 0),
"missing_publish_dir_count": int(payload.get("missing_publish_dir_count") or 0),
"missing_processor_count": int(payload.get("missing_processor_count") or 0),
"missing_runtime_count": int(payload.get("missing_runtime_count") or 0),
"missing_native_output_count": int(payload.get("missing_native_output_count") or 0),
}
def _sanitize_wsl_runtime_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
"broker_job_root_exists": bool(payload.get("broker_job_root_exists")),
"required_runtime_count": int(payload.get("required_runtime_count") or 0),
"healthy_runtime_count": int(payload.get("healthy_runtime_count") or 0),
"shared_distro": payload.get("shared_distro"),
"shared_conda_env_name": payload.get("shared_conda_env_name"),
}
def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
@@ -374,18 +404,26 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
worker = payload.get("worker", {}) or {}
result_catalog = payload.get("result_catalog", {}) or {}
dinsar_result_catalog = payload.get("dinsar_result_catalog", {}) or result_catalog
psinsar_result_catalog = payload.get("psinsar_result_catalog", {}) or {}
timeseries_result_catalog = (
payload.get("timeseries_result_catalog", {}) or payload.get("psinsar_result_catalog", {}) or {}
)
psinsar_result_catalog = timeseries_result_catalog
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
source_roots = payload.get("source_roots", {}) or {}
product_packages = payload.get("product_packages", {}) or {}
wsl_runtime = payload.get("wsl_runtime", {}) or {}
pairing_system = payload.get("pairing_system", {}) or {}
idl = payload.get("idl", {}) or {}
idl_status = idl.get("status", {}) or {}
ollama = payload.get("ollama", {}) or {}
nginx = payload.get("nginx", {}) or {}
sanitized_dinsar_catalog = _sanitize_catalog_status(dinsar_result_catalog)
sanitized_psinsar_catalog = _sanitize_catalog_status(psinsar_result_catalog)
sanitized_timeseries_catalog = _sanitize_catalog_status(timeseries_result_catalog)
sanitized_psinsar_catalog = sanitized_timeseries_catalog
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
sanitized_product_packages = _sanitize_product_package_status(product_packages)
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
return {
@@ -403,13 +441,17 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
},
"result_catalog": sanitized_dinsar_catalog,
"dinsar_result_catalog": sanitized_dinsar_catalog,
"timeseries_result_catalog": sanitized_timeseries_catalog,
"psinsar_result_catalog": sanitized_psinsar_catalog,
"catalogs": {
"dinsar": sanitized_dinsar_catalog,
"timeseries": sanitized_timeseries_catalog,
"psinsar": sanitized_psinsar_catalog,
},
"dinsar_bridge": sanitized_dinsar_bridge,
"source_roots": sanitized_source_roots,
"product_packages": sanitized_product_packages,
"wsl_runtime": sanitized_wsl_runtime,
"pairing_system": sanitized_pairing_system,
"idl": {
"ok": bool(idl.get("ok")),
@@ -787,6 +829,141 @@ async def _check_source_roots() -> Dict[str, Any]:
}
async def _check_product_packages() -> Dict[str, Any]:
status = {
"ok": False,
"canonical_schema": CANONICAL_PACKAGE_SCHEMA,
"total_count": 0,
"canonical_count": 0,
"missing_manifest_count": 0,
"missing_publish_dir_count": 0,
"missing_processor_count": 0,
"missing_runtime_count": 0,
"missing_native_output_count": 0,
"by_family": {},
"by_engine": {},
"error": None,
}
try:
session_factory = _get_session_factory()
async with session_factory() as db:
result = await db.execute(
select(
ResultProductORM.product_family,
ResultProductORM.engine_code,
ResultProductORM.package_schema,
ResultProductORM.processor_code,
ResultProductORM.runtime_id,
ResultProductORM.manifest_path,
ResultProductORM.publish_dir,
ResultProductORM.native_output_dir,
)
)
rows = result.all()
status["total_count"] = len(rows)
for (
product_family,
engine_code,
package_schema,
processor_code,
runtime_id,
manifest_path,
publish_dir,
native_output_dir,
) in rows:
family_key = str(product_family or "unknown").strip() or "unknown"
engine_key = str(engine_code or "unknown").strip() or "unknown"
status["by_family"][family_key] = int(status["by_family"].get(family_key, 0)) + 1
status["by_engine"][engine_key] = int(status["by_engine"].get(engine_key, 0)) + 1
if str(package_schema or "").strip() == CANONICAL_PACKAGE_SCHEMA:
status["canonical_count"] += 1
if not str(manifest_path or "").strip() or not os.path.isfile(str(manifest_path)):
status["missing_manifest_count"] += 1
if not str(publish_dir or "").strip() or not os.path.isdir(str(publish_dir)):
status["missing_publish_dir_count"] += 1
if not str(processor_code or "").strip():
status["missing_processor_count"] += 1
if engine_key in {"isce2", "pyint", "gamma"} and not str(runtime_id or "").strip():
status["missing_runtime_count"] += 1
if not str(native_output_dir or "").strip():
status["missing_native_output_count"] += 1
status["ok"] = all(
[
status["missing_manifest_count"] == 0,
status["missing_publish_dir_count"] == 0,
status["missing_processor_count"] == 0,
status["missing_runtime_count"] == 0,
status["missing_native_output_count"] == 0,
status["canonical_count"] == status["total_count"],
]
)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_wsl_runtime() -> Dict[str, Any]:
status = {
"ok": False,
"shared_distro": wsl_runtime_registry.shared_distro,
"shared_conda_env_name": wsl_runtime_registry.shared_conda_env_name,
"shared_python_path": wsl_runtime_registry.shared_python_path,
"broker_job_root_windows": wsl_runtime_registry.broker_job_root_windows,
"broker_job_root_exists": os.path.isdir(wsl_runtime_registry.broker_job_root_windows),
"required_runtime_count": 0,
"healthy_runtime_count": 0,
"runtimes": [],
"error": None,
}
try:
required_by_engine = {
"isce2": bool(settings.ISCE2_ENABLED or settings.TIMESERIES_ENABLED),
"pyint": bool(settings.PYINT_ENABLED),
}
for runtime in wsl_runtime_registry.runtimes.values():
required = bool(required_by_engine.get(runtime.engine_code, False))
runner_exists = os.path.isfile(runtime.runner_path_windows)
env_profile_exists = None
if str(runtime.env_profile_path_windows or "").strip():
env_profile_exists = os.path.isfile(runtime.env_profile_path_windows)
python_matches_shared = str(runtime.python_path or "").strip() == str(
wsl_runtime_registry.shared_python_path or ""
).strip()
distro_matches_shared = str(runtime.distro or "").strip() == str(
wsl_runtime_registry.shared_distro or ""
).strip()
runtime_ok = runner_exists and python_matches_shared and distro_matches_shared
if env_profile_exists is False and required:
runtime_ok = False
if required:
status["required_runtime_count"] += 1
if runtime_ok:
status["healthy_runtime_count"] += 1
status["runtimes"].append(
{
"runtime_id": runtime.runtime_id,
"engine_code": runtime.engine_code,
"display_name": runtime.display_name,
"required": required,
"ok": runtime_ok,
"runner_exists": runner_exists,
"env_profile_exists": env_profile_exists,
"python_matches_shared": python_matches_shared,
"distro_matches_shared": distro_matches_shared,
"allowed_operations": list(runtime.allowed_operations or ()),
}
)
status["ok"] = bool(status["broker_job_root_exists"]) and (
status["healthy_runtime_count"] >= status["required_runtime_count"]
)
except Exception as exc:
status["error"] = str(exc)
return status
async def get_health_status(
include_external: bool = True,
include_details: bool = False,
@@ -807,9 +984,12 @@ async def get_health_status(
ollama_status = await _check_ollama()
result_catalog_status = await _check_result_catalog()
psinsar_result_catalog_status = await _check_psinsar_result_catalog()
timeseries_result_catalog_status = await _check_timeseries_result_catalog()
psinsar_result_catalog_status = timeseries_result_catalog_status
dinsar_bridge_status = await _check_dinsar_bridge()
source_roots_status = await _check_source_roots()
product_packages_status = await _check_product_packages()
wsl_runtime_status = await _check_wsl_runtime()
pairing_system_status = await pairing_state_service.get_pairing_system_status()
engines_status = {"ok": None, "overall": None, "engines": []}
if full or include_details:
@@ -824,8 +1004,10 @@ async def get_health_status(
result_catalog_status.get("ok"),
dinsar_bridge_status.get("ok"),
source_roots_status.get("ok"),
product_packages_status.get("ok"),
wsl_runtime_status.get("ok"),
pairing_system_status.get("ok"),
(not settings.TIMESERIES_ENABLED) or psinsar_result_catalog_status.get("ok"),
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
]
)
@@ -836,13 +1018,17 @@ async def get_health_status(
"worker": worker_status,
"result_catalog": result_catalog_status,
"dinsar_result_catalog": result_catalog_status,
"timeseries_result_catalog": timeseries_result_catalog_status,
"psinsar_result_catalog": psinsar_result_catalog_status,
"catalogs": {
"dinsar": result_catalog_status,
"timeseries": timeseries_result_catalog_status,
"psinsar": psinsar_result_catalog_status,
},
"dinsar_bridge": dinsar_bridge_status,
"source_roots": source_roots_status,
"product_packages": product_packages_status,
"wsl_runtime": wsl_runtime_status,
"pairing_system": pairing_system_status,
"idl": {
"ok": idl_ok,
backend/app/services/isce2_result_validator.py
+162
View File
@@ -0,0 +1,162 @@
from __future__ import annotations
import os
from typing import Any, Dict, List, Sequence
def _normalize_path(value: Any) -> str:
text = str(value or "").strip()
if not text:
return ""
return os.path.normpath(os.path.abspath(text))
def _dedupe_existing_files(paths: Sequence[Any]) -> List[str]:
normalized_paths: List[str] = []
seen: set[str] = set()
for path in paths:
normalized = _normalize_path(path)
if not normalized or normalized in seen:
continue
seen.add(normalized)
normalized_paths.append(normalized)
return normalized_paths
def _inspect_raster(path: str) -> Dict[str, Any]:
gdal_error: Exception | None = None
try:
from osgeo import gdal
gdal.UseExceptions()
dataset = gdal.Open(path, gdal.GA_ReadOnly)
if dataset is None:
raise RuntimeError("GDAL returned no dataset")
band_count = int(dataset.RasterCount or 0)
width = int(dataset.RasterXSize or 0)
height = int(dataset.RasterYSize or 0)
if band_count <= 0 or width <= 0 or height <= 0:
raise ValueError(
f"invalid raster geometry bands={band_count} width={width} height={height}"
)
return {
"width": width,
"height": height,
"bands": band_count,
"driver": dataset.GetDriver().ShortName if dataset.GetDriver() is not None else "",
"projection_present": bool(dataset.GetProjection()),
"geo_transform_present": bool(dataset.GetGeoTransform(can_return_null=True)),
"reader": "gdal",
}
except Exception as exc:
gdal_error = exc
try:
import rasterio
with rasterio.open(path) as dataset:
band_count = int(dataset.count or 0)
width = int(dataset.width or 0)
height = int(dataset.height or 0)
if band_count <= 0 or width <= 0 or height <= 0:
raise ValueError(
f"invalid raster geometry bands={band_count} width={width} height={height}"
)
return {
"width": width,
"height": height,
"bands": band_count,
"driver": str(dataset.driver or ""),
"projection_present": bool(dataset.crs),
"geo_transform_present": dataset.transform is not None,
"reader": "rasterio",
}
except Exception as exc:
if gdal_error is not None:
raise RuntimeError(f"gdal={gdal_error}; rasterio={exc}") from exc
raise
def validate_isce2_result_files(
primary_file: Any,
source_files: Sequence[Any] | None = None,
) -> Dict[str, Any]:
normalized_primary = _normalize_path(primary_file)
normalized_sources = _dedupe_existing_files(source_files or [])
if normalized_primary and normalized_primary not in normalized_sources and os.path.isfile(normalized_primary):
normalized_sources.insert(0, normalized_primary)
issues: List[str] = []
accepted_sources: List[str] = []
metrics: Dict[str, Any] = {
"primary_exists": False,
"primary_non_empty": False,
"primary_readable": False,
"primary_size_bytes": 0,
"source_file_count": len(normalized_sources),
"coh_present": False,
}
primary_metadata: Dict[str, Any] = {}
if not normalized_primary:
issues.append("Primary ISCE2 displacement file path is empty.")
elif not os.path.isfile(normalized_primary):
issues.append(f"Primary ISCE2 displacement file not found: {normalized_primary}")
else:
metrics["primary_exists"] = True
try:
primary_size = int(os.path.getsize(normalized_primary))
except OSError:
primary_size = 0
metrics["primary_size_bytes"] = primary_size
if primary_size <= 0:
issues.append(f"Primary ISCE2 displacement file is empty: {normalized_primary}")
else:
metrics["primary_non_empty"] = True
try:
primary_metadata = _inspect_raster(normalized_primary)
metrics["primary_readable"] = True
metrics["primary_raster"] = primary_metadata
accepted_sources.append(normalized_primary)
except Exception as exc:
issues.append(
f"Primary ISCE2 displacement file is not a readable GeoTIFF: {normalized_primary}: {exc}"
)
coh_path = ""
coh_metadata: Dict[str, Any] = {}
for candidate in normalized_sources:
if candidate == normalized_primary or not os.path.isfile(candidate):
continue
try:
candidate_size = int(os.path.getsize(candidate))
except OSError:
candidate_size = 0
if candidate_size <= 0:
issues.append(f"Auxiliary ISCE2 output file is empty: {candidate}")
continue
try:
coh_metadata = _inspect_raster(candidate)
coh_path = candidate
accepted_sources.append(candidate)
break
except Exception as exc:
issues.append(
f"Auxiliary ISCE2 output file is not a readable GeoTIFF: {candidate}: {exc}"
)
if coh_path:
metrics["coh_present"] = True
metrics["coh_raster"] = coh_metadata
accepted = bool(metrics["primary_readable"])
return {
"accepted": accepted,
"primary_file": normalized_primary if accepted else "",
"source_files": accepted_sources if accepted_sources else ([normalized_primary] if accepted else []),
"coh_file": coh_path,
"issues": issues,
"metrics": metrics,
}
backend/app/services/job_handlers.py
+670 -32
View File
@@ -7,7 +7,6 @@ import logging
import os
import re
import subprocess
import sys
import tempfile
import time
import uuid
@@ -27,8 +26,14 @@ from .dinsar_compat_service import dinsar_compat_service
from .dinsar_naming import build_run_key
from .dinsar_production_service import dinsar_production_service
from .dinsar_read_service import dinsar_read_service
from .dinsar_result_layout_service import (
get_run_disp_asset_paths,
get_run_native_output_dir,
normalize_envi_run_layout,
)
from .dinsar_scan_service import dinsar_scan_service
from .engine_lock_service import engine_lock_service
from .envi_service import build_envi_runner_command, get_envi_runner_cwd, get_envi_runner_env
from .psinsar_catalog_service import psinsar_catalog_service
from .result_catalog_service import result_catalog_service
from .task_service import task_service
@@ -875,9 +880,9 @@ def _get_envi_progress_file(job_id: str) -> str:
def _get_envi_runtime_cwd() -> str:
runtime_dir = os.path.normpath(os.path.abspath(settings.IDL_WORKER_RUNTIME_DIR))
os.makedirs(runtime_dir, exist_ok=True)
return runtime_dir
# The ENVI runner is launched via `python -m backend.app.services.envi_runner_cli`,
# so its import root must be the project root rather than the runtime directory.
return get_envi_runner_cwd()
# Stale threshold: no progress file update AND no output file activity
@@ -992,18 +997,18 @@ def _clear_envi_progress_file(job_id: Optional[str]) -> None:
def _find_latest_envi_result(output_dir: str) -> Dict[str, Any]:
matches: List[tuple[float, str]] = []
try:
for entry in os.scandir(output_dir):
if not entry.is_file():
continue
if entry.name.lower().endswith((".hdr", ".sml")):
continue
if not _ENVI_RESULT_NAME_RE.match(entry.name):
continue
try:
stat = entry.stat()
matches.append((max(stat.st_mtime, stat.st_ctime), entry.path))
except OSError:
matches.append((0.0, entry.path))
for current_root, _dirs, files in os.walk(output_dir):
for name in files:
if name.lower().endswith((".hdr", ".sml")):
continue
if not _ENVI_RESULT_NAME_RE.match(name):
continue
path = os.path.join(current_root, name)
try:
stat = os.stat(path)
matches.append((max(stat.st_mtime, stat.st_ctime), path))
except OSError:
matches.append((0.0, path))
except OSError as exc:
raise RuntimeError(f"Failed to scan ENVI output directory: {output_dir}: {exc}") from exc
@@ -1023,6 +1028,77 @@ def _find_latest_envi_result(output_dir: str) -> Dict[str, Any]:
}
def _is_path_within(base_dir: str, candidate_path: str) -> bool:
try:
return os.path.commonpath(
[
os.path.normpath(os.path.abspath(str(base_dir or "").strip())),
os.path.normpath(os.path.abspath(str(candidate_path or "").strip())),
]
) == os.path.normpath(os.path.abspath(str(base_dir or "").strip()))
except ValueError:
return False
def _normalize_managed_envi_output_dir(output_dir: str) -> Dict[str, Any]:
normalized_output_dir = os.path.normpath(os.path.abspath(str(output_dir or "").strip()))
if not normalized_output_dir or not os.path.isdir(normalized_output_dir):
raise FileNotFoundError(f"Managed ENVI output directory not found: {output_dir}")
managed_root = os.path.normpath(os.path.abspath(str(settings.DINSAR_PRODUCT_DIR or "").strip()))
if not managed_root or not _is_path_within(managed_root, normalized_output_dir):
result_files = _find_latest_envi_result(normalized_output_dir)
return {
"run_dir": normalized_output_dir,
"native_output_dir": normalized_output_dir,
"primary_file": result_files["primary_file"],
"source_files": result_files["source_files"],
"promoted_files": [],
"moved_entries": [],
}
disp_paths = get_run_disp_asset_paths(normalized_output_dir)
if os.path.isfile(disp_paths["primary"]):
source_files = [disp_paths["primary"]]
for ext in (".hdr", ".sml"):
sidecar = disp_paths["primary"] + ext
if os.path.isfile(sidecar):
source_files.append(sidecar)
return {
"run_dir": normalized_output_dir,
"native_output_dir": get_run_native_output_dir(normalized_output_dir),
"primary_file": disp_paths["primary"],
"source_files": source_files,
"promoted_files": [],
"moved_entries": [],
}
native_output_dir = get_run_native_output_dir(normalized_output_dir)
search_dirs = []
if os.path.isdir(native_output_dir):
search_dirs.append(native_output_dir)
search_dirs.append(normalized_output_dir)
last_error: Optional[Exception] = None
result_files: Optional[Dict[str, Any]] = None
for search_dir in search_dirs:
try:
result_files = _find_latest_envi_result(search_dir)
break
except Exception as exc:
last_error = exc
if not result_files:
raise RuntimeError(
f"Failed to locate ENVI displacement result under managed run directory: {normalized_output_dir}"
) from last_error
return normalize_envi_run_layout(
normalized_output_dir,
primary_file=result_files["primary_file"],
source_files=result_files["source_files"],
)
async def _run_envi_runner_command(
job: SystemJobORM,
runner_cmd: List[str],
@@ -1065,6 +1141,7 @@ async def _run_envi_runner_command(
stdout=stdout_fd,
stderr=stderr_fd,
cwd=_get_envi_runtime_cwd(),
env=get_envi_runner_env(),
)
proc_state["pid"] = proc.pid
loop.call_soon_threadsafe(pid_ready.set)
@@ -1232,10 +1309,7 @@ async def _run_envi_workflow_job(
message=f"Launching ENVI worker subprocess... (tasks={task_folder_count}, timeout={effective_absolute_timeout}s)",
)
runner_cmd = [
sys.executable,
"-m",
"backend.app.services.envi_runner_cli",
runner_cmd = build_envi_runner_command(
"--workflow",
workflow,
"--root-dir",
@@ -1244,7 +1318,7 @@ async def _run_envi_workflow_job(
str(num_to_process),
"--job-id",
str(job.job_id),
]
)
if timeout_seconds is not None:
runner_cmd.extend(["--timeout-seconds", str(int(timeout_seconds))])
@@ -1304,6 +1378,7 @@ async def _run_envi_workflow_job(
stdout=stdout_fd,
stderr=stderr_fd,
cwd=_get_envi_runtime_cwd(),
env=get_envi_runner_env(),
)
# Close our copy of the fds; the child process has its own.
os.close(stdout_fd)
@@ -1443,6 +1518,11 @@ async def _run_envi_workflow_job(
if workflow in {"dinsar", "dinsar_custom"}:
output_dirs = _dedupe_existing_dirs(run_meta.get("output_dirs"))
if output_dirs:
normalized_output_dirs: List[str] = []
for output_dir in output_dirs:
layout_result = await asyncio.to_thread(_normalize_managed_envi_output_dir, output_dir)
normalized_output_dirs.append(layout_result["run_dir"])
output_dirs = _dedupe_existing_dirs(normalized_output_dirs)
await task_service.update_task(
job.task_id,
progress=92,
@@ -1575,10 +1655,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
db=db,
)
runner_cmd = [
sys.executable,
"-m",
"backend.app.services.envi_runner_cli",
runner_cmd = build_envi_runner_command(
"--workflow",
workflow,
"--task-dir",
@@ -1593,7 +1670,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
str(run_key),
"--profile-code",
str(run.profile_code),
]
)
if timeout_seconds is not None:
runner_cmd.extend(["--timeout-seconds", str(timeout_seconds)])
@@ -1638,7 +1715,10 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
keepalive_formatter=_keepalive_formatter,
register_pid=_register_pid,
)
result_files = await asyncio.to_thread(_find_latest_envi_result, execution.output_dir)
layout_result = await asyncio.to_thread(
_normalize_managed_envi_output_dir,
execution.output_dir,
)
metrics = {
"duration_seconds": run_meta.get("duration_seconds"),
"summary": run_meta.get("summary") or {},
@@ -1651,17 +1731,20 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
run=run,
item=item,
execution=execution,
primary_file=result_files["primary_file"],
source_files=result_files["source_files"],
primary_file=layout_result["primary_file"],
source_files=layout_result["source_files"],
native_output_dir=layout_result["native_output_dir"],
metrics=metrics,
)
await asyncio.to_thread(
dinsar_production_service.write_current_pointer,
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
primary_file=result_files["primary_file"],
source_files=result_files["source_files"],
primary_file=layout_result["primary_file"],
source_files=layout_result["source_files"],
native_output_dir=layout_result["native_output_dir"],
)
await dinsar_production_service.mark_item_completed(
run=run,
@@ -1894,6 +1977,8 @@ async def _handle_queued_engine_run(
timeout_seconds = payload.get("timeout_seconds")
extra = payload.get("extra", {})
selected_task_count = max(1, int(extra.get("__validated_task_count") or 0 or 1))
rerun_mode = str(payload.get("rerun_mode") or extra.get("__rerun_mode") or "rerun_all").strip()
skipped_completed_count = int(extra.get("__skipped_completed_count") or 0)
pair_timeout_seconds = int(timeout_seconds or fallback_timeout_seconds)
await task_service.start_task(
@@ -1903,7 +1988,10 @@ async def _handle_queued_engine_run(
await task_service.add_log(
job.task_id,
"INFO",
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, timeout={pair_timeout_seconds}s, extra={extra}",
(
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, "
f"rerun_mode={rerun_mode}, timeout={pair_timeout_seconds}s, extra={extra}"
),
)
await task_service.add_log(
job.task_id,
@@ -1911,6 +1999,7 @@ async def _handle_queued_engine_run(
(
f"{engine_title} batch contains {selected_task_count} pair task(s). "
f"Pairs run sequentially and each pair uses timeout={pair_timeout_seconds}s."
f"{f' Skipped completed={skipped_completed_count}.' if skipped_completed_count > 0 else ''}"
),
)
from ..dinsar_engines.base import RunRequest
@@ -2114,6 +2203,18 @@ async def _handle_queued_engine_run(
"INFO",
f"WSL command [{item.get('task_name')}]: {item.get('command')}",
)
if item.get("runtime_id"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL runtime [{item.get('task_name')}]: {item.get('runtime_id')}",
)
if item.get("manifest_path_windows"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL manifest [{item.get('task_name')}]: {item.get('manifest_path_windows')}",
)
if item.get("stdout_tail"):
await task_service.add_log(
job.task_id,
@@ -2163,6 +2264,18 @@ async def _handle_queued_engine_run(
"INFO",
f"WSL command: {detail['command']}",
)
if detail.get("runtime_id"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL runtime: {detail['runtime_id']}",
)
if detail.get("manifest_path_windows"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL manifest: {detail['manifest_path_windows']}",
)
if detail.get("stdout_tail"):
await task_service.add_log(
job.task_id,
@@ -2243,7 +2356,532 @@ async def _handle_queued_engine_run(
)
async def _run_wsl_dinsar_production_controller(
job: SystemJobORM,
*,
engine_code: str,
engine_title: str,
fallback_timeout_seconds: int,
) -> None:
if not job.task_id:
raise ValueError(f"{engine_title} production controller requires task_id.")
payload = job.payload or {}
production_run_id = str(payload.get("production_run_id") or "").strip()
if not production_run_id:
raise ValueError(f"{engine_title} production controller requires production_run_id.")
from ..dinsar_engines import registry
from ..dinsar_engines.base import RunRequest
engine = registry.get_engine(engine_code)
if engine is None:
raise RuntimeError(f"Engine '{engine_code}' is not registered.")
async with AsyncSessionLocal() as db:
run = await dinsar_production_service.get_run(production_run_id, db)
if run is None:
raise ValueError(f"D-InSAR production run not found: {production_run_id}")
items = await dinsar_production_service.list_run_items(run.run_id, db)
if not items:
raise ValueError(f"D-InSAR production run has no items: {production_run_id}")
params = run.params_json or {}
user_extra = dict(params.get("extra") or {})
timeout_seconds_raw = params.get("timeout_seconds")
if timeout_seconds_raw not in (None, ""):
per_task_timeout = int(timeout_seconds_raw)
else:
per_task_timeout = int(
getattr(engine, "default_timeout_seconds", None)
or fallback_timeout_seconds
or 0
)
total_items = len(items)
run_log = dinsar_production_service.append_run_log
async def _refresh_cancel_state() -> bool:
await db.refresh(run)
current_task = await task_service.get_task(job.task_id)
task_cancelled = bool(current_task and current_task.status == "CANCELLED")
if task_cancelled and not run.cancel_requested:
run.cancel_requested = True
await db.commit()
return bool(run.cancel_requested or task_cancelled)
await task_service.start_task(
job.task_id,
message=f"Starting {engine_title} D-InSAR production run {run.run_id} ({total_items} items)...",
)
await task_service.add_log(
job.task_id,
"INFO",
(
f"{engine_title} D-InSAR production controller started. run_id={run.run_id} "
f"profile={run.profile_code} items={total_items}"
),
)
await dinsar_production_service.mark_run_started(
run,
db=db,
message=f"Preparing {total_items} {engine_title} item(s)",
)
run_log(
run.run_id,
f"[start] engine={engine_code} items={total_items} source_root={run.source_root}",
)
successful_output_dirs: List[str] = []
async with engine_lock_service.acquire(f"wsl_dinsar_{engine_code}"):
await task_service.update_task(
job.task_id,
progress=5,
message=f"{engine_title} engine acquired. Preparing {total_items} item(s)...",
)
for item_index, item in enumerate(items, start=1):
if await _refresh_cancel_state():
await task_service.add_log(
job.task_id,
"WARNING",
f"Cancellation detected before item {item.task_alias or item.task_name}.",
)
break
await db.refresh(item)
if str(item.status or "").upper() in {"COMPLETED", "FAILED", "SKIPPED", "CANCELLED"}:
if item.status == "COMPLETED" and item.latest_output_dir and os.path.isdir(item.latest_output_dir):
successful_output_dirs.append(item.latest_output_dir)
continue
run_key = (
f"{build_run_key(engine_code, run.profile_code, started_at=datetime.utcnow())}"
f"_{item.id}_{uuid.uuid4().hex[:6]}"
)
execution = await dinsar_production_service.begin_item_execution(
run=run,
item=item,
run_key=run_key,
db=db,
)
managed_run_dir = os.path.normpath(execution.output_dir)
managed_native_output_dir = os.path.join(managed_run_dir, "native")
managed_work_dir = os.path.join(managed_native_output_dir, "workflow")
managed_export_dir = os.path.join(managed_native_output_dir, "export")
managed_orbit_output_dir = os.path.join(managed_work_dir, "orbits")
item_label = item.task_alias or item.task_name
base_progress = min(95, 5 + int(((item_index - 1) / max(1, total_items)) * 90))
progress_state: Dict[str, Any] = {
"progress": base_progress,
"message": f"[{engine_code}/{run.profile_code}] Running {item_index}/{total_items}: {item_label}",
"started_monotonic": time.monotonic(),
}
progress_queue: asyncio.Queue[Optional[Dict[str, Any]]] = asyncio.Queue()
loop = asyncio.get_running_loop()
def _emit_progress(event: Dict[str, Any]) -> None:
if not event:
return
try:
loop.call_soon_threadsafe(progress_queue.put_nowait, dict(event))
except RuntimeError:
return
async def _consume_progress() -> None:
while True:
event = await progress_queue.get()
if event is None:
return
event_type = str(event.get("event") or "").strip().lower()
if event_type == "pair_started":
progress_state["message"] = (
f"[{engine_code}/{run.profile_code}] Running "
f"{item_index}/{total_items}: {item_label}"
)
progress_state["started_monotonic"] = time.monotonic()
await task_service.add_log(
job.task_id,
"INFO",
f"[{item_index}/{total_items}] {engine_title} started {item_label}",
)
elif event_type == "pair_finished":
if bool(event.get("success")):
progress_state["progress"] = min(
98,
5 + int((item_index / max(1, total_items)) * 90),
)
progress_state["message"] = (
f"[{engine_code}/{run.profile_code}] Finished "
f"{item_index}/{total_items}: {item_label}"
)
else:
progress_state["message"] = (
f"[{engine_code}/{run.profile_code}] Failed "
f"{item_index}/{total_items}: {item_label}"
)
async def _task_keepalive() -> None:
while True:
await asyncio.sleep(30)
try:
message = str(progress_state.get("message") or "")
started_monotonic = progress_state.get("started_monotonic")
if isinstance(started_monotonic, (int, float)):
elapsed_seconds = max(0, int(time.monotonic() - float(started_monotonic)))
message = f"{message} (elapsed={elapsed_seconds}s)"
await task_service.update_task(
job.task_id,
progress=int(progress_state.get("progress") or base_progress),
message=message,
)
except Exception as exc:
logger.warning("keepalive update failed for %s item %s: %s", engine_title, item_label, exc)
progress_task = asyncio.create_task(_consume_progress())
keepalive_task = asyncio.create_task(_task_keepalive())
await task_service.add_log(
job.task_id,
"INFO",
f"[{item_index}/{total_items}] Launching {item_label} -> {managed_run_dir}",
)
run_log(
run.run_id,
f"[item-start] {item_index}/{total_items} {item_label} run_key={run_key} output={managed_run_dir}",
)
request = RunRequest(
engine_code=engine_code,
profile=run.profile_code,
root_dir=str(item.source_task_dir),
job_id=job.job_id,
num_to_process=1,
timeout_seconds=per_task_timeout or None,
extra={
**user_extra,
"__managed_run_dir": managed_run_dir,
"__managed_native_output_dir": managed_native_output_dir,
"__managed_work_dir": managed_work_dir,
"__managed_export_dir": managed_export_dir,
"__managed_orbit_output_dir": managed_orbit_output_dir,
"__managed_run_key": run_key,
"__source_root_override": run.source_root,
"__rerun_mode": "rerun_all",
},
progress_callback=_emit_progress,
)
result = None
run_exception_text = ""
try:
result = await asyncio.to_thread(engine.run, request)
except Exception as exc:
run_exception_text = str(exc)
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
await progress_queue.put(None)
await progress_task
detail = result.detail or {} if result else {}
task_result = ((detail.get("task_results") or [{}])[0]) if result else {}
try:
if not result or not result.success or not bool(task_result.get("success", result.success if result else False)):
error_message = (
str(task_result.get("error") or "").strip()
or str(result.error or "").strip()
or run_exception_text
or str(task_result.get("stderr_tail") or "").strip()
or f"{engine_title} run failed."
)
raise RuntimeError(error_message)
run_dir = os.path.normpath(
str(task_result.get("run_dir") or task_result.get("output_dir") or execution.output_dir)
)
if run_dir != managed_run_dir:
raise RuntimeError(
f"{engine_title} managed run dir mismatch: expected {managed_run_dir}, got {run_dir}"
)
primary_file = str(task_result.get("primary_file") or "").strip()
source_files = [
str(path)
for path in (task_result.get("source_files") or [])
if str(path or "").strip()
]
native_output_dir = str(
task_result.get("native_output_dir") or managed_native_output_dir
).strip() or managed_native_output_dir
if not primary_file or not os.path.isfile(primary_file):
raise RuntimeError(f"{engine_title} primary output is missing: {primary_file or '<empty>'}")
if not source_files:
source_files = [primary_file]
metrics = {
"result_detail": detail,
"task_result": task_result,
}
manifest_path = await asyncio.to_thread(
dinsar_production_service.build_execution_manifest,
run=run,
item=item,
execution=execution,
primary_file=primary_file,
source_files=source_files,
native_output_dir=native_output_dir,
metrics=metrics,
)
await asyncio.to_thread(
dinsar_production_service.write_current_pointer,
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
primary_file=primary_file,
source_files=source_files,
native_output_dir=native_output_dir,
)
await dinsar_production_service.mark_item_completed(
run=run,
item=item,
execution=execution,
manifest_path=manifest_path,
metrics=metrics,
db=db,
)
successful_output_dirs.append(managed_run_dir)
await task_service.add_log(
job.task_id,
"INFO",
f"[{item_index}/{total_items}] Completed {item_label}",
)
run_log(run.run_id, f"[item-ok] {item_index}/{total_items} {item_label}")
except Exception as exc:
cancelled = await _refresh_cancel_state()
error_message = str(exc)
if cancelled:
await dinsar_production_service.mark_item_cancelled(
run=run,
item=item,
execution=execution,
error_message=f"Cancelled while processing {item_label}",
db=db,
)
await task_service.add_log(
job.task_id,
"WARNING",
f"[{item_index}/{total_items}] Cancelled {item_label}: {error_message}",
)
run_log(
run.run_id,
f"[item-cancelled] {item_index}/{total_items} {item_label}: {error_message}",
)
break
await dinsar_production_service.mark_item_failed(
run=run,
item=item,
execution=execution,
error_message=error_message,
db=db,
)
await task_service.add_log(
job.task_id,
"WARNING",
f"[{item_index}/{total_items}] Failed {item_label}: {error_message}",
)
run_log(
run.run_id,
f"[item-failed] {item_index}/{total_items} {item_label}: {error_message}",
)
if task_result.get("command"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL command [{item_label}]: {task_result.get('command')}",
)
if task_result.get("stdout_tail"):
await task_service.add_log(
job.task_id,
"INFO",
f"WSL stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}",
)
if task_result.get("stderr_tail"):
await task_service.add_log(
job.task_id,
"WARNING",
f"WSL stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}",
)
publish_result = None
rebuild_result = None
publish_error = None
publish_dirs = _dedupe_existing_dirs(successful_output_dirs)
if publish_dirs:
await task_service.update_task(
job.task_id,
progress=99,
message=f"Publishing {len(publish_dirs)} successful {engine_title} result package(s)...",
)
try:
publish_result = await result_catalog_service.publish_from_sources(db, publish_dirs)
processed_count = int(publish_result.get("processed", 0) or 0)
failed_count = int(publish_result.get("failed", 0) or 0)
expected_count = len(publish_dirs)
if processed_count > 0:
rebuild_result = await result_catalog_service.rebuild_catalog(
db,
full_rebuild=True,
)
if processed_count != expected_count or failed_count != 0:
raise RuntimeError(
f"Expected to publish {expected_count} {engine_title} result package(s), "
f"but processed={processed_count}, failed={failed_count}"
)
await task_service.add_log(
job.task_id,
"INFO",
(
f"Published {publish_result.get('processed', 0)} {engine_title} result package(s). "
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
),
)
run_log(
run.run_id,
(
f"[publish] processed={publish_result.get('processed', 0)} "
f"failed={publish_result.get('failed', 0)} "
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
),
)
except Exception as exc:
publish_error = str(exc)
await task_service.add_log(
job.task_id,
"WARNING",
f"Result catalog publish failed: {publish_error}",
)
run_log(run.run_id, f"[publish-failed] {publish_error}")
cancelled = await _refresh_cancel_state()
await dinsar_production_service.refresh_run_counters(run, db=db)
if publish_error:
final_status = "FAILED"
latest_message = f"Result catalog publish failed: {publish_error}"
elif cancelled:
final_status = "CANCELLED"
latest_message = (
f"{engine_title} D-InSAR production cancelled. completed={run.completed_items} "
f"failed={run.failed_items} total={run.total_items}"
)
elif int(run.failed_items or 0) > 0:
final_status = "FAILED"
latest_message = (
f"{engine_title} D-InSAR production finished with failures. completed={run.completed_items} "
f"failed={run.failed_items} total={run.total_items}"
)
else:
final_status = "COMPLETED"
latest_message = (
f"{engine_title} D-InSAR production completed. completed={run.completed_items} "
f"failed={run.failed_items} total={run.total_items}"
)
summary_payload = {
"workflow": f"dinsar_{engine_code}",
"engine_code": run.engine_code,
"profile_code": run.profile_code,
"mode": run.mode,
"total_items": run.total_items,
"completed_items": run.completed_items,
"failed_items": run.failed_items,
"skipped_items": run.skipped_items,
"publish": publish_result,
"rebuild": rebuild_result,
"publish_error": publish_error,
"published_output_dirs": publish_dirs,
}
await dinsar_production_service.finalize_run(
run,
db=db,
status=final_status,
summary_payload=summary_payload,
latest_message=latest_message,
)
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
if final_status == "COMPLETED":
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=latest_message,
)
return
task_status = "CANCELLED" if final_status == "CANCELLED" else "FAILED"
await task_service.update_task(
job.task_id,
status=task_status,
progress=100,
message=latest_message,
)
raise RuntimeError(latest_message)
async def _handle_isce2_run(job: SystemJobORM) -> None:
payload = job.payload or {}
production_run_id = str(payload.get("production_run_id") or "").strip()
if production_run_id:
try:
await _run_wsl_dinsar_production_controller(
job,
engine_code="isce2",
engine_title="ISCE2",
fallback_timeout_seconds=settings.ISCE2_PER_TASK_TIMEOUT_SECONDS,
)
except Exception as exc:
latest_message = f"ISCE2 D-InSAR production controller failed: {exc}"
try:
async with AsyncSessionLocal() as db:
run = await dinsar_production_service.get_run(production_run_id, db)
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
summary_payload = dict(run.summary_json or {})
summary_payload["controller_error"] = str(exc)
await dinsar_production_service.finalize_run(
run,
db=db,
status="FAILED",
summary_payload=summary_payload,
latest_message=latest_message,
)
dinsar_production_service.append_run_log(
run.run_id,
f"[controller-failed] {exc}",
)
except Exception:
pass
try:
current_task = await task_service.get_task(job.task_id)
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
await task_service.add_log(job.task_id, "ERROR", latest_message)
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=latest_message,
)
except Exception:
pass
raise
return
await _handle_queued_engine_run(
job,
engine_title="ISCE2",
backend/app/services/manifest_inventory_service.py
+5
View File
@@ -59,12 +59,17 @@ def _read_manifest_metadata(path: str, *, root: ManagedRootORM) -> Dict[str, Any
for key in (
"schema_version",
"catalog_name",
"product_family",
"product_type",
"product_id",
"display_name",
"pair_key",
"stack_key",
"run_key",
"run_id",
"group_key",
"processor_code",
"runtime_id",
"reference_date",
"published_at",
"produced_at",
backend/app/services/product_package_schema.py
+400
View File
@@ -0,0 +1,400 @@
from __future__ import annotations
import copy
import os
from typing import Any, Dict, Iterable, List, Optional
CANONICAL_PACKAGE_SCHEMA = "insar.product-package/v1"
CANONICAL_PACKAGE_LAYOUT = "canonical.v1"
LEGACY_DINSAR_PACKAGE_SCHEMA = "dinsar-product/v1"
LEGACY_TIMESERIES_PACKAGE_SCHEMA = "psinsar.publish.v1"
TIMESERIES_ARTIFACT_ROLE_MAP = {
"timeseries_cube": "timeseries_cube",
"velocity_map": "velocity_map",
"velocity_geotiff": "velocity_geotiff",
"temporal_coherence": "temporal_coherence",
"temporal_coherence_geotiff": "temporal_coherence_geotiff",
"quality_mask": "quality_mask",
"quality_mask_geotiff": "quality_mask_geotiff",
"preview_png": "preview_png",
"diagnostic_png": "diagnostic_png",
}
_DINSAR_PRIMARY_ROLES = {"disp"}
_DINSAR_PREVIEW_ROLES = {"thumb"}
_TIMESERIES_PRIMARY_ROLES = {"velocity_geotiff", "timeseries_cube", "velocity_map"}
_TIMESERIES_PREVIEW_ROLES = {"preview_png"}
def _text(value: Any) -> str:
return str(value or "").strip()
def _copy_dict(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
return copy.deepcopy(payload) if isinstance(payload, dict) else {}
def _copy_list(payload: Optional[Iterable[Any]]) -> List[Any]:
return copy.deepcopy(list(payload or []))
def infer_asset_format(path: str) -> Optional[str]:
ext = os.path.splitext(str(path or "").lower())[1]
return {
".h5": "hdf5",
".hdr": "hdr",
".json": "json",
".png": "png",
".sml": "sml",
".tif": "geotiff",
".tiff": "geotiff",
".webp": "webp",
".xml": "xml",
"": None,
}.get(ext, ext.lstrip(".") or None)
def infer_asset_media_type(path: str) -> Optional[str]:
ext = os.path.splitext(str(path or "").lower())[1]
return {
".h5": "application/x-hdf5",
".hdr": "text/plain",
".json": "application/json",
".png": "image/png",
".sml": "text/plain",
".tif": "image/tiff",
".tiff": "image/tiff",
".webp": "image/webp",
".xml": "application/xml",
}.get(ext)
def build_asset_entry(
*,
role: str,
relative_path: str,
asset_name: Optional[str] = None,
format: Optional[str] = None,
media_type: Optional[str] = None,
is_required: bool = False,
is_primary: bool = False,
origin_role: Optional[str] = None,
native_path: Optional[str] = None,
extra: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
relative = _text(relative_path)
if not relative:
raise ValueError("relative_path is required")
payload: Dict[str, Any] = {
"role": _text(role) or "asset",
"asset_name": _text(asset_name) or os.path.basename(relative) or (_text(role) or "asset"),
"relative_path": relative,
"format": format or infer_asset_format(relative),
"media_type": media_type or infer_asset_media_type(relative),
"is_required": bool(is_required),
"is_primary": bool(is_primary),
}
if _text(origin_role):
payload["origin_role"] = _text(origin_role)
if _text(native_path):
payload["native_path"] = _text(native_path)
for key, value in (extra or {}).items():
if value is not None:
payload[key] = value
return payload
def canonicalize_timeseries_artifacts(artifacts: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
assets: List[Dict[str, Any]] = []
for artifact in artifacts or []:
relative_path = _text((artifact or {}).get("path"))
if not relative_path:
continue
product_type = _text((artifact or {}).get("product_type")) or "asset"
asset_role = TIMESERIES_ARTIFACT_ROLE_MAP.get(product_type, product_type or "asset")
assets.append(
build_asset_entry(
role=asset_role,
relative_path=relative_path,
asset_name=os.path.basename(relative_path),
format=infer_asset_format(relative_path),
media_type=infer_asset_media_type(relative_path),
is_required=asset_role in _TIMESERIES_PRIMARY_ROLES or asset_role in _TIMESERIES_PREVIEW_ROLES,
is_primary=asset_role in _TIMESERIES_PRIMARY_ROLES,
origin_role=product_type,
)
)
return assets
def _primary_roles_for_family(product_family: str) -> set[str]:
family = _text(product_family).lower()
if family == "timeseries":
return set(_TIMESERIES_PRIMARY_ROLES)
return set(_DINSAR_PRIMARY_ROLES)
def _preview_roles_for_family(product_family: str) -> set[str]:
family = _text(product_family).lower()
if family == "timeseries":
return set(_TIMESERIES_PREVIEW_ROLES)
return set(_DINSAR_PREVIEW_ROLES)
def build_canonical_descriptor(
assets: Iterable[Dict[str, Any]],
*,
product_family: str,
) -> Dict[str, Any]:
asset_items = list(assets or [])
available_roles = [
_text(asset.get("role"))
for asset in asset_items
if _text(asset.get("role"))
]
primary_role = None
preview_role = None
preferred_primary = _primary_roles_for_family(product_family)
preferred_preview = _preview_roles_for_family(product_family)
for asset in asset_items:
role = _text(asset.get("role"))
if not primary_role and (bool(asset.get("is_primary")) or role in preferred_primary):
primary_role = role
if not preview_role and role in preferred_preview:
preview_role = role
primary_relative = None
preview_relative = None
for asset in asset_items:
role = _text(asset.get("role"))
if primary_relative is None and role == primary_role:
primary_relative = _text(asset.get("relative_path"))
if preview_relative is None and role == preview_role:
preview_relative = _text(asset.get("relative_path"))
return {
"primary_asset_role": primary_role,
"preview_asset_role": preview_role,
"primary_asset_relative": primary_relative,
"preview_asset_relative": preview_relative,
"available_asset_roles": sorted({role for role in available_roles if role}),
}
def _normalize_canonical_manifest(document: Dict[str, Any]) -> Dict[str, Any]:
document["schema_version"] = CANONICAL_PACKAGE_SCHEMA
document["package_layout"] = _text(document.get("package_layout")) or CANONICAL_PACKAGE_LAYOUT
document["source_schema_version"] = (
_text(document.get("source_schema_version")) or CANONICAL_PACKAGE_SCHEMA
)
identity = _copy_dict(document.get("identity"))
if not _text(identity.get("pair_key")) and _text(document.get("pair_key")):
identity["pair_key"] = _text(document.get("pair_key"))
if not _text(identity.get("stack_key")) and _text(document.get("stack_key")):
identity["stack_key"] = _text(document.get("stack_key"))
if not _text(identity.get("run_key")):
identity["run_key"] = _text(document.get("run_key")) or _text(document.get("run_id"))
document["identity"] = identity
document["pair_key"] = _text(identity.get("pair_key")) or None
document["stack_key"] = _text(identity.get("stack_key")) or None
document["run_key"] = _text(identity.get("run_key")) or None
engine = _copy_dict(document.get("engine"))
if not _text(engine.get("code")):
engine["code"] = _text(document.get("engine_code")) or "unknown"
if not _text(engine.get("version")) and _text(document.get("engine_version")):
engine["version"] = _text(document.get("engine_version"))
document["engine"] = engine
processor = _copy_dict(document.get("processor"))
if not _text(processor.get("code")):
processor["code"] = _text(document.get("processor_code")) or _text(engine.get("code")) or "unknown"
if not _text(processor.get("profile_code")) and _text(document.get("profile_code")):
processor["profile_code"] = _text(document.get("profile_code"))
document["processor"] = processor
document["processor_code"] = _text(processor.get("code")) or None
runtime = _copy_dict(document.get("runtime"))
if not _text(runtime.get("runtime_id")) and _text(document.get("runtime_id")):
runtime["runtime_id"] = _text(document.get("runtime_id"))
document["runtime"] = runtime
document["runtime_id"] = _text(runtime.get("runtime_id")) or None
source = _copy_dict(document.get("source"))
if not _text(source.get("primary_path")) and _text(document.get("source_primary_path")):
source["primary_path"] = _text(document.get("source_primary_path"))
if not _text(source.get("publish_dir")) and _text(document.get("publish_dir")):
source["publish_dir"] = _text(document.get("publish_dir"))
if not _text(source.get("native_output_dir")):
source["native_output_dir"] = (
_text(source.get("output_dir"))
or _text(document.get("native_output_dir"))
or None
)
document["source"] = source
document["native_output_dir"] = _text(source.get("native_output_dir")) or None
temporal = _copy_dict(document.get("temporal"))
if not _text(temporal.get("reference_date")) and _text(document.get("reference_date")):
temporal["reference_date"] = _text(document.get("reference_date"))
if not temporal.get("stack_dates") and isinstance(document.get("stack_dates"), list):
temporal["stack_dates"] = [str(item).strip() for item in document.get("stack_dates") or [] if str(item).strip()]
if not _text(temporal.get("produced_at")) and _text(document.get("produced_at")):
temporal["produced_at"] = _text(document.get("produced_at"))
if not _text(temporal.get("published_at")) and _text(document.get("published_at")):
temporal["published_at"] = _text(document.get("published_at"))
document["temporal"] = temporal
document["produced_at"] = _text(temporal.get("produced_at")) or None
document["published_at"] = _text(temporal.get("published_at")) or None
spatial = _copy_dict(document.get("spatial"))
if not spatial and any(document.get(key) is not None for key in ("min_lon", "min_lat", "max_lon", "max_lat")):
spatial = {
"min_lon": document.get("min_lon"),
"min_lat": document.get("min_lat"),
"max_lon": document.get("max_lon"),
"max_lat": document.get("max_lat"),
"coverage_polygon": document.get("coverage_polygon"),
}
document["spatial"] = spatial
assets = _copy_list(document.get("assets"))
if not assets and isinstance(document.get("artifacts"), list):
assets = canonicalize_timeseries_artifacts(document.get("artifacts") or [])
document["assets"] = assets
product_family = _text(document.get("product_family")) or "dinsar"
canonical = _copy_dict(document.get("canonical"))
defaults = build_canonical_descriptor(assets, product_family=product_family)
for key, value in defaults.items():
if canonical.get(key) in (None, "", []):
canonical[key] = value
document["canonical"] = canonical
document["package_schema"] = CANONICAL_PACKAGE_SCHEMA
if not _text(document.get("catalog_name")):
document["catalog_name"] = "dinsar" if product_family == "dinsar" else "psinsar"
if not _text(document.get("product_type")):
document["product_type"] = "dinsar_interferogram" if product_family == "dinsar" else "timeseries_bundle"
if not _text(document.get("display_name")) and _text(document.get("product_id")):
document["display_name"] = _text(document.get("product_id"))
return document
def _normalize_legacy_dinsar_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
document = copy.deepcopy(payload)
run_payload = _copy_dict(document.get("run"))
source_payload = _copy_dict(document.get("source"))
engine_payload = _copy_dict(document.get("engine"))
document["source_schema_version"] = LEGACY_DINSAR_PACKAGE_SCHEMA
document["product_family"] = _text(document.get("product_family")) or "dinsar"
document["product_type"] = "dinsar_interferogram"
document["engine_code"] = _text(engine_payload.get("code")) or _text(document.get("engine_code")) or "unknown"
document["processor"] = {
"code": (
_text(document.get("processor_code"))
or _text(run_payload.get("profile_code"))
or _text(engine_payload.get("code"))
or "unknown"
),
"profile_code": _text(run_payload.get("profile_code")) or None,
}
runtime = _copy_dict(document.get("runtime"))
if not _text(runtime.get("kind")):
runtime["kind"] = "windows" if document["engine_code"] in {"envi", "sarscape"} else None
document["runtime"] = runtime
document["source"] = {
**source_payload,
"native_output_dir": (
_text(source_payload.get("native_output_dir"))
or _text(source_payload.get("output_dir"))
or None
),
}
document["canonical"] = build_canonical_descriptor(
document.get("assets") or [],
product_family="dinsar",
)
return _normalize_canonical_manifest(document)
def _normalize_legacy_timeseries_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
document = copy.deepcopy(payload)
runtime_payload = _copy_dict(document.get("runtime"))
source_summary = _copy_dict(document.get("source_summary"))
document["source_schema_version"] = LEGACY_TIMESERIES_PACKAGE_SCHEMA
document["product_family"] = _text(document.get("product_family")) or "timeseries"
document["product_type"] = "timeseries_bundle"
document["identity"] = {
**_copy_dict(document.get("identity")),
"stack_key": (
_text(_copy_dict(document.get("identity")).get("stack_key"))
or _text(document.get("stack_key"))
or _text(document.get("group_key"))
or None
),
"run_key": (
_text(_copy_dict(document.get("identity")).get("run_key"))
or _text(document.get("run_key"))
or _text(document.get("run_id"))
or None
),
}
document["engine"] = {
**_copy_dict(document.get("engine")),
"code": _text(document.get("engine_code")) or _text(_copy_dict(document.get("engine")).get("code")) or "unknown",
}
document["processor"] = {
"code": _text(document.get("processor_code")) or "unknown",
"profile_code": _text(document.get("processor_code")) or None,
}
if not _text(runtime_payload.get("kind")):
runtime_payload["kind"] = "wsl"
document["runtime"] = runtime_payload
document["source"] = {
**_copy_dict(document.get("source")),
"publish_dir": (
_text(_copy_dict(document.get("source")).get("publish_dir"))
or _text(source_summary.get("publish_dir_windows"))
or None
),
"native_output_dir": (
_text(_copy_dict(document.get("source")).get("native_output_dir"))
or _text(source_summary.get("mintpy_work_dir_windows"))
or None
),
"work_dir": _text(source_summary.get("generated_stack_manifest_path_windows")) or None,
"source_root": _text(source_summary.get("selected_manifest_path_windows")) or None,
}
document["temporal"] = {
**_copy_dict(document.get("temporal")),
"reference_date": _text(document.get("reference_date")) or None,
"stack_dates": [
str(item).strip()
for item in document.get("stack_dates") or []
if str(item).strip()
],
"published_at": _text(document.get("published_at")) or None,
"produced_at": _text(document.get("produced_at")) or None,
}
document["assets"] = canonicalize_timeseries_artifacts(document.get("artifacts") or [])
document["canonical"] = build_canonical_descriptor(
document.get("assets") or [],
product_family="timeseries",
)
return _normalize_canonical_manifest(document)
def normalize_package_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
schema_version = _text((payload or {}).get("schema_version")).lower()
if schema_version == CANONICAL_PACKAGE_SCHEMA:
return _normalize_canonical_manifest(copy.deepcopy(payload))
if schema_version == LEGACY_DINSAR_PACKAGE_SCHEMA:
return _normalize_legacy_dinsar_manifest(payload)
if schema_version == LEGACY_TIMESERIES_PACKAGE_SCHEMA:
return _normalize_legacy_timeseries_manifest(payload)
raise ValueError(f"Unsupported package schema_version: {schema_version or '<empty>'}")
backend/app/services/product_packaging.py
+231
View File
@@ -0,0 +1,231 @@
from __future__ import annotations
import hashlib
from typing import Any, Dict, List, Optional
from .product_package_schema import (
CANONICAL_PACKAGE_LAYOUT,
CANONICAL_PACKAGE_SCHEMA,
build_canonical_descriptor,
normalize_package_manifest,
)
def _clean_dict(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
return dict(payload or {})
def _kind_from_engine(engine_code: str) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized in {"envi", "sarscape"}:
return "windows"
if normalized in {"isce2", "pyint", "gamma"}:
return "wsl"
return None
def _stable_digest(*parts: Any, length: int = 20) -> str:
payload = "||".join(str(part or "") for part in parts)
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
def build_dinsar_package_manifest(
*,
product_id: str,
display_name: str,
task_name: Optional[str],
engine_code: str,
engine_version: Optional[str],
processor_code: Optional[str],
profile_code: Optional[str],
runtime_id: Optional[str],
source_primary_path: str,
source_dir: str,
publish_dir: str,
identity: Dict[str, Any],
source: Dict[str, Any],
run: Dict[str, Any],
temporal: Dict[str, Any],
spatial: Dict[str, Any],
dinsar_profile: Dict[str, Any],
pairing_trace: Dict[str, Any],
labels: Dict[str, Any],
summary: Dict[str, Any],
assets: List[Dict[str, Any]],
issues: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
resolved_processor_code = (
processor_code
or (f"{engine_code}_{profile_code}" if profile_code else None)
or engine_code
)
document = {
"schema_version": CANONICAL_PACKAGE_SCHEMA,
"package_layout": CANONICAL_PACKAGE_LAYOUT,
"catalog_name": "dinsar",
"product_family": "dinsar",
"product_type": "dinsar_interferogram",
"product_id": product_id,
"display_name": display_name,
"task_name": task_name or display_name,
"pair_key": identity.get("pair_key"),
"run_key": identity.get("run_key"),
"identity": {
"pair_key": identity.get("pair_key"),
"stack_key": identity.get("stack_key"),
"run_key": identity.get("run_key"),
"task_alias": identity.get("task_alias"),
},
"engine": {
"code": engine_code,
"version": engine_version,
},
"processor": {
"code": resolved_processor_code,
"profile_code": profile_code,
},
"runtime": {
"runtime_id": runtime_id,
"kind": _kind_from_engine(engine_code),
},
"source": {
**_clean_dict(source),
"primary_path": source_primary_path,
"source_dir": source_dir,
"publish_dir": publish_dir,
"native_output_dir": (
_clean_dict(source).get("native_output_dir")
or _clean_dict(source).get("output_dir")
),
},
"run": _clean_dict(run),
"temporal": _clean_dict(temporal),
"spatial": _clean_dict(spatial),
"dinsar_profile": _clean_dict(dinsar_profile),
"labels": _clean_dict(labels),
"pairing_trace": _clean_dict(pairing_trace),
"summary": _clean_dict(summary),
"assets": list(assets or []),
"canonical": build_canonical_descriptor(assets or [], product_family="dinsar"),
"issues": list(issues or []),
"published_at": _clean_dict(temporal).get("published_at"),
"produced_at": _clean_dict(temporal).get("produced_at"),
"engine_code": engine_code,
"processor_code": resolved_processor_code,
"runtime_id": runtime_id,
}
return normalize_package_manifest(document)
def upgrade_timeseries_package_manifest(
payload: Dict[str, Any],
*,
run_context: Dict[str, Any],
source_summary: Dict[str, Any],
) -> Dict[str, Any]:
document = normalize_package_manifest(payload)
engine_code = str(run_context.get("engine_code") or (document.get("engine") or {}).get("code") or "unknown")
processor_code = str(
run_context.get("processor_code")
or document.get("processor_code")
or ((document.get("processor") or {}).get("code"))
or "unknown"
)
runtime_id = run_context.get("runtime_id") or document.get("runtime_id")
document.update(
{
"schema_version": CANONICAL_PACKAGE_SCHEMA,
"package_layout": CANONICAL_PACKAGE_LAYOUT,
"catalog_name": "psinsar",
"product_family": "timeseries",
"product_type": "timeseries_bundle",
"run_id": run_context.get("run_id"),
"run_name": run_context.get("run_name"),
"batch_id": run_context.get("batch_id"),
"task_id": run_context.get("task_id"),
"workflow_run_id": run_context.get("workflow_run_id"),
"mode": run_context.get("mode"),
"engine_code": engine_code,
"processor_code": processor_code,
"runtime_id": runtime_id,
"stack_key": run_context.get("stack_key") or document.get("stack_key"),
"group_key": run_context.get("group_key") or document.get("group_key"),
"reference_date": run_context.get("reference_date") or document.get("reference_date"),
"stack_dates": run_context.get("stack_dates") or document.get("stack_dates") or [],
"published_at": run_context.get("published_at") or document.get("published_at"),
"produced_at": run_context.get("produced_at") or document.get("produced_at"),
"source_summary": {
**_clean_dict(document.get("source_summary")),
**_clean_dict(source_summary),
},
}
)
if not str(document.get("product_id") or "").strip():
document["product_id"] = "psinsar_" + _stable_digest(
run_context.get("run_id"),
document.get("stack_key") or document.get("group_key"),
document.get("reference_date"),
length=20,
)
document["identity"] = {
**_clean_dict(document.get("identity")),
"stack_key": document.get("stack_key") or document.get("group_key"),
"run_key": run_context.get("run_id") or _clean_dict(document.get("identity")).get("run_key"),
}
document["engine"] = {
**_clean_dict(document.get("engine")),
"code": engine_code,
}
document["processor"] = {
**_clean_dict(document.get("processor")),
"code": processor_code,
"profile_code": processor_code,
}
runtime_payload = {
**_clean_dict(document.get("runtime")),
**_clean_dict(run_context.get("runtime")),
}
runtime_payload["runtime_id"] = runtime_id
runtime_payload["kind"] = runtime_payload.get("kind") or _kind_from_engine(engine_code)
document["runtime"] = runtime_payload
source_payload = {
**_clean_dict(document.get("source")),
"publish_dir": run_context.get("publish_dir"),
"native_output_dir": (
run_context.get("native_output_dir")
or _clean_dict(source_summary).get("mintpy_work_dir_windows")
or _clean_dict(document.get("source")).get("native_output_dir")
),
"work_dir": (
run_context.get("work_dir")
or _clean_dict(source_summary).get("generated_stack_manifest_path_windows")
or _clean_dict(document.get("source")).get("work_dir")
),
"source_root": (
run_context.get("source_root")
or _clean_dict(source_summary).get("selected_manifest_path_windows")
or _clean_dict(document.get("source")).get("source_root")
),
}
document["source"] = source_payload
temporal_payload = {
**_clean_dict(document.get("temporal")),
"reference_date": document.get("reference_date"),
"stack_dates": [
str(item).strip()
for item in document.get("stack_dates") or []
if str(item).strip()
],
"published_at": document.get("published_at"),
"produced_at": document.get("produced_at"),
}
document["temporal"] = temporal_payload
document["canonical"] = build_canonical_descriptor(
document.get("assets") or [],
product_family="timeseries",
)
return normalize_package_manifest(document)
backend/app/services/psinsar_catalog_service.py
+85 -47
View File
@@ -19,26 +19,14 @@ from .manifest_snapshot_service import (
evaluate_manifest_reconcile,
iter_manifest_paths,
)
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
PSINSAR_CATALOG_NAME = "psinsar"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
TASK_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
_ARTIFACT_ROLE_MAP = {
"timeseries_cube": "timeseries_cube",
"velocity_map": "velocity_map",
"velocity_geotiff": "velocity_geotiff",
"temporal_coherence": "temporal_coherence",
"temporal_coherence_geotiff": "temporal_coherence_geotiff",
"quality_mask": "quality_mask",
"quality_mask_geotiff": "quality_mask_geotiff",
"preview_png": "preview_png",
"diagnostic_png": "diagnostic_png",
}
_PRIMARY_PRODUCT_TYPES = {"timeseries_cube", "velocity_geotiff"}
_PREFERRED_PRIMARY_PRODUCT_TYPES = ("velocity_geotiff", "timeseries_cube", "velocity_map")
_PREVIEW_PRODUCT_TYPES = {"preview_png"}
def _utcnow() -> datetime:
@@ -162,7 +150,7 @@ def _derive_bbox_from_summary(manifest: Dict[str, Any]) -> Dict[str, Optional[fl
class PsinsarCatalogService:
def get_publish_root(self, publish_root: Optional[str] = None) -> str:
root = publish_root or settings.PSINSAR_PRODUCT_DIR
root = publish_root or settings.TIMESERIES_PRODUCT_DIR
normalized = _normalize_path(root)
os.makedirs(normalized, exist_ok=True)
return normalized
@@ -183,6 +171,7 @@ class PsinsarCatalogService:
if state is None:
state = ResultCatalogStateORM(
catalog_name=PSINSAR_CATALOG_NAME,
product_family="timeseries",
storage_root=root,
status="READY",
needs_rebuild=False,
@@ -191,6 +180,8 @@ class PsinsarCatalogService:
await db.flush()
elif state.storage_root != root:
state.storage_root = root
if state.product_family != "timeseries":
state.product_family = "timeseries"
return state
def _iter_manifest_paths(self, publish_root: str) -> List[str]:
@@ -199,13 +190,13 @@ class PsinsarCatalogService:
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
with open(manifest_path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
schema_version = str(payload.get("schema_version") or "").strip().lower()
catalog_name = str(payload.get("catalog_name") or "").strip().lower()
if schema_version != "psinsar.publish.v1":
raise ValueError("manifest schema_version is not psinsar.publish.v1")
normalized = normalize_package_manifest(payload)
catalog_name = str(normalized.get("catalog_name") or "").strip().lower()
if str(normalized.get("product_family") or "").strip().lower() != "timeseries":
raise ValueError("manifest product_family is not timeseries")
if catalog_name != PSINSAR_CATALOG_NAME:
raise ValueError("manifest catalog_name is not psinsar")
return payload
return normalized
def _build_rows_from_manifest(
self,
@@ -215,11 +206,28 @@ class PsinsarCatalogService:
package_dir = _normalize_path(os.path.dirname(manifest_path))
basename = os.path.basename(package_dir)
group_key = str(manifest.get("group_key") or "").strip() or None
reference_date = str(manifest.get("reference_date") or "").strip() or None
stack_dates = [str(item).strip() for item in (manifest.get("stack_dates") or []) if str(item).strip()]
stack_key = str(manifest.get("stack_key") or "").strip() or group_key or basename
temporal = manifest.get("temporal") or {}
reference_date = str(
temporal.get("reference_date")
or manifest.get("reference_date")
or ""
).strip() or None
stack_dates = [
str(item).strip()
for item in (
temporal.get("stack_dates")
or manifest.get("stack_dates")
or []
)
if str(item).strip()
]
run_key = str(manifest.get("run_id") or "").strip() or basename
display_name = group_key or basename
product_id = "psinsar_" + _stable_digest(package_dir, run_key, reference_date, length=20)
display_name = stack_key or group_key or basename
product_id = (
str(manifest.get("product_id") or "").strip()
or "psinsar_" + _stable_digest(package_dir, run_key, reference_date, length=20)
)
bbox = _derive_bbox_from_summary(manifest)
poly = _build_bbox_polygon(
bbox.get("min_lon"),
@@ -227,19 +235,29 @@ class PsinsarCatalogService:
bbox.get("max_lon"),
bbox.get("max_lat"),
)
processor_payload = manifest.get("processor") or {}
runtime_payload = manifest.get("runtime") or {}
canonical_payload = manifest.get("canonical") or build_canonical_descriptor(
manifest.get("assets") or [],
product_family="timeseries",
)
summary_json = {
"product_family": "timeseries",
"stack_key": stack_key,
"group_key": group_key,
"reference_date": reference_date,
"reference_point": manifest.get("reference_point"),
"stack_dates": stack_dates,
"stack_size": len(stack_dates),
"mode": manifest.get("mode"),
"processor_code": manifest.get("processor_code"),
"processor_code": processor_payload.get("code") or manifest.get("processor_code"),
"quality": manifest.get("quality"),
"summaries": manifest.get("summaries"),
"canonical": canonical_payload,
"runtime": runtime_payload,
}
published_at = _parse_datetime(manifest.get("published_at"))
published_at = _parse_datetime(temporal.get("published_at") or manifest.get("published_at"))
if published_at is None:
try:
published_at = datetime.utcfromtimestamp(os.path.getmtime(manifest_path))
@@ -249,20 +267,27 @@ class PsinsarCatalogService:
product = ResultProductORM(
product_id=product_id,
catalog_name=PSINSAR_CATALOG_NAME,
product_type="psinsar_bundle",
product_family="timeseries",
product_type=str(manifest.get("product_type") or "timeseries_bundle").strip() or "timeseries_bundle",
display_name=display_name,
task_name=display_name,
task_alias=group_key or basename,
task_alias=stack_key or group_key or basename,
pair_key=None,
stack_key=stack_key,
run_key=run_key,
profile_code=str(manifest.get("processor_code") or "").strip() or None,
engine_code=str(manifest.get("engine_code") or "unknown"),
engine_version=None,
profile_code=str(processor_payload.get("profile_code") or manifest.get("processor_code") or "").strip() or None,
engine_code=str(((manifest.get("engine") or {}).get("code")) or manifest.get("engine_code") or "unknown"),
engine_version=str(((manifest.get("engine") or {}).get("version")) or "") or None,
package_schema=str(manifest.get("schema_version") or "").strip() or None,
package_layout=str(manifest.get("package_layout") or "").strip() or None,
processor_code=str(processor_payload.get("code") or manifest.get("processor_code") or "").strip() or None,
runtime_id=str(runtime_payload.get("runtime_id") or manifest.get("runtime_id") or "").strip() or None,
status="READY",
health_status="OK",
publish_dir=package_dir,
manifest_path=_normalize_path(manifest_path),
source_primary_path=None,
native_output_dir=((manifest.get("source") or {}).get("native_output_dir")),
preview_path=None,
primary_asset_path=None,
summary_json=summary_json,
@@ -282,25 +307,30 @@ class PsinsarCatalogService:
[bbox["min_lon"], bbox["min_lat"]],
]],
} if None not in (bbox["min_lon"], bbox["min_lat"], bbox["max_lon"], bbox["max_lat"]) else None,
produced_at=_parse_datetime(manifest.get("produced_at")) or published_at,
produced_at=_parse_datetime(temporal.get("produced_at") or manifest.get("produced_at")) or published_at,
published_at=published_at,
registered_at=_utcnow(),
)
has_warn = False
has_error = False
artifacts = manifest.get("artifacts") or []
assets_payload = manifest.get("assets") or []
chosen_primary_path = None
for candidate_type in _PREFERRED_PRIMARY_PRODUCT_TYPES:
for artifact in artifacts:
if str(artifact.get("product_type") or "").strip() == candidate_type:
chosen_primary_path = str(artifact.get("path") or "").strip()
preferred_primary_roles = [
str(canonical_payload.get("primary_asset_role") or "").strip(),
*_PREFERRED_PRIMARY_PRODUCT_TYPES,
]
for candidate_type in preferred_primary_roles:
for asset in assets_payload:
if str(asset.get("role") or "").strip() == candidate_type:
chosen_primary_path = str(asset.get("relative_path") or "").strip()
break
if chosen_primary_path:
break
for artifact in artifacts:
relative_path = str(artifact.get("path") or "").strip()
preview_role = str(canonical_payload.get("preview_asset_role") or "").strip()
for asset_payload in assets_payload:
relative_path = str(asset_payload.get("relative_path") or "").strip()
if not relative_path:
continue
absolute_path = _resolve_relative_path(package_dir, relative_path)
@@ -310,28 +340,27 @@ class PsinsarCatalogService:
except OSError:
file_size = None
product_type = str(artifact.get("product_type") or "asset").strip()
asset_role = _ARTIFACT_ROLE_MAP.get(product_type, product_type or "asset")
is_required = product_type in _PRIMARY_PRODUCT_TYPES or product_type in _PREVIEW_PRODUCT_TYPES
asset_role = str(asset_payload.get("role") or "asset").strip()
is_required = bool(asset_payload.get("is_required"))
is_primary = relative_path == chosen_primary_path
asset = ResultAssetORM(
asset_role=asset_role,
asset_name=os.path.basename(relative_path) or asset_role,
relative_path=relative_path,
absolute_path=absolute_path,
format=_artifact_format(relative_path),
media_type=_artifact_media_type(relative_path),
format=asset_payload.get("format") or _artifact_format(relative_path),
media_type=asset_payload.get("media_type") or _artifact_media_type(relative_path),
is_required=is_required,
is_primary=is_primary,
exists_flag=exists_flag,
file_size=file_size,
)
product.assets.append(asset)
if product_type == "timeseries_cube" and exists_flag:
if asset_role == "timeseries_cube" and exists_flag:
product.source_primary_path = absolute_path
if is_primary and exists_flag:
product.primary_asset_path = absolute_path
if product_type in _PREVIEW_PRODUCT_TYPES and exists_flag:
if preview_role and asset_role == preview_role and exists_flag:
product.preview_path = absolute_path
if is_required and not exists_flag:
has_error = True
@@ -394,12 +423,12 @@ class PsinsarCatalogService:
root = self.get_publish_root(publish_root)
normalized_manifest_path = _normalize_path(manifest_path)
if not os.path.isfile(normalized_manifest_path):
raise FileNotFoundError(f"PS-InSAR manifest not found: {normalized_manifest_path}")
raise FileNotFoundError(f"Timeseries manifest not found: {normalized_manifest_path}")
if not (
normalized_manifest_path == root
or normalized_manifest_path.startswith(root + os.sep)
):
raise ValueError("Manifest path is outside the configured PS-InSAR publish root.")
raise ValueError("Manifest path is outside the configured timeseries publish root.")
state = await self._get_or_create_catalog_state(db, storage_root=root)
state.status = "UPDATING"
@@ -598,6 +627,9 @@ class PsinsarCatalogService:
"run_key": item.run_key,
"profile_code": item.profile_code,
"engine_code": item.engine_code,
"package_schema": item.package_schema,
"processor_code": item.processor_code,
"runtime_id": item.runtime_id,
"status": item.status,
"health_status": item.health_status,
"preview_path": item.preview_path,
@@ -649,11 +681,16 @@ class PsinsarCatalogService:
"run_key": product.run_key,
"profile_code": product.profile_code,
"engine_code": product.engine_code,
"package_schema": product.package_schema,
"package_layout": product.package_layout,
"processor_code": product.processor_code,
"runtime_id": product.runtime_id,
"status": product.status,
"health_status": product.health_status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"source_primary_path": product.source_primary_path,
"native_output_dir": product.native_output_dir,
"preview_path": product.preview_path,
"primary_asset_path": product.primary_asset_path,
"reference_date": summary.get("reference_date"),
@@ -717,6 +754,7 @@ class PsinsarCatalogService:
db_count = int(db_count_result.scalar_one() or 0)
payload = {
"catalog_name": state.catalog_name,
"product_family": state.product_family,
"storage_root": state.storage_root,
"status": state.status,
"needs_rebuild": state.needs_rebuild,
backend/app/services/pyint_service.py
+16 -1
View File
@@ -48,6 +48,21 @@ def _read_bool_env(name: str, default: bool = False) -> bool:
return read_bool_env(name, default)
def default_gamma_env_script_windows() -> str:
return os.path.normpath(str(Path(settings.PROJECT_ROOT, "deploy", "wsl", "profiles", "gamma_env.sh")))
def resolve_gamma_env_script(gamma_env_script: Optional[str] = None) -> str:
explicit = str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")).strip()
if explicit:
return os.path.normpath(explicit)
candidate = default_gamma_env_script_windows()
if os.path.isfile(candidate):
return candidate
return ""
def normalize_date_text(value: Any) -> str:
text = str(value or "").strip()
if not text:
@@ -296,7 +311,7 @@ def check_pyint_environment(
work_root_wsl = to_wsl_path(str(work_root or _read_env("PYINT_WORK_ROOT", "")))
output_root_wsl = to_wsl_path(str(output_root or _read_env("PYINT_OUTPUT_ROOT", "")))
dem_root_wsl = to_wsl_path(str(dem_root or _read_env("PYINT_DEM_ROOT", "")))
gamma_env_wsl = to_wsl_path(str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")))
gamma_env_wsl = to_wsl_path(resolve_gamma_env_script(gamma_env_script))
smoke_enabled = _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) if smoke_test is None else bool(smoke_test)
precise_orbit_enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True)
backend/app/services/result_catalog_service.py
+201 -66
View File
@@ -38,6 +38,16 @@ from .dinsar_naming import (
build_fallback_pair_key,
find_json_sidecar,
)
from .dinsar_result_layout_service import (
RUN_CURRENT_DIRNAME,
RUN_NATIVE_DIRNAME,
RUN_PREVIEW_DIRNAME,
is_path_within_native_dir,
is_standard_envi_disp_file,
is_standard_isce2_disp_file,
)
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
from .product_packaging import build_dinsar_package_manifest
DINSAR_CATALOG_NAME = "dinsar"
@@ -120,6 +130,15 @@ def _coerce_optional_int(value: Any) -> Optional[int]:
return None
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized == "isce2":
return settings.ISCE2_RUNTIME_ID or None
if normalized in {"pyint", "gamma"}:
return settings.PYINT_RUNTIME_ID or None
return None
def _build_pairing_trace_payload(
candidate_meta: Dict[str, Any],
task_item: Optional[DinsarTaskItemORM],
@@ -194,11 +213,18 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
"task_dir": _first_text(run_meta.get("task_dir")),
"work_dir": _first_text(run_meta.get("work_dir")),
"output_dir": _first_text(run_meta.get("output_dir"), source_dir),
"native_output_dir": _first_text(run_meta.get("native_output_dir")),
"started_at": _first_text(run_meta.get("started_at")),
"finished_at": _first_text(run_meta.get("finished_at")),
"params": run_meta.get("params") if isinstance(run_meta.get("params"), dict) else {},
"metrics": run_meta.get("metrics") if isinstance(run_meta.get("metrics"), dict) else {},
}
if not resolved["native_output_dir"]:
native_dir = os.path.join(str(resolved["output_dir"] or source_dir), RUN_NATIVE_DIRNAME)
if os.path.isdir(native_dir):
resolved["native_output_dir"] = native_dir
else:
resolved["native_output_dir"] = resolved["output_dir"]
for field in (
"master_path",
@@ -247,7 +273,8 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
stack = [_normalize_path(root_dir)]
normalized_root = _normalize_path(root_dir)
stack = [normalized_root]
while stack:
current = stack.pop()
try:
@@ -255,12 +282,64 @@ def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
for entry in entries:
try:
if entry.is_dir(follow_symlinks=False):
rel_name = os.path.relpath(entry.path, normalized_root)
rel_parts = [part.lower() for part in rel_name.split(os.sep) if part]
if any(
part in {
RUN_NATIVE_DIRNAME,
RUN_CURRENT_DIRNAME,
RUN_PREVIEW_DIRNAME,
}
for part in rel_parts
):
continue
stack.append(entry.path)
continue
if not entry.is_file(follow_symlinks=False):
continue
if is_path_within_native_dir(normalized_root, entry.path):
continue
lower_name = entry.name.lower()
if is_standard_envi_disp_file(normalized_root, entry.path):
primary_file = os.path.join(os.path.dirname(entry.path), "disp")
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(primary_file)))
sidecars = []
if os.path.isfile(primary_file + ".hdr"):
sidecars.append(primary_file + ".hdr")
if os.path.isfile(primary_file + ".sml"):
sidecars.append(primary_file + ".sml")
yield {
"engine_code": "envi",
"name": "disp",
"task_name": "",
"source_dir": source_dir,
"primary_file": primary_file,
"source_files": [primary_file] + sidecars,
}
continue
if is_standard_isce2_disp_file(normalized_root, entry.path):
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(entry.path)))
source_files = [entry.path]
coh_candidates = (
os.path.join(source_dir, "assets", "coh", "coh.tif"),
os.path.join(source_dir, "assets", "coh", "coh.tiff"),
)
for coh_path in coh_candidates:
if os.path.isfile(coh_path):
source_files.append(coh_path)
break
yield {
"engine_code": "isce2",
"name": os.path.splitext(entry.name)[0],
"task_name": "",
"source_dir": source_dir,
"primary_file": entry.path,
"source_files": source_files,
}
continue
if lower_name.endswith(".hdr"):
base_name, _ = os.path.splitext(entry.name)
if not base_name.lower().endswith("_disp"):
@@ -344,6 +423,15 @@ def _resolve_relative_path(base_dir: str, relative_path: str) -> str:
return target
def _is_path_within(base_dir: str, candidate_path: str) -> bool:
base = _normalize_path(base_dir)
candidate = _normalize_path(candidate_path)
try:
return os.path.commonpath([base, candidate]) == base
except ValueError:
return False
def _build_bbox_polygon(
min_lon: Optional[float],
min_lat: Optional[float],
@@ -383,6 +471,7 @@ class ResultCatalogService:
if state is None:
state = ResultCatalogStateORM(
catalog_name=DINSAR_CATALOG_NAME,
product_family="dinsar",
storage_root=root,
status="READY",
needs_rebuild=False,
@@ -391,6 +480,8 @@ class ResultCatalogService:
await db.flush()
elif state.storage_root != root:
state.storage_root = root
if state.product_family != "dinsar":
state.product_family = "dinsar"
return state
async def _lookup_task_item(
@@ -565,57 +656,57 @@ class ResultCatalogService:
}
if pairing_trace:
summary_payload["pairing_trace"] = pairing_trace
return {
"schema_version": "dinsar-product/v1",
"catalog_name": DINSAR_CATALOG_NAME,
"product_id": product_id,
"product_type": "dinsar",
"display_name": display_name,
"task_name": candidate_meta.get("task_alias") or display_name,
"identity": {
return build_dinsar_package_manifest(
product_id=product_id,
display_name=display_name,
task_name=candidate_meta.get("task_alias") or display_name,
engine_code=engine_code,
engine_version=candidate_meta.get("engine_version") or "",
processor_code=candidate_meta.get("profile_code") or engine_code,
profile_code=candidate_meta.get("profile_code"),
runtime_id=_runtime_id_for_engine(engine_code),
source_primary_path=source_primary_path,
source_dir=source_dir,
publish_dir=package_dir,
identity={
"pair_key": candidate_meta.get("pair_key"),
"task_alias": candidate_meta.get("task_alias") or display_name,
"run_key": candidate_meta.get("run_key"),
},
"engine": {
"code": engine_code,
"version": candidate_meta.get("engine_version") or "",
},
"source": {
"primary_path": source_primary_path,
"source_dir": source_dir,
source={
"source_root": candidate_meta.get("source_root"),
"task_dir": candidate_meta.get("task_dir"),
"work_dir": candidate_meta.get("work_dir"),
"output_dir": candidate_meta.get("output_dir"),
"publish_dir": package_dir,
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
},
"run": {
run={
"engine_code": engine_code,
"profile_code": candidate_meta.get("profile_code"),
"source_root": candidate_meta.get("source_root"),
"task_dir": candidate_meta.get("task_dir"),
"work_dir": candidate_meta.get("work_dir"),
"output_dir": candidate_meta.get("output_dir"),
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
"started_at": candidate_meta.get("started_at"),
"finished_at": candidate_meta.get("finished_at"),
"params": profile_params,
"metrics": profile_metrics,
},
"temporal": {
temporal={
"master_imaging_date": master_date,
"slave_imaging_date": slave_date,
"produced_at": candidate_meta.get("finished_at") or candidate_meta.get("started_at"),
"published_at": published_at,
},
"spatial": {
spatial={
"min_lon": meta.get("min_lon"),
"min_lat": meta.get("min_lat"),
"max_lon": meta.get("max_lon"),
"max_lat": meta.get("max_lat"),
"coverage_polygon": meta.get("coverage_polygon"),
},
"dinsar_profile": {
dinsar_profile={
"master_path": getattr(task_item, "master_path", None) or candidate_meta.get("master_path"),
"slave_path": getattr(task_item, "slave_path", None) or candidate_meta.get("slave_path"),
"master_satellite": getattr(task_item, "master_satellite", None) or candidate_meta.get("master_satellite"),
@@ -637,15 +728,15 @@ class ResultCatalogService:
"params": profile_params,
"metrics": profile_metrics,
},
"labels": {
pairing_trace=pairing_trace,
labels={
"ai_score": None,
"user_label": None,
},
"pairing_trace": pairing_trace,
"summary": summary_payload,
"assets": asset_rows,
"issues": [],
}
summary=summary_payload,
assets=asset_rows,
issues=[],
)
async def publish_from_sources(
self,
@@ -691,7 +782,9 @@ class ResultCatalogService:
run_key,
primary_file,
)
package_dir = _ensure_directory(os.path.join(target_root, pair_key, run_key))
package_dir = _ensure_directory(os.path.join(target_root, pair_key, "runs", run_key))
source_dir = _normalize_path(candidate["source_dir"])
in_place_source = _is_path_within(package_dir, source_dir)
task_item = await self._lookup_task_item(
db,
pair_key=pair_key,
@@ -713,24 +806,28 @@ class ResultCatalogService:
)
continue
disp_dir = _ensure_directory(os.path.join(package_dir, "assets", "disp"))
disp_dir = os.path.join(package_dir, "assets", "disp")
preview_dir = _ensure_directory(os.path.join(package_dir, "preview"))
asset_rows: List[Dict[str, Any]] = []
if candidate["engine_code"] == "envi":
target_base = os.path.join(disp_dir, "disp")
target_primary = target_base
source_primary = candidate["source_files"][0]
for src_path in candidate["source_files"]:
suffix = src_path[len(source_primary):]
dst_path = target_base + suffix
op = _copy_file_if_needed(src_path, dst_path)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
source_primary = _normalize_path(candidate["source_files"][0])
if in_place_source:
target_primary = source_primary
else:
_ensure_directory(disp_dir)
target_base = os.path.join(disp_dir, "disp")
target_primary = target_base
for src_path in candidate["source_files"]:
suffix = src_path[len(source_primary):]
dst_path = target_base + suffix
op = _copy_file_if_needed(src_path, dst_path)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "disp",
@@ -769,14 +866,18 @@ class ResultCatalogService:
}
)
else:
target_primary = os.path.join(disp_dir, "disp.tif")
op = _copy_file_if_needed(primary_file, target_primary)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
if in_place_source:
target_primary = _normalize_path(primary_file)
else:
skipped += 1
_ensure_directory(disp_dir)
target_primary = os.path.join(disp_dir, "disp.tif")
op = _copy_file_if_needed(primary_file, target_primary)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "disp",
@@ -789,17 +890,20 @@ class ResultCatalogService:
}
)
if len(candidate["source_files"]) > 1:
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
source_coh = candidate["source_files"][1]
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
op = _copy_file_if_needed(source_coh, target_coh)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
if in_place_source:
target_coh = _normalize_path(source_coh)
else:
skipped += 1
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
op = _copy_file_if_needed(source_coh, target_coh)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "coh",
@@ -859,6 +963,7 @@ class ResultCatalogService:
"run_key": run_key,
"engine_code": candidate_meta["engine_code"],
"package_dir": package_dir,
"in_place": in_place_source,
"thumb_created": thumb_ok,
"status": "ok",
}
@@ -881,11 +986,12 @@ class ResultCatalogService:
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
with open(manifest_path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
if str(payload.get("product_type") or "").strip().lower() != "dinsar":
raise ValueError("manifest product_type is not dinsar")
if not str(payload.get("product_id") or "").strip():
normalized = normalize_package_manifest(payload)
if str(normalized.get("product_family") or "").strip().lower() != "dinsar":
raise ValueError("manifest product_family is not dinsar")
if not str(normalized.get("product_id") or "").strip():
raise ValueError("manifest product_id is empty")
return payload
return normalized
def _build_rows_from_manifest(
self,
@@ -921,39 +1027,58 @@ class ResultCatalogService:
profile_payload = manifest.get("dinsar_profile") or {}
labels = manifest.get("labels") or {}
pairing_trace = manifest.get("pairing_trace") or {}
processor_payload = manifest.get("processor") or {}
runtime_payload = manifest.get("runtime") or {}
canonical_payload = manifest.get("canonical") or build_canonical_descriptor(
assets_payload,
product_family="dinsar",
)
summary_json: Optional[Dict[str, Any]] = None
if summary or identity or run_payload or pairing_trace:
if summary or identity or run_payload or pairing_trace or canonical_payload:
summary_json = {
**summary,
"identity": identity,
"run": run_payload,
}
if processor_payload:
summary_json["processor"] = processor_payload
if runtime_payload:
summary_json["runtime"] = runtime_payload
if canonical_payload:
summary_json["canonical"] = canonical_payload
if pairing_trace:
summary_json["pairing_trace"] = pairing_trace
product = ResultProductORM(
product_id=str(manifest.get("product_id")).strip(),
catalog_name=str(manifest.get("catalog_name") or DINSAR_CATALOG_NAME).strip() or DINSAR_CATALOG_NAME,
product_type="dinsar",
product_family=str(manifest.get("product_family") or "dinsar").strip() or "dinsar",
product_type=str(manifest.get("product_type") or "dinsar_interferogram").strip() or "dinsar_interferogram",
display_name=str(manifest.get("display_name") or manifest.get("task_name") or manifest.get("product_id")),
task_name=str(manifest.get("task_name") or manifest.get("display_name") or "").strip() or None,
task_alias=str(identity.get("task_alias") or manifest.get("task_name") or "").strip() or None,
pair_key=str(identity.get("pair_key") or "").strip() or None,
stack_key=str(identity.get("stack_key") or "").strip() or None,
pair_uid=str(pairing_trace.get("pair_uid") or "").strip() or None,
run_key=str(identity.get("run_key") or "").strip() or None,
network_run_id=str(pairing_trace.get("network_run_id") or "").strip() or None,
network_edge_id=_coerce_optional_int(pairing_trace.get("network_edge_id")),
policy_version=str(pairing_trace.get("policy_version") or "").strip() or None,
selection_strategy=str(pairing_trace.get("selection_strategy") or "").strip() or None,
profile_code=str(run_payload.get("profile_code") or "").strip() or None,
profile_code=str(processor_payload.get("profile_code") or run_payload.get("profile_code") or "").strip() or None,
engine_code=str(((manifest.get("engine") or {}).get("code")) or "unknown"),
engine_version=str(((manifest.get("engine") or {}).get("version")) or "") or None,
package_schema=str(manifest.get("schema_version") or "").strip() or None,
package_layout=str(manifest.get("package_layout") or "").strip() or None,
processor_code=str(processor_payload.get("code") or manifest.get("processor_code") or "").strip() or None,
runtime_id=str(runtime_payload.get("runtime_id") or manifest.get("runtime_id") or "").strip() or None,
status="READY",
health_status="OK",
publish_dir=package_dir,
manifest_path=_normalize_path(manifest_path),
source_primary_path=source.get("primary_path"),
native_output_dir=source.get("native_output_dir"),
preview_path=None,
primary_asset_path=None,
summary_json=summary_json,
@@ -1000,6 +1125,7 @@ class ResultCatalogService:
has_warn = False
has_error = False
preview_role = str(canonical_payload.get("preview_asset_role") or "").strip() or "thumb"
for asset_payload in assets_payload:
relative_path = str(asset_payload.get("relative_path") or "").strip()
if not relative_path:
@@ -1031,7 +1157,7 @@ class ResultCatalogService:
product.assets.append(asset)
if asset.is_primary:
product.primary_asset_path = absolute_path
if asset.asset_role == "thumb":
if asset.asset_role == preview_role:
product.preview_path = absolute_path
if asset.is_required and not exists_flag:
has_error = True
@@ -1256,6 +1382,9 @@ class ResultCatalogService:
"selection_strategy": item.selection_strategy,
"profile_code": item.profile_code,
"engine_code": item.engine_code,
"package_schema": item.package_schema,
"processor_code": item.processor_code,
"runtime_id": item.runtime_id,
"status": item.status,
"health_status": item.health_status,
"preview_path": item.preview_path,
@@ -1325,11 +1454,16 @@ class ResultCatalogService:
"profile_code": product.profile_code,
"engine_code": product.engine_code,
"engine_version": product.engine_version,
"package_schema": product.package_schema,
"package_layout": product.package_layout,
"processor_code": product.processor_code,
"runtime_id": product.runtime_id,
"status": product.status,
"health_status": product.health_status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"source_primary_path": product.source_primary_path,
"native_output_dir": product.native_output_dir,
"preview_path": product.preview_path,
"primary_asset_path": product.primary_asset_path,
"summary_json": product.summary_json,
@@ -1440,6 +1574,7 @@ class ResultCatalogService:
db_count = int(db_count_result.scalar_one() or 0)
payload = {
"catalog_name": state.catalog_name,
"product_family": state.product_family,
"storage_root": state.storage_root,
"status": state.status,
"needs_rebuild": state.needs_rebuild,
backend/app/services/root_registry_service.py
+25 -5
View File
@@ -274,12 +274,12 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
)
specs.extend(
_iter_single_root_specs(
env_var="PSINSAR_PRODUCT_DIR",
path=settings.PSINSAR_PRODUCT_DIR,
root_role="publish_root_psinsar",
display_name="PS-InSAR Publish Root",
env_var="TIMESERIES_PRODUCT_DIR",
path=settings.TIMESERIES_PRODUCT_DIR,
root_role="publish_root_timeseries",
display_name="Timeseries Publish Root",
scan_mode="manifest_tree",
owner_engine="psinsar",
owner_engine="timeseries",
)
)
specs.extend(
@@ -311,6 +311,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
owner_engine="timeseries",
)
)
specs.extend(
_iter_single_root_specs(
env_var="WSL_BROKER_JOB_ROOT",
path=settings.WSL_BROKER_JOB_ROOT,
root_role="work_root_wsl_broker",
display_name="WSL Broker Root",
scan_mode="workspace",
owner_engine="wsl",
)
)
specs.extend(
_iter_single_root_specs(
env_var="IDL_WORKER_RUNTIME_DIR",
@@ -321,6 +331,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
owner_engine="idl",
)
)
specs.extend(
_iter_single_root_specs(
env_var="PYINT_WORK_ROOT",
path=settings.PYINT_WORK_ROOT,
root_role="work_root_pyint",
display_name="Gamma / PyINT Work Root",
scan_mode="workspace",
owner_engine="pyint",
)
)
return specs
backend/app/services/timeseries_service.py
+96 -9
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import os
import re
@@ -25,6 +26,7 @@ from ..models import (
WorkflowStepORM,
)
from .psinsar_catalog_service import psinsar_catalog_service
from .product_packaging import upgrade_timeseries_package_manifest
from .task_service import task_service
from .workflow_service import workflow_service
from .wsl_service import run_wsl_command
@@ -71,6 +73,7 @@ STACK_RUN_FILE_SEQUENCE = (
"run_07_grid_baseline",
"run_08_igram",
)
_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+")
def _utcnow() -> datetime:
@@ -104,6 +107,16 @@ def _normalize_lookup_key(path: Optional[str]) -> str:
return os.path.normcase(os.path.normpath(os.path.abspath(str(path or "").strip())))
def _stable_digest(*parts: Any, length: int = 10) -> str:
payload = "||".join(str(part or "") for part in parts)
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
def _slug_fragment(value: Optional[str], *, default: str) -> str:
text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._").lower()
return text or default
def _read_json(path: Path) -> Dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
@@ -190,6 +203,28 @@ def _build_stack_slug(
)
def _build_stack_key(group_key: Optional[str]) -> str:
parts = [str(item or "").strip() for item in str(group_key or "").split("|")]
while len(parts) < 5:
parts.append("")
satellite, imaging_mode, polarization, orbit_direction, tile_key = parts[:5]
return "_".join(
[
_slug_fragment(satellite, default="sat"),
_slug_fragment(imaging_mode, default="mode"),
_slug_fragment(polarization, default="pol"),
_slug_fragment(orbit_direction, default="dir"),
_slug_fragment(str(tile_key or "").replace(".", "p"), default="tile"),
_stable_digest(group_key, length=10),
]
)
def _compose_publish_dir(run_id: str, stack_key: Optional[str]) -> str:
stack_fragment = _slug_fragment(stack_key, default="unsorted")
return _normalize_path(os.path.join(settings.TIMESERIES_PRODUCT_DIR, stack_fragment, "runs", run_id))
def _common_source_root(paths: List[str]) -> Optional[str]:
normalized = [_normalize_path(path) for path in paths if str(path or "").strip()]
if not normalized:
@@ -207,10 +242,9 @@ def _write_step_logs(logs_dir: Path, step_name: str, stdout: str, stderr: str) -
class TimeseriesService:
def _derive_paths(self, run_id: str) -> Dict[str, str]:
year = _utcnow().strftime("%Y")
def _derive_paths(self, run_id: str, *, stack_key: Optional[str] = None) -> Dict[str, str]:
work_root_windows = _normalize_path(os.path.join(settings.TIMESERIES_WORK_ROOT, run_id))
publish_dir_windows = _normalize_path(os.path.join(settings.PSINSAR_PRODUCT_DIR, year, run_id))
publish_dir_windows = _compose_publish_dir(run_id, stack_key)
return {
"work_root_windows": work_root_windows,
"work_root_wsl": _windows_path_to_wsl_mount(work_root_windows) or "",
@@ -380,6 +414,7 @@ class TimeseriesService:
orbit_direction=first.get("orbit_direction"),
tile_key=manifest_tile_key,
)
stack_key = _build_stack_key(manifest_group_key)
slug = _build_stack_slug(
satellite=first.get("satellite"),
imaging_mode=first.get("imaging_mode"),
@@ -394,6 +429,7 @@ class TimeseriesService:
"source_root_windows": source_root,
"source_root_wsl": _windows_path_to_wsl_mount(source_root) if source_root else None,
"group_key": manifest_group_key,
"stack_key": stack_key,
"tile_key": manifest_tile_key,
"scene_count": len(scene_payloads),
"reference_strategy": REFERENCE_STRATEGY_MIDDLE_BY_DATE,
@@ -496,7 +532,7 @@ class TimeseriesService:
def _publish_manifest_path(self, run: PsTimeseriesRunORM) -> Path:
publish_dir = Path(
run.publish_dir_windows
or _normalize_path(os.path.join(settings.PSINSAR_PRODUCT_DIR, _utcnow().strftime("%Y"), run.run_id))
or _compose_publish_dir(run.run_id, run.stack_key)
)
return publish_dir / "manifest.json"
@@ -641,9 +677,12 @@ class TimeseriesService:
"mintpy_config_path_wsl": _windows_path_to_wsl_mount(str(mintpy_cfg_path)),
"mintpy_work_dir_windows": str(mintpy_work_dir),
"mintpy_work_dir_wsl": _windows_path_to_wsl_mount(str(mintpy_work_dir)),
"publish_dir_windows": str(run.publish_dir_windows or ""),
"publish_dir_wsl": _windows_path_to_wsl_mount(str(run.publish_dir_windows or "")),
}
payload.update(
{
payload = upgrade_timeseries_package_manifest(
payload,
run_context={
"run_id": run.run_id,
"run_name": run.run_name,
"batch_id": run.batch_id,
@@ -652,6 +691,8 @@ class TimeseriesService:
"mode": run.mode,
"engine_code": run.engine_code,
"processor_code": run.processor_code,
"runtime_id": run.runtime_id,
"stack_key": run.stack_key or payload.get("stack_key") or report.get("stack_key"),
"group_key": payload.get("group_key") or report.get("group_key"),
"reference_date": payload.get("reference_date") or run.reference_date,
"stack_dates": (
@@ -668,13 +709,16 @@ class TimeseriesService:
),
"published_at": payload.get("published_at") or now_text,
"produced_at": payload.get("produced_at") or now_text,
"publish_dir": run.publish_dir_windows,
"native_output_dir": str(mintpy_work_dir),
"runtime": {
"runtime_id": run.runtime_id,
"env_name": self._effective_env_name(run),
"wsl_distro": self._effective_wsl_distro(run),
"water_mask_mode": run.water_mask_mode,
},
"source_summary": source_summary,
}
},
source_summary=source_summary,
)
_write_json(manifest_path, payload)
return payload
@@ -750,11 +794,14 @@ class TimeseriesService:
return {
"run_id": run.run_id,
"batch_id": run.batch_id,
"product_family": run.product_family,
"run_name": run.run_name,
"catalog_name": run.catalog_name,
"stack_key": run.stack_key,
"mode": run.mode,
"engine_code": run.engine_code,
"processor_code": run.processor_code,
"runtime_id": run.runtime_id,
"env_name": run.env_name,
"wsl_distro": run.wsl_distro,
"status": run.status,
@@ -884,7 +931,18 @@ class TimeseriesService:
run_id = str(uuid.uuid4())
run_name_text = str(run_name or "").strip() or f"SBAS_{normalized_batch_id}_{run_id[:8]}"
paths = self._derive_paths(run_id)
work_root_windows = _normalize_path(os.path.join(settings.TIMESERIES_WORK_ROOT, run_id))
stack_preview = await self._resolve_stack_scene_records(
items=items,
batch_direction=batch.direction,
run_id=run_id,
work_root_windows=work_root_windows,
db=db,
)
stack_key = str(stack_preview.get("stack_key") or "").strip() or _build_stack_key(
stack_preview.get("group_key")
)
paths = self._derive_paths(run_id, stack_key=stack_key)
selected_manifest_path = Path(paths["work_root_windows"]) / "input" / "selected_stack_manifest.json"
task_name = f"SBAS timeseries run {run_name_text}"
task_id: Optional[str] = None
@@ -904,11 +962,14 @@ class TimeseriesService:
run = PsTimeseriesRunORM(
run_id=run_id,
batch_id=normalized_batch_id,
product_family="timeseries",
run_name=run_name_text,
catalog_name=CATALOG_NAME_PSINSAR,
stack_key=stack_key,
mode="sbas",
engine_code="isce2",
processor_code="isce2_stack_mintpy",
runtime_id=settings.ISCE2_RUNTIME_ID or None,
env_name=settings.TIMESERIES_ENV_NAME or None,
wsl_distro=settings.TIMESERIES_WSL_DISTRO or None,
status=STATUS_PENDING,
@@ -934,10 +995,14 @@ class TimeseriesService:
"water_mask_mode": normalized_water_mask_mode,
"notes": str(notes or "").strip() or None,
"stack_workflow": settings.TIMESERIES_STACK_WORKFLOW,
"group_key": stack_preview.get("group_key"),
"stack_key": stack_key,
},
summary_json={
"phase": "queued",
"workflow": settings.TIMESERIES_STACK_WORKFLOW,
"group_key": stack_preview.get("group_key"),
"stack_key": stack_key,
"stack_dates": stack_dates,
"task_name": task_name,
},
@@ -946,7 +1011,10 @@ class TimeseriesService:
"batch_name": batch.name,
"direction": batch.direction,
"scene_count": len(items),
"group_key": stack_preview.get("group_key"),
"stack_key": stack_key,
"stack_dates": stack_dates,
"source_root_windows": stack_preview.get("source_root_windows"),
"items": self._scene_payload(items),
},
orbit_summary_json={
@@ -976,8 +1044,10 @@ class TimeseriesService:
},
tags={
"catalog_name": CATALOG_NAME_PSINSAR,
"product_family": "timeseries",
"processor_code": "isce2_stack_mintpy",
"batch_id": normalized_batch_id,
"stack_key": stack_key,
},
created_by=created_by,
db=db,
@@ -1065,8 +1135,13 @@ class TimeseriesService:
_write_json(selected_manifest_path, selected_manifest)
run.status = STATUS_PREPARED
run.product_family = run.product_family or "timeseries"
run.stack_key = str(selected_manifest.get("stack_key") or run.stack_key or "").strip() or run.stack_key
run.reference_date = effective_reference_date
run.stack_size = len(stack_dates)
if run.stack_key:
run.publish_dir_windows = _compose_publish_dir(run.run_id, run.stack_key)
run.publish_dir_wsl = _windows_path_to_wsl_mount(run.publish_dir_windows)
run.manifest_path_windows = str(selected_manifest_path)
run.manifest_path_wsl = _windows_path_to_wsl_mount(str(selected_manifest_path))
run.input_snapshot_json = {
@@ -1075,6 +1150,7 @@ class TimeseriesService:
"scene_count": len(stack_dates),
"stack_dates": stack_dates,
"group_key": selected_manifest.get("group_key"),
"stack_key": selected_manifest.get("stack_key"),
"tile_key": selected_manifest.get("tile_key"),
"source_root_windows": selected_manifest.get("source_root_windows"),
"selected_manifest_path_windows": str(selected_manifest_path),
@@ -1097,6 +1173,7 @@ class TimeseriesService:
"phase": "prepared",
"workflow": settings.TIMESERIES_STACK_WORKFLOW,
"group_key": selected_manifest.get("group_key"),
"stack_key": selected_manifest.get("stack_key"),
"tile_key": selected_manifest.get("tile_key"),
"reference_date": effective_reference_date,
"stack_dates": stack_dates,
@@ -1223,6 +1300,10 @@ class TimeseriesService:
run.manifest_path_windows = str(generated_manifest_path)
run.manifest_path_wsl = _windows_path_to_wsl_mount(str(generated_manifest_path))
run.stack_key = str(report.get("stack_key") or run.stack_key or "").strip() or run.stack_key
if run.stack_key:
run.publish_dir_windows = _compose_publish_dir(run.run_id, run.stack_key)
run.publish_dir_wsl = _windows_path_to_wsl_mount(run.publish_dir_windows)
run.reference_date = str(report.get("reference_date") or run.reference_date or "").strip() or run.reference_date
run.stack_size = int(report.get("scene_count") or len(stack_dates) or run.stack_size or 0)
run.orbit_summary_json = {
@@ -1239,6 +1320,7 @@ class TimeseriesService:
"phase": "stack_ready" if (refresh and ready) else "stack_prepared",
"workflow": report.get("processing_workflow") or settings.TIMESERIES_STACK_WORKFLOW,
"group_key": report.get("group_key"),
"stack_key": report.get("stack_key"),
"tile_key": report.get("tile_key"),
"reference_date": report.get("reference_date"),
"stack_dates": stack_dates,
@@ -1655,6 +1737,7 @@ class TimeseriesService:
"publish_dir_wsl": publish_dir_wsl,
"manifest_path_windows": str(publish_manifest_path),
"manifest_path_wsl": _windows_path_to_wsl_mount(str(publish_manifest_path)),
"stack_key": publish_manifest.get("stack_key"),
"group_key": publish_manifest.get("group_key"),
"export_runner": export_result,
},
@@ -1812,10 +1895,14 @@ class TimeseriesService:
"product_id": product.product_id,
"display_name": product.display_name,
"run_key": product.run_key,
"package_schema": product.package_schema,
"processor_code": product.processor_code,
"runtime_id": product.runtime_id,
"status": product.status,
"health_status": product.health_status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"native_output_dir": product.native_output_dir,
"preview_path": product.preview_path,
"primary_asset_path": product.primary_asset_path,
"reference_date": summary.get("reference_date"),
backend/app/services/wsl_broker.py
+165
View File
@@ -0,0 +1,165 @@
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Mapping, Optional, Sequence
from .wsl_runtime_registry import WslRuntimeDefinition, get_wsl_runtime, wsl_runtime_registry
from .wsl_service import run_wsl_exec
_SLUG_RE = re.compile(r"[^a-zA-Z0-9_-]+")
def _utcnow_text() -> str:
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
def _slugify(value: str, *, fallback: str = "job") -> str:
text = _SLUG_RE.sub("_", str(value or "").strip()).strip("_")
return text or fallback
@dataclass(frozen=True)
class WslManifestRef:
runtime_id: str
job_id: str
operation: str
manifest_path_windows: str
manifest_path_wsl: str
def to_dict(self) -> Dict[str, Any]:
return {
"runtime_id": self.runtime_id,
"job_id": self.job_id,
"operation": self.operation,
"manifest_path_windows": self.manifest_path_windows,
"manifest_path_wsl": self.manifest_path_wsl,
}
@dataclass(frozen=True)
class WslBrokerResult:
runtime_id: str
distro: str
returncode: int
argv: Sequence[str]
manifest: WslManifestRef
stdout: str
stderr: str
def to_dict(self) -> Dict[str, Any]:
return {
"runtime_id": self.runtime_id,
"distro": self.distro,
"returncode": self.returncode,
"argv": list(self.argv),
"manifest": self.manifest.to_dict(),
"stdout": self.stdout,
"stderr": self.stderr,
}
class WslBroker:
def __init__(self, *, job_root_windows: Optional[str] = None) -> None:
self._job_root_windows = str(
job_root_windows or wsl_runtime_registry.broker_job_root_windows
).strip()
@property
def job_root_windows(self) -> str:
return self._job_root_windows
def stage_manifest(
self,
*,
runtime_id: str,
operation: str,
payload: Mapping[str, Any],
job_id: Optional[str] = None,
) -> WslManifestRef:
runtime = get_wsl_runtime(runtime_id)
safe_operation = _slugify(operation, fallback="operation")
safe_job_id = _slugify(job_id or f"{safe_operation}_{_utcnow_text()}")
manifest_dir = Path(self.job_root_windows, runtime.runtime_id, safe_operation)
manifest_dir.mkdir(parents=True, exist_ok=True)
manifest_path = manifest_dir / f"{safe_job_id}.json"
document = {
"job_id": safe_job_id,
"runtime_id": runtime.runtime_id,
"engine_code": runtime.engine_code,
"operation": operation,
"created_at": _utcnow_text(),
"payload": dict(payload or {}),
}
manifest_path.write_text(
json.dumps(document, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
return WslManifestRef(
runtime_id=runtime.runtime_id,
job_id=safe_job_id,
operation=operation,
manifest_path_windows=str(manifest_path),
manifest_path_wsl=self._manifest_to_wsl(str(manifest_path)),
)
def _manifest_to_wsl(self, manifest_path_windows: str) -> str:
drive = Path(manifest_path_windows).drive.rstrip(":").lower()
tail = Path(manifest_path_windows).as_posix().split(":", 1)[-1]
if not drive:
return Path(manifest_path_windows).as_posix()
return f"/mnt/{drive}{tail}"
def build_runner_argv(
self,
*,
runtime: WslRuntimeDefinition,
manifest: WslManifestRef,
extra_args: Optional[Sequence[str]] = None,
) -> list[str]:
argv = list(runtime.entrypoint_argv())
argv.extend(["--manifest", manifest.manifest_path_wsl])
argv.extend(str(item) for item in (extra_args or []) if str(item))
return argv
def run_manifest(
self,
*,
runtime_id: str,
operation: str,
payload: Mapping[str, Any],
job_id: Optional[str] = None,
extra_args: Optional[Sequence[str]] = None,
timeout_seconds: int = 30,
env: Optional[Dict[str, str]] = None,
) -> WslBrokerResult:
runtime = get_wsl_runtime(runtime_id)
manifest = self.stage_manifest(
runtime_id=runtime_id,
operation=operation,
payload=payload,
job_id=job_id,
)
argv = self.build_runner_argv(runtime=runtime, manifest=manifest, extra_args=extra_args)
rc, stdout, stderr = run_wsl_exec(
argv,
distro=runtime.distro,
timeout=max(30, int(timeout_seconds or 30)),
env=env,
)
return WslBrokerResult(
runtime_id=runtime.runtime_id,
distro=runtime.distro,
returncode=rc,
argv=argv,
manifest=manifest,
stdout=stdout,
stderr=stderr,
)
wsl_broker = WslBroker()
backend/app/services/wsl_runtime_registry.py
+170
View File
@@ -0,0 +1,170 @@
from __future__ import annotations
import os
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, Mapping, Tuple
from ..config import settings
def _windows_path_to_wsl_mount(path: str) -> str:
text = str(path or "").strip()
if not text:
return ""
drive, tail = os.path.splitdrive(os.path.normpath(text))
if not drive:
return text.replace("\\", "/")
drive_letter = drive.rstrip(":").lower()
normalized_tail = tail.replace("\\", "/")
return f"/mnt/{drive_letter}/{normalized_tail}"
def _project_file_windows(*relative_parts: str) -> str:
return os.path.normpath(str(Path(settings.PROJECT_ROOT, *relative_parts)))
@dataclass(frozen=True)
class WslRuntimeDefinition:
runtime_id: str
engine_code: str
display_name: str
distro: str
conda_env_name: str
python_path: str
runner_path_windows: str
runner_path_wsl: str
allowed_operations: Tuple[str, ...] = ()
env_profile_path_windows: str = ""
env_profile_path_wsl: str = ""
metadata_json: Dict[str, Any] = field(default_factory=dict)
def entrypoint_argv(self) -> list[str]:
return [self.python_path, self.runner_path_wsl]
def to_dict(self) -> Dict[str, Any]:
return {
"runtime_id": self.runtime_id,
"engine_code": self.engine_code,
"display_name": self.display_name,
"distro": self.distro,
"conda_env_name": self.conda_env_name,
"python_path": self.python_path,
"runner_path_windows": self.runner_path_windows,
"runner_path_wsl": self.runner_path_wsl,
"allowed_operations": list(self.allowed_operations),
"env_profile_path_windows": self.env_profile_path_windows,
"env_profile_path_wsl": self.env_profile_path_wsl,
"metadata_json": dict(self.metadata_json or {}),
}
@dataclass(frozen=True)
class WslRuntimeRegistry:
shared_distro: str
shared_conda_env_name: str
shared_python_path: str
broker_job_root_windows: str
broker_job_root_wsl: str
runtimes: Mapping[str, WslRuntimeDefinition]
def require(self, runtime_id: str) -> WslRuntimeDefinition:
key = str(runtime_id or "").strip()
runtime = self.runtimes.get(key)
if runtime is None:
known_ids = ", ".join(sorted(self.runtimes.keys()))
raise KeyError(f"Unknown WSL runtime_id '{runtime_id}'. Known: {known_ids}")
return runtime
def to_dict(self) -> Dict[str, Any]:
return {
"shared_distro": self.shared_distro,
"shared_conda_env_name": self.shared_conda_env_name,
"shared_python_path": self.shared_python_path,
"broker_job_root_windows": self.broker_job_root_windows,
"broker_job_root_wsl": self.broker_job_root_wsl,
"runtimes": {
runtime_id: runtime.to_dict()
for runtime_id, runtime in sorted(self.runtimes.items())
},
}
def build_wsl_runtime_registry() -> WslRuntimeRegistry:
shared_distro = str(
settings.WSL_DISTRO
or settings.ISCE2_WSL_DISTRO
or settings.PYINT_WSL_DISTRO
or "Ubuntu-24.04"
).strip()
shared_conda_env = str(settings.WSL_SHARED_CONDA_ENV or "insar_wsl_v1").strip() or "insar_wsl_v1"
shared_python_path = str(
settings.WSL_SHARED_PYTHON
or settings.ISCE2_PYTHON
or settings.PYINT_WSL_PYTHON
or f"/home/administrator/miniconda3/envs/{shared_conda_env}/bin/python"
).strip()
broker_job_root_windows = os.path.normpath(
settings.WSL_BROKER_JOB_ROOT
or os.path.join(settings.BACKEND_DIR, "runtime", "wsl_jobs")
)
broker_job_root_wsl = _windows_path_to_wsl_mount(broker_job_root_windows)
isce2_runner_windows = _project_file_windows("deploy", "wsl", "runners", "isce2_runner.py")
gamma_runner_windows = _project_file_windows("deploy", "wsl", "runners", "gamma_pyint_runner.py")
gamma_profile_windows = _project_file_windows("deploy", "wsl", "profiles", "gamma_env.sh")
runtimes = {
settings.ISCE2_RUNTIME_ID: WslRuntimeDefinition(
runtime_id=settings.ISCE2_RUNTIME_ID,
engine_code="isce2",
display_name="ISCE2 Runtime V1",
distro=shared_distro,
conda_env_name=shared_conda_env,
python_path=shared_python_path,
runner_path_windows=isce2_runner_windows,
runner_path_wsl=_windows_path_to_wsl_mount(isce2_runner_windows),
allowed_operations=("lt1_stripmap",),
metadata_json={
"shared_runtime": True,
"legacy_distro_env_var": "ISCE2_WSL_DISTRO",
"legacy_python_env_var": "ISCE2_PYTHON",
"legacy_pipeline_env_var": "ISCE2_PIPELINE_SCRIPT",
},
),
settings.PYINT_RUNTIME_ID: WslRuntimeDefinition(
runtime_id=settings.PYINT_RUNTIME_ID,
engine_code="pyint",
display_name="Gamma / PyINT Runtime V1",
distro=shared_distro,
conda_env_name=shared_conda_env,
python_path=shared_python_path,
runner_path_windows=gamma_runner_windows,
runner_path_wsl=_windows_path_to_wsl_mount(gamma_runner_windows),
allowed_operations=("lt1_gamma_dinsar", "gamma_refine"),
env_profile_path_windows=gamma_profile_windows,
env_profile_path_wsl=_windows_path_to_wsl_mount(gamma_profile_windows),
metadata_json={
"shared_runtime": True,
"legacy_distro_env_var": "PYINT_WSL_DISTRO",
"legacy_python_env_var": "PYINT_WSL_PYTHON",
"legacy_profile_env_var": "PYINT_GAMMA_ENV_SCRIPT",
},
),
}
return WslRuntimeRegistry(
shared_distro=shared_distro,
shared_conda_env_name=shared_conda_env,
shared_python_path=shared_python_path,
broker_job_root_windows=broker_job_root_windows,
broker_job_root_wsl=broker_job_root_wsl,
runtimes=runtimes,
)
def get_wsl_runtime(runtime_id: str) -> WslRuntimeDefinition:
return wsl_runtime_registry.require(runtime_id)
wsl_runtime_registry = build_wsl_runtime_registry()
backend/app/services/wsl_service.py
+32 -6
View File
@@ -12,7 +12,7 @@ import os
import shutil
import subprocess
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Sequence, Tuple
# ---------------------------------------------------------------------------
@@ -127,15 +127,41 @@ def run_wsl_command(
return -3, "", str(exc)
def run_wsl_exec(
argv: Sequence[str],
distro: Optional[str] = None,
timeout: int = 30,
env: Optional[Dict[str, str]] = None,
) -> Tuple[int, str, str]:
"""Execute a structured argv in WSL via ``wsl.exe --exec``."""
wsl_exe = _find_wsl_executable()
if not wsl_exe:
return -2, "", "wsl.exe not found"
normalized_argv = [str(part) for part in argv if str(part)]
if not normalized_argv:
return -3, "", "WSL argv is empty"
wsl_args = [wsl_exe]
if distro:
wsl_args += ["-d", distro]
wsl_args += ["--exec", *normalized_argv]
try:
return _run_windows_command(wsl_args, timeout=timeout, env=env)
except subprocess.TimeoutExpired:
return -1, "", f"command timed out ({timeout}s)"
except FileNotFoundError:
return -2, "", "wsl.exe not found"
except Exception as exc:
return -3, "", str(exc)
def windows_path_to_wsl(win_path: str, distro: Optional[str] = None) -> str:
"""将 Windows 路径转换为 WSL 路径（调用 wslpath）。"""
if not win_path:
return ""
rc, stdout, _ = run_wsl_command(
f"wslpath -u '{win_path.replace(chr(39), '')}'",
distro=distro,
timeout=10,
)
rc, stdout, _ = run_wsl_exec(["wslpath", "-u", win_path], distro=distro, timeout=10)
return stdout if rc == 0 else ""