diff --git a/.env.example b/.env.example index 399de61..aab4b21 100644 --- a/.env.example +++ b/.env.example @@ -221,6 +221,27 @@ PYINT_GAMMA_ENV_SCRIPT=D:\Code\Insar_management_system_v2\deploy\wsl\profiles\ga PYINT_DEFAULT_TIMEOUT_SECONDS=43200 PYINT_SMOKE_TEST_ENABLED=false +# ----------------------------------------------------------------------------- +# Gamma SBAS-InSAR +# ----------------------------------------------------------------------------- +GAMMA_SBAS_ENABLED=true +GAMMA_SBAS_RUNTIME_ID=gamma_sbas_runtime_v1 +GAMMA_SBAS_WSL_DISTRO=Ubuntu-24.04 +GAMMA_SBAS_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python +GAMMA_SBAS_ENV_SCRIPT=D:\Code\Insar_management_system_v2\deploy\wsl\profiles\gamma_env.sh +GAMMA_SBAS_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\sbas_insar_production +GAMMA_SBAS_PRODUCT_ROOT=D:\production_results\timeseries\sbas +GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\templates\gamma_sbas +GAMMA_SBAS_SOURCE_ROOTS=D:\LuTan1_Image_Pool +GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi +GAMMA_SBAS_DEFAULT_RLKS=8 +GAMMA_SBAS_DEFAULT_AZLKS=8 +GAMMA_SBAS_DEFAULT_MB_MODE=0 +GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW=16 +GAMMA_SBAS_AUTO_APPROVE_ITAB=true +GAMMA_SBAS_STEP_TIMEOUT_SECONDS=43200 +GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS=172800 + # ----------------------------------------------------------------------------- # 时序 InSAR（旧 ISCE2/MintPy 链路） diff --git a/LT1_GAMMA_SBAS_逐命令处理流程.docx b/LT1_GAMMA_SBAS_逐命令处理流程.docx new file mode 100644 index 0000000..c7cb167 Binary files /dev/null and b/LT1_GAMMA_SBAS_逐命令处理流程.docx differ diff --git a/backend/app/config.py b/backend/app/config.py index 5b3e733..8990197 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -308,6 +308,25 @@ class Settings(BaseSettings): PYINT_GAMMA_ENV_SCRIPT: str = "" PYINT_DEFAULT_TIMEOUT_SECONDS: int = 43200 PYINT_SMOKE_TEST_ENABLED: bool = False + + GAMMA_SBAS_ENABLED: bool = True + GAMMA_SBAS_RUNTIME_ID: str = "" + GAMMA_SBAS_WSL_DISTRO: str = "" + GAMMA_SBAS_PYTHON: str = "" + GAMMA_SBAS_ENV_SCRIPT: str = "" + GAMMA_SBAS_WORK_ROOT: str = "" + GAMMA_SBAS_PRODUCT_ROOT: str = "" + GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT: str = "" + GAMMA_SBAS_SOURCE_ROOTS: str = "" + GAMMA_SBAS_ORBIT_ROOTS: str = "" + GAMMA_SBAS_DEFAULT_RLKS: int = 8 + GAMMA_SBAS_DEFAULT_AZLKS: int = 8 + GAMMA_SBAS_DEFAULT_MB_MODE: int = 0 + GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW: int = 16 + GAMMA_SBAS_AUTO_APPROVE_ITAB: bool = True + GAMMA_SBAS_STEP_TIMEOUT_SECONDS: int = 43200 + GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS: int = 172800 + JOB_WORKER_HEALTH_TIMEOUT: int = 60 JOB_WORKER_JOB_HEARTBEAT_INTERVAL: float = 5.0 JOB_WORKER_STALE_RECOVER_INTERVAL: float = 15.0 @@ -623,6 +642,79 @@ class Settings(BaseSettings): ) if not self.PYINT_ORBIT_POOL_TXT: object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI) + if not self.GAMMA_SBAS_RUNTIME_ID: + object.__setattr__(self, "GAMMA_SBAS_RUNTIME_ID", "gamma_sbas_runtime_v1") + if not self.GAMMA_SBAS_WSL_DISTRO: + object.__setattr__( + self, + "GAMMA_SBAS_WSL_DISTRO", + self.WSL_DISTRO or self.PYINT_WSL_DISTRO or self.ISCE2_WSL_DISTRO, + ) + if not self.GAMMA_SBAS_PYTHON: + object.__setattr__( + self, + "GAMMA_SBAS_PYTHON", + self.WSL_SHARED_PYTHON or self.PYINT_WSL_PYTHON or self.ISCE2_PYTHON, + ) + if not self.GAMMA_SBAS_ENV_SCRIPT: + object.__setattr__(self, "GAMMA_SBAS_ENV_SCRIPT", self.PYINT_GAMMA_ENV_SCRIPT) + if not self.GAMMA_SBAS_WORK_ROOT: + object.__setattr__( + self, + "GAMMA_SBAS_WORK_ROOT", + os.path.join(backend_dir, "runtime", "sbas_insar_production"), + ) + if not self.GAMMA_SBAS_PRODUCT_ROOT: + object.__setattr__( + self, + "GAMMA_SBAS_PRODUCT_ROOT", + os.path.join(self.TIMESERIES_PRODUCT_DIR, "sbas"), + ) + if not self.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT: + object.__setattr__( + self, + "GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT", + os.path.join(backend_dir, "templates", "gamma_sbas"), + ) + if not self.GAMMA_SBAS_SOURCE_ROOTS: + def _local_split_paths(raw: str | None) -> list[str]: + items: list[str] = [] + for part in str(raw or "").replace(";", ",").split(","): + text = part.strip().strip('"').strip("'") + if text: + items.append(text) + return items + lt1_roots = [ + item + for value in (self.SOURCE_PRODUCT_DIRS, self.MONITOR_RADAR_DIRS, self.INSAR_STORAGE_DIRS) + for item in _local_split_paths(value) + if "lutan" in item.lower() or "lt1" in item.lower() + ] + lt1_roots = list(dict.fromkeys(lt1_roots)) + object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", ";".join(lt1_roots) or r"D:\LuTan1_Image_Pool") + if not self.GAMMA_SBAS_ORBIT_ROOTS: + object.__setattr__(self, "GAMMA_SBAS_ORBIT_ROOTS", self.PYINT_ORBIT_POOL_TXT or self.ORBIT_POOL_ENVI) + object.__setattr__(self, "GAMMA_SBAS_DEFAULT_RLKS", max(1, int(self.GAMMA_SBAS_DEFAULT_RLKS or 8))) + object.__setattr__(self, "GAMMA_SBAS_DEFAULT_AZLKS", max(1, int(self.GAMMA_SBAS_DEFAULT_AZLKS or 8))) + gamma_sbas_mb_mode = int(self.GAMMA_SBAS_DEFAULT_MB_MODE or 0) + if gamma_sbas_mb_mode not in {0, 1, 2}: + gamma_sbas_mb_mode = 0 + object.__setattr__(self, "GAMMA_SBAS_DEFAULT_MB_MODE", gamma_sbas_mb_mode) + object.__setattr__( + self, + "GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW", + max(1, int(self.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW or 16)), + ) + object.__setattr__( + self, + "GAMMA_SBAS_STEP_TIMEOUT_SECONDS", + max(60, int(self.GAMMA_SBAS_STEP_TIMEOUT_SECONDS or 43200)), + ) + object.__setattr__( + self, + "GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS", + max(self.GAMMA_SBAS_STEP_TIMEOUT_SECONDS, int(self.GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS or 172800)), + ) if self.TIMESERIES_ENABLED: if not self.TIMESERIES_WSL_DISTRO: object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO) @@ -755,6 +847,10 @@ class Settings(BaseSettings): os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True) os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True) os.makedirs(settings.PYINT_DEM_ROOT, exist_ok=True) + if settings.GAMMA_SBAS_ENABLED: + os.makedirs(settings.GAMMA_SBAS_WORK_ROOT, exist_ok=True) + os.makedirs(settings.GAMMA_SBAS_PRODUCT_ROOT, exist_ok=True) + os.makedirs(settings.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT, exist_ok=True) if settings.TIMESERIES_ENABLED and settings.TIMESERIES_WORK_ROOT: os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True) @@ -1033,6 +1129,42 @@ def validate_runtime_config() -> dict[str, Any]: expect_file=True, ) + if settings.GAMMA_SBAS_ENABLED: + if not settings.GAMMA_SBAS_RUNTIME_ID: + errors.append("GAMMA_SBAS_ENABLED=true but GAMMA_SBAS_RUNTIME_ID is not configured.") + if not settings.GAMMA_SBAS_WSL_DISTRO: + errors.append("GAMMA_SBAS_ENABLED=true but GAMMA_SBAS_WSL_DISTRO is not configured.") + if not settings.GAMMA_SBAS_PYTHON: + errors.append("GAMMA_SBAS_ENABLED=true but GAMMA_SBAS_PYTHON is not configured.") + _check_path( + label="GAMMA_SBAS_ENV_SCRIPT", + value=settings.GAMMA_SBAS_ENV_SCRIPT, + errors=errors, + warnings=warnings, + expect_file=True, + ) + _check_path( + label="GAMMA_SBAS_WORK_ROOT", + value=settings.GAMMA_SBAS_WORK_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="GAMMA_SBAS_PRODUCT_ROOT", + value=settings.GAMMA_SBAS_PRODUCT_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT", + value=settings.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + if settings.ISCE2_ENABLED or settings.PYINT_ENABLED: info.append( "WSL shared runtime: " diff --git a/backend/app/main.py b/backend/app/main.py index 63d8367..2289f05 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -190,6 +190,16 @@ async def lifespan(app: FastAPI): ) if pairing_bootstrap.get("error"): print(f">>> [Pairing] Startup bootstrap failed: {pairing_bootstrap['error']}") + print( + ">>> [Gamma SBAS] enabled={0} runtime={1} distro={2} python={3} work_root={4} product_root={5}".format( + "YES" if settings.GAMMA_SBAS_ENABLED else "NO", + settings.GAMMA_SBAS_RUNTIME_ID or "?", + settings.GAMMA_SBAS_WSL_DISTRO or "?", + settings.GAMMA_SBAS_PYTHON or "?", + settings.GAMMA_SBAS_WORK_ROOT or "?", + settings.GAMMA_SBAS_PRODUCT_ROOT or "?", + ) + ) # 当前项目默认 Manual-only 扫描模式，保留调度器关闭状态。 # scheduler_manager.start() diff --git a/backend/app/routers/sbas_insar_production.py b/backend/app/routers/sbas_insar_production.py index 824c400..86f14e0 100644 --- a/backend/app/routers/sbas_insar_production.py +++ b/backend/app/routers/sbas_insar_production.py @@ -88,6 +88,59 @@ class SbasCoregistrationJobRequest(BaseModel): timeout_seconds: int = Field(default=43200, ge=60, le=172800) +class SbasRdcDemRequest(BaseModel): + execute: bool = False + rlks: int = Field(default=8, ge=1, le=64) + + +class SbasRdcDemJobRequest(BaseModel): + rlks: int = Field(default=8, ge=1, le=64) + timeout_seconds: int = Field(default=43200, ge=60, le=172800) + + +class SbasInterferogramsRequest(BaseModel): + execute: bool = False + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + unwrap_threshold: float = Field(default=0.20, ge=0.01, le=0.95) + + +class SbasInterferogramsJobRequest(BaseModel): + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + unwrap_threshold: float = Field(default=0.20, ge=0.01, le=0.95) + timeout_seconds: int = Field(default=43200, ge=60, le=172800) + + +class SbasIptaTimeseriesRequest(BaseModel): + execute: bool = False + rlks: int = Field(default=8, ge=1, le=64) + reference_window: int = Field(default=16, ge=1, le=256) + mb_mode: int = Field(default=0, ge=0, le=2) + + +class SbasIptaTimeseriesJobRequest(BaseModel): + rlks: int = Field(default=8, ge=1, le=64) + reference_window: int = Field(default=16, ge=1, le=256) + mb_mode: int = Field(default=0, ge=0, le=2) + timeout_seconds: int = Field(default=43200, ge=60, le=172800) + + +class SbasWorkflowPrepareRequest(BaseModel): + force: bool = False + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + reference_window: int = Field(default=16, ge=1, le=256) + mb_mode: int = Field(default=0, ge=0, le=2) + + +class SbasWorkflowJobRequest(SbasWorkflowPrepareRequest): + from_step: str | None = Field(default=None, max_length=64) + to_step: str | None = Field(default=None, max_length=64) + only_steps: list[str] | None = None + timeout_seconds: int = Field(default=172800, ge=60, le=604800) + + @router.get("/capabilities") async def get_sbas_insar_capabilities(): return sbas_insar_production_service.get_capabilities() @@ -168,6 +221,76 @@ async def get_sbas_insar_run(run_id: str): raise HTTPException(status_code=400, detail=str(exc)) from exc +@router.post("/runs/{run_id}/workflow", status_code=202) +async def prepare_sbas_insar_workflow(run_id: str, request: SbasWorkflowPrepareRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.prepare_workflow, + run_id, + force=request.force, + rlks=request.rlks, + azlks=request.azlks, + reference_window=request.reference_window, + mb_mode=request.mb_mode, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/workflow/jobs", status_code=202) +async def submit_sbas_insar_workflow_job(run_id: str, request: SbasWorkflowJobRequest): + try: + await asyncio.to_thread( + sbas_insar_production_service.prepare_workflow, + run_id, + force=request.force, + rlks=request.rlks, + azlks=request.azlks, + reference_window=request.reference_window, + mb_mode=request.mb_mode, + ) + from ..services.job_handlers import JOB_TYPE_SBAS_GAMMA_WORKFLOW + + payload = { + "run_id": run_id, + "force": request.force, + "rlks": request.rlks, + "azlks": request.azlks, + "reference_window": request.reference_window, + "mb_mode": request.mb_mode, + "from_step": request.from_step, + "to_step": request.to_step, + "only_steps": request.only_steps or [], + "timeout_seconds": request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_GAMMA_WORKFLOW, + task_name=f"Gamma SBAS Workflow {run_id}", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_GAMMA_WORKFLOW, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "Gamma SBAS workflow job queued.", + "run_id": run_id, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_GAMMA_WORKFLOW, + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "already running" in message.lower() or "conflict" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + @router.post("/runs/{run_id}/baseline-audit", status_code=202) async def run_sbas_insar_baseline_audit(run_id: str, request: SbasBaselineAuditRequest): try: @@ -274,6 +397,219 @@ async def submit_sbas_insar_coregistration_job(run_id: str, request: SbasCoregis raise HTTPException(status_code=status_code, detail=message) from exc +@router.post("/runs/{run_id}/rdc-dem", status_code=202) +async def prepare_sbas_insar_rdc_dem(run_id: str, request: SbasRdcDemRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.prepare_rdc_dem, + run_id, + execute=request.execute, + rlks=request.rlks, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/rdc-dem/jobs", status_code=202) +async def submit_sbas_insar_rdc_dem_job(run_id: str, request: SbasRdcDemJobRequest): + try: + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status in {"COREGISTRATION_READY", "RDC_DEM_FAILED"}: + await asyncio.to_thread( + sbas_insar_production_service.prepare_rdc_dem, + run_id, + execute=False, + rlks=request.rlks, + ) + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status not in {"RDC_DEM_SCRIPT_READY", "RDC_DEM_RUNNING"}: + raise ValueError(f"run status does not allow RDC DEM job submission: {status}") + if status == "RDC_DEM_RUNNING": + raise ValueError("RDC DEM is already running for this run") + + from ..services.job_handlers import JOB_TYPE_SBAS_RDC_DEM + + payload = { + "run_id": run_id, + "rlks": request.rlks, + "timeout_seconds": request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_RDC_DEM, + task_name=f"SBAS-InSAR RDC DEM {run_id}", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_RDC_DEM, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "SBAS-InSAR RDC DEM job queued.", + "run_id": run_id, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_RDC_DEM, + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "already running" in message.lower() or "conflict" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + +@router.post("/runs/{run_id}/interferograms", status_code=202) +async def prepare_sbas_insar_interferograms(run_id: str, request: SbasInterferogramsRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.prepare_interferograms, + run_id, + execute=request.execute, + rlks=request.rlks, + azlks=request.azlks, + unwrap_threshold=request.unwrap_threshold, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/interferograms/jobs", status_code=202) +async def submit_sbas_insar_interferograms_job(run_id: str, request: SbasInterferogramsJobRequest): + try: + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status in {"RDC_DEM_READY", "INTERFEROGRAMS_FAILED"}: + await asyncio.to_thread( + sbas_insar_production_service.prepare_interferograms, + run_id, + execute=False, + rlks=request.rlks, + azlks=request.azlks, + unwrap_threshold=request.unwrap_threshold, + ) + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status not in {"INTERFEROGRAMS_SCRIPT_READY", "INTERFEROGRAMS_RUNNING"}: + raise ValueError(f"run status does not allow interferogram job submission: {status}") + if status == "INTERFEROGRAMS_RUNNING": + raise ValueError("interferograms are already running for this run") + + from ..services.job_handlers import JOB_TYPE_SBAS_INTERFEROGRAMS + + payload = { + "run_id": run_id, + "rlks": request.rlks, + "azlks": request.azlks, + "unwrap_threshold": request.unwrap_threshold, + "timeout_seconds": request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_INTERFEROGRAMS, + task_name=f"SBAS-InSAR Interferograms {run_id}", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_INTERFEROGRAMS, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "SBAS-InSAR interferogram job queued.", + "run_id": run_id, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_INTERFEROGRAMS, + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "already running" in message.lower() or "conflict" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + +@router.post("/runs/{run_id}/ipta-timeseries", status_code=202) +async def prepare_sbas_insar_ipta_timeseries(run_id: str, request: SbasIptaTimeseriesRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.prepare_ipta_timeseries, + run_id, + execute=request.execute, + rlks=request.rlks, + reference_window=request.reference_window, + mb_mode=request.mb_mode, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/ipta-timeseries/jobs", status_code=202) +async def submit_sbas_insar_ipta_timeseries_job(run_id: str, request: SbasIptaTimeseriesJobRequest): + try: + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status in {"INTERFEROGRAMS_READY", "IPTA_TIMESERIES_FAILED"}: + await asyncio.to_thread( + sbas_insar_production_service.prepare_ipta_timeseries, + run_id, + execute=False, + rlks=request.rlks, + reference_window=request.reference_window, + mb_mode=request.mb_mode, + ) + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status not in {"IPTA_TIMESERIES_SCRIPT_READY", "IPTA_TIMESERIES_RUNNING"}: + raise ValueError(f"run status does not allow IPTA time-series job submission: {status}") + if status == "IPTA_TIMESERIES_RUNNING": + raise ValueError("IPTA time-series is already running for this run") + + from ..services.job_handlers import JOB_TYPE_SBAS_IPTA_TIMESERIES + + payload = { + "run_id": run_id, + "rlks": request.rlks, + "reference_window": request.reference_window, + "mb_mode": request.mb_mode, + "timeout_seconds": request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_IPTA_TIMESERIES, + task_name=f"SBAS-InSAR IPTA Timeseries {run_id}", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_IPTA_TIMESERIES, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "SBAS-InSAR IPTA time-series job queued.", + "run_id": run_id, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_IPTA_TIMESERIES, + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "already running" in message.lower() or "conflict" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + @router.get("/runs/{run_id}/artifacts/{relative_path:path}") async def get_sbas_insar_run_artifact(run_id: str, relative_path: str): try: diff --git a/backend/app/services/health_service.py b/backend/app/services/health_service.py index 639bfc2..6e793ca 100644 --- a/backend/app/services/health_service.py +++ b/backend/app/services/health_service.py @@ -1272,6 +1272,7 @@ async def _check_wsl_runtime() -> Dict[str, Any]: required_by_engine = { "isce2": bool(settings.ISCE2_ENABLED or settings.TIMESERIES_ENABLED), "pyint": bool(settings.PYINT_ENABLED), + "gamma": bool(settings.GAMMA_SBAS_ENABLED), } for runtime in wsl_runtime_registry.runtimes.values(): required = bool(required_by_engine.get(runtime.engine_code, False)) @@ -1285,6 +1286,9 @@ async def _check_wsl_runtime() -> Dict[str, Any]: distro_matches_shared = str(runtime.distro or "").strip() == str( wsl_runtime_registry.shared_distro or "" ).strip() + if runtime.engine_code == "gamma": + python_matches_shared = bool(str(runtime.python_path or "").strip()) + distro_matches_shared = bool(str(runtime.distro or "").strip()) runtime_ok = runner_exists and python_matches_shared and distro_matches_shared if env_profile_exists is False and required: runtime_ok = False diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index e5da949..697454c 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -92,6 +92,10 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY" JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION" +JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM" +JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS" +JOB_TYPE_SBAS_IPTA_TIMESERIES = "SBAS_IPTA_TIMESERIES" +JOB_TYPE_SBAS_GAMMA_WORKFLOW = "SBAS_GAMMA_WORKFLOW" COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"} @@ -4263,6 +4267,346 @@ async def _handle_sbas_coregistration(job: SystemJobORM) -> None: ) +async def _handle_sbas_rdc_dem(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SBAS_RDC_DEM requires task_id for progress tracking.") + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + if not run_id: + raise ValueError("SBAS_RDC_DEM requires run_id payload.") + + rlks = _normalize_positive_int(payload.get("rlks")) or 8 + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200 + + await task_service.start_task(job.task_id, message="正在执行 SBAS-InSAR Gamma RDC DEM...") + await task_service.update_task( + job.task_id, + progress=5, + message=f"准备运行 Gamma gc_map1/gc_map_fine: run_id={run_id}", + ) + await task_service.add_log( + job.task_id, + "INFO", + f"SBAS RDC DEM queued: run_id={run_id}, rlks={rlks}, timeout={timeout_seconds}s", + ) + + from .sbas_insar_production_service import sbas_insar_production_service + + async def _task_keepalive() -> None: + progress = 12 + while True: + await asyncio.sleep(60) + progress = min(88, progress + 2) + await task_service.update_task( + job.task_id, + progress=progress, + message=f"Gamma RDC DEM 仍在运行: run_id={run_id}", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + sbas_insar_production_service.execute_rdc_dem, + run_id, + rlks=rlks, + timeout_seconds=timeout_seconds, + ) + ) + keepalive_task = asyncio.create_task(_task_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + manifest = result.get("manifest") or {} + run = result.get("run") or {} + summary = (manifest.get("rdc_dem") or {}).get("summary") or {} + status = str(run.get("status") or manifest.get("status") or "").strip() + if status != "RDC_DEM_READY": + raise RuntimeError( + "SBAS RDC DEM failed: " + f"status={status or 'UNKNOWN'}, " + f"missing_outputs={summary.get('missing_outputs') or []}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "SBAS-InSAR RDC DEM 完成: " + f"rdc_dem={((summary.get('outputs') or {}).get('rdc_dem') or {}).get('path') or '-'}" + ), + ) + + +async def _handle_sbas_interferograms(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SBAS_INTERFEROGRAMS requires task_id for progress tracking.") + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + if not run_id: + raise ValueError("SBAS_INTERFEROGRAMS requires run_id payload.") + + rlks = _normalize_positive_int(payload.get("rlks")) or 8 + azlks = _normalize_positive_int(payload.get("azlks")) or 8 + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200 + try: + unwrap_threshold = float(payload.get("unwrap_threshold") or 0.20) + except (TypeError, ValueError): + unwrap_threshold = 0.20 + + await task_service.start_task(job.task_id, message="正在执行 SBAS-InSAR Gamma 差分干涉图...") + await task_service.update_task( + job.task_id, + progress=5, + message=f"准备运行 Gamma phase_sim_orb/SLC_diff_intf/mcf: run_id={run_id}", + ) + await task_service.add_log( + job.task_id, + "INFO", + ( + f"SBAS interferograms queued: run_id={run_id}, rlks={rlks}, azlks={azlks}, " + f"unwrap_threshold={unwrap_threshold}, timeout={timeout_seconds}s" + ), + ) + + from .sbas_insar_production_service import sbas_insar_production_service + + async def _task_keepalive() -> None: + progress = 12 + while True: + await asyncio.sleep(60) + progress = min(88, progress + 2) + await task_service.update_task( + job.task_id, + progress=progress, + message=f"Gamma 差分干涉图仍在运行: run_id={run_id}", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + sbas_insar_production_service.execute_interferograms, + run_id, + rlks=rlks, + azlks=azlks, + unwrap_threshold=unwrap_threshold, + timeout_seconds=timeout_seconds, + ) + ) + keepalive_task = asyncio.create_task(_task_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + manifest = result.get("manifest") or {} + run = result.get("run") or {} + summary = (manifest.get("interferograms") or {}).get("summary") or {} + status = str(run.get("status") or manifest.get("status") or "").strip() + if status != "INTERFEROGRAMS_READY": + raise RuntimeError( + "SBAS interferograms failed: " + f"status={status or 'UNKNOWN'}, " + f"missing_pairs={summary.get('missing_pairs') or []}, " + f"missing_tabs={summary.get('missing_tabs') or []}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "SBAS-InSAR 差分干涉图完成: " + f"{summary.get('ready_pair_count', 0)}/{summary.get('pair_count', 0)} pairs ready" + ), + ) + + +async def _handle_sbas_ipta_timeseries(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SBAS_IPTA_TIMESERIES requires task_id for progress tracking.") + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + if not run_id: + raise ValueError("SBAS_IPTA_TIMESERIES requires run_id payload.") + + rlks = _normalize_positive_int(payload.get("rlks")) or 8 + reference_window = _normalize_positive_int(payload.get("reference_window")) or 16 + try: + mb_mode = int(payload.get("mb_mode") or 0) + except (TypeError, ValueError): + mb_mode = 0 + if mb_mode not in {0, 1, 2}: + mb_mode = 0 + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200 + + await task_service.start_task(job.task_id, message="Running SBAS-InSAR Gamma IPTA time-series inversion...") + await task_service.update_task( + job.task_id, + progress=5, + message=f"Preparing Gamma mb/ts_rate: run_id={run_id}", + ) + await task_service.add_log( + job.task_id, + "INFO", + ( + f"SBAS IPTA time-series queued: run_id={run_id}, rlks={rlks}, " + f"reference_window={reference_window}, mb_mode={mb_mode}, " + f"timeout={timeout_seconds}s" + ), + ) + + from .sbas_insar_production_service import sbas_insar_production_service + + async def _task_keepalive() -> None: + progress = 12 + while True: + await asyncio.sleep(60) + progress = min(88, progress + 2) + await task_service.update_task( + job.task_id, + progress=progress, + message=f"Gamma IPTA mb/ts_rate still running: run_id={run_id}", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + sbas_insar_production_service.execute_ipta_timeseries, + run_id, + rlks=rlks, + reference_window=reference_window, + mb_mode=mb_mode, + timeout_seconds=timeout_seconds, + ) + ) + keepalive_task = asyncio.create_task(_task_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + manifest = result.get("manifest") or {} + run = result.get("run") or {} + summary = (manifest.get("ipta_timeseries") or {}).get("summary") or {} + status = str(run.get("status") or manifest.get("status") or "").strip() + if status != "IPTA_TIMESERIES_READY": + raise RuntimeError( + "SBAS IPTA time-series failed: " + f"status={status or 'UNKNOWN'}, " + f"missing_outputs={summary.get('missing_outputs') or []}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "SBAS-InSAR IPTA time-series complete: " + f"ts_rate={((summary.get('outputs') or {}).get('ts_rate') or {}).get('path') or '-'}" + ), + ) + + +async def _handle_sbas_gamma_workflow(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SBAS_GAMMA_WORKFLOW requires task_id for progress tracking.") + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + if not run_id: + raise ValueError("SBAS_GAMMA_WORKFLOW requires run_id payload.") + + from_step = str(payload.get("from_step") or "").strip() or None + to_step = str(payload.get("to_step") or "").strip() or None + only_steps_raw = payload.get("only_steps") or [] + only_steps = [str(item).strip() for item in only_steps_raw if str(item).strip()] if isinstance(only_steps_raw, list) else [] + force = bool(payload.get("force", False)) + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or int(settings.GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS) + + await task_service.start_task(job.task_id, message="Running Gamma SBAS expert workflow...") + await task_service.update_task( + job.task_id, + progress=5, + message=f"Preparing Gamma SBAS manifest runner: run_id={run_id}", + ) + await task_service.add_log( + job.task_id, + "INFO", + ( + f"SBAS Gamma workflow queued: run_id={run_id}, from={from_step or '-'}, " + f"to={to_step or '-'}, only={only_steps or '-'}, force={force}, timeout={timeout_seconds}s" + ), + ) + + from .sbas_insar_production_service import sbas_insar_production_service + + async def _task_keepalive() -> None: + progress = 10 + while True: + await asyncio.sleep(60) + progress = min(92, progress + 2) + await task_service.update_task( + job.task_id, + progress=progress, + message="Gamma SBAS workflow is still running...", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + sbas_insar_production_service.execute_workflow, + run_id, + from_step=from_step, + to_step=to_step, + only_steps=only_steps, + force=force, + timeout_seconds=timeout_seconds, + ) + ) + keepalive_task = asyncio.create_task(_task_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + run = result.get("run") or {} + manifest = result.get("manifest") or {} + workflow = manifest.get("workflow") or {} + summary = workflow.get("summary") or {} + status = str(run.get("status") or manifest.get("status") or "").strip() + if status not in {"WORKFLOW_COMPLETED", "WORKFLOW_PARTIAL"}: + raise RuntimeError( + "SBAS Gamma workflow failed: " + f"status={status or 'UNKNOWN'}, failed_steps={summary.get('failed_count') or 0}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + f"Gamma SBAS expert workflow {status.lower()}: " + f"completed={summary.get('completed_count', 0)}, " + f"skipped={summary.get('skipped_count', 0)}, " + f"planned={summary.get('planned_count', 0)}" + ), + ) + + _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, @@ -4301,6 +4645,10 @@ _HANDLERS = { JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack, JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process, JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration, + JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem, + JOB_TYPE_SBAS_INTERFEROGRAMS: _handle_sbas_interferograms, + JOB_TYPE_SBAS_IPTA_TIMESERIES: _handle_sbas_ipta_timeseries, + JOB_TYPE_SBAS_GAMMA_WORKFLOW: _handle_sbas_gamma_workflow, } diff --git a/backend/app/services/sbas_insar_production_service.py b/backend/app/services/sbas_insar_production_service.py index af99605..63b9617 100644 --- a/backend/app/services/sbas_insar_production_service.py +++ b/backend/app/services/sbas_insar_production_service.py @@ -2,9 +2,11 @@ from __future__ import annotations import hashlib import json +import math import os import re import shutil +import struct import subprocess from datetime import datetime from pathlib import Path @@ -15,6 +17,60 @@ from ..config import settings PRODUCT_DEFINITIONS = ( + { + "key": "los_rate_toward_m_per_year_hls_geo_preview_png", + "label": "Expert HLS LOS velocity geocoded RGB preview, toward radar positive", + "role": "primary_geocoded_preview", + "relative_path": "publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png", + }, + { + "key": "los_rate_toward_m_per_year_hls_rgb_tif", + "label": "Expert HLS LOS velocity geocoded RGB GeoTIFF, toward radar positive", + "role": "primary_rgb_geotiff", + "relative_path": "publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif", + }, + { + "key": "los_rate_toward_m_per_year_hls_bmp", + "label": "Expert HLS LOS velocity RDC browse BMP, toward radar positive", + "role": "rdc_processing_preview", + "relative_path": "publish/geotiff/los_rate_toward_m_per_year.hls.bmp", + }, + { + "key": "los_sigma_m_per_year_cc_geo_preview_png", + "label": "Expert CC LOS velocity sigma geocoded RGB preview", + "role": "quality_geocoded_preview", + "relative_path": "publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png", + }, + { + "key": "los_sigma_m_per_year_cc_rgb_tif", + "label": "Expert CC LOS velocity sigma geocoded RGB GeoTIFF", + "role": "quality_rgb_geotiff", + "relative_path": "publish/geotiff/los_sigma_m_per_year.cc.geo_rgb.tif", + }, + { + "key": "los_sigma_m_per_year_cc_bmp", + "label": "Expert CC LOS velocity sigma RDC browse BMP", + "role": "rdc_processing_preview", + "relative_path": "publish/geotiff/los_sigma_m_per_year.cc.bmp", + }, + { + "key": "los_rate_toward_m_per_year_tif", + "label": "LOS velocity GeoTIFF in meters per year, toward radar positive", + "role": "primary_geotiff", + "relative_path": "publish/geotiff/los_rate_toward_m_per_year.tif", + }, + { + "key": "los_rate_away_m_per_year_tif", + "label": "LOS velocity GeoTIFF in meters per year, away from radar positive", + "role": "alternate_geotiff", + "relative_path": "publish/geotiff/los_rate_away_m_per_year.tif", + }, + { + "key": "los_sigma_m_per_year_tif", + "label": "LOS velocity sigma GeoTIFF in meters per year", + "role": "quality_geotiff", + "relative_path": "publish/geotiff/los_sigma_m_per_year.tif", + }, { "key": "los_rate_toward_mm_per_year_geo_preview_png", "label": "LOS velocity geocoded preview, toward radar positive", @@ -90,6 +146,13 @@ MONITOR_ARTIFACT_SUFFIXES = ( ("metadata_json", "Monitoring point metadata", ".json"), ) +DEFAULT_IPTA_MB_MODE = 0 +IPTA_MB_MODE_DESCRIPTIONS = { + 0: "valid unwrapped phase values required in all layers", + 1: "allow missing unwrapped phase values with network connectivity", + 2: "allow missing unwrapped phase values without network connectivity requirement", +} + GAMMA_STAGE_PLAN = ( { "stage_id": "prepare_slc", @@ -121,11 +184,17 @@ GAMMA_STAGE_PLAN = ( "gamma_tools": ["phase_sim_orb", "SLC_diff_intf", "adf", "mcf"], "status": "PLANNED_AFTER_BASELINE_AUDIT", }, + { + "stage_id": "detrend_atm", + "label": "Detrend and atmospheric phase correction", + "gamma_tools": ["quad_fit", "quad_sub", "atm_mod_2d", "atm_sim_2d", "sub_phase"], + "status": "PLANNED_AFTER_INTERFEROGRAMS", + }, { "stage_id": "ipta_timeseries", "label": "IPTA SBAS time-series inversion", "gamma_tools": ["mb", "ts_rate"], - "status": "PLANNED_AFTER_BASELINE_AUDIT", + "status": "PLANNED_AFTER_DETREND_ATM", }, { "stage_id": "publish_products", @@ -141,6 +210,319 @@ GAMMA_STAGE_PLAN = ( }, ) +EXPERT_WORKSPACE_DIRS = ( + "RAW", + "SLC", + "dem", + "rslc_prep", + "mli_dir", + "diff_dir", + "diff1_dir", + "sbas", + "publish", + "logs", + "scripts", + "state", +) + +GAMMA_SBAS_WORKFLOW_STEPS = ( + { + "id": "01_workspace_data", + "name": "Directory and LT1 data preparation", + "legacy_stage": "workspace", + "script_name": "01_workspace_data.sh", + "status": "PENDING", + "expert_tools": ["mkdir", "ls"], + }, + { + "id": "02_import_lt1_slc", + "name": "Import every LT1 SLC", + "legacy_stage": "baseline_audit", + "script_name": "02_import_lt1_slc.sh", + "status": "PENDING", + "expert_tools": ["par_LT1_SLC", "ORB_filt_spline.py", "SLC_corners", "disSLC", "dismph_fft"], + }, + { + "id": "03_reference_mli", + "name": "Reference MLI and footprint checks", + "legacy_stage": "baseline_audit", + "script_name": "03_reference_mli.sh", + "status": "PENDING", + "expert_tools": ["multi_look", "grep", "ras_dB", "SLC_corners"], + }, + { + "id": "04_dem_lookup", + "name": "DEM import and lookup table", + "legacy_stage": "rdc_dem", + "script_name": "04_dem_lookup.sh", + "status": "PENDING", + "expert_tools": ["dem_import", "fill_gaps", "gc_map2", "pixel_area", "gc_map_fine", "geocode"], + }, + { + "id": "05_coreg_prep", + "name": "SLC coregistration preparation", + "legacy_stage": "coregistration", + "script_name": "05_coreg_prep.sh", + "status": "PENDING", + "expert_tools": ["cp", "rslc_tab"], + }, + { + "id": "06_coregister_scenes", + "name": "Coregister every SLC to reference", + "legacy_stage": "coregistration", + "script_name": "06_coregister_scenes.sh", + "status": "PENDING", + "expert_tools": ["create_offset", "init_offset_orbit", "init_offset", "offset_pwr", "offset_fit", "SLC_interp"], + }, + { + "id": "07_rmli_average", + "name": "RMLI stack and average intensity", + "legacy_stage": "coregistration", + "script_name": "07_rmli_average.sh", + "status": "PENDING", + "expert_tools": ["mk_mli_all", "grep", "ras_dB"], + }, + { + "id": "08_diff_network", + "name": "Interferogram network and differential phase", + "legacy_stage": "interferograms", + "script_name": "08_diff_network.sh", + "status": "PENDING", + "expert_tools": ["base_calc", "base_plot", "mk_diff_2d"], + }, + { + "id": "09_filter_unwrap", + "name": "Adaptive filtering, coherence mask and unwrap", + "legacy_stage": "interferograms", + "script_name": "09_filter_unwrap.sh", + "status": "PENDING", + "expert_tools": ["mk_adf_2d", "ave_image", "rascc_mask", "mk_unw_2d"], + }, + { + "id": "10_detrend_atm", + "name": "Detrend and atmospheric correction", + "legacy_stage": "quality_correction", + "script_name": "10_detrend_atm.sh", + "status": "PENDING", + "optional": False, + "expert_tools": ["quad_fit", "quad_sub", "atm_mod_2d", "atm_sim_2d", "sub_phase"], + }, + { + "id": "11_sbas_inversion", + "name": "Gamma IPTA SBAS inversion", + "legacy_stage": "ipta_timeseries", + "script_name": "11_sbas_inversion.sh", + "status": "PENDING", + "expert_tools": ["mb", "unw_to_cpx", "unw_model", "ts_rate"], + }, + { + "id": "12_outputs_points", + "name": "Output, geocode and point time-series", + "legacy_stage": "publish_products+monitor_points", + "script_name": "12_outputs_points.sh", + "status": "PENDING", + "expert_tools": ["replace_values", "mask_data", "dispmap", "ts_rate", "rasdt_pwr", "geocode_back", "data2geotiff", "disp_prt_2d"], + }, +) + +GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( + { + "id": "expert_01_workspace_data", + "order": 1, + "title": "Directory and LT1 data preparation", + "document_section": "1. Directory and data preparation", + "workflow_steps": ["01_workspace_data"], + "implementation_status": "implemented_bridge", + "commands": [ + "mkdir -p RAW SLC dem rslc_prep mli_dir diff_dir diff1_dir sbas", + "ls RAW//*.tiff", + "ls RAW//*.meta.xml", + ], + }, + { + "id": "expert_02_import_slc", + "order": 2, + "title": "Import every LT1 SLC", + "document_section": "2. Import LT1 SLC scenes", + "workflow_steps": ["02_import_lt1_slc"], + "implementation_status": "implemented", + "commands": [ + "par_LT1_SLC .tiff .meta.xml .slc.par .slc 0", + "cp .slc.par .slc.par.orig", + "ORB_filt_spline.py .slc.par.orig .slc.par --ignore_start 3 --ignore_end 17 --degree 5", + "SLC_corners .slc.par", + "disSLC .slc ...", + "dismph_fft .slc ...", + ], + }, + { + "id": "expert_03_reference_mli", + "order": 3, + "title": "Reference MLI and footprint checks", + "document_section": "3. Reference multilook and range check", + "workflow_steps": ["03_reference_mli"], + "implementation_status": "implemented_bridge", + "commands": [ + "multi_look .slc .slc.par __.mli __.mli.par ", + "grep range_samples .mli.par", + "grep azimuth_lines .mli.par", + "ras_dB .mli ... gray.cm .mli.bmp", + "SLC_corners .mli.par", + ], + }, + { + "id": "expert_04_dem_lookup", + "order": 4, + "title": "DEM import and lookup table", + "document_section": "4. DEM import and geocoding lookup table", + "workflow_steps": ["04_dem_lookup"], + "implementation_status": "implemented_bridge", + "commands": [ + "dem_import .tif SRTM.dem SRTM.dem.par ...", + "fill_gaps SRTM.dem SRTM_dem_fill", + "gc_map2 .mli.par SRTM.dem.par SRTM_dem_fill _seg.dem_par _seg.dem .lt ...", + "pixel_area .mli.par _seg.dem_par _seg.dem .lt ...", + "create_diff_par .mli.par - .diff_par 1 0", + "offset_pwrm .gamma0 .mli .diff_par ...", + "offset_fitm .offs .snr .diff_par ...", + "gc_map_fine .lt .diff_par .lt_fine 1", + "geocode .lt_fine _seg.dem .hgt ", + "geocode_back .mli .lt_fine .geo 5 0", + ], + }, + { + "id": "expert_05_coreg_prep", + "order": 5, + "title": "SLC coregistration preparation", + "document_section": "5. SLC coregistration preparation", + "workflow_steps": ["05_coreg_prep"], + "implementation_status": "implemented_bridge", + "commands": [ + "cp SLC/dates rslc_prep/dates", + "cp .slc .rslc", + "cp .slc.par .rslc.par", + ], + }, + { + "id": "expert_06_coregister_scenes", + "order": 6, + "title": "Coregister every SLC to reference", + "document_section": "6. Coregister scenes to reference geometry", + "workflow_steps": ["06_coregister_scenes"], + "implementation_status": "implemented_bridge", + "commands": [ + "create_offset .rslc.par .slc.par _.off 1", + "init_offset_orbit .rslc.par .slc.par _.off", + "init_offset .rslc .slc .rslc.par .slc.par _.off ", + "offset_pwr .rslc .slc .rslc.par .slc.par _.off ...", + "offset_fit _.offs _.snr _.off ...", + "SLC_interp .slc .rslc.par .slc.par _.off .rslc .rslc.par", + "echo '.rslc .rslc.par' >> rslc_tab", + ], + }, + { + "id": "expert_07_rmli_average", + "order": 7, + "title": "RMLI stack and average intensity", + "document_section": "7. Generate RMLI and average intensity", + "workflow_steps": ["07_rmli_average"], + "implementation_status": "implemented_bridge", + "commands": [ + "mk_mli_all rslc_tab . 1 1.0 0.4 mli.ave", + "grep range_samples mli.ave.par", + "grep azimuth_lines mli.ave.par", + "ras_dB mli.ave ... gray.cm mli.ave.bmp", + ], + }, + { + "id": "expert_08_diff_network", + "order": 8, + "title": "Interferogram network and differential phase", + "document_section": "8. Interferogram generation and differential interferometry", + "workflow_steps": ["08_diff_network"], + "implementation_status": "implemented_bridge", + "commands": [ + "base_calc rslc_tab .rslc.par bprep_file itab 1 1 -", + "base_plot rslc_tab .rslc.par itab bprep_file 1", + "mk_diff_2d rslc_tab itab 0 .hgt - mli.ave mli_dir . 3 1 1 0 -u", + "ls *.diff", + "ls *.diff.bmp", + ], + }, + { + "id": "expert_09_filter_unwrap", + "order": 9, + "title": "Adaptive filtering, coherence mask and unwrap", + "document_section": "9. Adaptive filtering, coherence mask and phase unwrapping", + "workflow_steps": ["09_filter_unwrap"], + "implementation_status": "implemented_bridge", + "commands": [ + "mk_adf_2d rslc_tab itab mli.ave . 5 0.6 32 8 -u", + "ls *.adf.diff", + "ls *.adf.cc", + "ave_image cc.list mean.cc", + "rascc_mask mean.cc - 1 1 - 1 1 ", + "mk_unw_2d rslc_tab itab mli.ave . 0 1 1 1 1 1 -u", + "mk_unw_2d rslc_tab itab mli.ave . - - 1 1 1 1 1 mean.cc_mask.bmp -u", + ], + }, + { + "id": "expert_10_detrend_atm", + "order": 10, + "title": "Detrend and atmospheric phase removal", + "document_section": "10. Detrending and atmospheric phase removal", + "workflow_steps": ["10_detrend_atm"], + "implementation_status": "implemented_bridge", + "commands": [ + "create_diff_par .off .off .diff_par 0 0", + "quad_fit .adf.unw .diff_par 5 5 - - 3 .unw_linear", + "quad_sub .adf.unw .diff_par .unw_sub_linear 0 0", + "rasdt_pwr .unw_sub_linear mli.ave 1 - 1 1 -6.28 6.28 1 rmg.cm ...", + "atm_mod_2d .unw_sub_linear .hgt .adf.cc .diff_par - 0 .a0 .a1 ...", + "fill_gaps .a0 .a0_fill ...", + "fill_gaps .a1 .a1_fill ...", + "atm_sim_2d .diff_par .hgt .a0_fill .a1_fill .atm_model", + "sub_phase .unw_sub_linear .atm_model .diff_par .unw.atmsub 0", + ], + }, + { + "id": "expert_11_sbas_inversion", + "order": 11, + "title": "SBAS inversion", + "document_section": "11. SBAS inversion", + "workflow_steps": ["11_sbas_inversion"], + "implementation_status": "implemented_bridge", + "commands": [ + "mb unw_atmsub_tab RMLI_tab itab - itab_ts ras/diff1 1 diff1.sigma_ts 1 - 15 15 0.0 mli.ave.par", + "unw_to_cpx .unw.atmsub .unw.atmsub.cpx ", + "unw_model .unw.atmsub.cpx .unw.atmsub_sim .unw.atmsub_1 ", + "mb unw.atmsub_1_tab RMLI_tab itab - itab_ts ras/diff2 1 diff2.sigma_ts 0 - 15 15 0.0 mli.ave.par", + "mb final_unw_tab RMLI_tab itab - itab_ts ras/diff 0 diff.sigma_ts 0 - 15 15 0.5 mli.ave.par", + ], + }, + { + "id": "expert_12_outputs_points", + "order": 12, + "title": "Output, geocode and point time-series", + "document_section": "12. Output, geocoding and point time-series", + "workflow_steps": ["12_outputs_points"], + "implementation_status": "implemented_bridge", + "commands": [ + "replace_values diff.sigma_ts 0.5 0.0 diff.sigma_ts.masked 1 2 0", + "rasdt_pwr diff.sigma_ts.masked - 1 0 1 1 0.0 1.5 1 cc.cm diff.sigma_ts.masked.bmp 1.0 0.35 8", + "mask_data ras/diff_ ras/diff_.masked diff.sigma_ts.masked.bmp 0", + "dispmap ras/.disp.phase - mli.ave.par - ras/.disp 0 0", + "ts_rate disp.TS_tab RMLI_tab itab_ts - los_def_rate los_def_const los_def_sigma 0", + "rasdt_pwr los_def_rate mli.ave 1 0 1 1 -0.08 0.08 0 hls.cm los_def_rate.bmp 1.0 0.35 24", + "geocode_back los_def_rate .lt_fine geo_los_def_rate 5 0", + "data2geotiff _seg.dem_par geo_los_def_rate 2 geo_los_def_rate.tif", + "geocode_back los_def_rate.bmp .lt_fine geo_los_def_rate.bmp 0 2", + "data2geotiff _seg.dem_par geo_los_def_rate.bmp 0 geo_los_def_rate_rgb.tif", + "disp_prt_2d disp_geo.TS_tab RMLI_tab itab_ts - 3 disp_point.txt - geo_los_def_rate geo_diff.sigma_ts items.txt disp_tab.txt 3 1 0", + ], + }, +) + LT1_SCENE_RE = re.compile( r"^(?PLT1[AB])_" r"(?P[A-Z0-9]+)_" @@ -157,25 +539,154 @@ LT1_SCENE_RE = re.compile( class SbasInsarProductionService: + _WORKFLOW_BASELINE_DONE_STATUSES = { + "BASELINE_AUDIT_READY", + "ITAB_APPROVED", + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_RUNNING", + "COREGISTRATION_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + "INTERFEROGRAMS_SCRIPT_READY", + "INTERFEROGRAMS_RUNNING", + "INTERFEROGRAMS_READY", + "DETREND_ATM_SCRIPT_READY", + "DETREND_ATM_RUNNING", + "DETREND_ATM_READY", + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_COREG_DONE_STATUSES = { + "COREGISTRATION_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + "INTERFEROGRAMS_SCRIPT_READY", + "INTERFEROGRAMS_RUNNING", + "INTERFEROGRAMS_READY", + "DETREND_ATM_SCRIPT_READY", + "DETREND_ATM_RUNNING", + "DETREND_ATM_READY", + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_RDC_DEM_DONE_STATUSES = { + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + "INTERFEROGRAMS_SCRIPT_READY", + "INTERFEROGRAMS_RUNNING", + "INTERFEROGRAMS_READY", + "DETREND_ATM_SCRIPT_READY", + "DETREND_ATM_RUNNING", + "DETREND_ATM_READY", + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_INTERFEROGRAMS_DONE_STATUSES = { + "INTERFEROGRAMS_READY", + "DETREND_ATM_SCRIPT_READY", + "DETREND_ATM_RUNNING", + "DETREND_ATM_READY", + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_DETREND_DONE_STATUSES = { + "DETREND_ATM_READY", + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_IPTA_DONE_STATUSES = { + "IPTA_TIMESERIES_READY", + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_PUBLISH_DONE_STATUSES = { + "PRODUCTS_READY", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + } + _WORKFLOW_MONITOR_DONE_STATUSES = { + "MONITOR_POINTS_READY", + } + def __init__(self) -> None: self.trial_root = Path(settings.BACKEND_DIR) / "runtime" / "gamma_ipta_trials" - self.production_root = Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production" + self.production_root = Path(settings.GAMMA_SBAS_WORK_ROOT or (Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production")) def get_capabilities(self) -> dict[str, Any]: return { "workflow_code": "sbas_insar", "processor_code": "gamma_ipta_sbas", "engine_code": "gamma", - "implementation_state": "baseline_audit_and_coregistration_queue", + "implementation_state": "expert_manifest_script_runner_primary", "trial_root": str(self.trial_root), "production_root": str(self.production_root), + "workflow_runner": { + "enabled": bool(settings.GAMMA_SBAS_ENABLED), + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "wsl_distro": settings.GAMMA_SBAS_WSL_DISTRO, + "python": settings.GAMMA_SBAS_PYTHON, + "env_script": settings.GAMMA_SBAS_ENV_SCRIPT, + "work_root": settings.GAMMA_SBAS_WORK_ROOT, + "product_root": settings.GAMMA_SBAS_PRODUCT_ROOT, + "style": "expert_document_manifest_and_scripts", + }, + "workflow_node_count": len(GAMMA_SBAS_WORKFLOW_STEPS), "supported_sensors": ["LT1"], "supported_products": [item["key"] for item in PRODUCT_DEFINITIONS], "run_submission": { "enabled": True, "execution_enabled": True, - "status_after_submit": "PLANNED_GAMMA_BASELINE_AUDIT", - "description": "Creates reproducible filesystem manifests and can execute the Gamma SLC preparation plus base_calc baseline-audit stage.", + "status_after_submit": "WORKFLOW_READY", + "description": "Creates the expert-document workspace, manifest, scripts, and a queued Gamma SBAS workflow runner job.", + }, + "expert_workspace": { + "schema": "insar.gamma-sbas-workflow/v1", + "directories": list(EXPERT_WORKSPACE_DIRS), + "steps": [dict(item) for item in GAMMA_SBAS_WORKFLOW_STEPS], + "expert_document_steps": [dict(item) for item in GAMMA_SBAS_EXPERT_DOCUMENT_STEPS], }, "baseline_audit": { "enabled": True, @@ -192,6 +703,48 @@ class SbasInsarProductionService: "default_strategy": "common_reference_to_stack_reference_date", "requires_status": "ITAB_APPROVED", }, + "rdc_dem": { + "enabled": True, + "execution_enabled": True, + "execution_mode": "queued_background_task", + "job_type": "SBAS_RDC_DEM", + "default_strategy": "gamma_gc_map_fine_reference_geometry", + "requires_status": "COREGISTRATION_READY", + }, + "interferograms": { + "enabled": True, + "execution_enabled": True, + "execution_mode": "queued_background_task", + "job_type": "SBAS_INTERFEROGRAMS", + "default_strategy": "approved_itab_common_reference_diff_unwrap", + "requires_status": "RDC_DEM_READY", + }, + "detrend_atm": { + "enabled": True, + "execution_enabled": True, + "execution_mode": "workflow_or_direct_stage", + "default_strategy": "expert_quad_fit_quad_sub_atm_mod_2d_sub_phase", + "requires_status": "INTERFEROGRAMS_READY", + "stage_status_after_success": "DETREND_ATM_READY", + }, + "ipta_timeseries": { + "enabled": True, + "execution_enabled": True, + "execution_mode": "queued_background_task", + "job_type": "SBAS_IPTA_TIMESERIES", + "default_strategy": "gamma_mb_ts_rate_common_reference", + "default_mb_mode": DEFAULT_IPTA_MB_MODE, + "mb_mode_description": IPTA_MB_MODE_DESCRIPTIONS[DEFAULT_IPTA_MB_MODE], + "requires_status": "DETREND_ATM_READY", + }, + "publish_products": { + "enabled": True, + "execution_enabled": True, + "requires_status": "IPTA_TIMESERIES_READY", + "status_after_success": "PRODUCTS_READY", + "default_strategy": "gamma_geocode_back_data2geotiff_los_sign_conversion", + "geocoded_preview_source": "EPSG:4326 GeoTIFF", + }, "monitor_point_modes": ["auto_low_sigma_high_rate", "manual_lonlat"], "default_los_convention": { "key": "los_rate_toward_mm_per_year", @@ -210,7 +763,7 @@ class SbasInsarProductionService: "description": "toward radar positive; Gamma dispmap default sflg=0", }, ], - "next_enabled_operation": "gamma_coregistration_background_job", + "next_enabled_operation": "gamma_ipta_timeseries_background_job", } def discover_stacks( @@ -225,9 +778,26 @@ class SbasInsarProductionService: platform: str | None = None, relative_orbit: str | None = None, orbit_direction: str | None = None, + force_refresh: bool = False, ) -> dict[str, Any]: source_paths = self._resolve_source_roots(source_roots) orbit_paths = self._resolve_orbit_roots(orbit_roots) + cache_key = self._discovery_cache_key( + source_paths=source_paths, + orbit_paths=orbit_paths, + min_scenes=min_scenes, + require_orbits=require_orbits, + include_scenes=include_scenes, + limit=limit, + platform=platform, + relative_orbit=relative_orbit, + orbit_direction=orbit_direction, + ) + if not force_refresh: + cached = self._read_discovery_cache(cache_key) + if cached is not None: + return cached + scenes: list[dict[str, Any]] = [] errors: list[dict[str, str]] = [] @@ -293,6 +863,7 @@ class SbasInsarProductionService: snapshot, ) snapshot["snapshot_path"] = str(snapshot_path) + self._write_discovery_cache(cache_key, snapshot) return snapshot def audit_stack( @@ -415,9 +986,6 @@ class SbasInsarProductionService: monitor_point_strategy: str = "auto_low_sigma_high_rate", dry_run: bool = True, ) -> dict[str, Any]: - if not dry_run: - raise ValueError("Gamma SBAS execution is not wired yet; submit with dry_run=true.") - audit = self.audit_stack( stack_id, source_roots=source_roots, @@ -437,6 +1005,7 @@ class SbasInsarProductionService: log_dir = run_dir / "logs" for path in (work_dir, publish_dir, log_dir): path.mkdir(parents=True, exist_ok=True) + expert_workspace = self._ensure_expert_workspace(run_dir) monitor_config = self._build_monitor_point_config( monitor_points=monitor_points, @@ -450,32 +1019,39 @@ class SbasInsarProductionService: "workflow_code": "sbas_insar", "processor_code": "gamma_ipta_sbas", "engine_code": "gamma", - "execution_mode": "dry_run_plan", - "status": "PLANNED_GAMMA_BASELINE_AUDIT", + "execution_mode": "expert_manifest_script_workflow", + "status": "WORKFLOW_READY", "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", "stack_id": stack_id, "stack_manifest_path": audit["manifest_path"], "pair_network_path": audit["pair_network_path"], + "workflow_manifest_path": str(run_dir / "manifest.json"), + "workflow_state_path": str(run_dir / "state" / "step_status.json"), "work_root": str(work_dir), "publish_root": str(publish_dir), "log_root": str(log_dir), + "expert_workspace": expert_workspace, "stack": manifest.get("stack") or {}, "scene_count": len(manifest.get("scenes") or []), "pair_count": len(((manifest.get("pair_network") or {}).get("pairs")) or []), - "next_stage": "baseline_audit", + "next_stage": "workflow", "requires_user_action": [ "Review Gamma base_calc baseline table before approving final itab.", "Confirm monitoring-point source: manual points, imported layer, or automatic sampler.", "Confirm geocoded preview products are published from EPSG:4326 GeoTIFFs.", ], "monitor_points": monitor_config, - "dry_run": dry_run, + "planning_only": True, + "legacy_dry_run_request": bool(dry_run), } command_manifest = self._build_command_manifest(run_manifest, manifest) + workflow_manifest = self._build_workflow_manifest(run_dir, run_manifest, manifest) run_manifest_path = self._write_json(run_dir / "run_manifest.json", run_manifest) command_manifest_path = self._write_json(run_dir / "gamma_command_manifest.json", command_manifest) + workflow_manifest_path = self._write_json(run_dir / "manifest.json", workflow_manifest) monitor_config_path = self._write_json(run_dir / "monitor_points.json", monitor_config) + self._write_json(run_dir / "state" / "step_status.json", self._initial_workflow_state(run_manifest, workflow_manifest)) self._write_json(run_dir / "stack_manifest.json", manifest) self._write_json(run_dir / "pair_network.json", manifest.get("pair_network") or {}) @@ -483,12 +1059,14 @@ class SbasInsarProductionService: **self._build_run_card(run_dir, run_manifest), "run_manifest_path": str(run_manifest_path), "gamma_command_manifest_path": str(command_manifest_path), + "workflow_manifest_path": str(workflow_manifest_path), "monitor_config_path": str(monitor_config_path), } return { "run": index_item, "manifest": run_manifest, "command_manifest": command_manifest, + "workflow_manifest": workflow_manifest, "monitor_points": monitor_config, } @@ -518,11 +1096,22 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest = self._read_json(run_dir / "run_manifest.json") command_manifest = self._read_optional_json(run_dir / "gamma_command_manifest.json") + workflow_manifest = self._read_optional_json(run_dir / "manifest.json") + if workflow_manifest and not workflow_manifest.get("expert_document"): + workflow_manifest["expert_document"] = { + "schema": "insar.gamma-sbas-expert-document/v1", + "source": "LT1_GAMMA_SBAS_逐命令处理流程.docx", + "section_count": len(GAMMA_SBAS_EXPERT_DOCUMENT_STEPS), + "steps": self._build_expert_document_step_manifest(workflow_manifest.get("steps") or []), + } + workflow_state = self._read_optional_json(run_dir / "state" / "step_status.json") monitor_points = self._read_optional_json(run_dir / "monitor_points.json") return { "run": self._build_run_card(run_dir, manifest), "manifest": manifest, "command_manifest": command_manifest, + "workflow_manifest": workflow_manifest, + "workflow_state": workflow_state, "monitor_points": monitor_points, "artifacts": self._build_run_artifacts(run_dir), } @@ -543,6 +1132,8 @@ class SbasInsarProductionService: stack_manifest = self._read_json(run_dir / "stack_manifest.json") if manifest.get("status") not in { "PLANNED_GAMMA_BASELINE_AUDIT", + "WORKFLOW_READY", + "WORKFLOW_RUNNING", "BASELINE_AUDIT_SCRIPT_READY", "BASELINE_AUDIT_FAILED", "BASELINE_AUDIT_READY", @@ -654,11 +1245,29 @@ class SbasInsarProductionService: normalized_decision = str(decision or "").strip().lower() if normalized_decision not in {"approve", "reject"}: raise ValueError("decision must be approve or reject") - if manifest.get("status") in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING", "COREGISTRATION_READY"}: + if manifest.get("status") in { + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_RUNNING", + "COREGISTRATION_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + }: existing_decision = ((manifest.get("baseline_audit") or {}).get("itab_decision") or {}).get("decision") if normalized_decision == "approve" and existing_decision == "approve": return self.get_run_detail(run_id) - if manifest.get("status") not in {"BASELINE_AUDIT_READY", "ITAB_APPROVED", "ITAB_REJECTED"}: + previous_status = str(manifest.get("status") or "").strip() + if previous_status not in { + "BASELINE_AUDIT_READY", + "ITAB_APPROVED", + "ITAB_REJECTED", + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_RUNNING", + "COREGISTRATION_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + }: raise ValueError(f"run status does not allow itab decision: {manifest.get('status')}") baseline_summary = self._read_optional_json(run_dir / "baseline_audit_summary.json") @@ -691,8 +1300,15 @@ class SbasInsarProductionService: baseline_state["approved_for_next_stage"] = True baseline_state["itab_decision"] = decision_payload baseline_state["approved_itab_path"] = str(approved_itab) - manifest["status"] = "ITAB_APPROVED" - manifest["next_stage"] = "coregistration" + if previous_status in {"RDC_DEM_SCRIPT_READY", "RDC_DEM_RUNNING", "RDC_DEM_READY"}: + manifest["status"] = previous_status + manifest["next_stage"] = "coregistration" + elif previous_status in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING", "COREGISTRATION_READY"}: + manifest["status"] = previous_status + manifest["next_stage"] = self._next_stage_for_status(previous_status) + else: + manifest["status"] = "ITAB_APPROVED" + manifest["next_stage"] = "coregistration" else: self._write_json(run_dir / "itab_decision.json", decision_payload) baseline_state["approved_for_next_stage"] = False @@ -717,7 +1333,13 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) - if manifest.get("status") not in {"ITAB_APPROVED", "COREGISTRATION_SCRIPT_READY", "COREGISTRATION_FAILED"}: + if manifest.get("status") not in { + "ITAB_APPROVED", + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_FAILED", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_READY", + }: raise ValueError(f"run status does not allow coregistration preparation: {manifest.get('status')}") approved_itab = run_dir / "work" / "gamma" / "diff" / "itab_approved" if not approved_itab.is_file(): @@ -759,8 +1381,12 @@ class SbasInsarProductionService: }, } manifest["coregistration"] = coregistration - manifest["status"] = "COREGISTRATION_SCRIPT_READY" - manifest["next_stage"] = "execute_coregistration" + if self._stage_execution_completed(manifest.get("rdc_dem")): + manifest["status"] = "RDC_DEM_READY" + manifest["next_stage"] = "execute_coregistration" + else: + manifest["status"] = "COREGISTRATION_SCRIPT_READY" + manifest["next_stage"] = "execute_coregistration" self._write_json(run_dir / "coregistration_plan.json", coregistration) self._write_json(manifest_path, manifest) self._refresh_command_manifest_after_coregistration(run_dir, manifest) @@ -780,11 +1406,11 @@ class SbasInsarProductionService: status = str(manifest.get("status") or "").strip() if status == "COREGISTRATION_READY": return self.get_run_detail(run_id) - if status in {"ITAB_APPROVED", "COREGISTRATION_FAILED"}: + if status in {"ITAB_APPROVED", "COREGISTRATION_FAILED", "RDC_DEM_SCRIPT_READY", "RDC_DEM_READY"}: self.prepare_coregistration(run_id, execute=False, rlks=rlks, azlks=azlks) manifest = self._read_json(manifest_path) status = str(manifest.get("status") or "").strip() - if status not in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING"}: + if status not in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING", "RDC_DEM_READY"}: raise ValueError(f"run status does not allow coregistration execution: {manifest.get('status')}") coregistration = dict(manifest.get("coregistration") or {}) @@ -856,6 +1482,9 @@ class SbasInsarProductionService: if completed.returncode == 0 and summary.get("ready"): manifest["status"] = "COREGISTRATION_READY" manifest["next_stage"] = "rdc_dem" + if self._stage_execution_completed(manifest.get("rdc_dem")): + manifest["status"] = "RDC_DEM_READY" + manifest["next_stage"] = "interferograms" else: manifest["status"] = "COREGISTRATION_FAILED" manifest["next_stage"] = "fix_coregistration" @@ -865,6 +1494,1324 @@ class SbasInsarProductionService: self._refresh_command_manifest_after_coregistration(run_dir, manifest) return self.get_run_detail(run_id) + def prepare_rdc_dem( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + ) -> dict[str, Any]: + if execute: + raise ValueError("RDC DEM execution is submitted through the background job endpoint.") + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "RDC_DEM_READY": + return self.get_run_detail(run_id) + if status not in { + "BASELINE_AUDIT_READY", + "ITAB_APPROVED", + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_FAILED", + }: + raise ValueError(f"run status does not allow RDC DEM preparation: {manifest.get('status')}") + + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + reference_date = str( + ((manifest.get("coregistration") or {}).get("reference_date")) + or ((manifest.get("coregistration") or {}).get("summary") or {}).get("reference_date") + or (stack_manifest.get("stack") or {}).get("reference_date") + or "" + ).strip() + if not reference_date: + raise ValueError("RDC DEM requires a reference date") + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + rmli_path, rmli_par_path = self._find_reference_rmli_paths(run_dir, reference_date) + if not rmli_path.is_file() or not rmli_par_path.is_file(): + raise FileNotFoundError(f"reference RMLI is missing for {reference_date}: {rmli_path}") + + dem_source = self._resolve_rdc_dem_source(stack_manifest) + script_path = self._write_rdc_dem_script( + run_dir, + reference_date=reference_date, + rlks=rlks, + dem_source=dem_source, + ) + gamma_dem_dir = run_dir / "work" / "gamma" / "dem" + rdc_dem = { + "schema": "insar.gamma-rdc-dem-stage/v1", + "strategy": "gamma_gc_map_fine_reference_geometry", + "script_path": str(script_path), + "reference_date": reference_date, + "rlks": rlks, + "dem_source": dem_source, + "reference_rmli": { + "mli": str(rmli_path), + "mli_par": str(rmli_par_path), + }, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "outputs": { + "dem_dir": str(gamma_dem_dir), + "utm_dem": str(gamma_dem_dir / f"{reference_date}_{rlks}rlks.utm.dem"), + "utm_dem_par": str(gamma_dem_dir / f"{reference_date}_{rlks}rlks.utm.dem.par"), + "lookup_table": str(gamma_dem_dir / f"{reference_date}_{rlks}rlks.UTM_TO_RDC"), + "rdc_dem": str(gamma_dem_dir / f"{reference_date}_{rlks}rlks.rdc.dem"), + "diff_par": str(gamma_dem_dir / f"{reference_date}_{rlks}rlks.diff_par"), + }, + } + manifest["rdc_dem"] = rdc_dem + manifest["status"] = "RDC_DEM_SCRIPT_READY" + manifest["next_stage"] = "execute_rdc_dem" + self._write_json(run_dir / "rdc_dem_plan.json", rdc_dem) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_rdc_dem(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_rdc_dem( + self, + run_id: str, + *, + rlks: int = 8, + timeout_seconds: int = 43200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "RDC_DEM_READY": + return self.get_run_detail(run_id) + if status in {"BASELINE_AUDIT_READY", "ITAB_APPROVED", "COREGISTRATION_SCRIPT_READY", "COREGISTRATION_READY", "RDC_DEM_FAILED"}: + self.prepare_rdc_dem(run_id, execute=False, rlks=rlks) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"RDC_DEM_SCRIPT_READY", "RDC_DEM_RUNNING"}: + raise ValueError(f"run status does not allow RDC DEM execution: {manifest.get('status')}") + + rdc_dem = dict(manifest.get("rdc_dem") or {}) + script_path = Path(self._path_to_windows(str(rdc_dem.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"RDC DEM script not found: {script_path}") + + reference_date = str(rdc_dem.get("reference_date") or "").strip() + rlks = self._bounded_int(rdc_dem.get("rlks") or rlks, default=8, minimum=1, maximum=64) + timeout_seconds = self._bounded_int(timeout_seconds, default=43200, minimum=60, maximum=172800) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + + rdc_dem["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["rdc_dem"] = rdc_dem + manifest["status"] = "RDC_DEM_RUNNING" + manifest["next_stage"] = "rdc_dem" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_rdc_dem(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_rdc_dem_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + dem_source=rdc_dem.get("dem_source") or {}, + ) + execution = { + **rdc_dem.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + rdc_dem = {**rdc_dem, "execution": execution, "summary": summary} + manifest["rdc_dem"] = rdc_dem + manifest["status"] = "RDC_DEM_FAILED" + manifest["next_stage"] = "fix_rdc_dem" + self._write_json(run_dir / "rdc_dem_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_rdc_dem(run_dir, manifest) + raise + + summary = self._build_rdc_dem_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + dem_source=rdc_dem.get("dem_source") or {}, + ) + execution = { + **rdc_dem.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + rdc_dem = {**rdc_dem, "execution": execution, "summary": summary} + manifest["rdc_dem"] = rdc_dem + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "RDC_DEM_READY" + manifest["next_stage"] = "interferograms" + else: + manifest["status"] = "RDC_DEM_FAILED" + manifest["next_stage"] = "fix_rdc_dem" + + self._write_json(run_dir / "rdc_dem_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_rdc_dem(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_interferograms( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + azlks: int = 8, + unwrap_threshold: float = 0.20, + ) -> dict[str, Any]: + if execute: + raise ValueError("Interferogram execution is submitted through the background job endpoint.") + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "INTERFEROGRAMS_READY": + return self.get_run_detail(run_id) + if status not in { + "COREGISTRATION_READY", + "RDC_DEM_READY", + "INTERFEROGRAMS_SCRIPT_READY", + "INTERFEROGRAMS_FAILED", + }: + raise ValueError(f"run status does not allow interferogram preparation: {manifest.get('status')}") + + rdc_dem_summary = ((manifest.get("rdc_dem") or {}).get("summary")) or self._read_optional_json(run_dir / "rdc_dem_summary.json") or {} + if not rdc_dem_summary.get("ready"): + raise ValueError("RDC DEM summary is not ready; run RDC DEM generation first") + coreg_summary = ((manifest.get("coregistration") or {}).get("summary")) or self._read_optional_json(run_dir / "coregistration_summary.json") or {} + if not coreg_summary.get("ready"): + raise ValueError("coregistration summary is not ready; run common-reference coregistration first") + + reference_date = str( + (manifest.get("rdc_dem") or {}).get("reference_date") + or rdc_dem_summary.get("reference_date") + or ((manifest.get("coregistration") or {}).get("reference_date")) + or ((manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + if not reference_date: + raise ValueError("interferogram stage requires a reference date") + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + azlks = self._bounded_int(azlks, default=8, minimum=1, maximum=64) + unwrap_threshold = self._bounded_float(unwrap_threshold, default=0.20, minimum=0.01, maximum=0.95) + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + approved_itab = common_dir / "itab_approved" + if not approved_itab.is_file(): + approved_itab = run_dir / "work" / "gamma" / "diff" / "itab_approved" + if not approved_itab.is_file(): + raise FileNotFoundError(f"approved itab not found: {approved_itab}") + + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + dates = self._stack_dates(stack_manifest) + pair_plan = self._build_interferogram_pair_plan( + run_dir, + reference_date=reference_date, + approved_itab=approved_itab, + dates=dates, + rlks=rlks, + ) + if not pair_plan: + raise ValueError("approved itab produced no interferogram pairs") + + script_path = self._write_interferogram_script( + run_dir, + reference_date=reference_date, + pair_plan=pair_plan, + rlks=rlks, + azlks=azlks, + unwrap_threshold=unwrap_threshold, + ) + interferograms = { + "schema": "insar.gamma-interferograms-stage/v1", + "strategy": "approved_itab_common_reference_diff_unwrap", + "script_path": str(script_path), + "reference_date": reference_date, + "rlks": rlks, + "azlks": azlks, + "unwrap_threshold": unwrap_threshold, + "approved_itab_path": str(approved_itab), + "pair_count": len(pair_plan), + "pairs": pair_plan, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "outputs": { + "diff_dir": str(common_dir / "diff"), + "diff_tab": str(common_dir / "DIFF_tab"), + "itab_common_ref": str(common_dir / "itab_common_ref"), + }, + } + manifest["interferograms"] = interferograms + manifest["status"] = "INTERFEROGRAMS_SCRIPT_READY" + manifest["next_stage"] = "execute_interferograms" + self._write_json(run_dir / "interferogram_plan.json", interferograms) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_interferograms(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_interferograms( + self, + run_id: str, + *, + rlks: int = 8, + azlks: int = 8, + unwrap_threshold: float = 0.20, + timeout_seconds: int = 43200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "INTERFEROGRAMS_READY": + return self.get_run_detail(run_id) + if status in {"COREGISTRATION_READY", "RDC_DEM_READY", "INTERFEROGRAMS_FAILED"}: + self.prepare_interferograms( + run_id, + execute=False, + rlks=rlks, + azlks=azlks, + unwrap_threshold=unwrap_threshold, + ) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"INTERFEROGRAMS_SCRIPT_READY", "INTERFEROGRAMS_RUNNING"}: + raise ValueError(f"run status does not allow interferogram execution: {manifest.get('status')}") + + interferograms = dict(manifest.get("interferograms") or {}) + script_path = Path(self._path_to_windows(str(interferograms.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"interferogram script not found: {script_path}") + + reference_date = str(interferograms.get("reference_date") or "").strip() + pair_plan = list(interferograms.get("pairs") or []) + rlks = self._bounded_int(interferograms.get("rlks") or rlks, default=8, minimum=1, maximum=64) + azlks = self._bounded_int(interferograms.get("azlks") or azlks, default=8, minimum=1, maximum=64) + timeout_seconds = self._bounded_int(timeout_seconds, default=43200, minimum=60, maximum=172800) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + + interferograms["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["interferograms"] = interferograms + manifest["status"] = "INTERFEROGRAMS_RUNNING" + manifest["next_stage"] = "interferograms" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_interferograms(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_interferogram_summary( + run_dir, + reference_date=reference_date, + pair_plan=pair_plan, + rlks=rlks, + ) + execution = { + **interferograms.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + interferograms = {**interferograms, "execution": execution, "summary": summary} + manifest["interferograms"] = interferograms + manifest["status"] = "INTERFEROGRAMS_FAILED" + manifest["next_stage"] = "fix_interferograms" + self._write_json(run_dir / "interferogram_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_interferograms(run_dir, manifest) + raise + + summary = self._build_interferogram_summary( + run_dir, + reference_date=reference_date, + pair_plan=pair_plan, + rlks=rlks, + ) + execution = { + **interferograms.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + interferograms = {**interferograms, "execution": execution, "summary": summary} + manifest["interferograms"] = interferograms + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "INTERFEROGRAMS_READY" + manifest["next_stage"] = "detrend_atm" + else: + manifest["status"] = "INTERFEROGRAMS_FAILED" + manifest["next_stage"] = "fix_interferograms" + + self._write_json(run_dir / "interferogram_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_interferograms(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_detrend_atm( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + reference_window: int = 16, + coherence_min: float = 0.15, + ) -> dict[str, Any]: + if execute: + return self.execute_detrend_atm( + run_id, + rlks=rlks, + reference_window=reference_window, + coherence_min=coherence_min, + ) + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "DETREND_ATM_READY": + return self.get_run_detail(run_id) + if status not in {"INTERFEROGRAMS_READY", "DETREND_ATM_SCRIPT_READY", "DETREND_ATM_FAILED"}: + raise ValueError(f"run status does not allow detrend/atm preparation: {manifest.get('status')}") + + interferogram_summary = ( + ((manifest.get("interferograms") or {}).get("summary")) + or self._read_optional_json(run_dir / "interferogram_summary.json") + or {} + ) + if not interferogram_summary.get("ready"): + raise ValueError("interferogram summary is not ready; run differential interferograms first") + + reference_date = str( + (manifest.get("interferograms") or {}).get("reference_date") + or interferogram_summary.get("reference_date") + or ((manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + if not reference_date: + raise ValueError("detrend/atm stage requires a reference date") + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + reference_window = self._bounded_int(reference_window, default=16, minimum=1, maximum=256) + coherence_min = self._bounded_float(coherence_min, default=0.15, minimum=0.0, maximum=1.0) + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + diff_tab = common_dir / "DIFF_tab" + itab = common_dir / "itab_common_ref" + rmli_path, rmli_par_path = self._find_reference_rmli_paths(run_dir, reference_date) + hgt_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.rdc.dem" + if not hgt_path.is_file(): + hgt_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.hgt" + for label, path in { + "DIFF_tab": diff_tab, + "itab_common_ref": itab, + "reference_mli": rmli_path, + "reference_mli_par": rmli_par_path, + "rdc_dem_height": hgt_path, + }.items(): + if not path.is_file() or path.stat().st_size <= 0: + raise FileNotFoundError(f"{label} is missing or empty: {path}") + + pair_plan = self._detrend_pair_plan_from_diff_tab(diff_tab, rlks=rlks) + if not pair_plan: + raise ValueError("DIFF_tab produced no detrend/atm pair plan") + reference_region = self._select_ipta_reference_region( + run_dir, + reference_date=reference_date, + rlks=rlks, + reference_window=reference_window, + geom_ref_mli_par=rmli_par_path, + ) + script_path = self._write_detrend_atm_script( + run_dir, + reference_date=reference_date, + rlks=rlks, + reference_window=reference_window, + reference_region=reference_region, + coherence_min=coherence_min, + diff_tab=diff_tab, + itab=itab, + rmli_path=rmli_path, + rmli_par_path=rmli_par_path, + hgt_path=hgt_path, + pair_plan=pair_plan, + ) + detrend_atm = { + "schema": "insar.gamma-detrend-atm-stage/v1", + "strategy": "expert_quad_fit_quad_sub_atm_mod_2d_sub_phase", + "script_path": str(script_path), + "reference_date": reference_date, + "rlks": rlks, + "reference_window": reference_window, + "reference_region": reference_region, + "coherence_min": coherence_min, + "pair_count": len(pair_plan), + "pairs": pair_plan, + "inputs": { + "diff_tab": str(diff_tab), + "itab": str(itab), + "reference_mli": str(rmli_path), + "reference_mli_par": str(rmli_par_path), + "hgt": str(hgt_path), + }, + "outputs": { + "detrend_dir": str(common_dir / "detrend_atm"), + "diff_atmsub_tab": str(common_dir / "DIFF_atmsub_tab"), + "itab_atmsub": str(common_dir / "itab_atmsub"), + }, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + manifest["detrend_atm"] = detrend_atm + manifest["status"] = "DETREND_ATM_SCRIPT_READY" + manifest["next_stage"] = "execute_detrend_atm" + self._write_json(run_dir / "detrend_atm_plan.json", detrend_atm) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_detrend_atm(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_detrend_atm( + self, + run_id: str, + *, + rlks: int = 8, + reference_window: int = 16, + coherence_min: float = 0.15, + timeout_seconds: int = 43200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "DETREND_ATM_READY": + return self.get_run_detail(run_id) + if status in {"INTERFEROGRAMS_READY", "DETREND_ATM_FAILED"}: + self.prepare_detrend_atm( + run_id, + execute=False, + rlks=rlks, + reference_window=reference_window, + coherence_min=coherence_min, + ) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"DETREND_ATM_SCRIPT_READY", "DETREND_ATM_RUNNING"}: + raise ValueError(f"run status does not allow detrend/atm execution: {manifest.get('status')}") + + detrend_atm = dict(manifest.get("detrend_atm") or {}) + script_path = Path(self._path_to_windows(str(detrend_atm.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"detrend/atm script not found: {script_path}") + + reference_date = str(detrend_atm.get("reference_date") or "").strip() + pair_plan = list(detrend_atm.get("pairs") or []) + rlks = self._bounded_int(detrend_atm.get("rlks") or rlks, default=8, minimum=1, maximum=64) + timeout_seconds = self._bounded_int(timeout_seconds, default=43200, minimum=60, maximum=172800) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + + detrend_atm["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["detrend_atm"] = detrend_atm + manifest["status"] = "DETREND_ATM_RUNNING" + manifest["next_stage"] = "detrend_atm" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_detrend_atm(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_detrend_atm_summary( + run_dir, + reference_date=reference_date, + pair_plan=pair_plan, + rlks=rlks, + inputs=detrend_atm.get("inputs") or {}, + ) + execution = { + **detrend_atm.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + detrend_atm = {**detrend_atm, "execution": execution, "summary": summary} + manifest["detrend_atm"] = detrend_atm + manifest["status"] = "DETREND_ATM_FAILED" + manifest["next_stage"] = "fix_detrend_atm" + self._write_json(run_dir / "detrend_atm_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_detrend_atm(run_dir, manifest) + raise + + summary = self._build_detrend_atm_summary( + run_dir, + reference_date=reference_date, + pair_plan=pair_plan, + rlks=rlks, + inputs=detrend_atm.get("inputs") or {}, + ) + execution = { + **detrend_atm.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + detrend_atm = {**detrend_atm, "execution": execution, "summary": summary} + manifest["detrend_atm"] = detrend_atm + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "DETREND_ATM_READY" + manifest["next_stage"] = "ipta_timeseries" + else: + manifest["status"] = "DETREND_ATM_FAILED" + manifest["next_stage"] = "fix_detrend_atm" + + self._write_json(run_dir / "detrend_atm_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_detrend_atm(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_ipta_timeseries( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + reference_window: int = 16, + mb_mode: int = DEFAULT_IPTA_MB_MODE, + ) -> dict[str, Any]: + if execute: + raise ValueError("IPTA time-series execution is submitted through the background job endpoint.") + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "IPTA_TIMESERIES_READY": + return self.get_run_detail(run_id) + if status not in {"DETREND_ATM_READY", "IPTA_TIMESERIES_SCRIPT_READY", "IPTA_TIMESERIES_FAILED"}: + raise ValueError(f"run status does not allow IPTA time-series preparation: {manifest.get('status')}") + + detrend_summary = ( + ((manifest.get("detrend_atm") or {}).get("summary")) + or self._read_optional_json(run_dir / "detrend_atm_summary.json") + or {} + ) + if not detrend_summary.get("ready"): + raise ValueError("detrend/atm summary is not ready; run expert section 10 first") + + reference_date = str( + (manifest.get("detrend_atm") or {}).get("reference_date") + or detrend_summary.get("reference_date") + or ((manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + if not reference_date: + raise ValueError("IPTA time-series stage requires a reference date") + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + reference_window = self._bounded_int(reference_window, default=16, minimum=1, maximum=256) + mb_mode = self._normalize_ipta_mb_mode(mb_mode) + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + diff_tab = common_dir / "DIFF_atmsub_tab" + rmli_tab = common_dir / "RMLI_tab" + itab = common_dir / "itab_atmsub" + for label, path in {"DIFF_atmsub_tab": diff_tab, "RMLI_tab": rmli_tab, "itab_atmsub": itab}.items(): + if not path.is_file() or path.stat().st_size <= 0: + raise FileNotFoundError(f"{label} is missing or empty: {path}") + + geom_ref_mli, geom_ref_mli_par = self._find_reference_rmli_paths(run_dir, reference_date) + if not geom_ref_mli_par.is_file(): + raise FileNotFoundError(f"reference MLI parameter file is missing: {geom_ref_mli_par}") + mb_ref_mli, mb_ref_mli_par = self._select_ipta_mb_reference_mli( + run_dir, + reference_date=reference_date, + rmli_tab=rmli_tab, + ) + if not mb_ref_mli_par.is_file(): + raise FileNotFoundError(f"IPTA mb reference MLI parameter file is missing: {mb_ref_mli_par}") + + reference_region = self._select_ipta_reference_region( + run_dir, + reference_date=reference_date, + rlks=rlks, + reference_window=reference_window, + geom_ref_mli_par=geom_ref_mli_par, + ) + script_path = self._write_ipta_timeseries_script( + run_dir, + reference_date=reference_date, + rlks=rlks, + reference_window=reference_window, + diff_tab=diff_tab, + rmli_tab=rmli_tab, + itab=itab, + geom_ref_mli_par=geom_ref_mli_par, + mb_ref_mli_par=mb_ref_mli_par, + reference_region=reference_region, + mb_mode=mb_mode, + ) + timeseries_dir = common_dir / "timeseries" + ipta_timeseries = { + "schema": "insar.gamma-ipta-timeseries-stage/v1", + "strategy": "gamma_mb_ts_rate_atmsub_expert_section_10", + "script_path": str(script_path), + "reference_date": reference_date, + "rlks": rlks, + "reference_window": reference_window, + "reference_region": reference_region, + "mb_mode": mb_mode, + "mb_mode_description": IPTA_MB_MODE_DESCRIPTIONS[mb_mode], + "inputs": { + "diff_tab": str(diff_tab), + "diff_tab_source": "detrend_atm", + "rmli_tab": str(rmli_tab), + "itab": str(itab), + "geometry_reference_mli": str(geom_ref_mli), + "geometry_reference_mli_par": str(geom_ref_mli_par), + "mb_reference_mli": str(mb_ref_mli), + "mb_reference_mli_par": str(mb_ref_mli_par), + }, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "outputs": { + "timeseries_dir": str(timeseries_dir), + "diff_ts_tab": str(timeseries_dir / "diff_ts.tab"), + "itab_ts": str(timeseries_dir / "itab_ts"), + "sigma_ts": str(timeseries_dir / "sigma_ts"), + "hgt_correction": str(timeseries_dir / "hgt_correction"), + "ts_rate": str(timeseries_dir / "ts_rate"), + "ts_const": str(timeseries_dir / "ts_const"), + "sigma_rate": str(timeseries_dir / "sigma_rate"), + }, + } + manifest["ipta_timeseries"] = ipta_timeseries + manifest["status"] = "IPTA_TIMESERIES_SCRIPT_READY" + manifest["next_stage"] = "execute_ipta_timeseries" + self._write_json(run_dir / "ipta_timeseries_plan.json", ipta_timeseries) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_ipta_timeseries(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_ipta_timeseries( + self, + run_id: str, + *, + rlks: int = 8, + reference_window: int = 16, + mb_mode: int = DEFAULT_IPTA_MB_MODE, + timeout_seconds: int = 43200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "IPTA_TIMESERIES_READY": + return self.get_run_detail(run_id) + if status in {"DETREND_ATM_READY", "IPTA_TIMESERIES_FAILED"}: + self.prepare_ipta_timeseries( + run_id, + execute=False, + rlks=rlks, + reference_window=reference_window, + mb_mode=mb_mode, + ) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"IPTA_TIMESERIES_SCRIPT_READY", "IPTA_TIMESERIES_RUNNING"}: + raise ValueError(f"run status does not allow IPTA time-series execution: {manifest.get('status')}") + + ipta_timeseries = dict(manifest.get("ipta_timeseries") or {}) + if status == "IPTA_TIMESERIES_SCRIPT_READY" and ipta_timeseries.get("mb_mode") is None: + self.prepare_ipta_timeseries( + run_id, + execute=False, + rlks=rlks, + reference_window=reference_window, + mb_mode=mb_mode, + ) + manifest = self._read_json(manifest_path) + ipta_timeseries = dict(manifest.get("ipta_timeseries") or {}) + script_path = Path(self._path_to_windows(str(ipta_timeseries.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"IPTA time-series script not found: {script_path}") + + reference_date = str(ipta_timeseries.get("reference_date") or "").strip() + rlks = self._bounded_int(ipta_timeseries.get("rlks") or rlks, default=8, minimum=1, maximum=64) + mb_mode = self._normalize_ipta_mb_mode(ipta_timeseries.get("mb_mode", mb_mode)) + timeout_seconds = self._bounded_int(timeout_seconds, default=43200, minimum=60, maximum=172800) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + + ipta_timeseries["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["ipta_timeseries"] = ipta_timeseries + manifest["status"] = "IPTA_TIMESERIES_RUNNING" + manifest["next_stage"] = "ipta_timeseries" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_ipta_timeseries(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_ipta_timeseries_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + inputs=ipta_timeseries.get("inputs") or {}, + reference_region=ipta_timeseries.get("reference_region") or {}, + mb_mode=mb_mode, + ) + execution = { + **ipta_timeseries.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + ipta_timeseries = {**ipta_timeseries, "execution": execution, "summary": summary} + manifest["ipta_timeseries"] = ipta_timeseries + manifest["status"] = "IPTA_TIMESERIES_FAILED" + manifest["next_stage"] = "fix_ipta_timeseries" + self._write_json(run_dir / "ipta_timeseries_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_ipta_timeseries(run_dir, manifest) + raise + + summary = self._build_ipta_timeseries_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + inputs=ipta_timeseries.get("inputs") or {}, + reference_region=ipta_timeseries.get("reference_region") or {}, + mb_mode=mb_mode, + ) + execution = { + **ipta_timeseries.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + ipta_timeseries = {**ipta_timeseries, "execution": execution, "summary": summary} + manifest["ipta_timeseries"] = ipta_timeseries + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "IPTA_TIMESERIES_READY" + manifest["next_stage"] = "publish_products" + else: + manifest["status"] = "IPTA_TIMESERIES_FAILED" + manifest["next_stage"] = "fix_ipta_timeseries" + + self._write_json(run_dir / "ipta_timeseries_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_ipta_timeseries(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_publish_products( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + ) -> dict[str, Any]: + if execute: + raise ValueError("publish product execution is submitted through the workflow/background job path.") + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + recovered = self._recover_workflow_resume_status(dict(manifest)) + if recovered.get("status") != manifest.get("status"): + manifest = recovered + self._write_json(manifest_path, manifest) + status = str(manifest.get("status") or "").strip() + if status in {"PRODUCTS_READY", "MONITOR_POINTS_SCRIPT_READY", "MONITOR_POINTS_RUNNING", "MONITOR_POINTS_READY"}: + return self.get_run_detail(run_id) + if status not in {"IPTA_TIMESERIES_READY", "PUBLISH_PRODUCTS_SCRIPT_READY", "PUBLISH_PRODUCTS_FAILED"}: + raise ValueError(f"run status does not allow product publishing preparation: {manifest.get('status')}") + + ipta_summary = ( + ((manifest.get("ipta_timeseries") or {}).get("summary")) + or self._read_optional_json(run_dir / "ipta_timeseries_summary.json") + or {} + ) + if not ipta_summary.get("ready"): + raise ValueError("IPTA time-series summary is not ready; run IPTA inversion first") + + reference_date = str( + (manifest.get("ipta_timeseries") or {}).get("reference_date") + or ipta_summary.get("reference_date") + or ((manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + if not reference_date: + raise ValueError("publish products stage requires a reference date") + + rlks = self._bounded_int(rlks, default=settings.GAMMA_SBAS_DEFAULT_RLKS or 8, minimum=1, maximum=64) + rmli_path, rmli_par_path = self._find_reference_rmli_paths(run_dir, reference_date) + slc_par_path = run_dir / "work" / "gamma" / "slc" / f"{reference_date}.slc.par" + if not slc_par_path.is_file(): + slc_par_path = rmli_par_path + dem_par_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.utm.dem.par" + lookup_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.UTM_TO_RDC" + timeseries_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" / "timeseries" + for label, path in { + "reference_mli": rmli_path, + "reference_mli_par": rmli_par_path, + "slc_par": slc_par_path, + "utm_dem_par": dem_par_path, + "lookup_table": lookup_path, + "ts_rate": timeseries_dir / "ts_rate", + "sigma_rate": timeseries_dir / "sigma_rate", + }.items(): + if not path.is_file() or path.stat().st_size <= 0: + raise FileNotFoundError(f"{label} is missing or empty: {path}") + + wavelength = self._resolve_radar_wavelength_m(slc_par_path, rmli_par_path) + script_path = self._write_publish_products_script( + run_dir, + reference_date=reference_date, + rlks=rlks, + timeseries_dir=timeseries_dir, + rmli_path=rmli_path, + rmli_par_path=rmli_par_path, + slc_par_path=slc_par_path, + dem_par_path=dem_par_path, + lookup_path=lookup_path, + wavelength=wavelength, + ) + export_dir = run_dir / "publish" / "geotiff" + publish_products = { + "schema": "insar.gamma-sbas-publish-products-stage/v1", + "strategy": "gamma_geocode_back_data2geotiff_los_sign_conversion", + "script_path": str(script_path), + "reference_date": reference_date, + "rlks": rlks, + "wavelength_m": wavelength, + "los_sign_convention": { + "default": "los_rate_toward_m_per_year", + "toward_positive": "positive means motion toward radar", + "away_positive": "positive means motion away from radar", + "formulas": { + "away_m_per_year": "phase_rate_rad_per_year * wavelength / (4*pi)", + "toward_m_per_year": "-phase_rate_rad_per_year * wavelength / (4*pi)", + "away_mm_per_year": "phase_rate_rad_per_year * wavelength / (4*pi) * 1000", + "toward_mm_per_year": "-phase_rate_rad_per_year * wavelength / (4*pi) * 1000", + }, + }, + "expert_color_conventions": { + "velocity": "hls.cm with -0.08 to 0.08 m/year as in the expert document", + "sigma": "cc.cm; production uses 0.0 to 0.06 m/year for LOS sigma-rate browse products", + "phase_and_atmosphere": "rmg.cm with -6.28 to 6.28 radians for detrend/atmosphere browse products", + }, + "inputs": { + "timeseries_dir": str(timeseries_dir), + "ts_rate": str(timeseries_dir / "ts_rate"), + "sigma_rate": str(timeseries_dir / "sigma_rate"), + "sigma_ts": str(timeseries_dir / "sigma_ts"), + "hgt_correction": str(timeseries_dir / "hgt_correction"), + "reference_mli": str(rmli_path), + "reference_mli_par": str(rmli_par_path), + "slc_par": str(slc_par_path), + "utm_dem_par": str(dem_par_path), + "lookup_table": str(lookup_path), + }, + "outputs": { + "export_dir": str(export_dir), + "product_summary": str(run_dir / "product_summary.json"), + "quality_summary": str(run_dir / "quality_summary.json"), + }, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + manifest["publish_products"] = publish_products + manifest["status"] = "PUBLISH_PRODUCTS_SCRIPT_READY" + manifest["next_stage"] = "execute_publish_products" + self._write_json(run_dir / "publish_product_plan.json", publish_products) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_publish_products(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_publish_products( + self, + run_id: str, + *, + rlks: int = 8, + timeout_seconds: int = 7200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + recovered = self._recover_workflow_resume_status(dict(manifest)) + if recovered.get("status") != manifest.get("status"): + manifest = recovered + self._write_json(manifest_path, manifest) + status = str(manifest.get("status") or "").strip() + if status in {"PRODUCTS_READY", "MONITOR_POINTS_SCRIPT_READY", "MONITOR_POINTS_RUNNING", "MONITOR_POINTS_READY"}: + return self.get_run_detail(run_id) + if status in {"IPTA_TIMESERIES_READY", "PUBLISH_PRODUCTS_FAILED"}: + self.prepare_publish_products(run_id, execute=False, rlks=rlks) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"PUBLISH_PRODUCTS_SCRIPT_READY", "PUBLISH_PRODUCTS_RUNNING"}: + raise ValueError(f"run status does not allow product publishing execution: {manifest.get('status')}") + + publish_products = dict(manifest.get("publish_products") or {}) + script_path = Path(self._path_to_windows(str(publish_products.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"publish products script not found: {script_path}") + + reference_date = str(publish_products.get("reference_date") or "").strip() + rlks = self._bounded_int(publish_products.get("rlks") or rlks, default=8, minimum=1, maximum=64) + timeout_seconds = self._bounded_int(timeout_seconds, default=7200, minimum=60, maximum=86400) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + publish_products["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["publish_products"] = publish_products + manifest["status"] = "PUBLISH_PRODUCTS_RUNNING" + manifest["next_stage"] = "publish_products" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_publish_products(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_publish_products_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + inputs=publish_products.get("inputs") or {}, + wavelength=publish_products.get("wavelength_m"), + ) + execution = { + **publish_products.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + publish_products = {**publish_products, "execution": execution, "summary": summary} + manifest["publish_products"] = publish_products + manifest["status"] = "PUBLISH_PRODUCTS_FAILED" + manifest["next_stage"] = "fix_publish_products" + self._write_json(run_dir / "publish_product_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_publish_products(run_dir, manifest) + raise + + summary = self._build_publish_products_summary( + run_dir, + reference_date=reference_date, + rlks=rlks, + inputs=publish_products.get("inputs") or {}, + wavelength=publish_products.get("wavelength_m"), + ) + execution = { + **publish_products.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + publish_products = {**publish_products, "execution": execution, "summary": summary} + manifest["publish_products"] = publish_products + manifest["publish_artifacts"] = self._build_run_artifacts(run_dir) + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "PRODUCTS_READY" + manifest["next_stage"] = "monitor_points" + else: + manifest["status"] = "PUBLISH_PRODUCTS_FAILED" + manifest["next_stage"] = "fix_publish_products" + + self._write_json(run_dir / "publish_product_summary.json", summary) + self._write_json(run_dir / "product_summary.json", summary.get("product_summary") or summary) + self._write_json(run_dir / "quality_summary.json", summary.get("quality_summary") or {}) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_publish_products(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_monitor_points( + self, + run_id: str, + *, + execute: bool = False, + ) -> dict[str, Any]: + if execute: + raise ValueError("monitor point execution is submitted through the workflow/background job path.") + + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + recovered = self._recover_workflow_resume_status(dict(manifest)) + if recovered.get("status") != manifest.get("status"): + manifest = recovered + self._write_json(manifest_path, manifest) + status = str(manifest.get("status") or "").strip() + if status == "MONITOR_POINTS_READY": + return self.get_run_detail(run_id) + if status not in {"PRODUCTS_READY", "MONITOR_POINTS_SCRIPT_READY", "MONITOR_POINTS_FAILED"}: + raise ValueError(f"run status does not allow monitor point preparation: {manifest.get('status')}") + + publish_summary = ( + ((manifest.get("publish_products") or {}).get("summary")) + or self._read_optional_json(run_dir / "publish_product_summary.json") + or {} + ) + if not publish_summary.get("ready"): + raise ValueError("published LOS products are not ready; run publish products first") + + reference_date = str( + (manifest.get("publish_products") or {}).get("reference_date") + or publish_summary.get("reference_date") + or ((manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + rlks = self._bounded_int( + (manifest.get("publish_products") or {}).get("rlks") or settings.GAMMA_SBAS_DEFAULT_RLKS, + default=8, + minimum=1, + maximum=64, + ) + rmli_path, rmli_par_path = self._find_reference_rmli_paths(run_dir, reference_date) + slc_par_path = Path(self._path_to_windows(str(((manifest.get("publish_products") or {}).get("inputs") or {}).get("slc_par") or "")) or "") + if not slc_par_path.is_file(): + slc_par_path = run_dir / "work" / "gamma" / "slc" / f"{reference_date}.slc.par" + if not slc_par_path.is_file(): + slc_par_path = rmli_par_path + dem_par_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.utm.dem.par" + lookup_path = run_dir / "work" / "gamma" / "dem" / f"{reference_date}_{rlks}rlks.UTM_TO_RDC" + timeseries_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" / "timeseries" + export_dir = run_dir / "publish" / "geotiff" + point_dir = run_dir / "publish" / "monitor_points" + for label, path in { + "reference_mli_par": rmli_par_path, + "slc_par": slc_par_path, + "dem_par": dem_par_path, + "lookup": lookup_path, + "los_rate_toward_rdc": export_dir / "los_rate_toward_mm_per_year.rdc", + "los_sigma_rdc": export_dir / "los_sigma_mm_per_year.rdc", + "diff_ts_tab": timeseries_dir / "diff_ts.tab", + }.items(): + if not path.is_file() or path.stat().st_size <= 0: + raise FileNotFoundError(f"{label} is missing or empty: {path}") + + stack_manifest = self._read_optional_json(run_dir / "stack_manifest.json") or {} + dates = self._stack_dates(stack_manifest) + script_path = self._write_monitor_points_script( + run_dir, + reference_date=reference_date, + dates=dates, + timeseries_dir=timeseries_dir, + export_dir=export_dir, + point_dir=point_dir, + rmli_par_path=rmli_par_path, + slc_par_path=slc_par_path, + dem_par_path=dem_par_path, + lookup_path=lookup_path, + ) + monitor_points = { + "schema": "insar.gamma-sbas-monitor-points-stage/v1", + "strategy": "sample_or_configured_points_from_gamma_diff_ts", + "script_path": str(script_path), + "reference_date": reference_date, + "dates": dates, + "inputs": { + "timeseries_dir": str(timeseries_dir), + "export_dir": str(export_dir), + "monitor_config": str(run_dir / "monitor_points.json"), + "reference_mli": str(rmli_path), + "reference_mli_par": str(rmli_par_path), + "slc_par": str(slc_par_path), + "dem_par": str(dem_par_path), + "lookup": str(lookup_path), + }, + "outputs": { + "point_dir": str(point_dir), + "summary": str(run_dir / "monitor_points_summary.json"), + }, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + manifest["monitor_point_products"] = monitor_points + manifest["status"] = "MONITOR_POINTS_SCRIPT_READY" + manifest["next_stage"] = "execute_monitor_points" + self._write_json(run_dir / "monitor_points_plan.json", monitor_points) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_monitor_points(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_monitor_points( + self, + run_id: str, + *, + timeout_seconds: int = 1800, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + recovered = self._recover_workflow_resume_status(dict(manifest)) + if recovered.get("status") != manifest.get("status"): + manifest = recovered + self._write_json(manifest_path, manifest) + status = str(manifest.get("status") or "").strip() + if status == "MONITOR_POINTS_READY": + return self.get_run_detail(run_id) + if status in {"PRODUCTS_READY", "MONITOR_POINTS_FAILED"}: + self.prepare_monitor_points(run_id, execute=False) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"MONITOR_POINTS_SCRIPT_READY", "MONITOR_POINTS_RUNNING"}: + raise ValueError(f"run status does not allow monitor point execution: {manifest.get('status')}") + + monitor_points = dict(manifest.get("monitor_point_products") or {}) + script_path = Path(self._path_to_windows(str(monitor_points.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"monitor point script not found: {script_path}") + + timeout_seconds = self._bounded_int(timeout_seconds, default=1800, minimum=60, maximum=86400) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + monitor_points["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["monitor_point_products"] = monitor_points + manifest["status"] = "MONITOR_POINTS_RUNNING" + manifest["next_stage"] = "monitor_points" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_monitor_points(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_monitor_points_summary(run_dir, monitor_points=monitor_points) + execution = { + **monitor_points.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + monitor_points = {**monitor_points, "execution": execution, "summary": summary} + manifest["monitor_point_products"] = monitor_points + manifest["status"] = "MONITOR_POINTS_FAILED" + manifest["next_stage"] = "fix_monitor_points" + self._write_json(run_dir / "monitor_points_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_monitor_points(run_dir, manifest) + raise + + summary = self._build_monitor_points_summary(run_dir, monitor_points=monitor_points) + execution = { + **monitor_points.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + monitor_points = {**monitor_points, "execution": execution, "summary": summary} + manifest["monitor_point_products"] = monitor_points + manifest["publish_artifacts"] = self._build_run_artifacts(run_dir) + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "MONITOR_POINTS_READY" + manifest["next_stage"] = "review_publish_products" + else: + manifest["status"] = "MONITOR_POINTS_FAILED" + manifest["next_stage"] = "fix_monitor_points" + + self._write_json(run_dir / "monitor_points_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_monitor_points(run_dir, manifest) + return self.get_run_detail(run_id) + def list_trial_runs(self) -> dict[str, Any]: items: list[dict[str, Any]] = [] if not self.trial_root.exists(): @@ -956,25 +2903,758 @@ class SbasInsarProductionService: raise FileNotFoundError(f"artifact not found: {normalized}") return candidate - def _resolve_source_roots(self, roots: list[str] | None) -> list[Path]: - raw_values = roots or self._split_config_paths( - settings.SOURCE_PRODUCT_DIRS, - settings.MONITOR_RADAR_DIRS, - settings.INSAR_STORAGE_DIRS, + def prepare_workflow( + self, + run_id: str, + *, + force: bool = False, + rlks: int | None = None, + azlks: int | None = None, + mb_mode: int | None = None, + reference_window: int | None = None, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + run_manifest = self._read_json(manifest_path) + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + self._ensure_expert_workspace(run_dir) + run_manifest = self._recover_workflow_resume_status(run_manifest) + self._write_json(manifest_path, run_manifest) + params = { + "rlks": self._bounded_int(rlks or settings.GAMMA_SBAS_DEFAULT_RLKS, default=8, minimum=1, maximum=64), + "azlks": self._bounded_int(azlks or settings.GAMMA_SBAS_DEFAULT_AZLKS, default=8, minimum=1, maximum=64), + "mb_mode": self._normalize_ipta_mb_mode(mb_mode if mb_mode is not None else settings.GAMMA_SBAS_DEFAULT_MB_MODE), + "reference_window": self._bounded_int( + reference_window or settings.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW, + default=16, + minimum=1, + maximum=256, + ), + } + self._prepare_reusable_stage_scripts(run_id, run_dir, run_manifest, params) + run_manifest = self._read_json(manifest_path) + resume_stage_status = str(run_manifest.get("status") or "").strip() + run_manifest["workflow"] = { + **(run_manifest.get("workflow") or {}), + "schema": "insar.gamma-sbas-workflow-binding/v1", + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "params": params, + "force": bool(force), + "prepared_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "manifest_path": str(run_dir / "manifest.json"), + "state_path": str(run_dir / "state" / "step_status.json"), + "resume_stage_status": resume_stage_status, + } + run_manifest["status"] = "WORKFLOW_READY" + run_manifest["next_stage"] = "submit_workflow_job" + + workflow_manifest = self._build_workflow_manifest(run_dir, run_manifest, stack_manifest, params=params) + self._write_json(run_dir / "manifest.json", workflow_manifest) + state_path = run_dir / "state" / "step_status.json" + if force or not state_path.is_file(): + self._write_json(state_path, self._initial_workflow_state(run_manifest, workflow_manifest)) + self._write_json(manifest_path, run_manifest) + return self.get_run_detail(run_id) + + @classmethod + def _recover_workflow_resume_status(cls, run_manifest: dict[str, Any]) -> dict[str, Any]: + current_status = str(run_manifest.get("status") or "").strip() + if current_status not in {"WORKFLOW_READY", "WORKFLOW_RUNNING", "WORKFLOW_FAILED", "WORKFLOW_PARTIAL"}: + return run_manifest + inferred_status = cls._infer_stage_status_from_manifest(run_manifest) + if inferred_status: + run_manifest["status"] = inferred_status + run_manifest["next_stage"] = cls._next_stage_for_status(inferred_status) + return run_manifest + workflow = run_manifest.get("workflow") or {} + candidates = [ + workflow.get("resume_stage_status"), + workflow.get("previous_status"), + ] + for candidate in candidates: + stage_status = str(candidate or "").strip() + if stage_status and not stage_status.startswith("WORKFLOW_"): + run_manifest["status"] = stage_status + run_manifest["next_stage"] = cls._next_stage_for_status(stage_status) + return run_manifest + run_manifest["status"] = "PLANNED_GAMMA_BASELINE_AUDIT" + run_manifest["next_stage"] = "baseline_audit" + return run_manifest + + @staticmethod + def _stage_execution_completed(stage: dict[str, Any] | None) -> bool: + payload = stage or {} + execution = payload.get("execution") or {} + summary = payload.get("summary") or {} + return ( + str(execution.get("status") or "").upper() == "COMPLETED" + and int(execution.get("returncode") or 0) == 0 + and (summary.get("ready") is not False) ) + + @classmethod + def _infer_stage_status_from_manifest(cls, run_manifest: dict[str, Any]) -> str: + if cls._stage_execution_completed(run_manifest.get("monitor_point_products")): + return "MONITOR_POINTS_READY" + if cls._stage_execution_completed(run_manifest.get("publish_products")): + return "PRODUCTS_READY" + if cls._stage_execution_completed(run_manifest.get("ipta_timeseries")): + return "IPTA_TIMESERIES_READY" + if cls._stage_execution_completed(run_manifest.get("detrend_atm")): + return "DETREND_ATM_READY" + if cls._stage_execution_completed(run_manifest.get("interferograms")): + return "INTERFEROGRAMS_READY" + if cls._stage_execution_completed(run_manifest.get("coregistration")): + if cls._stage_execution_completed(run_manifest.get("rdc_dem")): + return "RDC_DEM_READY" + return "COREGISTRATION_READY" + if cls._stage_execution_completed(run_manifest.get("rdc_dem")): + return "RDC_DEM_READY" + if (run_manifest.get("coregistration") or {}).get("script_path"): + return "COREGISTRATION_SCRIPT_READY" + if (run_manifest.get("baseline_audit") or {}).get("summary"): + return "BASELINE_AUDIT_READY" + if (run_manifest.get("baseline_audit") or {}).get("script_path"): + return "BASELINE_AUDIT_SCRIPT_READY" + return "" + + def _prepare_reusable_stage_scripts( + self, + run_id: str, + run_dir: Path, + run_manifest: dict[str, Any], + params: dict[str, Any], + ) -> None: + status = str(run_manifest.get("status") or "").strip() + if status in {"PLANNED_GAMMA_BASELINE_AUDIT", "WORKFLOW_READY", "BASELINE_AUDIT_FAILED", "BASELINE_AUDIT_READY"}: + self.run_baseline_audit( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + azlks=int(params.get("azlks") or 8), + max_delta_n=1, + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + + if status == "BASELINE_AUDIT_READY" and settings.GAMMA_SBAS_AUTO_APPROVE_ITAB: + try: + self.decide_itab( + run_id, + decision="approve", + reviewer="system", + note="Auto-approved for Gamma SBAS expert workflow after baseline audit summary was present.", + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + except Exception: + pass + + if status in { + "ITAB_APPROVED", + "COREGISTRATION_FAILED", + "COREGISTRATION_SCRIPT_READY", + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_READY", + }: + try: + self.prepare_coregistration( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + azlks=int(params.get("azlks") or 8), + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + except Exception: + pass + + if status in { + "BASELINE_AUDIT_READY", + "ITAB_APPROVED", + "COREGISTRATION_SCRIPT_READY", + "COREGISTRATION_READY", + "RDC_DEM_FAILED", + "RDC_DEM_SCRIPT_READY", + }: + try: + self.prepare_rdc_dem( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + except Exception: + pass + + if status in {"RDC_DEM_READY", "INTERFEROGRAMS_FAILED", "INTERFEROGRAMS_SCRIPT_READY"}: + try: + self.prepare_interferograms( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + azlks=int(params.get("azlks") or 8), + unwrap_threshold=0.20, + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + except Exception: + pass + + if status in {"INTERFEROGRAMS_READY", "DETREND_ATM_FAILED", "DETREND_ATM_SCRIPT_READY"}: + try: + self.prepare_detrend_atm( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + reference_window=int(params.get("reference_window") or 16), + ) + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + except Exception: + pass + + if status in {"DETREND_ATM_READY", "IPTA_TIMESERIES_FAILED", "IPTA_TIMESERIES_SCRIPT_READY"}: + try: + self.prepare_ipta_timeseries( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + reference_window=int(params.get("reference_window") or 16), + mb_mode=int(params.get("mb_mode") or 0), + ) + except Exception: + pass + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + + if status in {"IPTA_TIMESERIES_READY", "PUBLISH_PRODUCTS_FAILED", "PUBLISH_PRODUCTS_SCRIPT_READY"}: + try: + self.prepare_publish_products( + run_id, + execute=False, + rlks=int(params.get("rlks") or 8), + ) + except Exception: + pass + run_manifest = self._read_json(run_dir / "run_manifest.json") + status = str(run_manifest.get("status") or "").strip() + + if status in {"PRODUCTS_READY", "MONITOR_POINTS_FAILED", "MONITOR_POINTS_SCRIPT_READY"}: + try: + self.prepare_monitor_points(run_id, execute=False) + except Exception: + pass + + def execute_workflow( + self, + run_id: str, + *, + from_step: str | None = None, + to_step: str | None = None, + only_steps: list[str] | None = None, + force: bool = False, + timeout_seconds: int | None = None, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + run_manifest = self._read_json(manifest_path) + if not (run_dir / "manifest.json").is_file(): + self.prepare_workflow(run_id, force=force) + run_manifest = self._read_json(manifest_path) + + workflow_manifest = self._read_json(run_dir / "manifest.json") + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + previous_status = str(run_manifest.get("status") or "") + run_manifest["status"] = "WORKFLOW_RUNNING" + run_manifest["next_stage"] = "workflow" + run_manifest["workflow"] = { + **(run_manifest.get("workflow") or {}), + "started_at": started_at, + "previous_status": previous_status, + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "from_step": from_step, + "to_step": to_step, + "only_steps": only_steps or [], + "force": bool(force), + } + self._write_json(manifest_path, run_manifest) + + execution_results = self._execute_workflow_bridge( + run_id, + run_dir, + workflow_manifest=workflow_manifest, + from_step=from_step, + to_step=to_step, + only_steps=only_steps or [], + force=force, + timeout_seconds=timeout_seconds or settings.GAMMA_SBAS_STEP_TIMEOUT_SECONDS, + ) + state = self._read_optional_json(run_dir / "state" / "step_status.json") or {} + summary = self._summarize_workflow_state(workflow_manifest, state) + returncode = 0 if summary.get("failed_count") == 0 else 1 + execution = { + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "returncode": returncode, + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "distro": settings.GAMMA_SBAS_WSL_DISTRO, + "mode": "managed_python_bridge_to_expert_scripts", + "results": execution_results, + "summary": summary, + } + run_manifest = self._read_json(manifest_path) + run_manifest["workflow"] = { + **(run_manifest.get("workflow") or {}), + "execution": execution, + "summary": summary, + } + if returncode == 0 and summary.get("ready"): + run_manifest["status"] = "WORKFLOW_COMPLETED" + run_manifest["next_stage"] = "review_publish_products" + elif returncode == 0: + run_manifest["status"] = "WORKFLOW_PARTIAL" + run_manifest["next_stage"] = "continue_workflow" + else: + run_manifest["status"] = "WORKFLOW_FAILED" + run_manifest["next_stage"] = "fix_workflow" + self._write_json(manifest_path, run_manifest) + self._write_json(run_dir / "workflow_summary.json", summary) + return self.get_run_detail(run_id) + + @staticmethod + def _workflow_runner_step_args( + *, + from_step: str | None, + to_step: str | None, + only_steps: list[str] | None, + ) -> list[str]: + args: list[str] = [] + if from_step: + args.extend(["--from-step", str(from_step)]) + if to_step: + args.extend(["--to-step", str(to_step)]) + if only_steps: + args.extend(["--only-steps", ",".join(str(item) for item in only_steps if str(item).strip())]) + return args + + def _execute_workflow_bridge( + self, + run_id: str, + run_dir: Path, + *, + workflow_manifest: dict[str, Any], + from_step: str | None, + to_step: str | None, + only_steps: list[str], + force: bool, + timeout_seconds: int, + ) -> list[dict[str, Any]]: + selected = self._select_workflow_steps( + workflow_manifest.get("steps") or [], + from_step=from_step, + to_step=to_step, + only_steps=only_steps, + ) + state_path = run_dir / "state" / "step_status.json" + state = self._read_optional_json(state_path) or self._initial_workflow_state( + self._read_json(run_dir / "run_manifest.json"), + workflow_manifest, + ) + state.setdefault("steps", {}) + results: list[dict[str, Any]] = [] + + for step in selected: + step_id = str(step.get("id") or "") + if not step.get("enabled"): + result = self._workflow_step_result(step, status="PLANNED", skipped_reason="step planned but not enabled") + state["steps"][step_id] = result + results.append(result) + continue + previous = state["steps"].get(step_id) or {} + if previous.get("status") == "COMPLETED" and not force: + result = {**previous, "status": "SKIPPED", "skipped_reason": "already completed"} + state["steps"][step_id] = result + results.append(result) + continue + + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + try: + detail = self._execute_workflow_step_bridge( + run_id, + step_id, + timeout_seconds=timeout_seconds, + ) + result = { + "id": step_id, + "name": step.get("name") or step_id, + "status": "COMPLETED", + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "returncode": 0, + "detail": self._workflow_step_detail_summary(step_id, detail), + } + except Exception as exc: + result = { + "id": step_id, + "name": step.get("name") or step_id, + "status": "FAILED", + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "returncode": 1, + "error": str(exc), + } + state["steps"][step_id] = result + state["updated_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z" + self._write_json(state_path, state) + results.append(result) + break + + state["steps"][step_id] = result + state["updated_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z" + self._write_json(state_path, state) + results.append(result) + return results + + def _execute_workflow_step_bridge(self, run_id: str, step_id: str, *, timeout_seconds: int) -> dict[str, Any]: + self._restore_stage_status_for_workflow_step(run_id) + if step_id in {"01_workspace_data"}: + return self.get_run_detail(run_id) + if step_id in {"01_import_slc", "02_import_lt1_slc", "03_reference_mli"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + if status not in self._WORKFLOW_BASELINE_DONE_STATUSES: + return self.run_baseline_audit( + run_id, + execute=True, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + azlks=settings.GAMMA_SBAS_DEFAULT_AZLKS, + max_delta_n=1, + timeout_seconds=timeout_seconds, + ) + return detail + if step_id in {"02_coregister_stack", "05_coreg_prep", "06_coregister_scenes", "07_rmli_average"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("coregistration")): + return detail + run_dir = self._resolve_run_dir(run_id) + approved_itab = run_dir / "work" / "gamma" / "diff" / "itab_approved" + if not approved_itab.is_file() and settings.GAMMA_SBAS_AUTO_APPROVE_ITAB: + self.decide_itab( + run_id, + decision="approve", + reviewer="system", + note="Auto-approved for Gamma SBAS expert workflow execution.", + ) + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + return self.execute_coregistration( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + azlks=settings.GAMMA_SBAS_DEFAULT_AZLKS, + timeout_seconds=timeout_seconds, + ) + if step_id in {"03_prepare_dem", "04_dem_lookup"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("rdc_dem")): + return detail + if status not in {"COREGISTRATION_READY", "RDC_DEM_SCRIPT_READY", "RDC_DEM_RUNNING", "RDC_DEM_FAILED"}: + pass + return self.execute_rdc_dem( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + timeout_seconds=timeout_seconds, + ) + if step_id in {"04_build_network_diff", "08_diff_network", "09_filter_unwrap"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("interferograms")): + return detail + if not self._stage_execution_completed(manifest.get("coregistration")): + self._execute_workflow_step_bridge(run_id, "07_rmli_average", timeout_seconds=timeout_seconds) + detail = self.get_run_detail(run_id) + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if not self._stage_execution_completed(manifest.get("rdc_dem")): + self._execute_workflow_step_bridge(run_id, "04_dem_lookup", timeout_seconds=timeout_seconds) + return self.execute_interferograms( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + azlks=settings.GAMMA_SBAS_DEFAULT_AZLKS, + unwrap_threshold=0.20, + timeout_seconds=timeout_seconds, + ) + if step_id in {"05_detrend_atm", "10_detrend_atm"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("detrend_atm")): + return detail + if status in self._WORKFLOW_DETREND_DONE_STATUSES: + return detail + if status not in {"INTERFEROGRAMS_READY", "DETREND_ATM_SCRIPT_READY", "DETREND_ATM_RUNNING", "DETREND_ATM_FAILED"}: + self._execute_workflow_step_bridge(run_id, "09_filter_unwrap", timeout_seconds=timeout_seconds) + return self.execute_detrend_atm( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + reference_window=settings.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW, + timeout_seconds=timeout_seconds, + ) + if step_id in {"06_sbas_inversion", "11_sbas_inversion"}: + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("ipta_timeseries")): + return detail + if status in self._WORKFLOW_IPTA_DONE_STATUSES: + return detail + if status not in {"DETREND_ATM_READY", "IPTA_TIMESERIES_SCRIPT_READY", "IPTA_TIMESERIES_RUNNING", "IPTA_TIMESERIES_FAILED"}: + self._execute_workflow_step_bridge(run_id, "10_detrend_atm", timeout_seconds=timeout_seconds) + return self.execute_ipta_timeseries( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + reference_window=settings.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW, + mb_mode=settings.GAMMA_SBAS_DEFAULT_MB_MODE, + timeout_seconds=timeout_seconds, + ) + if step_id == "12_outputs_points": + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + manifest = detail.get("manifest") if isinstance(detail.get("manifest"), dict) else {} + if self._stage_execution_completed(manifest.get("monitor_point_products")): + return detail + if status not in self._WORKFLOW_IPTA_DONE_STATUSES: + detail = self._execute_workflow_step_bridge(run_id, "11_sbas_inversion", timeout_seconds=timeout_seconds) + status = str((detail.get("run") or {}).get("status") or "").strip() + if status not in self._WORKFLOW_PUBLISH_DONE_STATUSES: + detail = self.execute_publish_products( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + timeout_seconds=min(timeout_seconds, 86400), + ) + status = str((detail.get("run") or {}).get("status") or "").strip() + if status not in self._WORKFLOW_MONITOR_DONE_STATUSES: + return self.execute_monitor_points( + run_id, + timeout_seconds=min(timeout_seconds, 86400), + ) + return detail + if step_id == "07_publish_products": + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + if status in self._WORKFLOW_PUBLISH_DONE_STATUSES: + return detail + return self.execute_publish_products( + run_id, + rlks=settings.GAMMA_SBAS_DEFAULT_RLKS, + timeout_seconds=min(timeout_seconds, 86400), + ) + if step_id == "08_point_timeseries": + detail = self.get_run_detail(run_id) + status = str((detail.get("run") or {}).get("status") or "").strip() + if status in self._WORKFLOW_MONITOR_DONE_STATUSES: + return detail + return self.execute_monitor_points( + run_id, + timeout_seconds=min(timeout_seconds, 86400), + ) + return {"status": "planned_only", "step_id": step_id} + + @staticmethod + def _workflow_step_detail_summary(step_id: str, detail: dict[str, Any]) -> dict[str, Any]: + run = detail.get("run") if isinstance(detail, dict) else {} + run = run if isinstance(run, dict) else {} + summary: dict[str, Any] = { + "step_id": step_id, + "run_id": run.get("run_id"), + "run_status": run.get("status"), + "next_stage": run.get("next_stage"), + } + stage_by_step = { + "01_workspace_data": "stack", + "01_import_slc": "baseline_audit", + "02_import_lt1_slc": "baseline_audit", + "03_reference_mli": "baseline_audit", + "02_coregister_stack": "coregistration", + "05_coreg_prep": "coregistration", + "06_coregister_scenes": "coregistration", + "07_rmli_average": "coregistration", + "03_prepare_dem": "rdc_dem", + "04_dem_lookup": "rdc_dem", + "04_build_network_diff": "interferograms", + "08_diff_network": "interferograms", + "09_filter_unwrap": "interferograms", + "10_detrend_atm": "detrend_atm", + "06_sbas_inversion": "ipta_timeseries", + "11_sbas_inversion": "ipta_timeseries", + "07_publish_products": "publish_products", + "08_point_timeseries": "monitor_point_products", + "12_outputs_points": "publish_products", + } + stage_key = stage_by_step.get(step_id) + stage = run.get(stage_key) if stage_key else None + if isinstance(stage, dict): + execution = stage.get("execution") if isinstance(stage.get("execution"), dict) else {} + stage_summary = stage.get("summary") if isinstance(stage.get("summary"), dict) else {} + summary["stage"] = { + "key": stage_key, + "script_path": stage.get("script_path"), + "reference_date": stage.get("reference_date"), + "returncode": execution.get("returncode"), + "execution_status": execution.get("status"), + "ready": stage_summary.get("ready"), + "outputs": stage.get("outputs") if isinstance(stage.get("outputs"), dict) else None, + } + return summary + + def _restore_stage_status_for_workflow_step(self, run_id: str) -> None: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + if manifest.get("status") != "WORKFLOW_RUNNING": + return + workflow = manifest.get("workflow") or {} + stage_status = str( + workflow.get("resume_stage_status") + or workflow.get("previous_status") + or "" + ).strip() + if not stage_status or stage_status == "WORKFLOW_READY": + stage_status = "PLANNED_GAMMA_BASELINE_AUDIT" + manifest["status"] = stage_status + manifest["next_stage"] = self._next_stage_for_status(stage_status) + self._write_json(manifest_path, manifest) + + @staticmethod + def _next_stage_for_status(status: str) -> str: + return { + "PLANNED_GAMMA_BASELINE_AUDIT": "baseline_audit", + "BASELINE_AUDIT_READY": "approve_itab", + "ITAB_APPROVED": "coregistration", + "COREGISTRATION_SCRIPT_READY": "execute_coregistration", + "COREGISTRATION_READY": "rdc_dem", + "RDC_DEM_SCRIPT_READY": "execute_rdc_dem", + "RDC_DEM_READY": "interferograms", + "INTERFEROGRAMS_SCRIPT_READY": "execute_interferograms", + "INTERFEROGRAMS_READY": "detrend_atm", + "DETREND_ATM_SCRIPT_READY": "execute_detrend_atm", + "DETREND_ATM_READY": "ipta_timeseries", + "IPTA_TIMESERIES_SCRIPT_READY": "execute_ipta_timeseries", + "IPTA_TIMESERIES_READY": "publish_products", + "PUBLISH_PRODUCTS_SCRIPT_READY": "execute_publish_products", + "PRODUCTS_READY": "monitor_points", + "MONITOR_POINTS_SCRIPT_READY": "execute_monitor_points", + "MONITOR_POINTS_READY": "review_publish_products", + }.get(str(status or "").strip(), "workflow") + + @staticmethod + def _workflow_step_result(step: dict[str, Any], *, status: str, skipped_reason: str | None = None) -> dict[str, Any]: + payload = { + "id": step.get("id"), + "name": step.get("name") or step.get("id"), + "status": status, + "started_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + if skipped_reason: + payload["skipped_reason"] = skipped_reason + return payload + + @staticmethod + def _select_workflow_steps( + steps: list[dict[str, Any]], + *, + from_step: str | None, + to_step: str | None, + only_steps: list[str], + ) -> list[dict[str, Any]]: + only = {str(item).strip() for item in only_steps or [] if str(item).strip()} + if only: + return [step for step in steps if str(step.get("id") or "") in only] + if not from_step and not to_step: + return steps + selected: list[dict[str, Any]] = [] + active = from_step is None + for step in steps: + step_id = str(step.get("id") or "") + if step_id == from_step: + active = True + if active: + selected.append(step) + if step_id == to_step: + break + return selected + + def _resolve_source_roots(self, roots: list[str] | None) -> list[Path]: + raw_values = roots or self._split_config_paths(settings.GAMMA_SBAS_SOURCE_ROOTS) if not raw_values: raw_values = [r"D:\LuTan1_Image_Pool"] return self._dedupe_existing_dirs(raw_values) def _resolve_orbit_roots(self, roots: list[str] | None) -> list[Path]: - raw_values = roots or self._split_config_paths( - settings.PYINT_ORBIT_POOL_TXT, - settings.ORBIT_POOL_ENVI, - ) + raw_values = roots or self._split_config_paths(settings.GAMMA_SBAS_ORBIT_ROOTS) if not raw_values: raw_values = [r"D:\orbit_pools\envi"] return self._dedupe_existing_dirs(raw_values) + @staticmethod + def _discovery_cache_key( + *, + source_paths: list[Path], + orbit_paths: list[Path], + min_scenes: int, + require_orbits: bool, + include_scenes: bool, + limit: int, + platform: str | None, + relative_orbit: str | None, + orbit_direction: str | None, + ) -> str: + payload = { + "source_paths": [os.path.normcase(str(path.resolve())) for path in source_paths], + "orbit_paths": [os.path.normcase(str(path.resolve())) for path in orbit_paths], + "source_mtime_ns": [ + int(path.stat().st_mtime_ns) if path.exists() else 0 + for path in source_paths + ], + "orbit_mtime_ns": [ + int(path.stat().st_mtime_ns) if path.exists() else 0 + for path in orbit_paths + ], + "min_scenes": int(min_scenes), + "require_orbits": bool(require_orbits), + "include_scenes": bool(include_scenes), + "limit": int(limit), + "platform": str(platform or "").strip().upper(), + "relative_orbit": str(relative_orbit or "").strip(), + "orbit_direction": str(orbit_direction or "").strip().upper(), + } + return hashlib.sha1(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()[:16] + + def _discovery_cache_path(self, cache_key: str) -> Path: + return self.production_root / "discoveries" / "cache" / f"{cache_key}.json" + + def _read_discovery_cache(self, cache_key: str) -> dict[str, Any] | None: + path = self._discovery_cache_path(cache_key) + if not path.is_file(): + return None + try: + payload = self._read_json(path) + except Exception: + return None + payload["cache_hit"] = True + payload["cache_path"] = str(path) + return payload + + def _write_discovery_cache(self, cache_key: str, snapshot: dict[str, Any]) -> None: + payload = {**snapshot, "cache_key": cache_key, "cache_hit": False} + self._write_json(self._discovery_cache_path(cache_key), payload) + @staticmethod def _split_config_paths(*values: str) -> list[str]: paths: list[str] = [] @@ -1251,6 +3931,524 @@ class SbasInsarProductionService: return f"{drive}:\\{tail}" return os.path.normpath(text) + def _resolve_rdc_dem_source(self, stack_manifest: dict[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + explicit_candidates = [ + ("PYINT_PREPARED_DEM_PATH", settings.PYINT_PREPARED_DEM_PATH), + ("ISCE2_DEM_PATH", settings.ISCE2_DEM_PATH), + ("IDL_DINSAR_DEM_BASE_FILE", settings.IDL_DINSAR_DEM_BASE_FILE), + ] + for label, raw_path in explicit_candidates: + for candidate in self._gamma_dem_candidate_paths(raw_path): + source = self._build_dem_source_record(candidate, source_label=label, stack_manifest=stack_manifest) + if source: + return source + if str(raw_path or "").strip(): + errors.append(f"{label} does not point to an existing Gamma DEM + .par pair: {raw_path}") + + cache_roots = [ + Path(settings.BACKEND_DIR) / "runtime" / "pyint_dem", + Path(settings.BACKEND_DIR) / "runtime" / "pyint_dem_cache", + ] + stack_bbox = self._stack_bbox_union(stack_manifest) + cached_sources: list[dict[str, Any]] = [] + for root in cache_roots: + if not root.is_dir(): + continue + for dem_path in root.glob("**/*.dem"): + source = self._build_dem_source_record( + dem_path, + source_label=f"runtime_cache:{root.name}", + stack_manifest=stack_manifest, + ) + if not source: + continue + coverage = source.get("coverage") or {} + if stack_bbox and not ( + self._bbox_contains(coverage, stack_bbox, margin_degrees=0.05) + or self._bbox_contains_point(coverage, self._stack_center(stack_manifest), margin_degrees=0.05) + ): + continue + cached_sources.append(source) + + if cached_sources: + cached_sources.sort( + key=lambda item: self._dem_source_sort_key(item, stack_manifest) + ) + selected = cached_sources[0] + selected["selection_note"] = "Selected existing PyINT Gamma DEM cache covering the SBAS stack extent." + return selected + + detail = "; ".join(errors) if errors else "no runtime Gamma DEM cache covers the selected stack" + raise FileNotFoundError( + "No usable Gamma DEM source was found for RDC DEM generation. " + "Configure PYINT_PREPARED_DEM_PATH to a .dem file with .dem.par, " + "or generate a PyINT Gamma DEM cache for this LT1 stack. " + f"Details: {detail}" + ) + + def _gamma_dem_candidate_paths(self, raw_path: str | None) -> list[Path]: + text = str(raw_path or "").strip() + if not text: + return [] + win_text = self._path_to_windows(text) or text + base = Path(win_text) + candidates = [base] + if base.suffix.lower() != ".dem": + candidates.append(Path(f"{win_text}.dem")) + if base.is_dir(): + candidates.extend(sorted(base.glob("*.dem"))) + deduped: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + key = str(candidate) + if key in seen: + continue + seen.add(key) + deduped.append(candidate) + return deduped + + def _build_dem_source_record( + self, + dem_path: Path, + *, + source_label: str, + stack_manifest: dict[str, Any], + ) -> dict[str, Any] | None: + if not dem_path.is_file(): + return None + par_path = Path(f"{dem_path}.par") + if not par_path.is_file(): + return None + params = self._parse_gamma_params(par_path) + width = self._as_int(params.get("width")) + nlines = self._as_int(params.get("nlines")) + coverage = self._dem_coverage_from_params(params) + stack_bbox = self._stack_bbox_union(stack_manifest) + return { + "source_label": source_label, + "windows_path": str(dem_path), + "windows_par_path": str(par_path), + "wsl_path": self._windows_path_to_wsl_mount(str(dem_path)), + "wsl_par_path": self._windows_path_to_wsl_mount(str(par_path)), + "data_format": params.get("data_format"), + "width": width, + "nlines": nlines, + "coverage": coverage, + "covers_stack_bbox": self._bbox_contains(coverage, stack_bbox, margin_degrees=0.05) if stack_bbox else None, + "covers_stack_center": self._bbox_contains_point(coverage, self._stack_center(stack_manifest), margin_degrees=0.05), + "stack_bbox": stack_bbox, + } + + def _dem_source_sort_key(self, source: dict[str, Any], stack_manifest: dict[str, Any]) -> tuple[Any, ...]: + path_text = str(source.get("windows_path") or source.get("wsl_path") or "").replace("\\", "/").lower() + stack_satellite = str((stack_manifest.get("stack") or {}).get("satellite") or "").lower() + same_family = bool(stack_satellite.startswith("lt1") and "/lt1_" in f"/{path_text}") + return ( + 0 if source.get("covers_stack_bbox") else 1, + 0 if same_family else 1, + self._dem_center_distance(source.get("coverage") or {}, self._stack_center(stack_manifest)), + str(source.get("windows_path") or source.get("wsl_path") or ""), + ) + + @staticmethod + def _dem_center_distance(coverage: dict[str, Any], point: dict[str, float] | None) -> float: + if not coverage or not point: + return float("inf") + try: + lon = float(point["lon"]) + lat = float(point["lat"]) + center_lon = (float(coverage["min_lon"]) + float(coverage["max_lon"])) / 2 + center_lat = (float(coverage["min_lat"]) + float(coverage["max_lat"])) / 2 + return ((center_lon - lon) ** 2 + (center_lat - lat) ** 2) ** 0.5 + except (KeyError, TypeError, ValueError): + return float("inf") + + def _find_reference_rmli_paths(self, run_dir: Path, reference_date: str) -> tuple[Path, Path]: + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + rmli_tab = common_dir / "RMLI_tab" + if rmli_tab.is_file(): + for line in rmli_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if len(parts) < 2: + continue + if Path(parts[0]).name == f"{reference_date}.mli": + return Path(self._path_to_windows(parts[0]) or parts[0]), Path(self._path_to_windows(parts[1]) or parts[1]) + return ( + run_dir / "work" / "gamma" / "mli" / f"{reference_date}.mli", + run_dir / "work" / "gamma" / "mli" / f"{reference_date}.mli.par", + ) + + def _select_ipta_mb_reference_mli( + self, + run_dir: Path, + *, + reference_date: str, + rmli_tab: Path, + ) -> tuple[Path, Path]: + reference_dt = None + try: + reference_dt = datetime.strptime(reference_date, "%Y%m%d") + except ValueError: + pass + + candidates: list[tuple[tuple[Any, ...], Path, Path]] = [] + if rmli_tab.is_file(): + for index, line in enumerate(rmli_tab.read_text(encoding="utf-8", errors="ignore").splitlines()): + parts = line.split() + if len(parts) < 2: + continue + mli = Path(self._path_to_windows(parts[0]) or parts[0]) + mli_par = Path(self._path_to_windows(parts[1]) or parts[1]) + date = mli.stem + if date == reference_date: + continue + if reference_dt is not None: + try: + delta_days = abs((datetime.strptime(date, "%Y%m%d") - reference_dt).days) + except ValueError: + delta_days = 999999 + else: + delta_days = index + candidates.append(((delta_days, index), mli, mli_par)) + + if candidates: + _, mli, mli_par = sorted(candidates, key=lambda item: item[0])[0] + return mli, mli_par + return self._find_reference_rmli_paths(run_dir, reference_date) + + def _select_ipta_reference_region( + self, + run_dir: Path, + *, + reference_date: str, + rlks: int, + reference_window: int, + geom_ref_mli_par: Path, + ) -> dict[str, Any]: + params = self._parse_gamma_params(geom_ref_mli_par) + width = self._as_int(params.get("range_samples")) + lines = self._as_int(params.get("azimuth_lines")) + if not width or not lines: + raise ValueError(f"cannot parse reference geometry from {geom_ref_mli_par}") + + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + diff_tab = common_dir / "DIFF_tab" + pair_paths = [ + Path(self._path_to_windows(row) or row) + for row in self._read_text_rows(diff_tab) + ] + pair_paths = [path for path in pair_paths if path.is_file()] + if not pair_paths: + raise FileNotFoundError(f"DIFF_tab has no readable unwrapped interferograms: {diff_tab}") + + half = max(1, reference_window // 2) + window = half * 2 + search_step = max(8, min(64, window * 2)) + center_x = width // 2 + center_y = lines // 2 + best: dict[str, Any] | None = None + for y in range(half, max(half + 1, lines - half), search_step): + for x in range(half, max(half + 1, width - half), search_step): + metrics = self._score_ipta_reference_region( + pair_paths, + width=width, + lines=lines, + x=x, + y=y, + half=half, + ) + score = ( + metrics["min_valid_pixel_count"], + metrics["median_mean_coherence"], + metrics["total_valid_pixel_count"], + -abs(x - center_x) - abs(y - center_y), + ) + if best is None or score > best["score"]: + best = { + **metrics, + "score": score, + "range_pixel": x, + "azimuth_line": y, + } + + if best is None: + raise ValueError("could not select an IPTA reference region") + return { + "strategy": "auto_valid_unwrapped_high_coherence_window", + "range_pixel": int(best["range_pixel"]), + "azimuth_line": int(best["azimuth_line"]), + "window_width": window, + "window_height": window, + "search_step": search_step, + "pair_count": len(pair_paths), + "min_valid_pixel_count": int(best["min_valid_pixel_count"]), + "total_valid_pixel_count": int(best["total_valid_pixel_count"]), + "median_mean_coherence": float(best["median_mean_coherence"]), + "mean_coherence_by_pair": best["mean_coherence_by_pair"], + "valid_pixel_count_by_pair": best["valid_pixel_count_by_pair"], + } + + @staticmethod + def _normalize_ipta_mb_mode(value: Any) -> int: + try: + mode = int(value) + except (TypeError, ValueError): + mode = DEFAULT_IPTA_MB_MODE + if mode not in IPTA_MB_MODE_DESCRIPTIONS: + mode = DEFAULT_IPTA_MB_MODE + return mode + + def _resolve_radar_wavelength_m(self, *parameter_paths: Path) -> float: + for path in parameter_paths: + params = self._parse_gamma_params(path) + radar_frequency = self._as_float(params.get("radar_frequency")) + if radar_frequency and radar_frequency > 0: + return 299792458.0 / radar_frequency + return 0.23793052222222222 + + def _score_ipta_reference_region( + self, + pair_paths: list[Path], + *, + width: int, + lines: int, + x: int, + y: int, + half: int, + ) -> dict[str, Any]: + y0 = max(0, y - half) + y1 = min(lines, y + half) + x0 = max(0, x - half) + x1 = min(width, x + half) + valid_counts: list[int] = [] + coherence_means: list[float] = [] + for unw_path in pair_paths: + cor_path = unw_path.with_name(unw_path.name.replace(".diff_filt.unw", ".diff_filt.cor")) + unw = self._read_gamma_float32_window(unw_path, width=width, lines=lines, x0=x0, x1=x1, y0=y0, y1=y1) + cor = self._read_gamma_float32_window(cor_path, width=width, lines=lines, x0=x0, x1=x1, y0=y0, y1=y1) + valid = [value for value in unw if math.isfinite(value) and value != 0.0] + finite_cor = [value for value in cor if math.isfinite(value)] + valid_counts.append(len(valid)) + coherence_means.append(sum(finite_cor) / len(finite_cor) if finite_cor else 0.0) + sorted_coh = sorted(coherence_means) + if sorted_coh: + mid = len(sorted_coh) // 2 + median_coh = sorted_coh[mid] if len(sorted_coh) % 2 else (sorted_coh[mid - 1] + sorted_coh[mid]) / 2 + else: + median_coh = 0.0 + return { + "min_valid_pixel_count": min(valid_counts) if valid_counts else 0, + "total_valid_pixel_count": sum(valid_counts), + "median_mean_coherence": median_coh, + "mean_coherence_by_pair": coherence_means, + "valid_pixel_count_by_pair": valid_counts, + } + + @staticmethod + def _read_gamma_float32_window( + path: Path, + *, + width: int, + lines: int, + x0: int, + x1: int, + y0: int, + y1: int, + ) -> list[float]: + if not path.is_file(): + return [] + values: list[float] = [] + row_bytes = width * 4 + count = max(0, x1 - x0) + with path.open("rb") as fh: + for y in range(y0, y1): + if y < 0 or y >= lines: + continue + fh.seek(y * row_bytes + x0 * 4) + chunk = fh.read(count * 4) + if len(chunk) != count * 4: + continue + values.extend(struct.unpack(f">{count}f", chunk)) + return values + + @staticmethod + def _gamma_float32_stats(path: Path, *, width: int, lines: int) -> dict[str, Any]: + if not path.is_file() or not width or not lines: + return {"exists": path.is_file(), "valid_count": 0} + try: + import numpy as np + except Exception as exc: + return {"exists": True, "error": f"numpy unavailable: {exc}"} + expected = width * lines + try: + data = np.fromfile(path, dtype=">f4", count=expected) + except Exception as exc: + return {"exists": True, "error": str(exc)} + finite = data[np.isfinite(data)] + nonzero = finite[finite != 0.0] + sample = nonzero if nonzero.size else finite + if sample.size == 0: + return { + "exists": True, + "pixel_count": int(data.size), + "valid_count": 0, + "nonzero_count": 0, + } + percentiles = np.percentile(sample, [1, 5, 50, 95, 99]) + return { + "exists": True, + "pixel_count": int(data.size), + "expected_pixel_count": int(expected), + "valid_count": int(finite.size), + "nonzero_count": int(nonzero.size), + "min": float(np.nanmin(sample)), + "p01": float(percentiles[0]), + "p05": float(percentiles[1]), + "median": float(percentiles[2]), + "p95": float(percentiles[3]), + "p99": float(percentiles[4]), + "max": float(np.nanmax(sample)), + "mean": float(np.nanmean(sample)), + "std": float(np.nanstd(sample)), + } + + @staticmethod + def _parse_gamma_params(path: Path) -> dict[str, str]: + if not path.is_file(): + return {} + params: dict[str, str] = {} + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + key = key.strip() + value = value.strip().split()[0] if value.strip() else "" + if key: + params[key] = value + return params + + def _dem_coverage_from_params(self, params: dict[str, str]) -> dict[str, Any]: + width = self._as_int(params.get("width")) + nlines = self._as_int(params.get("nlines")) + corner_lon = self._as_float(params.get("corner_lon")) + corner_lat = self._as_float(params.get("corner_lat")) + post_lon = self._as_float(params.get("post_lon")) + post_lat = self._as_float(params.get("post_lat")) + coverage: dict[str, Any] = { + "width": width, + "nlines": nlines, + "corner_lon": corner_lon, + "corner_lat": corner_lat, + "post_lon": post_lon, + "post_lat": post_lat, + } + if None in {width, nlines, corner_lon, corner_lat, post_lon, post_lat}: + return coverage + east = float(corner_lon) + float(post_lon) * int(width) + south = float(corner_lat) + float(post_lat) * int(nlines) + min_lon = min(float(corner_lon), east) + max_lon = max(float(corner_lon), east) + min_lat = min(float(corner_lat), south) + max_lat = max(float(corner_lat), south) + coverage.update( + { + "min_lon": min_lon, + "max_lon": max_lon, + "min_lat": min_lat, + "max_lat": max_lat, + "area_sq_deg": max(0.0, (max_lon - min_lon) * (max_lat - min_lat)), + } + ) + return coverage + + @staticmethod + def _bbox_contains( + outer: dict[str, Any] | None, + inner: dict[str, Any] | None, + *, + margin_degrees: float = 0.0, + ) -> bool: + if not outer or not inner: + return False + try: + return ( + float(outer["min_lon"]) <= float(inner["min_lon"]) + margin_degrees + and float(outer["max_lon"]) >= float(inner["max_lon"]) - margin_degrees + and float(outer["min_lat"]) <= float(inner["min_lat"]) + margin_degrees + and float(outer["max_lat"]) >= float(inner["max_lat"]) - margin_degrees + ) + except (KeyError, TypeError, ValueError): + return False + + @staticmethod + def _bbox_contains_point( + outer: dict[str, Any] | None, + point: dict[str, float] | None, + *, + margin_degrees: float = 0.0, + ) -> bool: + if not outer or not point: + return False + try: + lon = float(point["lon"]) + lat = float(point["lat"]) + return ( + float(outer["min_lon"]) - margin_degrees <= lon <= float(outer["max_lon"]) + margin_degrees + and float(outer["min_lat"]) - margin_degrees <= lat <= float(outer["max_lat"]) + margin_degrees + ) + except (KeyError, TypeError, ValueError): + return False + + def _stack_center(self, stack_manifest: dict[str, Any]) -> dict[str, float] | None: + scenes = stack_manifest.get("scenes") or [] + lons = [self._as_float(scene.get("center_lon")) for scene in scenes] + lats = [self._as_float(scene.get("center_lat")) for scene in scenes] + lons = [value for value in lons if value is not None] + lats = [value for value in lats if value is not None] + if lons and lats: + return {"lon": sum(lons) / len(lons), "lat": sum(lats) / len(lats)} + bbox = self._stack_bbox_union(stack_manifest) + if not bbox: + return None + return { + "lon": (bbox["min_lon"] + bbox["max_lon"]) / 2, + "lat": (bbox["min_lat"] + bbox["max_lat"]) / 2, + } + + def _stack_bbox_union(self, stack_manifest: dict[str, Any]) -> dict[str, float] | None: + boxes = [ + scene.get("bbox") for scene in (stack_manifest.get("scenes") or []) + if isinstance(scene.get("bbox"), dict) + ] + if not boxes: + return None + try: + return { + "min_lon": min(float(item["min_lon"]) for item in boxes), + "min_lat": min(float(item["min_lat"]) for item in boxes), + "max_lon": max(float(item["max_lon"]) for item in boxes), + "max_lat": max(float(item["max_lat"]) for item in boxes), + } + except (KeyError, TypeError, ValueError): + return None + + @staticmethod + def _file_record(path: Path) -> dict[str, Any]: + exists = path.is_file() + return { + "path": str(path), + "exists": exists, + "size_bytes": path.stat().st_size if exists else 0, + } + + @staticmethod + def _as_int(value: Any) -> int | None: + try: + return int(float(str(value).strip())) + except (TypeError, ValueError): + return None + @staticmethod def _find_lt1_orbit(orbit_roots: list[Path], satellite: str, date: str) -> Path | None: if not satellite or not date: @@ -1408,6 +4606,19 @@ class SbasInsarProductionService: path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") return path + @staticmethod + def _write_script(path: Path, lines: list[str]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + text = "\n".join(lines) + try: + path.write_text(text, encoding="utf-8", newline="\n") + return path + except PermissionError: + suffix = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + fallback = path.with_name(f"{path.stem}_{suffix}{path.suffix}") + fallback.write_text(text, encoding="utf-8", newline="\n") + return fallback + def _resolve_trial_dir(self, trial_id: str) -> Path: clean_id = str(trial_id or "").strip() if not clean_id or Path(clean_id).name != clean_id: @@ -1445,6 +4656,410 @@ class SbasInsarProductionService: return None return self._read_json(path) + def _ensure_expert_workspace(self, run_dir: Path) -> dict[str, Any]: + created: dict[str, str] = {} + for dirname in EXPERT_WORKSPACE_DIRS: + path = run_dir / dirname + path.mkdir(parents=True, exist_ok=True) + created[dirname] = str(path) + + work_gamma = run_dir / "work" / "gamma" + aliases = { + "RAW": work_gamma / "raw", + "SLC": work_gamma / "slc", + "dem": work_gamma / "dem", + "rslc_prep": work_gamma / "rslc_prep", + "mli_dir": work_gamma / "mli", + "diff_dir": work_gamma / "diff", + "diff1_dir": work_gamma / "diff1", + "sbas": work_gamma / "sbas", + } + for path in aliases.values(): + path.mkdir(parents=True, exist_ok=True) + + workspace = { + "schema": "insar.gamma-sbas-expert-workspace/v1", + "run_root": str(run_dir), + "directories": created, + "gamma_work_aliases": {key: str(value) for key, value in aliases.items()}, + "layout_source": "LT1_GAMMA_SBAS_expert_document", + } + self._write_json(run_dir / "workspace.json", workspace) + return workspace + + def _build_workflow_manifest( + self, + run_dir: Path, + run_manifest: dict[str, Any], + stack_manifest: dict[str, Any], + *, + params: dict[str, Any] | None = None, + ) -> dict[str, Any]: + resolved_params = { + "rlks": settings.GAMMA_SBAS_DEFAULT_RLKS, + "azlks": settings.GAMMA_SBAS_DEFAULT_AZLKS, + "mb_mode": settings.GAMMA_SBAS_DEFAULT_MB_MODE, + "reference_window": settings.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW, + **(params or {}), + } + reference_date = str( + ((run_manifest.get("coregistration") or {}).get("reference_date")) + or ((run_manifest.get("stack") or {}).get("reference_date")) + or ((stack_manifest.get("stack") or {}).get("reference_date")) + or "" + ).strip() + script_records = self._materialize_workflow_scripts( + run_dir, + run_manifest=run_manifest, + stack_manifest=stack_manifest, + params=resolved_params, + reference_date=reference_date, + ) + steps: list[dict[str, Any]] = [] + for template in GAMMA_SBAS_WORKFLOW_STEPS: + step_id = template["id"] + script_record = script_records.get(step_id) or {} + step_status = template.get("status") or "PENDING" + enabled = step_status != "PLANNED" + steps.append( + { + "id": step_id, + "name": template["name"], + "status": step_status, + "enabled": enabled, + "optional": bool(template.get("optional")), + "legacy_stage": template.get("legacy_stage"), + "script": script_record.get("script"), + "script_wsl": script_record.get("script_wsl"), + "log": str(run_dir / "logs" / f"{step_id}.log"), + "log_wsl": self._windows_path_to_wsl_mount(str(run_dir / "logs" / f"{step_id}.log")), + "expert_tools": list(template.get("expert_tools") or []), + "notes": script_record.get("notes") or [], + } + ) + expert_steps = self._build_expert_document_step_manifest(steps) + return { + "schema": "insar.gamma-sbas-workflow/v1", + "run_id": run_manifest.get("run_id") or run_dir.name, + "workflow_code": "sbas_insar", + "processor_code": "gamma_ipta_sbas", + "engine_code": "gamma", + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "run_root": str(run_dir), + "run_root_wsl": self._windows_path_to_wsl_mount(str(run_dir)), + "state": { + "step_status_path": self._windows_path_to_wsl_mount(str(run_dir / "state" / "step_status.json")), + "step_status_path_windows": str(run_dir / "state" / "step_status.json"), + }, + "params": resolved_params, + "stack": stack_manifest.get("stack") or {}, + "scenes": stack_manifest.get("scenes") or [], + "pair_network": stack_manifest.get("pair_network") or {}, + "directories": { + dirname: str(run_dir / dirname) + for dirname in EXPERT_WORKSPACE_DIRS + }, + "directories_wsl": { + dirname: self._windows_path_to_wsl_mount(str(run_dir / dirname)) + for dirname in EXPERT_WORKSPACE_DIRS + }, + "steps": steps, + "expert_document": { + "schema": "insar.gamma-sbas-expert-document/v1", + "source": "LT1_GAMMA_SBAS_逐命令处理流程.docx", + "section_count": len(expert_steps), + "steps": expert_steps, + }, + } + + def _initial_workflow_state(self, run_manifest: dict[str, Any], workflow_manifest: dict[str, Any]) -> dict[str, Any]: + return { + "schema": "insar.gamma-sbas-step-status/v1", + "run_id": run_manifest.get("run_id"), + "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "steps": { + str(step.get("id")): { + "id": step.get("id"), + "name": step.get("name"), + "status": "PENDING" if step.get("enabled") else "PLANNED", + "script": step.get("script_wsl") or step.get("script"), + } + for step in workflow_manifest.get("steps") or [] + }, + } + + @staticmethod + def _build_expert_document_step_manifest(workflow_steps: list[dict[str, Any]]) -> list[dict[str, Any]]: + workflow_by_id = {str(step.get("id") or ""): step for step in workflow_steps} + expert_steps: list[dict[str, Any]] = [] + for template in GAMMA_SBAS_EXPERT_DOCUMENT_STEPS: + mapped_workflow_steps = [] + enabled = False + optional = False + planned = False + scripts: list[str] = [] + logs: list[str] = [] + for workflow_step_id in template.get("workflow_steps") or []: + workflow_step = workflow_by_id.get(str(workflow_step_id)) + if not workflow_step: + continue + mapped_workflow_steps.append( + { + "id": workflow_step.get("id"), + "name": workflow_step.get("name"), + "status": workflow_step.get("status"), + "enabled": bool(workflow_step.get("enabled")), + "optional": bool(workflow_step.get("optional")), + "script": workflow_step.get("script"), + "script_wsl": workflow_step.get("script_wsl"), + } + ) + enabled = enabled or bool(workflow_step.get("enabled")) + optional = optional or bool(workflow_step.get("optional")) + planned = planned or str(workflow_step.get("status") or "") == "PLANNED" + if workflow_step.get("script"): + scripts.append(str(workflow_step.get("script"))) + if workflow_step.get("log"): + logs.append(str(workflow_step.get("log"))) + status = str(template.get("implementation_status") or "planned") + if planned and status.startswith("implemented"): + status = "planned_bridge" + expert_steps.append( + { + "id": template.get("id"), + "order": template.get("order"), + "title": template.get("title"), + "document_section": template.get("document_section"), + "implementation_status": status, + "workflow_steps": list(template.get("workflow_steps") or []), + "mapped_workflow_steps": mapped_workflow_steps, + "enabled": enabled, + "optional": optional, + "command_count": len(template.get("commands") or []), + "commands": list(template.get("commands") or []), + "scripts": scripts, + "logs": logs, + } + ) + return expert_steps + + def _summarize_workflow_state(self, workflow_manifest: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]: + state_steps = state.get("steps") or {} + steps = [] + completed_count = 0 + failed_count = 0 + skipped_count = 0 + planned_count = 0 + blocking_planned_count = 0 + for step in workflow_manifest.get("steps") or []: + step_id = str(step.get("id") or "") + record = state_steps.get(step_id) or {} + status = str(record.get("status") or ("PLANNED" if not step.get("enabled") else "PENDING")) + if status == "COMPLETED": + completed_count += 1 + elif status == "FAILED": + failed_count += 1 + elif status == "SKIPPED": + skipped_count += 1 + elif status == "PLANNED": + planned_count += 1 + if step.get("enabled") or not step.get("optional"): + blocking_planned_count += 1 + steps.append( + { + "id": step_id, + "name": step.get("name"), + "enabled": bool(step.get("enabled")), + "optional": bool(step.get("optional")), + "status": status, + "returncode": record.get("returncode"), + "log": record.get("log") or step.get("log"), + } + ) + enabled_count = sum(1 for step in workflow_manifest.get("steps") or [] if step.get("enabled")) + return { + "schema": "insar.gamma-sbas-workflow-summary/v1", + "run_id": workflow_manifest.get("run_id"), + "step_count": len(steps), + "enabled_count": enabled_count, + "completed_count": completed_count, + "failed_count": failed_count, + "skipped_count": skipped_count, + "planned_count": planned_count, + "blocking_planned_count": blocking_planned_count, + "ready": ( + enabled_count > 0 + and failed_count == 0 + and blocking_planned_count == 0 + and completed_count + skipped_count >= enabled_count + ), + "steps": steps, + } + + def _materialize_workflow_scripts( + self, + run_dir: Path, + *, + run_manifest: dict[str, Any], + stack_manifest: dict[str, Any], + params: dict[str, Any], + reference_date: str, + ) -> dict[str, dict[str, Any]]: + script_records: dict[str, dict[str, Any]] = {} + + workspace_script = run_dir / "scripts" / "01_workspace_data.sh" + workspace_script.parent.mkdir(parents=True, exist_ok=True) + workspace_script.write_text( + "\n".join( + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + 'mkdir -p "${RUN_ROOT}"/{RAW,SLC,dem,rslc_prep,mli_dir,diff_dir,diff1_dir,sbas,publish,logs,scripts,state}', + 'find "${RUN_ROOT}" -maxdepth 1 -type d -printf "%f\\n" | sort', + "", + ] + ), + encoding="utf-8", + newline="\n", + ) + script_records["01_workspace_data"] = self._script_record( + workspace_script, + notes=["Expert section 1 workspace/data-layout check."], + ) + + try: + baseline_script = self._write_baseline_audit_script( + run_dir, + stack_manifest=stack_manifest, + rlks=int(params.get("rlks") or 8), + azlks=int(params.get("azlks") or 8), + max_delta_n=1, + ) + for step_id, filename in ( + ("01_import_slc", "01_import_slc.sh"), + ("02_import_lt1_slc", "02_import_lt1_slc.sh"), + ("03_reference_mli", "03_reference_mli.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(baseline_script, target) + script_records[step_id] = self._script_record( + target, + notes=["Current bridge reuses verified baseline-audit import/multilook/base_calc script."], + ) + except Exception as exc: + script_records["01_import_slc"] = {"notes": [f"script not ready: {exc}"]} + script_records["02_import_lt1_slc"] = {"notes": [f"script not ready: {exc}"]} + script_records["03_reference_mli"] = {"notes": [f"script not ready: {exc}"]} + + coreg = run_manifest.get("coregistration") or {} + coreg_script = coreg.get("script_path") + if coreg_script: + source = Path(self._path_to_windows(coreg_script) or coreg_script) + for step_id, filename in ( + ("02_coregister_stack", "02_coregister_stack.sh"), + ("05_coreg_prep", "05_coreg_prep.sh"), + ("06_coregister_scenes", "06_coregister_scenes.sh"), + ("07_rmli_average", "07_rmli_average.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(source, target) + script_records[step_id] = self._script_record(target) + + rdc_dem = run_manifest.get("rdc_dem") or {} + rdc_script = rdc_dem.get("script_path") + if rdc_script: + source = Path(self._path_to_windows(rdc_script) or rdc_script) + for step_id, filename in ( + ("03_prepare_dem", "03_prepare_dem.sh"), + ("04_dem_lookup", "04_dem_lookup.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(source, target) + script_records[step_id] = self._script_record(target) + + interferograms = run_manifest.get("interferograms") or {} + intf_script = interferograms.get("script_path") + if intf_script: + source = Path(self._path_to_windows(intf_script) or intf_script) + for step_id, filename in ( + ("04_build_network_diff", "04_build_network_diff.sh"), + ("08_diff_network", "08_diff_network.sh"), + ("09_filter_unwrap", "09_filter_unwrap.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(source, target) + script_records[step_id] = self._script_record(target) + + detrend_atm = run_manifest.get("detrend_atm") or {} + detrend_script = detrend_atm.get("script_path") + if detrend_script: + source = Path(self._path_to_windows(detrend_script) or detrend_script) + for step_id, filename in ( + ("05_detrend_atm", "05_detrend_atm.sh"), + ("10_detrend_atm", "10_detrend_atm.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(source, target) + script_records[step_id] = self._script_record(target) + + ipta = run_manifest.get("ipta_timeseries") or {} + ipta_script = ipta.get("script_path") + if ipta_script: + source = Path(self._path_to_windows(ipta_script) or ipta_script) + for step_id, filename in ( + ("06_sbas_inversion", "06_sbas_inversion.sh"), + ("11_sbas_inversion", "11_sbas_inversion.sh"), + ): + target = run_dir / "scripts" / filename + self._copy_script_alias(source, target) + script_records[step_id] = self._script_record(target) + + publish = run_manifest.get("publish_products") or {} + publish_script = publish.get("script_path") + if publish_script: + target = run_dir / "scripts" / "07_publish_products.sh" + self._copy_script_alias(Path(self._path_to_windows(publish_script) or publish_script), target) + script_records["07_publish_products"] = self._script_record(target) + + monitor = run_manifest.get("monitor_point_products") or {} + monitor_script = monitor.get("script_path") + if monitor_script: + target = run_dir / "scripts" / "08_point_timeseries.sh" + self._copy_script_alias(Path(self._path_to_windows(monitor_script) or monitor_script), target) + script_records["08_point_timeseries"] = self._script_record(target) + if publish_script or monitor_script: + target = run_dir / "scripts" / "12_outputs_points.sh" + wrapper_lines = ["#!/usr/bin/env bash", "set -euo pipefail"] + if publish_script: + wrapper_lines.append(f'bash "{self._windows_path_to_wsl_mount(str(Path(self._path_to_windows(publish_script) or publish_script)))}"') + if monitor_script: + wrapper_lines.append(f'bash "{self._windows_path_to_wsl_mount(str(Path(self._path_to_windows(monitor_script) or monitor_script)))}"') + wrapper_lines.append("") + target.write_text("\n".join(wrapper_lines), encoding="utf-8", newline="\n") + script_records["12_outputs_points"] = self._script_record( + target, + notes=["Expert section 12 wrapper runs publish products followed by monitoring-point extraction when both scripts are available."], + ) + return script_records + + def _script_record(self, path: Path, *, notes: list[str] | None = None) -> dict[str, Any]: + return { + "script": str(path), + "script_wsl": self._windows_path_to_wsl_mount(str(path)), + "notes": notes or [], + } + + def _copy_script_alias(self, source: Path, target: Path) -> None: + if not source.is_file(): + raise FileNotFoundError(source) + target.parent.mkdir(parents=True, exist_ok=True) + if source.resolve() == target.resolve(): + return + target.write_text(source.read_text(encoding="utf-8", errors="ignore"), encoding="utf-8", newline="\n") + def _build_monitor_point_config( self, *, @@ -1502,6 +5117,16 @@ class SbasInsarProductionService: number = default return max(minimum, min(maximum, number)) + @staticmethod + def _bounded_float(value: Any, *, default: float, minimum: float, maximum: float) -> float: + try: + number = float(value) + except (TypeError, ValueError): + number = default + if not math.isfinite(number): + number = default + return max(minimum, min(maximum, number)) + def _write_baseline_audit_script( self, run_dir: Path, @@ -1648,8 +5273,7 @@ class SbasInsarProductionService: ] ) scripts_dir.mkdir(parents=True, exist_ok=True) - script_path.write_text("\n".join(lines), encoding="utf-8", newline="\n") - return script_path + return self._write_script(script_path, lines) def _write_coregistration_script( self, @@ -1675,6 +5299,9 @@ class SbasInsarProductionService: self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" ) + source_itab = diff_dir / "itab_approved" + if not source_itab.is_file(): + source_itab = common_dir / "itab_approved" dates = [str(scene.get("date") or "") for scene in scenes if scene.get("date")] lines = [ "#!/usr/bin/env bash", @@ -1695,7 +5322,7 @@ class SbasInsarProductionService: "", f'source "{env_script}" >/dev/null 2>&1', 'SLC_COREG="${GAMMA_HOME}/DIFF/scripts/SLC_coreg.py"', - 'APPROVED_ITAB="${DIFF_DIR}/itab_approved"', + f'APPROVED_ITAB="{self._windows_path_to_wsl_mount(str(source_itab))}"', 'test -s "${APPROVED_ITAB}"', 'mkdir -p "${COMMON_RSLC_DIR}" "${COMMON_RMLI_DIR}" "${LOG_DIR}"', "", @@ -1708,6 +5335,14 @@ class SbasInsarProductionService: "", 'REF_SLC="${SLC_DIR}/${REF_DATE}.slc"', 'REF_PAR="${SLC_DIR}/${REF_DATE}.slc.par"', + 'REF_MLI_SRC="${MLI_DIR}/${REF_DATE}.mli"', + 'REF_MLI_PAR_SRC="${MLI_DIR}/${REF_DATE}.mli.par"', + 'REF_MLI="${COMMON_RMLI_DIR}/${REF_DATE}.mli"', + 'REF_MLI_PAR="${COMMON_RMLI_DIR}/${REF_DATE}.mli.par"', + 'test -s "${REF_MLI_SRC}"', + 'test -s "${REF_MLI_PAR_SRC}"', + 'cp -f "${REF_MLI_SRC}" "${REF_MLI}"', + 'cp -f "${REF_MLI_PAR_SRC}" "${REF_MLI_PAR}"', "", "coreg_to_ref() {", ' local date="$1"', @@ -1719,6 +5354,8 @@ class SbasInsarProductionService: ' local rmli_par="${COMMON_RMLI_DIR}/${date}.mli.par"', ' local gamma_off="${SLC_DIR}/${date}.slc.off"', ' local off="${COMMON_RSLC_DIR}/${date}_to_${REF_DATE}.off"', + ' local base_mli="${MLI_DIR}/${date}.mli"', + ' local base_mli_par="${MLI_DIR}/${date}.mli.par"', ' {', ' echo "== common-reference coreg ${date} -> ${REF_DATE} =="', ' test -s "${slc}"', @@ -1726,11 +5363,17 @@ class SbasInsarProductionService: ' test -s "${REF_SLC}"', ' test -s "${REF_PAR}"', ' if [ "${date}" = "${REF_DATE}" ]; then', + ' test -s "${REF_MLI}"', + ' test -s "${REF_MLI_PAR}"', ' echo "reference date, no resampling needed"', ' return', ' fi', ' if [ ! -s "${rslc}" ] || [ ! -s "${rslc_par}" ] || [ ! -s "${rmli}" ] || [ ! -s "${rmli_par}" ] || [ ! -s "${off}" ]; then', ' rm -f "${rslc}" "${rslc_par}" "${rmli}" "${rmli_par}" "${off}"', + ' if [ -s "${base_mli}" ] && [ -s "${base_mli_par}" ]; then', + ' cp -f "${base_mli}" "${rmli}"', + ' cp -f "${base_mli_par}" "${rmli_par}"', + ' fi', ' "${PYTHON_BIN}" "${SLC_COREG}" \\', ' "${slc}" "${par}" \\', ' "${rslc}" "${rslc_par}" \\', @@ -1764,7 +5407,7 @@ class SbasInsarProductionService: "rmli_path() {", ' local date="$1"', ' if [ "${date}" = "${REF_DATE}" ]; then', - ' printf "%s %s\\n" "${MLI_DIR}/${date}.mli" "${MLI_DIR}/${date}.mli.par"', + ' printf "%s %s\\n" "${COMMON_RMLI_DIR}/${date}.mli" "${COMMON_RMLI_DIR}/${date}.mli.par"', " else", ' printf "%s %s\\n" "${COMMON_RMLI_DIR}/${date}.mli" "${COMMON_RMLI_DIR}/${date}.mli.par"', " fi", @@ -1788,8 +5431,867 @@ class SbasInsarProductionService: ] ) scripts_dir.mkdir(parents=True, exist_ok=True) - script_path.write_text("\n".join(lines), encoding="utf-8", newline="\n") - return script_path + return self._write_script(script_path, lines) + + def _write_rdc_dem_script( + self, + run_dir: Path, + *, + reference_date: str, + rlks: int, + dem_source: dict[str, Any], + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "03_prepare_rdc_dem.sh" + gamma_root = run_dir / "work" / "gamma" + common_dir = gamma_root / f"common_{reference_date}" + dem_dir = gamma_root / "dem" + log_dir = run_dir / "logs" + env_script = ( + self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + dem_wsl = str(dem_source.get("wsl_path") or "").strip() + dem_par_wsl = str(dem_source.get("wsl_par_path") or "").strip() + if not dem_wsl or not dem_par_wsl: + raise ValueError("RDC DEM source requires WSL dem and dem.par paths") + + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'COMMON_DIR="{self._windows_path_to_wsl_mount(str(common_dir))}"', + f'DEM_DIR="{self._windows_path_to_wsl_mount(str(dem_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'DEM_SRC="{dem_wsl}"', + f'DEM_SRC_PAR="{dem_par_wsl}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${DEM_DIR}" "${LOG_DIR}"', + "", + 'REF_MLI=""', + 'REF_MLI_PAR=""', + 'if [ -s "${COMMON_DIR}/RMLI_tab" ]; then', + ' REF_MLI="$(awk -v d="${REF_DATE}" \'$1 ~ d "\\\\.mli$" {print $1; exit}\' "${COMMON_DIR}/RMLI_tab")"', + ' REF_MLI_PAR="$(awk -v d="${REF_DATE}" \'$2 ~ d "\\\\.mli\\\\.par$" {print $2; exit}\' "${COMMON_DIR}/RMLI_tab")"', + "fi", + 'if [ -z "${REF_MLI}" ]; then', + ' REF_MLI="${RUN_ROOT}/work/gamma/mli/${REF_DATE}.mli"', + 'fi', + 'if [ -z "${REF_MLI_PAR}" ]; then', + ' REF_MLI_PAR="${RUN_ROOT}/work/gamma/mli/${REF_DATE}.mli.par"', + "fi", + "", + 'DEM_CLEAN="${DEM_DIR}/source_dem_clean.dem"', + 'DEM_CLEAN_PAR="${DEM_CLEAN}.par"', + 'UTMDEM_PAR="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.utm.dem.par"', + 'UTMDEM="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.utm.dem"', + 'UTM2RDC="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.utm_to_rdc0"', + 'SIMSARUTM="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.sim_sar_utm"', + 'PIX="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.pix"', + 'LSMAP="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.ls_map"', + 'SIMSARRDC="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.sim_sar_rdc"', + 'SIMDIFF_PAR="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.diff_par"', + 'SIMOFFS="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.offs"', + 'SIMSNR="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.snr"', + 'SIMOFFSET="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.offset"', + 'SIMCOFF="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.coff"', + 'SIMCOFFSETS="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.coffsets"', + 'UTM_TO_RDC_FINE="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.UTM_TO_RDC"', + 'HGT_RDC="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.rdc.dem"', + 'BLANK="${DEM_DIR}/${REF_DATE}.blank"', + 'OFFSTD="${DEM_DIR}/${REF_DATE}_dem.off_std"', + "", + "{", + ' echo "== prepare RDC DEM for ${REF_DATE} =="', + ' test -s "${REF_MLI}"', + ' test -s "${REF_MLI_PAR}"', + ' test -s "${DEM_SRC}"', + ' test -s "${DEM_SRC_PAR}"', + "", + ' cp -f "${DEM_SRC_PAR}" "${DEM_CLEAN_PAR}"', + ' dem_width="$(awk \'$1 == "width:" {print $2; exit}\' "${DEM_SRC_PAR}")"', + ' dem_format="$(awk \'$1 == "data_format:" {print $2; exit}\' "${DEM_SRC_PAR}")"', + ' if [ -z "${dem_width}" ]; then', + ' echo "DEM width missing in ${DEM_SRC_PAR}"', + " exit 2", + " fi", + ' if [ "${dem_format}" = "INTEGER*2" ]; then', + ' dem_dtype="4"', + " else", + ' dem_dtype="2"', + " fi", + "", + ' rm -f "${DEM_CLEAN}"', + ' replace_values "${DEM_SRC}" -32767 0 "${DEM_CLEAN}" "${dem_width}" 2 "${dem_dtype}"', + "", + ' : >"${BLANK}"', + "", + ' gc_map1 "${REF_MLI_PAR}" - "${DEM_CLEAN_PAR}" "${DEM_CLEAN}" \\', + ' "${UTMDEM_PAR}" "${UTMDEM}" "${UTM2RDC}" \\', + ' 1 1 "${SIMSARUTM}" - - - - "${PIX}" "${LSMAP}" - 3 128', + "", + ' utm_width="$(awk \'$1 == "width:" {print $2; exit}\' "${UTMDEM_PAR}")"', + ' rdc_width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${REF_MLI_PAR}")"', + ' rdc_lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${REF_MLI_PAR}")"', + ' test -n "${utm_width}"', + ' test -n "${rdc_width}"', + ' test -n "${rdc_lines}"', + "", + ' geocode "${UTM2RDC}" "${SIMSARUTM}" "${utm_width}" "${SIMSARRDC}" \\', + ' "${rdc_width}" "${rdc_lines}" 0 0 - - 2 64 1', + "", + ' create_diff_par "${REF_MLI_PAR}" "${REF_MLI_PAR}" "${SIMDIFF_PAR}" 1 <"${BLANK}"', + "", + ' if ! init_offsetm "${SIMSARRDC}" "${REF_MLI}" "${SIMDIFF_PAR}" 2 2 - -; then', + ' echo "WARNING: init_offsetm returned non-zero; continuing with offset refinement"', + " fi", + "", + ' offset_pwrm "${SIMSARRDC}" "${REF_MLI}" "${SIMDIFF_PAR}" \\', + ' "${SIMOFFS}" "${SIMSNR}" 256 256 "${SIMOFFSET}"', + "", + ' offset_fitm "${SIMOFFS}" "${SIMSNR}" "${SIMDIFF_PAR}" \\', + ' "${SIMCOFF}" "${SIMCOFFSETS}" - >"${OFFSTD}"', + "", + ' gc_map_fine "${UTM2RDC}" "${utm_width}" "${SIMDIFF_PAR}" "${UTM_TO_RDC_FINE}" 1', + "", + ' geocode "${UTM_TO_RDC_FINE}" "${UTMDEM}" "${utm_width}" "${HGT_RDC}" \\', + ' "${rdc_width}" "${rdc_lines}" 0 0 - - 2 64 1', + "", + ' test -s "${HGT_RDC}"', + ' ls -lh "${HGT_RDC}" "${UTM_TO_RDC_FINE}" "${UTMDEM_PAR}"', + '} >"${LOG_DIR}/${REF_DATE}_rdc_dem.log" 2>&1', + "", + 'echo "RDC DEM complete: ${HGT_RDC}"', + "", + ] + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) + + def _write_interferogram_script( + self, + run_dir: Path, + *, + reference_date: str, + pair_plan: list[dict[str, Any]], + rlks: int, + azlks: int, + unwrap_threshold: float, + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "04_diff_unwrap_common_ref.sh" + gamma_root = run_dir / "work" / "gamma" + slc_dir = gamma_root / "slc" + mli_dir = gamma_root / "mli" + common_dir = gamma_root / f"common_{reference_date}" + dem_dir = gamma_root / "dem" + diff_dir = common_dir / "diff" + log_dir = run_dir / "logs" + python_bin = settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + env_script = ( + self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'SLC_DIR="{self._windows_path_to_wsl_mount(str(slc_dir))}"', + f'MLI_DIR="{self._windows_path_to_wsl_mount(str(mli_dir))}"', + f'COMMON_DIR="{self._windows_path_to_wsl_mount(str(common_dir))}"', + f'DEM_DIR="{self._windows_path_to_wsl_mount(str(dem_dir))}"', + f'DIFF_DIR="{self._windows_path_to_wsl_mount(str(diff_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'PYTHON_BIN="{python_bin}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'AZLKS="{azlks}"', + f'UNWRAP_THRESHOLD="{unwrap_threshold:.3f}"', + 'SPS_FLAG="${SPS_FLAG:-1}"', + 'AZF_FLAG="${AZF_FLAG:-0}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${DIFF_DIR}" "${LOG_DIR}"', + 'HGT="${DEM_DIR}/${REF_DATE}_${RLKS}rlks.rdc.dem"', + "", + "slc_for_date() {", + ' local date="$1"', + ' if [ "${date}" = "${REF_DATE}" ]; then', + ' printf "%s %s\\n" "${SLC_DIR}/${date}.slc" "${SLC_DIR}/${date}.slc.par"', + " else", + ' printf "%s %s\\n" "${COMMON_DIR}/rslc/${date}.rslc" "${COMMON_DIR}/rslc/${date}.rslc.par"', + " fi", + "}", + "", + "mli_for_date() {", + ' local date="$1"', + ' if [ "${date}" = "${REF_DATE}" ]; then', + ' printf "%s %s\\n" "${MLI_DIR}/${date}.mli" "${MLI_DIR}/${date}.mli.par"', + " else", + ' printf "%s %s\\n" "${COMMON_DIR}/rmli/${date}.mli" "${COMMON_DIR}/rmli/${date}.mli.par"', + " fi", + "}", + "", + "cc_stats() {", + ' local cc="$1"', + ' local out="$2"', + ' "${PYTHON_BIN}" - "${cc}" >"${out}" <<\'PY\'', + "import json", + "import sys", + "from pathlib import Path", + "import numpy as np", + "", + "path = Path(sys.argv[1])", + "data = np.fromfile(path, dtype='>f4')", + "finite = data[np.isfinite(data)]", + "nonzero = finite[finite != 0]", + "payload = {", + " 'path': str(path),", + " 'pixels': int(data.size),", + " 'finite_pixels': int(finite.size),", + " 'nonzero_pixels': int(nonzero.size),", + "}", + "if finite.size:", + " payload.update({", + " 'min': float(np.min(finite)),", + " 'median': float(np.median(finite)),", + " 'max': float(np.max(finite)),", + " })", + "print(json.dumps(payload, ensure_ascii=False, indent=2))", + "PY", + "}", + "", + "diff_unwrap_pair() {", + ' local master_date="$1"', + ' local slave_date="$2"', + ' local pair="${master_date}_${slave_date}"', + ' local slc1 slc1_par slc2 slc2_par mli1 mli1_par mli2 mli2_par', + ' read -r slc1 slc1_par < <(slc_for_date "${master_date}")', + ' read -r slc2 slc2_par < <(slc_for_date "${slave_date}")', + ' read -r mli1 mli1_par < <(mli_for_date "${master_date}")', + ' read -r mli2 mli2_par < <(mli_for_date "${slave_date}")', + "", + ' local work_dir="${DIFF_DIR}/${pair}"', + ' local off="${work_dir}/${pair}_${RLKS}rlks.off"', + ' local sim_unw="${work_dir}/${pair}.sim_unw"', + ' local diff="${work_dir}/${pair}_${RLKS}rlks.diff"', + ' local diff_filt="${work_dir}/${pair}_${RLKS}rlks.diff_filt"', + ' local cc="${work_dir}/${pair}_${RLKS}rlks.diff_filt.cor"', + ' local mask="${work_dir}/${pair}_${RLKS}rlks.diff_filt.cor_mask.bmp"', + ' local unw="${work_dir}/${pair}_${RLKS}rlks.diff_filt.unw"', + ' local width lines r_ref a_ref', + "", + ' mkdir -p "${work_dir}"', + ' width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${mli1_par}")"', + ' lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${mli1_par}")"', + ' r_ref="$(( width / 2 ))"', + ' a_ref="$(( lines / 2 ))"', + "", + " {", + ' echo "== differential unwrap ${pair} =="', + ' echo "width=${width} lines=${lines} threshold=${UNWRAP_THRESHOLD}"', + ' test -s "${slc1}"', + ' test -s "${slc1_par}"', + ' test -s "${slc2}"', + ' test -s "${slc2_par}"', + ' test -s "${mli1}"', + ' test -s "${mli2}"', + ' test -s "${HGT}"', + "", + ' create_offset "${slc1_par}" "${slc2_par}" "${off}" 1 "${RLKS}" "${AZLKS}" 0', + ' phase_sim_orb "${slc1_par}" "${slc2_par}" "${off}" "${HGT}" "${sim_unw}" "${SLC_DIR}/${REF_DATE}.slc.par" - - 1 1', + ' SLC_diff_intf "${slc1}" "${slc2}" "${slc1_par}" "${slc2_par}" "${off}" "${sim_unw}" \\', + ' "${diff}" "${RLKS}" "${AZLKS}" "${SPS_FLAG}" "${AZF_FLAG}" - 1 1', + ' adf "${diff}" "${diff_filt}" "${cc}" "${width}" 0.4 - 5', + ' cc_wave "${diff_filt}" "${mli1}" "${mli2}" "${cc}" "${width}" 5 5', + ' rasmph_pwr "${diff_filt}" "${mli1}" "${width}" - - - - - - - - - "${cc}" - 0.1', + ' rasdt_pwr "${cc}" "${mli1}" "${width}" 1 0 1 1 0.1 1.0 1', + ' rascc_mask "${cc}" "${mli1}" "${width}" 1 1 0 1 1 "${UNWRAP_THRESHOLD}" 0.0 0.1 0.9 1 .35 1 "${mask}"', + ' mcf "${diff_filt}" "${cc}" "${mask}" "${unw}" "${width}" 2 0 0 "${width}" "${lines}" 1 1 - "${r_ref}" "${a_ref}" 1', + ' rasdt_pwr "${unw}" "${mli1}" "${width}" 1 0 1 1 -3.14 3.14 1', + ' ls -lh "${sim_unw}" "${diff}" "${diff_filt}" "${cc}" "${mask}" "${unw}"', + ' } >"${LOG_DIR}/${pair}_diff_unwrap_common.log" 2>&1', + "", + ( + ' cc_stats "${cc}" "${LOG_DIR}/${pair}_diff_filt_cc_stats.json" || ' + 'printf \'{"path":"%s","error":"cc_stats_failed"}\\n\' "${cc}" >"${LOG_DIR}/${pair}_diff_filt_cc_stats.json"' + ), + ' echo "completed ${pair}"', + "}", + "", + "PAIR_ROWS=(", + ] + for pair in pair_plan: + lines.append( + " " + f'"{pair.get("master_date")} {pair.get("slave_date")} {pair.get("itab_row", [None, None, None, None])[2]}"' + ) + lines.extend( + [ + ")", + "", + 'for row in "${PAIR_ROWS[@]}"; do', + " read -r master_date slave_date pair_index <<<\"${row}\"", + ' diff_unwrap_pair "${master_date}" "${slave_date}"', + "done", + "", + 'DIFF_TAB="${COMMON_DIR}/DIFF_tab"', + 'ITAB="${COMMON_DIR}/itab_common_ref"', + ': >"${DIFF_TAB}"', + ': >"${ITAB}"', + ] + ) + for pair in pair_plan: + pair_id = str(pair.get("pair_id") or "") + itab_row = pair.get("itab_row") or [] + lines.append(f'echo "${{DIFF_DIR}}/{pair_id}/{pair_id}_${{RLKS}}rlks.diff_filt.unw" >>"${{DIFF_TAB}}"') + if len(itab_row) >= 4: + lines.append(f'echo "{itab_row[0]} {itab_row[1]} {itab_row[2]} {itab_row[3]}" >>"${{ITAB}}"') + lines.extend( + [ + "", + 'test "$(wc -l <"${DIFF_TAB}")" -eq "${#PAIR_ROWS[@]}"', + 'test "$(wc -l <"${ITAB}")" -eq "${#PAIR_ROWS[@]}"', + 'echo "Common-reference differential/unwrapped stack complete: ${COMMON_DIR}"', + "", + ] + ) + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) + + def _write_ipta_timeseries_script( + self, + run_dir: Path, + *, + reference_date: str, + rlks: int, + reference_window: int, + diff_tab: Path, + rmli_tab: Path, + itab: Path, + geom_ref_mli_par: Path, + mb_ref_mli_par: Path, + reference_region: dict[str, Any], + mb_mode: int, + ) -> Path: + mb_mode = self._normalize_ipta_mb_mode(mb_mode) + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "05_mb_ts_rate.sh" + gamma_root = run_dir / "work" / "gamma" + common_dir = gamma_root / f"common_{reference_date}" + timeseries_dir = common_dir / "timeseries" + log_dir = run_dir / "logs" + env_script = ( + self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'COMMON_DIR="{self._windows_path_to_wsl_mount(str(common_dir))}"', + f'TS_DIR="{self._windows_path_to_wsl_mount(str(timeseries_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'REFERENCE_WINDOW="{reference_window}"', + f'R_REF="{int(reference_region.get("range_pixel") or 0)}"', + f'A_REF="{int(reference_region.get("azimuth_line") or 0)}"', + f'MB_MODE="{mb_mode}"', + f'DIFF_TAB="{self._windows_path_to_wsl_mount(str(diff_tab))}"', + f'RMLI_TAB="{self._windows_path_to_wsl_mount(str(rmli_tab))}"', + f'ITAB="{self._windows_path_to_wsl_mount(str(itab))}"', + f'GEOM_REF_MLI_PAR="{self._windows_path_to_wsl_mount(str(geom_ref_mli_par))}"', + f'REF_MLI_PAR="{self._windows_path_to_wsl_mount(str(mb_ref_mli_par))}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${TS_DIR}" "${LOG_DIR}"', + "", + 'ITAB_TS="${TS_DIR}/itab_ts"', + 'DIFF_TS="${TS_DIR}/diff_ts"', + 'SIGMA_TS="${TS_DIR}/sigma_ts"', + 'HGT_OUT="${TS_DIR}/hgt_correction"', + 'RATE="${TS_DIR}/ts_rate"', + 'CONST="${TS_DIR}/ts_const"', + 'SIGMA_RATE="${TS_DIR}/sigma_rate"', + 'WIDTH="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${GEOM_REF_MLI_PAR}")"', + 'LINES="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${GEOM_REF_MLI_PAR}")"', + 'rm -f "${ITAB_TS}" "${DIFF_TS}.tab" "${DIFF_TS}"_*.diff "${DIFF_TS}"_*.diff_sim \\', + ' "${SIGMA_TS}" "${HGT_OUT}" "${RATE}" "${CONST}" "${SIGMA_RATE}"', + "", + "{", + ' echo "== Gamma mb time-series =="', + ' echo "width=${WIDTH} lines=${LINES} ref_region=${R_REF},${A_REF}"', + ' echo "geometry_reference=${GEOM_REF_MLI_PAR}"', + ' echo "mb_reference=${REF_MLI_PAR}"', + ' echo "mb_mode=${MB_MODE}"', + ' test "${R_REF}" -gt 0', + ' test "${A_REF}" -gt 0', + ' test -s "${DIFF_TAB}"', + ' test -s "${RMLI_TAB}"', + ' test -s "${ITAB}"', + ' test -s "${GEOM_REF_MLI_PAR}"', + ' test -s "${REF_MLI_PAR}"', + ' mb "${DIFF_TAB}" "${RMLI_TAB}" "${ITAB}" - \\', + ' "${ITAB_TS}" "${DIFF_TS}" 1 "${SIGMA_TS}" 1 "${HGT_OUT}" \\', + ' "${R_REF}" "${A_REF}" "${REFERENCE_WINDOW}" "${REFERENCE_WINDOW}" 1.0 "${GEOM_REF_MLI_PAR}" "${REF_MLI_PAR}" "${MB_MODE}"', + ' test -s "${DIFF_TS}.tab"', + ' test -s "${ITAB_TS}"', + ' test -s "${SIGMA_TS}"', + ' test -s "${HGT_OUT}"', + ' ls -lh "${DIFF_TS}.tab" "${ITAB_TS}" "${SIGMA_TS}" "${HGT_OUT}"', + "", + ' echo "== Gamma ts_rate =="', + ' ts_rate "${DIFF_TS}.tab" "${RMLI_TAB}" "${ITAB_TS}" \\', + ' - "${RATE}" "${CONST}" "${SIGMA_RATE}" 1', + ' test -s "${RATE}"', + ' test -s "${CONST}"', + ' test -s "${SIGMA_RATE}"', + ' ls -lh "${RATE}" "${CONST}" "${SIGMA_RATE}"', + '} >"${LOG_DIR}/mb_ts_rate.log" 2>&1', + "", + 'echo "Gamma mb/ts_rate complete: ${TS_DIR}"', + "", + ] + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) + + def _write_detrend_atm_script( + self, + run_dir: Path, + *, + reference_date: str, + rlks: int, + reference_window: int, + reference_region: dict[str, Any], + coherence_min: float, + diff_tab: Path, + itab: Path, + rmli_path: Path, + rmli_par_path: Path, + hgt_path: Path, + pair_plan: list[dict[str, Any]], + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "05_detrend_atm.sh" + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + detrend_dir = common_dir / "detrend_atm" + log_dir = run_dir / "logs" + env_script = ( + self._windows_path_to_wsl_mount(settings.GAMMA_SBAS_ENV_SCRIPT or settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + python_bin = settings.GAMMA_SBAS_PYTHON or settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + r_ref = int(reference_region.get("range_pixel") or 0) + a_ref = int(reference_region.get("azimuth_line") or 0) + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'COMMON_DIR="{self._windows_path_to_wsl_mount(str(common_dir))}"', + f'DETREND_DIR="{self._windows_path_to_wsl_mount(str(detrend_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'DIFF_TAB="{self._windows_path_to_wsl_mount(str(diff_tab))}"', + f'ITAB="{self._windows_path_to_wsl_mount(str(itab))}"', + f'MLI="{self._windows_path_to_wsl_mount(str(rmli_path))}"', + f'MLI_PAR="{self._windows_path_to_wsl_mount(str(rmli_par_path))}"', + f'HGT="{self._windows_path_to_wsl_mount(str(hgt_path))}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'REFERENCE_WINDOW="{reference_window}"', + f'R_REF="{r_ref}"', + f'A_REF="{a_ref}"', + f'CC_MIN="{coherence_min:.6g}"', + f'PYTHON_BIN="{python_bin}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${DETREND_DIR}" "${LOG_DIR}"', + 'WIDTH="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_PAR}")"', + 'LINES="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${MLI_PAR}")"', + 'test -n "${WIDTH}"', + 'test -n "${LINES}"', + 'test -s "${DIFF_TAB}"', + 'test -s "${ITAB}"', + 'test -s "${MLI}"', + 'test -s "${MLI_PAR}"', + 'test -s "${HGT}"', + 'ATMSUB_TAB="${COMMON_DIR}/DIFF_atmsub_tab"', + 'ITAB_ATMSUB="${COMMON_DIR}/itab_atmsub"', + ': >"${ATMSUB_TAB}"', + 'cp -f "${ITAB}" "${ITAB_ATMSUB}"', + "", + "infer_model_width() {", + ' local model_file="$1"', + ' local byte_count', + ' byte_count="$(wc -c <"${model_file}")"', + ' local pixels=$((byte_count / 4))', + ' if [ "${pixels}" -le 0 ]; then', + ' echo 0', + ' return', + ' fi', + ' "${PYTHON_BIN}" - "$pixels" "$WIDTH" "$LINES" <<\'PY\'', + "import math, sys", + "pixels = int(sys.argv[1])", + "width = max(1, int(float(sys.argv[2])))", + "lines = max(1, int(float(sys.argv[3])))", + "target = width / lines", + "best = None", + "for w in range(1, int(math.sqrt(pixels)) + 2):", + " if pixels % w:", + " continue", + " for cand in (w, pixels // w):", + " h = pixels // cand", + " score = abs((cand / h) - target)", + " if best is None or score < best[0]:", + " best = (score, cand)", + "print(best[1] if best else 0)", + "PY", + "}", + "", + "fill_model_if_possible() {", + ' local in_file="$1"', + ' local out_file="$2"', + ' local model_width', + ' model_width="$(infer_model_width "${in_file}")"', + ' if [ "${model_width}" -gt 0 ]; then', + ' if fill_gaps "${in_file}" "${model_width}" "${out_file}" 0 4 0 0; then', + ' return', + ' fi', + ' echo "fill_gaps failed for ${in_file}; using raw model coefficients" >&2', + ' else', + ' echo "could not infer model width for ${in_file}; using raw model coefficients" >&2', + ' fi', + ' cp -f "${in_file}" "${out_file}"', + "}", + "", + "run_pair() {", + ' local pair="$1"', + ' local unw="$2"', + ' local cor="$3"', + ' local off="$4"', + ' local pair_dir="${DETREND_DIR}/${pair}"', + ' local diff_par="${pair_dir}/${pair}.diff_par"', + ' local linear="${pair_dir}/${pair}.unw_linear"', + ' local sub_linear="${pair_dir}/${pair}.unw_sub_linear"', + ' local a0="${pair_dir}/${pair}.a0"', + ' local a1="${pair_dir}/${pair}.a1"', + ' local a0_fill="${pair_dir}/${pair}.a0_fill"', + ' local a1_fill="${pair_dir}/${pair}.a1_fill"', + ' local sigma="${pair_dir}/${pair}.atm_sigma"', + ' local sigma_h="${pair_dir}/${pair}.atm_sigma_h"', + ' local s1="${pair_dir}/${pair}.atm_s1"', + ' local atm_model="${pair_dir}/${pair}.atm_model"', + ' local atmsub="${pair_dir}/${pair}_${RLKS}rlks.diff_filt.unw.atmsub"', + ' local log="${LOG_DIR}/${pair}_detrend_atm.log"', + ' mkdir -p "${pair_dir}"', + ' {', + ' echo "== detrend/atm ${pair} =="', + ' echo "unw=${unw}"', + ' echo "cor=${cor}"', + ' echo "off=${off}"', + ' test -s "${unw}"', + ' test -s "${cor}"', + ' test -s "${off}"', + ' create_diff_par "${off}" "${off}" "${diff_par}" 0 0', + ' quad_fit "${unw}" "${diff_par}" 5 5 - - 3 "${linear}"', + ' quad_sub "${unw}" "${diff_par}" "${sub_linear}" 0 0', + ' rasdt_pwr "${sub_linear}" "${MLI}" "${WIDTH}" 1 - 1 1 -6.28 6.28 1 rmg.cm "${sub_linear}.bmp" 1.0 0.35 24', + ' atm_mod_2d "${sub_linear}" "${HGT}" "${cor}" "${diff_par}" - 0 "${a0}" "${a1}" "${sigma}" "${sigma_h}" "${s1}" 512 512 64 64 7000 - "${CC_MIN}" 0.20 "${R_REF}" "${A_REF}" 1', + ' test -s "${a0}"', + ' test -s "${a1}"', + ' fill_model_if_possible "${a0}" "${a0_fill}"', + ' fill_model_if_possible "${a1}" "${a1_fill}"', + ' atm_sim_2d "${diff_par}" "${HGT}" "${a0_fill}" "${a1_fill}" "${atm_model}" -', + ' sub_phase "${sub_linear}" "${atm_model}" "${diff_par}" "${atmsub}" 0 0 0', + ' rasdt_pwr "${atmsub}" "${MLI}" "${WIDTH}" 1 - 1 1 -6.28 6.28 1 rmg.cm "${atmsub}.bmp" 1.0 0.35 24', + ' test -s "${atmsub}"', + ' printf "%s\\n" "${atmsub}" >>"${ATMSUB_TAB}"', + ' ls -lh "${diff_par}" "${linear}" "${sub_linear}" "${a0}" "${a1}" "${atm_model}" "${atmsub}"', + ' } >"${log}" 2>&1', + "}", + "", + ] + for pair in pair_plan: + lines.append( + "run_pair " + f'"{pair.get("pair_id")}" ' + f'"{self._windows_path_to_wsl_mount(str(pair.get("unw") or ""))}" ' + f'"{self._windows_path_to_wsl_mount(str(pair.get("cor") or ""))}" ' + f'"{self._windows_path_to_wsl_mount(str(pair.get("offset") or ""))}"' + ) + lines.extend( + [ + "", + 'test "$(wc -l <"${ATMSUB_TAB}")" -eq "$(wc -l <"${DIFF_TAB}")"', + 'du -h "${DETREND_DIR}"/*/* "${ATMSUB_TAB}" "${ITAB_ATMSUB}" 2>/dev/null | sort -h >"${LOG_DIR}/detrend_atm_inventory.txt"', + 'echo "detrend/atm complete: ${ATMSUB_TAB}"', + "", + ] + ) + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) + + def _write_publish_products_script( + self, + run_dir: Path, + *, + reference_date: str, + rlks: int, + timeseries_dir: Path, + rmli_path: Path, + rmli_par_path: Path, + slc_par_path: Path, + dem_par_path: Path, + lookup_path: Path, + wavelength: float, + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "07_publish_products.sh" + export_dir = run_dir / "publish" / "geotiff" + log_dir = run_dir / "logs" + env_script = ( + self._windows_path_to_wsl_mount(settings.GAMMA_SBAS_ENV_SCRIPT or settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + python_bin = settings.GAMMA_SBAS_PYTHON or settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + tool_script = Path(settings.PROJECT_ROOT) / "deploy" / "wsl" / "runners" / "gamma_sbas_product_tools.py" + phase_to_los = wavelength / (4.0 * math.pi) + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'TS_DIR="{self._windows_path_to_wsl_mount(str(timeseries_dir))}"', + f'EXPORT_DIR="{self._windows_path_to_wsl_mount(str(export_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'MLI="{self._windows_path_to_wsl_mount(str(rmli_path))}"', + f'MLI_PAR="{self._windows_path_to_wsl_mount(str(rmli_par_path))}"', + f'SLC_PAR="{self._windows_path_to_wsl_mount(str(slc_par_path))}"', + f'DEM_PAR="{self._windows_path_to_wsl_mount(str(dem_par_path))}"', + f'LOOKUP="{self._windows_path_to_wsl_mount(str(lookup_path))}"', + f'PYTHON_BIN="{python_bin}"', + f'TOOL_SCRIPT="{self._windows_path_to_wsl_mount(str(tool_script))}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'WAVELENGTH="{wavelength:.12g}"', + f'PHASE_TO_LOS="{phase_to_los:.12g}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${EXPORT_DIR}" "${LOG_DIR}"', + "", + 'RDC_WIDTH="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_PAR}")"', + 'GEO_WIDTH="$(awk \'$1 == "width:" {print $2; exit}\' "${DEM_PAR}")"', + 'GEO_LINES="$(awk \'$1 == "nlines:" {print $2; exit}\' "${DEM_PAR}")"', + "", + "geo_float() {", + ' local in_file="$1"', + ' local out_root="$2"', + ' local geo_bin="${EXPORT_DIR}/${out_root}.geo"', + ' local tif="${EXPORT_DIR}/${out_root}.tif"', + ' geocode_back "${in_file}" "${RDC_WIDTH}" "${LOOKUP}" "${geo_bin}" "${GEO_WIDTH}" "${GEO_LINES}" 1 0', + ' data2geotiff "${DEM_PAR}" "${geo_bin}" 2 "${tif}" - 1', + ' if command -v gdalinfo >/dev/null 2>&1; then', + ' gdalinfo "${tif}" >"${EXPORT_DIR}/${out_root}.gdalinfo.txt"', + " fi", + "}", + "", + "geo_bmp_rgb() {", + ' local in_bmp="$1"', + ' local out_root="$2"', + ' local geo_bmp="${EXPORT_DIR}/${out_root}.geo.bmp"', + ' local rgb_tif="${EXPORT_DIR}/${out_root}.geo_rgb.tif"', + ' local png="${EXPORT_DIR}/${out_root}.geo_preview.png"', + ' geocode_back "${in_bmp}" "${RDC_WIDTH}" "${LOOKUP}" "${geo_bmp}" "${GEO_WIDTH}" "${GEO_LINES}" 0 2', + ' data2geotiff "${DEM_PAR}" "${geo_bmp}" 0 "${rgb_tif}"', + ' if command -v gdal_translate >/dev/null 2>&1; then', + ' gdal_translate -of PNG -outsize 1400 0 "${rgb_tif}" "${png}" >/dev/null', + " fi", + "}", + "", + "make_preview() {", + ' local tif="$1"', + ' local cmap="$2"', + ' local png="$3"', + ' local tmp_tif="${png%.png}.rgba.tif"', + ' if command -v gdaldem >/dev/null 2>&1 && command -v gdal_translate >/dev/null 2>&1; then', + ' gdaldem color-relief -alpha -nearest_color_entry "${tif}" "${cmap}" "${tmp_tif}"', + ' gdal_translate -of PNG -outsize 1400 0 "${tmp_tif}" "${png}" >/dev/null', + ' rm -f "${tmp_tif}"', + " else", + ' "${PYTHON_BIN}" - "${tif}" "${png}" <<\'PY\'', + "import sys", + "from pathlib import Path", + "import matplotlib", + "matplotlib.use('Agg')", + "import matplotlib.pyplot as plt", + "import numpy as np", + "try:", + " import rasterio", + " with rasterio.open(sys.argv[1]) as src:", + " arr = src.read(1)", + "except Exception:", + " arr = np.fromfile(sys.argv[1], dtype='>f4')", + "arr = np.where(np.isfinite(arr), arr, np.nan)", + "plt.figure(figsize=(10, 7), dpi=140)", + "plt.imshow(arr, cmap='RdYlBu_r')", + "plt.colorbar(shrink=0.75)", + "plt.axis('off')", + "Path(sys.argv[2]).parent.mkdir(parents=True, exist_ok=True)", + "plt.tight_layout(pad=0)", + "plt.savefig(sys.argv[2], bbox_inches='tight', pad_inches=0.02)", + "PY", + " fi", + "}", + "", + 'RATE_CMAP="${EXPORT_DIR}/los_rate_toward_mm_per_year.preview.cmap.txt"', + 'SIGMA_CMAP="${EXPORT_DIR}/los_sigma_mm_per_year.preview.cmap.txt"', + 'cat >"${RATE_CMAP}" <<\'EOF\'', + "-100 49 54 149 255", + "-75 69 117 180 255", + "-50 116 173 209 255", + "-25 224 243 248 255", + "0 255 255 255 255", + "25 254 224 144 255", + "50 253 174 97 255", + "75 215 48 39 255", + "100 165 0 38 255", + "nv 0 0 0 0", + "EOF", + 'cat >"${SIGMA_CMAP}" <<\'EOF\'', + "0 247 252 245 255", + "5 229 245 249 255", + "10 204 236 230 255", + "20 153 216 201 255", + "30 102 194 164 255", + "45 44 162 95 255", + "60 0 109 44 255", + "90 84 39 136 255", + "nv 0 0 0 0", + "EOF", + "", + "{", + ' echo "== publish Gamma SBAS products =="', + ' echo "REF_DATE=${REF_DATE} RLKS=${RLKS}"', + ' echo "RDC_WIDTH=${RDC_WIDTH} GEO_WIDTH=${GEO_WIDTH} GEO_LINES=${GEO_LINES}"', + ' echo "WAVELENGTH=${WAVELENGTH} PHASE_TO_LOS=${PHASE_TO_LOS}"', + ' test -s "${MLI}"', + ' test -s "${MLI_PAR}"', + ' test -s "${SLC_PAR}"', + ' test -s "${DEM_PAR}"', + ' test -s "${LOOKUP}"', + ' test -s "${TS_DIR}/ts_rate"', + ' test -s "${TS_DIR}/sigma_rate"', + "", + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${TS_DIR}/ts_rate" "${EXPORT_DIR}/los_rate_m_per_year.rdc" "${PHASE_TO_LOS}"', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${TS_DIR}/sigma_rate" "${EXPORT_DIR}/los_sigma_m_per_year.rdc" "${PHASE_TO_LOS}"', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${EXPORT_DIR}/los_rate_m_per_year.rdc" "${EXPORT_DIR}/los_rate_away_m_per_year.rdc" 1.0', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${EXPORT_DIR}/los_rate_m_per_year.rdc" "${EXPORT_DIR}/los_rate_toward_m_per_year.rdc" -1.0', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${EXPORT_DIR}/los_rate_m_per_year.rdc" "${EXPORT_DIR}/los_rate_away_mm_per_year.rdc" 1000.0', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${EXPORT_DIR}/los_rate_m_per_year.rdc" "${EXPORT_DIR}/los_rate_toward_mm_per_year.rdc" -1000.0', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" phase-to-los "${EXPORT_DIR}/los_sigma_m_per_year.rdc" "${EXPORT_DIR}/los_sigma_mm_per_year.rdc" 1000.0', + "", + ' geo_float "${TS_DIR}/ts_rate" "ts_rate_rad_per_year"', + ' geo_float "${TS_DIR}/sigma_rate" "sigma_rate_rad_per_year"', + ' if [ -s "${TS_DIR}/sigma_ts" ]; then geo_float "${TS_DIR}/sigma_ts" "sigma_ts_rad"; fi', + ' if [ -s "${TS_DIR}/hgt_correction" ]; then geo_float "${TS_DIR}/hgt_correction" "hgt_correction_m"; fi', + ' geo_float "${EXPORT_DIR}/los_rate_away_m_per_year.rdc" "los_rate_away_m_per_year"', + ' geo_float "${EXPORT_DIR}/los_rate_toward_m_per_year.rdc" "los_rate_toward_m_per_year"', + ' geo_float "${EXPORT_DIR}/los_sigma_m_per_year.rdc" "los_sigma_m_per_year"', + ' geo_float "${EXPORT_DIR}/los_rate_away_mm_per_year.rdc" "los_rate_away_mm_per_year"', + ' geo_float "${EXPORT_DIR}/los_rate_toward_mm_per_year.rdc" "los_rate_toward_mm_per_year"', + ' geo_float "${EXPORT_DIR}/los_sigma_mm_per_year.rdc" "los_sigma_mm_per_year"', + "", + ' rasdt_pwr "${EXPORT_DIR}/los_rate_toward_m_per_year.rdc" "${MLI}" "${RDC_WIDTH}" 1 0 1 1 -0.08 0.08 0 hls.cm "${EXPORT_DIR}/los_rate_toward_m_per_year.hls.bmp" 1.0 0.35 24', + ' rasdt_pwr "${EXPORT_DIR}/los_rate_away_m_per_year.rdc" "${MLI}" "${RDC_WIDTH}" 1 0 1 1 -0.08 0.08 0 hls.cm "${EXPORT_DIR}/los_rate_away_m_per_year.hls.bmp" 1.0 0.35 24', + ' rasdt_pwr "${EXPORT_DIR}/los_sigma_m_per_year.rdc" "${MLI}" "${RDC_WIDTH}" 1 0 1 1 0.0 0.06 1 cc.cm "${EXPORT_DIR}/los_sigma_m_per_year.cc.bmp" 1.0 0.35 8', + ' rasdt_pwr "${EXPORT_DIR}/los_rate_away_mm_per_year.rdc" "${MLI}" "${RDC_WIDTH}" - - 4 4 -100 100 0 hls.cm "${EXPORT_DIR}/los_rate_away_mm_per_year.bmp" - - 24', + ' rasdt_pwr "${EXPORT_DIR}/los_rate_toward_mm_per_year.rdc" "${MLI}" "${RDC_WIDTH}" - - 4 4 -100 100 0 hls.cm "${EXPORT_DIR}/los_rate_toward_mm_per_year.bmp" - - 24', + ' rasdt_pwr "${EXPORT_DIR}/los_sigma_mm_per_year.rdc" "${MLI}" "${RDC_WIDTH}" - - 4 4 0 60 1 cc.cm "${EXPORT_DIR}/los_sigma_mm_per_year.bmp" - - 8', + ' geo_bmp_rgb "${EXPORT_DIR}/los_rate_toward_m_per_year.hls.bmp" "los_rate_toward_m_per_year.hls"', + ' geo_bmp_rgb "${EXPORT_DIR}/los_sigma_m_per_year.cc.bmp" "los_sigma_m_per_year.cc"', + "", + ' make_preview "${EXPORT_DIR}/los_rate_toward_mm_per_year.tif" "${RATE_CMAP}" "${EXPORT_DIR}/los_rate_toward_mm_per_year.geo_preview.png"', + ' make_preview "${EXPORT_DIR}/los_sigma_mm_per_year.tif" "${SIGMA_CMAP}" "${EXPORT_DIR}/los_sigma_mm_per_year.geo_preview.png"', + ' ls -lh "${EXPORT_DIR}"', + '} >"${LOG_DIR}/publish_products.log" 2>&1', + "", + 'echo "Published Gamma SBAS products: ${EXPORT_DIR}"', + "", + ] + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) + + def _write_monitor_points_script( + self, + run_dir: Path, + *, + reference_date: str, + dates: list[str], + timeseries_dir: Path, + export_dir: Path, + point_dir: Path, + rmli_par_path: Path, + slc_par_path: Path, + dem_par_path: Path, + lookup_path: Path, + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "08_point_timeseries.sh" + log_dir = run_dir / "logs" + python_bin = settings.GAMMA_SBAS_PYTHON or settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + tool_script = Path(settings.PROJECT_ROOT) / "deploy" / "wsl" / "runners" / "gamma_sbas_product_tools.py" + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'TS_DIR="{self._windows_path_to_wsl_mount(str(timeseries_dir))}"', + f'EXPORT_DIR="{self._windows_path_to_wsl_mount(str(export_dir))}"', + f'POINT_DIR="{self._windows_path_to_wsl_mount(str(point_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'MLI_PAR="{self._windows_path_to_wsl_mount(str(rmli_par_path))}"', + f'SLC_PAR="{self._windows_path_to_wsl_mount(str(slc_par_path))}"', + f'DEM_PAR="{self._windows_path_to_wsl_mount(str(dem_par_path))}"', + f'LOOKUP="{self._windows_path_to_wsl_mount(str(lookup_path))}"', + f'PYTHON_BIN="{python_bin}"', + f'TOOL_SCRIPT="{self._windows_path_to_wsl_mount(str(tool_script))}"', + f'REF_DATE="{reference_date}"', + f'DATES="{",".join(dates)}"', + 'MONITOR_CONFIG="${RUN_ROOT}/monitor_points.json"', + 'SUMMARY="${RUN_ROOT}/monitor_points_summary.json"', + "", + 'mkdir -p "${POINT_DIR}" "${LOG_DIR}"', + "", + "{", + ' echo "== extract monitoring point time-series =="', + ' test -s "${TS_DIR}/diff_ts.tab"', + ' test -s "${EXPORT_DIR}/los_rate_toward_mm_per_year.rdc"', + ' test -s "${EXPORT_DIR}/los_sigma_mm_per_year.rdc"', + ' "${PYTHON_BIN}" "${TOOL_SCRIPT}" monitor-points \\', + ' --monitor-config "${MONITOR_CONFIG}" \\', + ' --timeseries-dir "${TS_DIR}" \\', + ' --export-dir "${EXPORT_DIR}" \\', + ' --point-dir "${POINT_DIR}" \\', + ' --mli-par "${MLI_PAR}" \\', + ' --slc-par "${SLC_PAR}" \\', + ' --dem-par "${DEM_PAR}" \\', + ' --lookup "${LOOKUP}" \\', + ' --dates "${DATES}" \\', + ' --reference-date "${REF_DATE}" \\', + ' --summary-path "${SUMMARY}"', + '} >"${LOG_DIR}/monitor_points.log" 2>&1', + "", + 'echo "Monitoring point products complete: ${POINT_DIR}"', + "", + ] + scripts_dir.mkdir(parents=True, exist_ok=True) + return self._write_script(script_path, lines) def _build_baseline_summary(self, run_dir: Path) -> dict[str, Any]: diff_dir = run_dir / "work" / "gamma" / "diff" @@ -1932,6 +6434,588 @@ class SbasInsarProductionService: }, } + def _build_rdc_dem_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + rlks: int, + dem_source: dict[str, Any], + ) -> dict[str, Any]: + reference = str(reference_date or "").strip() + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + dem_dir = run_dir / "work" / "gamma" / "dem" + prefix = f"{reference}_{rlks}rlks" + required_outputs = { + "utm_dem": dem_dir / f"{prefix}.utm.dem", + "utm_dem_par": dem_dir / f"{prefix}.utm.dem.par", + "lookup_table": dem_dir / f"{prefix}.UTM_TO_RDC", + "rdc_dem": dem_dir / f"{prefix}.rdc.dem", + "diff_par": dem_dir / f"{prefix}.diff_par", + } + optional_outputs = { + "utm_to_rdc_initial": dem_dir / f"{prefix}.utm_to_rdc0", + "sim_sar_rdc": dem_dir / f"{prefix}.sim_sar_rdc", + "offset_std": dem_dir / f"{reference}_dem.off_std", + "source_dem_clean": dem_dir / "source_dem_clean.dem", + "source_dem_clean_par": dem_dir / "source_dem_clean.dem.par", + } + missing_outputs = [ + name for name, path in required_outputs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + + rmli_path, rmli_par_path = self._find_reference_rmli_paths(run_dir, reference) + rmli_params = self._parse_gamma_params(rmli_par_path) + utm_params = self._parse_gamma_params(required_outputs["utm_dem_par"]) + rdc_width = self._as_int(rmli_params.get("range_samples")) + rdc_lines = self._as_int(rmli_params.get("azimuth_lines")) + expected_rdc_bytes = (rdc_width * rdc_lines * 4) if rdc_width and rdc_lines else None + rdc_size = required_outputs["rdc_dem"].stat().st_size if required_outputs["rdc_dem"].is_file() else None + log_path = run_dir / "logs" / f"{reference}_rdc_dem.log" + size_matches_reference_geometry = ( + expected_rdc_bytes is None + or (rdc_size is not None and rdc_size == expected_rdc_bytes) + ) + ready = not missing_outputs and bool(reference) and size_matches_reference_geometry + return { + "schema": "insar.gamma-rdc-dem-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "rlks": rlks, + "ready": ready, + "missing_outputs": missing_outputs, + "dem_source": dem_source, + "reference_rmli": { + "mli": str(rmli_path), + "mli_par": str(rmli_par_path), + "range_samples": rdc_width, + "azimuth_lines": rdc_lines, + }, + "utm_dem": { + "width": self._as_int(utm_params.get("width")), + "nlines": self._as_int(utm_params.get("nlines")), + "corner_lon": self._as_float(utm_params.get("corner_lon")), + "corner_lat": self._as_float(utm_params.get("corner_lat")), + "post_lon": self._as_float(utm_params.get("post_lon")), + "post_lat": self._as_float(utm_params.get("post_lat")), + }, + "rdc_dem": { + "size_bytes": rdc_size, + "expected_float32_bytes": expected_rdc_bytes, + "size_matches_reference_geometry": size_matches_reference_geometry, + }, + "outputs": { + name: self._file_record(path) + for name, path in {**required_outputs, **optional_outputs}.items() + }, + "log": self._file_record(log_path), + "log_tail": self._tail_text(log_path.read_text(encoding="utf-8", errors="replace")) if log_path.is_file() else "", + } + + def _build_interferogram_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + pair_plan: list[dict[str, Any]], + rlks: int, + ) -> dict[str, Any]: + reference = str(reference_date or "").strip() + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + common_dir = run_dir / "work" / "gamma" / f"common_{reference}" + diff_dir = common_dir / "diff" + diff_tab = common_dir / "DIFF_tab" + itab_common_ref = common_dir / "itab_common_ref" + + per_pair: list[dict[str, Any]] = [] + missing_pairs: list[str] = [] + for pair in pair_plan: + pair_id = str(pair.get("pair_id") or "").strip() + pair_dir = diff_dir / pair_id + required_outputs = { + "offset": pair_dir / f"{pair_id}_{rlks}rlks.off", + "sim_unw": pair_dir / f"{pair_id}.sim_unw", + "diff": pair_dir / f"{pair_id}_{rlks}rlks.diff", + "diff_filt": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt", + "cor": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.cor", + "mask": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.cor_mask.bmp", + "unw": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.unw", + } + missing = [ + name for name, path in required_outputs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + if missing: + missing_pairs.append(pair_id) + log_path = run_dir / "logs" / f"{pair_id}_diff_unwrap_common.log" + cc_stats_path = run_dir / "logs" / f"{pair_id}_diff_filt_cc_stats.json" + per_pair.append( + { + **pair, + "ready": not missing, + "missing": missing, + "outputs": {name: self._file_record(path) for name, path in required_outputs.items()}, + "log": self._file_record(log_path), + "cc_stats": self._read_optional_json(cc_stats_path) or self._file_record(cc_stats_path), + } + ) + + diff_tab_rows = self._read_text_rows(diff_tab) + itab_rows = self._parse_itab(itab_common_ref) + missing_tabs = [ + name for name, path in {"diff_tab": diff_tab, "itab_common_ref": itab_common_ref}.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + ready_pair_count = len([item for item in per_pair if item.get("ready")]) + ready = ( + ready_pair_count == len(pair_plan) + and not missing_pairs + and not missing_tabs + and len(diff_tab_rows) == len(pair_plan) + and len(itab_rows) == len(pair_plan) + and bool(pair_plan) + ) + return { + "schema": "insar.gamma-interferogram-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "rlks": rlks, + "pair_count": len(pair_plan), + "ready_pair_count": ready_pair_count, + "missing_pairs": missing_pairs, + "missing_tabs": missing_tabs, + "diff_tab_row_count": len(diff_tab_rows), + "itab_common_ref_row_count": len(itab_rows), + "ready": ready, + "per_pair": per_pair, + "outputs": { + "diff_dir": str(diff_dir), + "diff_tab": self._file_record(diff_tab), + "itab_common_ref": self._file_record(itab_common_ref), + }, + } + + def _build_detrend_atm_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + pair_plan: list[dict[str, Any]], + rlks: int, + inputs: dict[str, Any], + ) -> dict[str, Any]: + reference = str(reference_date or "").strip() + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + common_dir = run_dir / "work" / "gamma" / f"common_{reference}" + detrend_dir = common_dir / "detrend_atm" + diff_atmsub_tab = common_dir / "DIFF_atmsub_tab" + itab_atmsub = common_dir / "itab_atmsub" + rmli_par_path = Path(self._path_to_windows(str(inputs.get("reference_mli_par") or "")) or "") + rmli_params = self._parse_gamma_params(rmli_par_path) + width = self._as_int(rmli_params.get("range_samples")) + lines = self._as_int(rmli_params.get("azimuth_lines")) + expected_float32_bytes = (width * lines * 4) if width and lines else None + + per_pair: list[dict[str, Any]] = [] + missing_pairs: list[str] = [] + for pair in pair_plan: + pair_id = str(pair.get("pair_id") or "").strip() + pair_dir = detrend_dir / pair_id + outputs = { + "diff_par": pair_dir / f"{pair_id}.diff_par", + "unw_linear": pair_dir / f"{pair_id}.unw_linear", + "unw_sub_linear": pair_dir / f"{pair_id}.unw_sub_linear", + "a0": pair_dir / f"{pair_id}.a0", + "a1": pair_dir / f"{pair_id}.a1", + "a0_fill": pair_dir / f"{pair_id}.a0_fill", + "a1_fill": pair_dir / f"{pair_id}.a1_fill", + "atm_model": pair_dir / f"{pair_id}.atm_model", + "atmsub": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.unw.atmsub", + "atmsub_bmp": pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.unw.atmsub.bmp", + } + missing = [ + name for name, path in outputs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + if missing: + missing_pairs.append(pair_id) + atmsub_size = outputs["atmsub"].stat().st_size if outputs["atmsub"].is_file() else 0 + per_pair.append( + { + **pair, + "ready": not missing, + "missing": missing, + "size_checks": { + "atmsub": { + "size_bytes": atmsub_size, + "expected_float32_bytes": expected_float32_bytes, + "size_matches_reference_geometry": ( + expected_float32_bytes is None + or (atmsub_size > 0 and atmsub_size == expected_float32_bytes) + ), + } + }, + "outputs": {name: self._file_record(path) for name, path in outputs.items()}, + "log": self._file_record(run_dir / "logs" / f"{pair_id}_detrend_atm.log"), + } + ) + + diff_rows = self._read_text_rows(diff_atmsub_tab) + itab_rows = self._parse_itab(itab_atmsub) + missing_tabs = [ + name for name, path in {"diff_atmsub_tab": diff_atmsub_tab, "itab_atmsub": itab_atmsub}.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + ready_pair_count = len( + [ + item for item in per_pair + if item.get("ready") + and ((item.get("size_checks") or {}).get("atmsub") or {}).get("size_matches_reference_geometry") + ] + ) + ready = ( + ready_pair_count == len(pair_plan) + and not missing_pairs + and not missing_tabs + and len(diff_rows) == len(pair_plan) + and len(itab_rows) == len(pair_plan) + and bool(pair_plan) + ) + log_path = run_dir / "logs" / "detrend_atm_inventory.txt" + return { + "schema": "insar.gamma-detrend-atm-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "rlks": rlks, + "ready": ready, + "pair_count": len(pair_plan), + "ready_pair_count": ready_pair_count, + "missing_pairs": missing_pairs, + "missing_tabs": missing_tabs, + "diff_atmsub_tab_row_count": len(diff_rows), + "itab_atmsub_row_count": len(itab_rows), + "reference_geometry": { + "mli_par": str(rmli_par_path) if str(rmli_par_path) else None, + "range_samples": width, + "azimuth_lines": lines, + "expected_float32_bytes": expected_float32_bytes, + }, + "inputs": { + key: self._file_record(Path(self._path_to_windows(str(value)) or str(value))) + for key, value in inputs.items() + if value + }, + "outputs": { + "detrend_dir": str(detrend_dir), + "diff_atmsub_tab": self._file_record(diff_atmsub_tab), + "itab_atmsub": self._file_record(itab_atmsub), + }, + "per_pair": per_pair, + "log": self._file_record(log_path), + "log_tail": self._tail_text(log_path.read_text(encoding="utf-8", errors="replace")) if log_path.is_file() else "", + } + + def _build_ipta_timeseries_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + rlks: int, + inputs: dict[str, Any], + reference_region: dict[str, Any] | None = None, + mb_mode: int = DEFAULT_IPTA_MB_MODE, + ) -> dict[str, Any]: + reference = str(reference_date or "").strip() + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + mb_mode = self._normalize_ipta_mb_mode(mb_mode) + common_dir = run_dir / "work" / "gamma" / f"common_{reference}" + timeseries_dir = common_dir / "timeseries" + required_outputs = { + "diff_ts_tab": timeseries_dir / "diff_ts.tab", + "itab_ts": timeseries_dir / "itab_ts", + "sigma_ts": timeseries_dir / "sigma_ts", + "hgt_correction": timeseries_dir / "hgt_correction", + "ts_rate": timeseries_dir / "ts_rate", + "ts_const": timeseries_dir / "ts_const", + "sigma_rate": timeseries_dir / "sigma_rate", + } + missing_outputs = [ + name for name, path in required_outputs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + diff_ts_rows = self._read_text_rows(required_outputs["diff_ts_tab"]) + itab_ts_rows = self._parse_itab(required_outputs["itab_ts"]) + rmli_par_path = Path(self._path_to_windows(str(inputs.get("geometry_reference_mli_par") or "")) or "") + rmli_params = self._parse_gamma_params(rmli_par_path) + width = self._as_int(rmli_params.get("range_samples")) + lines = self._as_int(rmli_params.get("azimuth_lines")) + expected_float32_bytes = (width * lines * 4) if width and lines else None + size_checks = {} + for key in ("sigma_ts", "hgt_correction", "ts_rate", "ts_const", "sigma_rate"): + path = required_outputs[key] + size = path.stat().st_size if path.is_file() else 0 + size_checks[key] = { + "size_bytes": size, + "expected_float32_bytes": expected_float32_bytes, + "size_matches_reference_geometry": ( + expected_float32_bytes is None + or (size > 0 and size == expected_float32_bytes) + ), + } + log_path = run_dir / "logs" / "mb_ts_rate.log" + ready = ( + not missing_outputs + and bool(diff_ts_rows) + and bool(itab_ts_rows) + and all(item.get("size_matches_reference_geometry") for item in size_checks.values()) + ) + return { + "schema": "insar.gamma-ipta-timeseries-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "rlks": rlks, + "mb_mode": mb_mode, + "mb_mode_description": IPTA_MB_MODE_DESCRIPTIONS[mb_mode], + "ready": ready, + "missing_outputs": missing_outputs, + "diff_ts_row_count": len(diff_ts_rows), + "itab_ts_row_count": len(itab_ts_rows), + "reference_geometry": { + "mli_par": str(rmli_par_path) if str(rmli_par_path) else None, + "range_samples": width, + "azimuth_lines": lines, + "expected_float32_bytes": expected_float32_bytes, + }, + "reference_region": reference_region or {}, + "inputs": { + key: self._file_record(Path(self._path_to_windows(str(value)) or str(value))) + for key, value in inputs.items() + if value + }, + "outputs": { + "timeseries_dir": str(timeseries_dir), + **{name: self._file_record(path) for name, path in required_outputs.items()}, + }, + "size_checks": size_checks, + "log": self._file_record(log_path), + "log_tail": self._tail_text(log_path.read_text(encoding="utf-8", errors="replace")) if log_path.is_file() else "", + } + + def _build_publish_products_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + rlks: int, + inputs: dict[str, Any], + wavelength: Any, + ) -> dict[str, Any]: + reference = str(reference_date or "").strip() + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + export_dir = run_dir / "publish" / "geotiff" + rmli_par_path = Path(self._path_to_windows(str(inputs.get("reference_mli_par") or "")) or "") + if not rmli_par_path.is_file(): + rmli_par_path = run_dir / "work" / "gamma" / "mli" / f"{reference}.mli.par" + rmli_params = self._parse_gamma_params(rmli_par_path) + width = self._as_int(rmli_params.get("range_samples")) + lines = self._as_int(rmli_params.get("azimuth_lines")) + expected_float32_bytes = (width * lines * 4) if width and lines else None + required_outputs = { + "los_rate_toward_m_per_year_rdc": export_dir / "los_rate_toward_m_per_year.rdc", + "los_rate_away_m_per_year_rdc": export_dir / "los_rate_away_m_per_year.rdc", + "los_sigma_m_per_year_rdc": export_dir / "los_sigma_m_per_year.rdc", + "los_rate_toward_m_per_year_tif": export_dir / "los_rate_toward_m_per_year.tif", + "los_rate_away_m_per_year_tif": export_dir / "los_rate_away_m_per_year.tif", + "los_sigma_m_per_year_tif": export_dir / "los_sigma_m_per_year.tif", + "los_rate_toward_m_per_year_hls_bmp": export_dir / "los_rate_toward_m_per_year.hls.bmp", + "los_rate_toward_m_per_year_hls_rgb_tif": export_dir / "los_rate_toward_m_per_year.hls.geo_rgb.tif", + "los_rate_toward_m_per_year_hls_geo_preview": export_dir / "los_rate_toward_m_per_year.hls.geo_preview.png", + "los_sigma_m_per_year_cc_bmp": export_dir / "los_sigma_m_per_year.cc.bmp", + "los_sigma_m_per_year_cc_rgb_tif": export_dir / "los_sigma_m_per_year.cc.geo_rgb.tif", + "los_sigma_m_per_year_cc_geo_preview": export_dir / "los_sigma_m_per_year.cc.geo_preview.png", + "los_rate_toward_mm_per_year_rdc": export_dir / "los_rate_toward_mm_per_year.rdc", + "los_rate_away_mm_per_year_rdc": export_dir / "los_rate_away_mm_per_year.rdc", + "los_sigma_mm_per_year_rdc": export_dir / "los_sigma_mm_per_year.rdc", + "los_rate_toward_mm_per_year_tif": export_dir / "los_rate_toward_mm_per_year.tif", + "los_rate_away_mm_per_year_tif": export_dir / "los_rate_away_mm_per_year.tif", + "los_sigma_mm_per_year_tif": export_dir / "los_sigma_mm_per_year.tif", + "los_rate_toward_mm_per_year_geo_preview": export_dir / "los_rate_toward_mm_per_year.geo_preview.png", + "los_sigma_mm_per_year_geo_preview": export_dir / "los_sigma_mm_per_year.geo_preview.png", + "los_rate_toward_mm_per_year_bmp": export_dir / "los_rate_toward_mm_per_year.bmp", + "los_sigma_mm_per_year_bmp": export_dir / "los_sigma_mm_per_year.bmp", + "ts_rate_rad_per_year_tif": export_dir / "ts_rate_rad_per_year.tif", + "sigma_rate_rad_per_year_tif": export_dir / "sigma_rate_rad_per_year.tif", + } + optional_outputs = { + "sigma_ts_rad_tif": export_dir / "sigma_ts_rad.tif", + "hgt_correction_m_tif": export_dir / "hgt_correction_m.tif", + "los_rate_m_per_year_tif": export_dir / "los_rate_m_per_year.tif", + "los_rate_away_m_per_year_hls_bmp": export_dir / "los_rate_away_m_per_year.hls.bmp", + } + missing_outputs = [ + name for name, path in required_outputs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + rdc_size_checks = {} + for key in ( + "los_rate_toward_m_per_year_rdc", + "los_rate_away_m_per_year_rdc", + "los_sigma_m_per_year_rdc", + "los_rate_toward_mm_per_year_rdc", + "los_rate_away_mm_per_year_rdc", + "los_sigma_mm_per_year_rdc", + ): + path = required_outputs[key] + size = path.stat().st_size if path.is_file() else 0 + rdc_size_checks[key] = { + "size_bytes": size, + "expected_float32_bytes": expected_float32_bytes, + "size_matches_reference_geometry": ( + expected_float32_bytes is None + or (size > 0 and size == expected_float32_bytes) + ), + } + quality_stats = {} + if width and lines: + quality_stats = { + "los_rate_toward_mm_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_rate_toward_mm_per_year_rdc"], + width=width, + lines=lines, + ), + "los_rate_toward_m_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_rate_toward_m_per_year_rdc"], + width=width, + lines=lines, + ), + "los_rate_away_mm_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_rate_away_mm_per_year_rdc"], + width=width, + lines=lines, + ), + "los_rate_away_m_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_rate_away_m_per_year_rdc"], + width=width, + lines=lines, + ), + "los_sigma_mm_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_sigma_mm_per_year_rdc"], + width=width, + lines=lines, + ), + "los_sigma_m_per_year_rdc": self._gamma_float32_stats( + required_outputs["los_sigma_m_per_year_rdc"], + width=width, + lines=lines, + ), + "ts_rate_rad_per_year_rdc": self._gamma_float32_stats( + Path(self._path_to_windows(str(inputs.get("ts_rate") or "")) or ""), + width=width, + lines=lines, + ), + "sigma_rate_rad_per_year_rdc": self._gamma_float32_stats( + Path(self._path_to_windows(str(inputs.get("sigma_rate") or "")) or ""), + width=width, + lines=lines, + ), + } + artifacts = self._build_run_artifacts(run_dir) + log_path = run_dir / "logs" / "publish_products.log" + ready = ( + not missing_outputs + and all(item.get("size_matches_reference_geometry") for item in rdc_size_checks.values()) + ) + product_summary = { + "schema": "insar.gamma-sbas-product-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "default_los_product": "los_rate_toward_m_per_year", + "los_sign_convention": "toward radar positive; away from radar negative", + "expert_color_conventions": { + "velocity": "rasdt_pwr ... -0.08 0.08 ... hls.cm, geocoded RGB browse from Gamma BMP", + "sigma": "rasdt_pwr ... cc.cm; production adapts the range to LOS sigma rate units", + "phase_and_atmosphere": "rasdt_pwr ... -6.28 6.28 ... rmg.cm", + }, + "geocoded_preview_rule": "primary web previews prefer expert Gamma geocoded RGB browse products; legacy PNG previews are retained for comparison", + "artifact_count": len(artifacts), + "artifacts": artifacts, + } + return { + "schema": "insar.gamma-sbas-publish-products-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "rlks": rlks, + "ready": ready, + "missing_outputs": missing_outputs, + "wavelength_m": self._as_float(wavelength), + "reference_geometry": { + "mli_par": str(rmli_par_path) if str(rmli_par_path) else None, + "range_samples": width, + "azimuth_lines": lines, + "expected_float32_bytes": expected_float32_bytes, + }, + "inputs": { + key: self._file_record(Path(self._path_to_windows(str(value)) or str(value))) + for key, value in inputs.items() + if value and key != "timeseries_dir" + }, + "outputs": { + "export_dir": str(export_dir), + **{name: self._file_record(path) for name, path in {**required_outputs, **optional_outputs}.items()}, + }, + "rdc_size_checks": rdc_size_checks, + "quality_summary": quality_stats, + "product_summary": product_summary, + "log": self._file_record(log_path), + "log_tail": self._tail_text(log_path.read_text(encoding="utf-8", errors="replace")) if log_path.is_file() else "", + } + + def _build_monitor_points_summary( + self, + run_dir: Path, + *, + monitor_points: dict[str, Any], + ) -> dict[str, Any]: + summary_path = run_dir / "monitor_points_summary.json" + summary = self._read_optional_json(summary_path) or {} + point_dir = run_dir / "publish" / "monitor_points" + monitor_outputs = [] + if point_dir.is_dir(): + for metadata_path in sorted(point_dir.glob("*_metadata.json")): + metadata = self._read_optional_json(metadata_path) or {} + point_id = str(metadata.get("point_id") or metadata_path.name.replace("_metadata.json", "")) + png_path = point_dir / f"{point_id}_timeseries.png" + csv_path = point_dir / f"{point_id}_timeseries.csv" + monitor_outputs.append( + { + "point_id": point_id, + "metadata": metadata, + "files": { + "png": self._file_record(png_path), + "csv": self._file_record(csv_path), + "metadata": self._file_record(metadata_path), + }, + } + ) + if not summary: + summary = { + "schema": "insar.gamma-sbas-monitor-points-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "mode": ((self._read_optional_json(run_dir / "monitor_points.json") or {}).get("mode")), + "reference_date": monitor_points.get("reference_date"), + } + summary["monitor_outputs"] = monitor_outputs + summary["ready"] = bool(monitor_outputs) and all( + (item.get("files") or {}).get("png", {}).get("exists") + and (item.get("files") or {}).get("csv", {}).get("exists") + and (item.get("files") or {}).get("metadata", {}).get("exists") + for item in monitor_outputs + ) + log_path = run_dir / "logs" / "monitor_points.log" + summary["log"] = self._file_record(log_path) + summary["log_tail"] = self._tail_text(log_path.read_text(encoding="utf-8", errors="replace")) if log_path.is_file() else "" + return summary + @staticmethod def _tail_text(value: Any, length: int = 4000) -> str: if value is None: @@ -1942,6 +7026,47 @@ class SbasInsarProductionService: text = str(value) return text[-length:] + @staticmethod + def _read_text_rows(path: Path) -> list[str]: + if not path.is_file(): + return [] + return [ + line.strip() + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() + if line.strip() + ] + + def _detrend_pair_plan_from_diff_tab(self, diff_tab: Path, *, rlks: int) -> list[dict[str, Any]]: + pairs: list[dict[str, Any]] = [] + for row in self._read_text_rows(diff_tab): + unw = Path(self._path_to_windows(row.split()[0]) or row.split()[0]) + pair_dir = unw.parent + name = unw.name + suffix = f"_{rlks}rlks.diff_filt.unw" + pair_id = name[:-len(suffix)] if name.endswith(suffix) else name.replace(".diff_filt.unw", "") + parts = pair_id.split("_") + master_date = parts[0] if len(parts) >= 2 else "" + slave_date = parts[1] if len(parts) >= 2 else "" + cor = pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.cor" + offset = pair_dir / f"{pair_id}_{rlks}rlks.off" + pairs.append( + { + "pair_id": pair_id, + "master_date": master_date, + "slave_date": slave_date, + "unw": str(unw), + "cor": str(cor), + "offset": str(offset), + "expected_atmsub": str( + diff_tab.parent + / "detrend_atm" + / pair_id + / f"{pair_id}_{rlks}rlks.diff_filt.unw.atmsub" + ), + } + ) + return pairs + @staticmethod def _parse_bperp_table(path: Path) -> list[dict[str, Any]]: if not path.is_file(): @@ -1984,6 +7109,51 @@ class SbasInsarProductionService: continue return rows + @staticmethod + def _stack_dates(stack_manifest: dict[str, Any]) -> list[str]: + return [ + str(scene.get("date") or "").strip() + for scene in sorted(stack_manifest.get("scenes") or [], key=lambda item: str(item.get("date") or "")) + if str(scene.get("date") or "").strip() + ] + + def _build_interferogram_pair_plan( + self, + run_dir: Path, + *, + reference_date: str, + approved_itab: Path, + dates: list[str], + rlks: int, + ) -> list[dict[str, Any]]: + itab_rows = self._parse_itab(approved_itab) + pair_plan: list[dict[str, Any]] = [] + common_dir = run_dir / "work" / "gamma" / f"common_{reference_date}" + for row in itab_rows: + if len(row) < 4: + continue + master_index = row[0] - 1 + slave_index = row[1] - 1 + if master_index < 0 or slave_index < 0 or master_index >= len(dates) or slave_index >= len(dates): + continue + master_date = dates[master_index] + slave_date = dates[slave_index] + pair_id = f"{master_date}_{slave_date}" + pair_dir = common_dir / "diff" / pair_id + pair_plan.append( + { + "pair_id": pair_id, + "master_date": master_date, + "slave_date": slave_date, + "itab_row": row, + "pair_index": row[2], + "expected_unw": str(pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.unw"), + "expected_cor": str(pair_dir / f"{pair_id}_{rlks}rlks.diff_filt.cor"), + "log_path": str(run_dir / "logs" / f"{pair_id}_diff_unwrap_common.log"), + } + ) + return pair_plan + def _refresh_command_manifest_after_baseline( self, run_dir: Path, @@ -2059,6 +7229,233 @@ class SbasInsarProductionService: command_manifest["next_stage"] = run_manifest.get("next_stage") self._write_json(path, command_manifest) + def _refresh_command_manifest_after_rdc_dem( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "coregistration" and status in { + "RDC_DEM_SCRIPT_READY", + "RDC_DEM_RUNNING", + "RDC_DEM_READY", + "RDC_DEM_FAILED", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "rdc_dem": + if status == "RDC_DEM_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "RDC_DEM_RUNNING": + stage["status"] = "RUNNING" + elif status == "RDC_DEM_READY": + stage["status"] = "COMPLETED" + elif status == "RDC_DEM_FAILED": + stage["status"] = "FAILED" + elif status in {"BASELINE_AUDIT_READY", "ITAB_APPROVED", "COREGISTRATION_SCRIPT_READY", "COREGISTRATION_READY"}: + stage["status"] = "READY" + if stage.get("stage_id") == "interferograms" and status == "RDC_DEM_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["rdc_dem"] = run_manifest.get("rdc_dem") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_interferograms( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "rdc_dem" and status in { + "INTERFEROGRAMS_SCRIPT_READY", + "INTERFEROGRAMS_RUNNING", + "INTERFEROGRAMS_READY", + "INTERFEROGRAMS_FAILED", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "interferograms": + if status == "INTERFEROGRAMS_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "INTERFEROGRAMS_RUNNING": + stage["status"] = "RUNNING" + elif status == "INTERFEROGRAMS_READY": + stage["status"] = "COMPLETED" + elif status == "INTERFEROGRAMS_FAILED": + stage["status"] = "FAILED" + elif status == "RDC_DEM_READY": + stage["status"] = "READY" + if stage.get("stage_id") == "detrend_atm" and status == "INTERFEROGRAMS_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["rdc_dem"] = run_manifest.get("rdc_dem") + command_manifest["interferograms"] = run_manifest.get("interferograms") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_detrend_atm( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "interferograms" and status in { + "DETREND_ATM_SCRIPT_READY", + "DETREND_ATM_RUNNING", + "DETREND_ATM_READY", + "DETREND_ATM_FAILED", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "detrend_atm": + if status == "DETREND_ATM_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "DETREND_ATM_RUNNING": + stage["status"] = "RUNNING" + elif status == "DETREND_ATM_READY": + stage["status"] = "COMPLETED" + elif status == "DETREND_ATM_FAILED": + stage["status"] = "FAILED" + elif status == "INTERFEROGRAMS_READY": + stage["status"] = "READY" + if stage.get("stage_id") == "ipta_timeseries" and status == "DETREND_ATM_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["rdc_dem"] = run_manifest.get("rdc_dem") + command_manifest["interferograms"] = run_manifest.get("interferograms") + command_manifest["detrend_atm"] = run_manifest.get("detrend_atm") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_ipta_timeseries( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "detrend_atm" and status in { + "IPTA_TIMESERIES_SCRIPT_READY", + "IPTA_TIMESERIES_RUNNING", + "IPTA_TIMESERIES_READY", + "IPTA_TIMESERIES_FAILED", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "ipta_timeseries": + if status == "IPTA_TIMESERIES_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "IPTA_TIMESERIES_RUNNING": + stage["status"] = "RUNNING" + elif status == "IPTA_TIMESERIES_READY": + stage["status"] = "COMPLETED" + elif status == "IPTA_TIMESERIES_FAILED": + stage["status"] = "FAILED" + elif status == "DETREND_ATM_READY": + stage["status"] = "READY" + if stage.get("stage_id") == "publish_products" and status == "IPTA_TIMESERIES_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["rdc_dem"] = run_manifest.get("rdc_dem") + command_manifest["interferograms"] = run_manifest.get("interferograms") + command_manifest["detrend_atm"] = run_manifest.get("detrend_atm") + command_manifest["ipta_timeseries"] = run_manifest.get("ipta_timeseries") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_publish_products( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "ipta_timeseries" and status in { + "PUBLISH_PRODUCTS_SCRIPT_READY", + "PUBLISH_PRODUCTS_RUNNING", + "PRODUCTS_READY", + "PUBLISH_PRODUCTS_FAILED", + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "publish_products": + if status == "PUBLISH_PRODUCTS_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "PUBLISH_PRODUCTS_RUNNING": + stage["status"] = "RUNNING" + elif status == "PRODUCTS_READY": + stage["status"] = "COMPLETED" + elif status == "PUBLISH_PRODUCTS_FAILED": + stage["status"] = "FAILED" + elif status == "IPTA_TIMESERIES_READY": + stage["status"] = "READY" + if stage.get("stage_id") == "monitor_points" and status == "PRODUCTS_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["rdc_dem"] = run_manifest.get("rdc_dem") + command_manifest["interferograms"] = run_manifest.get("interferograms") + command_manifest["detrend_atm"] = run_manifest.get("detrend_atm") + command_manifest["ipta_timeseries"] = run_manifest.get("ipta_timeseries") + command_manifest["publish_products"] = run_manifest.get("publish_products") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_monitor_points( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "publish_products" and status in { + "MONITOR_POINTS_SCRIPT_READY", + "MONITOR_POINTS_RUNNING", + "MONITOR_POINTS_READY", + "MONITOR_POINTS_FAILED", + }: + stage["status"] = "COMPLETED" + if stage.get("stage_id") == "monitor_points": + if status == "MONITOR_POINTS_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif status == "MONITOR_POINTS_RUNNING": + stage["status"] = "RUNNING" + elif status == "MONITOR_POINTS_READY": + stage["status"] = "COMPLETED" + elif status == "MONITOR_POINTS_FAILED": + stage["status"] = "FAILED" + elif status == "PRODUCTS_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["publish_products"] = run_manifest.get("publish_products") + command_manifest["monitor_point_products"] = run_manifest.get("monitor_point_products") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + def _build_command_manifest(self, run_manifest: dict[str, Any], stack_manifest: dict[str, Any]) -> dict[str, Any]: scenes = stack_manifest.get("scenes") or [] pair_network = stack_manifest.get("pair_network") or {} @@ -2070,6 +7467,7 @@ class SbasInsarProductionService: "execution_enabled": False, "reason_execution_disabled": "The managed Gamma runner is intentionally not attached in this planning slice.", "stage_plan": [dict(item) for item in GAMMA_STAGE_PLAN], + "expert_document_steps": [dict(item) for item in GAMMA_SBAS_EXPERT_DOCUMENT_STEPS], "inputs": { "scene_count": len(scenes), "scenes": [ @@ -2123,10 +7521,43 @@ class SbasInsarProductionService: ("itab_decision.json", "Approved/rejected itab decision", "itab_decision"), ("coregistration_plan.json", "Coregistration stage plan", "coregistration_plan"), ("coregistration_summary.json", "Coregistration execution summary", "coregistration_summary"), + ("rdc_dem_plan.json", "RDC DEM stage plan", "rdc_dem_plan"), + ("rdc_dem_summary.json", "RDC DEM execution summary", "rdc_dem_summary"), + ("interferogram_plan.json", "Interferogram stage plan", "interferogram_plan"), + ("interferogram_summary.json", "Interferogram execution summary", "interferogram_summary"), + ("detrend_atm_plan.json", "Detrend/atmospheric correction stage plan", "detrend_atm_plan"), + ("detrend_atm_summary.json", "Detrend/atmospheric correction execution summary", "detrend_atm_summary"), + ("ipta_timeseries_plan.json", "IPTA time-series stage plan", "ipta_timeseries_plan"), + ("ipta_timeseries_summary.json", "IPTA time-series execution summary", "ipta_timeseries_summary"), + ("publish_product_plan.json", "Publish product stage plan", "publish_product_plan"), + ("publish_product_summary.json", "Publish product execution summary", "publish_product_summary"), + ("product_summary.json", "Published SBAS product summary", "product_summary"), + ("quality_summary.json", "Published SBAS quality summary", "quality_summary"), + ("monitor_points_plan.json", "Monitoring-point extraction plan", "monitor_points_plan"), + ("monitor_points_summary.json", "Monitoring-point extraction summary", "monitor_points_summary"), + ("workflow_summary.json", "Gamma SBAS workflow summary", "workflow_summary"), ("gamma_command_manifest.json", "Gamma command manifest", "command_manifest"), ("monitor_points.json", "Monitoring-point configuration", "monitor_points"), ("scripts/01_baseline_audit.sh", "Gamma baseline audit script", "baseline_audit_script"), ("scripts/02_coreg_common_ref.sh", "Gamma common-reference coregistration script", "coregistration_script"), + ("scripts/03_prepare_rdc_dem.sh", "Gamma RDC DEM script", "rdc_dem_script"), + ("scripts/04_diff_unwrap_common_ref.sh", "Gamma differential interferogram script", "interferogram_script"), + ("scripts/05_detrend_atm.sh", "Gamma detrend/atmospheric correction script", "detrend_atm_script"), + ("scripts/05_mb_ts_rate.sh", "Gamma IPTA mb/ts_rate script", "ipta_timeseries_script"), + ("scripts/07_publish_products.sh", "Gamma product publishing script", "publish_products_script"), + ("scripts/08_point_timeseries.sh", "Monitoring-point time-series script", "monitor_points_script"), + ("scripts/01_workspace_data.sh", "Expert section 1 workspace/data script", "expert_workflow_script"), + ("scripts/02_import_lt1_slc.sh", "Expert section 2 LT1 SLC import script", "expert_workflow_script"), + ("scripts/03_reference_mli.sh", "Expert section 3 reference MLI script", "expert_workflow_script"), + ("scripts/04_dem_lookup.sh", "Expert section 4 DEM lookup script", "expert_workflow_script"), + ("scripts/05_coreg_prep.sh", "Expert section 5 coregistration prep script", "expert_workflow_script"), + ("scripts/06_coregister_scenes.sh", "Expert section 6 coregister scenes script", "expert_workflow_script"), + ("scripts/07_rmli_average.sh", "Expert section 7 RMLI average script", "expert_workflow_script"), + ("scripts/08_diff_network.sh", "Expert section 8 differential network script", "expert_workflow_script"), + ("scripts/09_filter_unwrap.sh", "Expert section 9 filter and unwrap script", "expert_workflow_script"), + ("scripts/10_detrend_atm.sh", "Expert section 10 detrend/ATM script", "expert_workflow_script"), + ("scripts/11_sbas_inversion.sh", "Expert section 11 SBAS inversion script", "expert_workflow_script"), + ("scripts/12_outputs_points.sh", "Expert section 12 outputs and points script", "expert_workflow_script"), ]: path = run_dir / relative_path if path.is_file(): @@ -2139,6 +7570,34 @@ class SbasInsarProductionService: "size_bytes": path.stat().st_size, } ) + for item in PRODUCT_DEFINITIONS: + if item["key"] == "trial_summary_json": + continue + path = run_dir / item["relative_path"] + if path.is_file(): + artifacts.append( + { + **item, + "size_bytes": path.stat().st_size, + } + ) + monitor_dir = run_dir / "publish" / "monitor_points" + if monitor_dir.is_dir(): + for path in sorted(monitor_dir.iterdir()): + if not path.is_file(): + continue + for suffix_key, label, ext in MONITOR_ARTIFACT_SUFFIXES: + if path.name.endswith(ext): + artifacts.append( + { + "key": f"monitor_{path.stem}_{suffix_key}", + "label": label, + "role": "monitor_point", + "relative_path": str(path.relative_to(run_dir)).replace("\\", "/"), + "size_bytes": path.stat().st_size, + } + ) + break return artifacts def _build_trial_card(self, trial_dir: Path, summary: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/app/services/wsl_runtime_registry.py b/backend/app/services/wsl_runtime_registry.py index 60f1543..4fbca40 100644 --- a/backend/app/services/wsl_runtime_registry.py +++ b/backend/app/services/wsl_runtime_registry.py @@ -112,6 +112,7 @@ def build_wsl_runtime_registry() -> WslRuntimeRegistry: isce2_runner_windows = _project_file_windows("deploy", "wsl", "runners", "isce2_runner.py") gamma_runner_windows = _project_file_windows("deploy", "wsl", "runners", "gamma_pyint_runner.py") + gamma_sbas_runner_windows = _project_file_windows("deploy", "wsl", "runners", "gamma_sbas_runner.py") gamma_profile_windows = _project_file_windows("deploy", "wsl", "profiles", "gamma_env.sh") runtimes = { @@ -151,6 +152,33 @@ def build_wsl_runtime_registry() -> WslRuntimeRegistry: "legacy_profile_env_var": "PYINT_GAMMA_ENV_SCRIPT", }, ), + settings.GAMMA_SBAS_RUNTIME_ID: WslRuntimeDefinition( + runtime_id=settings.GAMMA_SBAS_RUNTIME_ID, + engine_code="gamma", + display_name="Gamma SBAS Runtime V1", + distro=str(settings.GAMMA_SBAS_WSL_DISTRO or shared_distro).strip() or shared_distro, + conda_env_name=shared_conda_env, + python_path=str(settings.GAMMA_SBAS_PYTHON or shared_python_path).strip() or shared_python_path, + runner_path_windows=gamma_sbas_runner_windows, + runner_path_wsl=_windows_path_to_wsl_mount(gamma_sbas_runner_windows), + allowed_operations=("lt1_gamma_sbas_workflow", "lt1_gamma_sbas_step"), + env_profile_path_windows=str(settings.GAMMA_SBAS_ENV_SCRIPT or gamma_profile_windows).strip(), + env_profile_path_wsl=_windows_path_to_wsl_mount( + str(settings.GAMMA_SBAS_ENV_SCRIPT or gamma_profile_windows).strip() + ), + metadata_json={ + "shared_runtime": True, + "workflow_code": "sbas_insar", + "processor_code": "gamma_ipta_sbas", + "env_vars": { + "runtime_id": "GAMMA_SBAS_RUNTIME_ID", + "python": "GAMMA_SBAS_PYTHON", + "env_profile": "GAMMA_SBAS_ENV_SCRIPT", + "work_root": "GAMMA_SBAS_WORK_ROOT", + "product_root": "GAMMA_SBAS_PRODUCT_ROOT", + }, + }, + ), } return WslRuntimeRegistry( diff --git a/deploy/wsl/runners/gamma_sbas_product_tools.py b/deploy/wsl/runners/gamma_sbas_product_tools.py new file mode 100644 index 0000000..e41ae20 --- /dev/null +++ b/deploy/wsl/runners/gamma_sbas_product_tools.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import csv +import json +import math +import re +import struct +from datetime import datetime +from pathlib import Path +from typing import Any + +import numpy as np + + +def read_gamma_value(path: Path, key: str) -> str: + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + parts = line.split() + if parts and parts[0].rstrip(":") == key.rstrip(":"): + return parts[1] + raise KeyError(f"{key} not found in {path}") + + +def read_float32(path: Path, shape: tuple[int, int] | None = None) -> np.ndarray: + data = np.fromfile(path, dtype=">f4") + if shape is not None: + data = data.reshape(shape) + return data + + +def read_float32_pixel(path: Path, width: int, x: int, y: int) -> float: + with path.open("rb") as handle: + handle.seek((y * width + x) * 4) + chunk = handle.read(4) + if len(chunk) != 4: + return float("nan") + return float(struct.unpack(">f", chunk)[0]) + + +def write_scaled_float32(input_path: Path, output_path: Path, scale: float) -> None: + data = np.fromfile(input_path, dtype=">f4") + output_path.parent.mkdir(parents=True, exist_ok=True) + (data * float(scale)).astype(">f4", copy=False).tofile(output_path) + + +def pick_auto_point(rate: np.ndarray, sigma: np.ndarray) -> tuple[int, int]: + lines, width = rate.shape + yy, xx = np.indices(rate.shape) + edge_mask = ( + (xx > width * 0.1) + & (xx < width * 0.9) + & (yy > lines * 0.1) + & (yy < lines * 0.9) + ) + finite = np.isfinite(rate) & np.isfinite(sigma) + valid = finite & edge_mask & (rate != 0.0) & (sigma > 0.0) + if not valid.any(): + raise RuntimeError("No valid pixels available for monitor point selection") + + abs_rate = np.abs(rate[valid]) + sig = sigma[valid] + rate_min = np.percentile(abs_rate, 85) + rate_max = np.percentile(abs_rate, 99) + sigma_max = np.percentile(sig, 40) + candidate = valid & (np.abs(rate) >= rate_min) & (np.abs(rate) <= rate_max) & (sigma <= sigma_max) + if not candidate.any(): + candidate = valid + + score = np.zeros(rate.shape, dtype=np.float32) + score[candidate] = np.abs(rate[candidate]) / (sigma[candidate] + 1.0e-6) + y, x = np.unravel_index(int(np.argmax(score)), rate.shape) + return int(x), int(y) + + +def dem_grid(dem_par: Path) -> dict[str, float | int]: + return { + "width": int(read_gamma_value(dem_par, "width")), + "nlines": int(read_gamma_value(dem_par, "nlines")), + "corner_lon": float(read_gamma_value(dem_par, "corner_lon")), + "corner_lat": float(read_gamma_value(dem_par, "corner_lat")), + "post_lon": float(read_gamma_value(dem_par, "post_lon")), + "post_lat": float(read_gamma_value(dem_par, "post_lat")), + } + + +def radar_to_lonlat(x: int, y: int, dem_par: Path, lookup: Path) -> tuple[float | None, float | None]: + grid = dem_grid(dem_par) + width = int(grid["width"]) + lines = int(grid["nlines"]) + lut = np.fromfile(lookup, dtype=">c8").reshape((lines, width)) + rng = lut.real + az = lut.imag + valid = np.isfinite(rng) & np.isfinite(az) & (rng > 0.0) & (az > 0.0) + if not valid.any(): + return None, None + + distance = np.full(rng.shape, np.inf, dtype=np.float32) + distance[valid] = (rng[valid] - float(x)) ** 2 + (az[valid] - float(y)) ** 2 + gy, gx = np.unravel_index(int(np.argmin(distance)), distance.shape) + lon = float(grid["corner_lon"]) + (gx + 0.5) * float(grid["post_lon"]) + lat = float(grid["corner_lat"]) + (gy + 0.5) * float(grid["post_lat"]) + return float(lon), float(lat) + + +def lonlat_to_radar(lon: float, lat: float, dem_par: Path, lookup: Path) -> tuple[int, int]: + grid = dem_grid(dem_par) + width = int(grid["width"]) + lines = int(grid["nlines"]) + gx = int(round((lon - float(grid["corner_lon"])) / float(grid["post_lon"]) - 0.5)) + gy = int(round((lat - float(grid["corner_lat"])) / float(grid["post_lat"]) - 0.5)) + gx = max(0, min(width - 1, gx)) + gy = max(0, min(lines - 1, gy)) + lut = np.fromfile(lookup, dtype=">c8").reshape((lines, width)) + value = lut[gy, gx] + if not (np.isfinite(value.real) and np.isfinite(value.imag) and value.real > 0 and value.imag > 0): + raise RuntimeError(f"manual lon/lat maps to invalid lookup pixel: lon={lon}, lat={lat}") + return int(round(float(value.real))), int(round(float(value.imag))) + + +def safe_point_id(value: str, fallback: str) -> str: + text = str(value or "").strip() or fallback + text = re.sub(r"[^A-Za-z0-9_.-]+", "_", text)[:64] + return text or fallback + + +def load_diff_files(timeseries_dir: Path) -> list[Path]: + tab = timeseries_dir / "diff_ts.tab" + if tab.is_file(): + rows = [line.strip() for line in tab.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()] + files = [Path(row.split()[0]) for row in rows if row.split()] + files = [path for path in files if path.is_file()] + if files: + return files + return sorted(timeseries_dir.glob("diff_ts_*.diff")) + + +def point_records( + diff_files: list[Path], + *, + dates: list[str], + width: int, + x: int, + y: int, + scale_mm: float, +) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for index, path in enumerate(diff_files): + phase = read_float32_pixel(path, width, x, y) + away_mm = float(phase * scale_mm) if math.isfinite(phase) else float("nan") + records.append( + { + "date": dates[index] if index < len(dates) else f"step_{index + 1:03d}", + "phase_rad": float(phase), + "los_away_mm": away_mm, + "los_toward_mm": -away_mm if math.isfinite(away_mm) else float("nan"), + } + ) + return records + + +def write_point_outputs( + point_dir: Path, + point: dict[str, Any], + *, + records: list[dict[str, Any]], + rate_value: float, + sigma_value: float, + wavelength: float, + reference_date: str, +) -> dict[str, str]: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + point_id = str(point["point_id"]) + csv_path = point_dir / f"{point_id}_timeseries.csv" + json_path = point_dir / f"{point_id}_metadata.json" + png_path = point_dir / f"{point_id}_timeseries.png" + + with csv_path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=["date", "phase_rad", "los_away_mm", "los_toward_mm"]) + writer.writeheader() + writer.writerows(records) + + metadata = { + "schema": "insar.sbas-monitor-point/v1", + "point_id": point_id, + "selection": point.get("selection"), + "radar_pixel": {"range": int(point["range_pixel"]), "azimuth": int(point["azimuth_line"])}, + "approx_lonlat": {"lon": point.get("lon"), "lat": point.get("lat")}, + "reference_date": reference_date, + "los_convention": "toward radar positive; away from radar negative", + "los_rate_toward_mm_per_year": rate_value, + "los_sigma_mm_per_year": sigma_value, + "wavelength_m": wavelength, + "records": records, + } + json_path.write_text(json.dumps(metadata, indent=2, ensure_ascii=False), encoding="utf-8") + + dates = [record["date"] for record in records] + disp = [record["los_toward_mm"] for record in records] + plt.figure(figsize=(8.0, 4.6), dpi=160) + plt.plot(dates, disp, marker="o", linewidth=2.0, color="#1f77b4") + plt.axhline(0, color="#666666", linewidth=0.8) + plt.grid(True, color="#dddddd", linewidth=0.7) + plt.title( + f"LOS displacement time series ({point_id})\n" + f"toward radar positive, rate={rate_value:.2f} mm/yr, sigma={sigma_value:.2f} mm/yr", + fontsize=10, + ) + plt.xlabel("Date") + plt.ylabel("LOS displacement (mm)") + plt.tight_layout() + plt.savefig(png_path) + plt.close() + return {"png": str(png_path), "csv": str(csv_path), "metadata": str(json_path)} + + +def run_phase_to_los(args: argparse.Namespace) -> int: + write_scaled_float32(Path(args.input), Path(args.output), float(args.scale)) + return 0 + + +def run_monitor_points(args: argparse.Namespace) -> int: + timeseries_dir = Path(args.timeseries_dir) + export_dir = Path(args.export_dir) + point_dir = Path(args.point_dir) + mli_par = Path(args.mli_par) + slc_par = Path(args.slc_par) if args.slc_par else mli_par + dem_par = Path(args.dem_par) + lookup = Path(args.lookup) + monitor_config_path = Path(args.monitor_config) + summary_path = Path(args.summary_path) + point_dir.mkdir(parents=True, exist_ok=True) + + width = int(read_gamma_value(mli_par, "range_samples")) + lines = int(read_gamma_value(mli_par, "azimuth_lines")) + shape = (lines, width) + dates = [item.strip() for item in str(args.dates or "").split(",") if item.strip()] + reference_date = str(args.reference_date or "").strip() + + radar_freq = float(read_gamma_value(slc_par, "radar_frequency")) + wavelength = 299792458.0 / radar_freq + scale_mm = wavelength / (4.0 * math.pi) * 1000.0 + + rate_toward = read_float32(export_dir / "los_rate_toward_mm_per_year.rdc", shape) + sigma = read_float32(export_dir / "los_sigma_mm_per_year.rdc", shape) + diff_files = load_diff_files(timeseries_dir) + if not diff_files: + raise RuntimeError(f"No diff_ts files found in {timeseries_dir}") + + config = {} + if monitor_config_path.is_file(): + config = json.loads(monitor_config_path.read_text(encoding="utf-8")) + mode = str(config.get("mode") or "auto_low_sigma_high_rate") + selected_points: list[dict[str, Any]] = [] + + if mode == "manual_lonlat" and config.get("points"): + for index, raw in enumerate(config.get("points") or []): + lon = float(raw["lon"]) + lat = float(raw["lat"]) + x, y = lonlat_to_radar(lon, lat, dem_par, lookup) + selected_points.append( + { + "point_id": safe_point_id(raw.get("point_id"), f"manual_{index + 1:03d}"), + "selection": "manual_lonlat_nearest_lookup_pixel", + "range_pixel": x, + "azimuth_line": y, + "lon": lon, + "lat": lat, + } + ) + else: + x, y = pick_auto_point(rate_toward, sigma) + lon, lat = radar_to_lonlat(x, y, dem_par, lookup) + selected_points.append( + { + "point_id": "auto_low_sigma_high_rate", + "selection": "automatic_low_sigma_high_rate_non_edge", + "range_pixel": x, + "azimuth_line": y, + "lon": lon, + "lat": lat, + } + ) + + outputs: list[dict[str, Any]] = [] + for point in selected_points: + x = int(point["range_pixel"]) + y = int(point["azimuth_line"]) + if not (0 <= x < width and 0 <= y < lines): + raise ValueError(f"pixel out of bounds: x={x}, y={y}, width={width}, lines={lines}") + records = point_records(diff_files, dates=dates, width=width, x=x, y=y, scale_mm=scale_mm) + rate_value = float(rate_toward[y, x]) + sigma_value = float(sigma[y, x]) + files = write_point_outputs( + point_dir, + point, + records=records, + rate_value=rate_value, + sigma_value=sigma_value, + wavelength=wavelength, + reference_date=reference_date, + ) + outputs.append( + { + **point, + "los_rate_toward_mm_per_year": rate_value, + "los_sigma_mm_per_year": sigma_value, + "record_count": len(records), + "files": files, + } + ) + + summary = { + "schema": "insar.gamma-sbas-monitor-points-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "ready": bool(outputs), + "mode": mode, + "reference_date": reference_date, + "width": width, + "lines": lines, + "wavelength_m": wavelength, + "diff_ts_count": len(diff_files), + "date_count": len(dates), + "monitor_points": outputs, + } + summary_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False), encoding="utf-8") + print(json.dumps(summary, indent=2, ensure_ascii=False)) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Gamma SBAS product helper tools") + subparsers = parser.add_subparsers(dest="command", required=True) + + phase = subparsers.add_parser("phase-to-los") + phase.add_argument("input") + phase.add_argument("output") + phase.add_argument("scale", type=float) + phase.set_defaults(func=run_phase_to_los) + + monitor = subparsers.add_parser("monitor-points") + monitor.add_argument("--monitor-config", required=True) + monitor.add_argument("--timeseries-dir", required=True) + monitor.add_argument("--export-dir", required=True) + monitor.add_argument("--point-dir", required=True) + monitor.add_argument("--mli-par", required=True) + monitor.add_argument("--slc-par", required=True) + monitor.add_argument("--dem-par", required=True) + monitor.add_argument("--lookup", required=True) + monitor.add_argument("--dates", default="") + monitor.add_argument("--reference-date", default="") + monitor.add_argument("--summary-path", required=True) + monitor.set_defaults(func=run_monitor_points) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return int(args.func(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/wsl/runners/gamma_sbas_runner.py b/deploy/wsl/runners/gamma_sbas_runner.py new file mode 100644 index 0000000..dd486a2 --- /dev/null +++ b/deploy/wsl/runners/gamma_sbas_runner.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import argparse +import json +import os +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +TERMINAL_STATUSES = {"COMPLETED", "FAILED", "SKIPPED"} + + +def _utcnow() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _load_json(path: str | Path) -> dict[str, Any]: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def _write_json(path: str | Path, payload: dict[str, Any]) -> None: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _tail(text: str, limit: int = 4000) -> str: + if not text: + return "" + return text[-limit:] + + +def _normalize_step_ids(value: str | None) -> set[str]: + text = str(value or "").strip() + if not text: + return set() + return {item.strip() for item in text.replace(";", ",").split(",") if item.strip()} + + +def _load_workflow_manifest(broker_manifest: dict[str, Any]) -> tuple[dict[str, Any], Path]: + payload = broker_manifest.get("payload") or {} + workflow_manifest_path = payload.get("workflow_manifest_wsl") or payload.get("workflow_manifest") + if not workflow_manifest_path: + raise ValueError("payload.workflow_manifest_wsl is required") + manifest_path = Path(str(workflow_manifest_path)) + return _load_json(manifest_path), manifest_path + + +def _state_path(workflow_manifest: dict[str, Any]) -> Path: + state = workflow_manifest.get("state") or {} + explicit = state.get("step_status_path") + if explicit: + return Path(str(explicit)) + run_root = Path(str(workflow_manifest.get("run_root_wsl") or workflow_manifest.get("run_root") or ".")) + return run_root / "state" / "step_status.json" + + +def _script_env(workflow_manifest: dict[str, Any], step: dict[str, Any]) -> dict[str, str]: + env = os.environ.copy() + run_root = str(workflow_manifest.get("run_root_wsl") or workflow_manifest.get("run_root") or "") + params = workflow_manifest.get("params") or {} + env.update( + { + "GAMMA_SBAS_RUN_ROOT": run_root, + "GAMMA_SBAS_MANIFEST": str(workflow_manifest.get("manifest_path_wsl") or ""), + "GAMMA_SBAS_STEP_ID": str(step.get("id") or ""), + "GAMMA_SBAS_STEP_NAME": str(step.get("name") or step.get("id") or ""), + "GAMMA_SBAS_RLKS": str(params.get("rlks") or ""), + "GAMMA_SBAS_AZLKS": str(params.get("azlks") or ""), + "GAMMA_SBAS_MB_MODE": str(params.get("mb_mode") or ""), + "GAMMA_SBAS_REFERENCE_WINDOW": str(params.get("reference_window") or ""), + } + ) + for key, value in (step.get("env") or {}).items(): + env[str(key)] = str(value) + return env + + +def _selected_steps(steps: list[dict[str, Any]], only_steps: set[str], from_step: str | None, to_step: str | None) -> list[dict[str, Any]]: + if only_steps: + return [step for step in steps if str(step.get("id") or "") in only_steps] + if not from_step and not to_step: + return steps + + selected: list[dict[str, Any]] = [] + active = from_step is None + for step in steps: + step_id = str(step.get("id") or "") + if step_id == from_step: + active = True + if active: + selected.append(step) + if step_id == to_step: + break + return selected + + +def _run_step( + workflow_manifest: dict[str, Any], + step: dict[str, Any], + *, + state: dict[str, Any], + force: bool, + dry_run: bool, + timeout_seconds: int, +) -> dict[str, Any]: + step_id = str(step.get("id") or "").strip() + if not step_id: + raise ValueError("workflow step id must not be empty") + + if step.get("enabled") is False: + return { + "id": step_id, + "name": step.get("name") or step_id, + "status": "SKIPPED", + "skipped_reason": "step disabled in workflow manifest", + "started_at": _utcnow(), + "ended_at": _utcnow(), + } + + previous = (state.get("steps") or {}).get(step_id) or {} + if previous.get("status") == "COMPLETED" and not force: + return {**previous, "status": "SKIPPED", "skipped_reason": "already completed"} + + script = Path(str(step.get("script_wsl") or step.get("script") or "")) + if not script.is_file(): + raise FileNotFoundError(f"step script not found: {script}") + + log_path = Path(str(step.get("log_wsl") or step.get("log") or "")) + if not log_path: + run_root = Path(str(workflow_manifest.get("run_root_wsl") or ".")) + log_path = run_root / "logs" / f"{step_id}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + + started_at = _utcnow() + if dry_run: + return { + "id": step_id, + "name": step.get("name") or step_id, + "status": "DRY_RUN", + "script": str(script), + "log": str(log_path), + "started_at": started_at, + "ended_at": _utcnow(), + "returncode": None, + } + + proc = subprocess.run( + ["bash", str(script)], + cwd=str(script.parent), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + env=_script_env(workflow_manifest, step), + ) + log_path.write_text( + "\n".join( + [ + f"# step={step_id}", + f"# started_at={started_at}", + f"# ended_at={_utcnow()}", + f"# returncode={proc.returncode}", + "", + "## stdout", + proc.stdout or "", + "", + "## stderr", + proc.stderr or "", + ] + ), + encoding="utf-8", + ) + return { + "id": step_id, + "name": step.get("name") or step_id, + "status": "COMPLETED" if proc.returncode == 0 else "FAILED", + "script": str(script), + "log": str(log_path), + "started_at": started_at, + "ended_at": _utcnow(), + "returncode": proc.returncode, + "stdout_tail": _tail(proc.stdout), + "stderr_tail": _tail(proc.stderr), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Gamma SBAS manifest runner.") + parser.add_argument("--manifest", required=True, help="WSL path to broker manifest.") + parser.add_argument("--from-step", default="", help="First workflow step id to execute.") + parser.add_argument("--to-step", default="", help="Last workflow step id to execute.") + parser.add_argument("--only-steps", default="", help="Comma-separated step ids to execute.") + parser.add_argument("--force", action="store_true", help="Run completed steps again.") + parser.add_argument("--dry-run", action="store_true", help="Validate manifest and write dry-run state.") + parser.add_argument("--timeout-seconds", type=int, default=43200) + args = parser.parse_args() + + broker_manifest = _load_json(args.manifest) + workflow_manifest, workflow_manifest_path = _load_workflow_manifest(broker_manifest) + workflow_manifest["manifest_path_wsl"] = str(workflow_manifest_path) + + state_file = _state_path(workflow_manifest) + state = _load_json(state_file) if state_file.is_file() else { + "schema": "insar.gamma-sbas-step-status/v1", + "run_id": workflow_manifest.get("run_id"), + "steps": {}, + } + state.setdefault("steps", {}) + state["updated_at"] = _utcnow() + state["runner_manifest"] = args.manifest + + all_steps = list(workflow_manifest.get("steps") or []) + selected = _selected_steps( + all_steps, + _normalize_step_ids(args.only_steps), + str(args.from_step or "").strip() or None, + str(args.to_step or "").strip() or None, + ) + if not selected: + raise ValueError("no workflow steps selected") + + overall_rc = 0 + executed: list[str] = [] + for step in selected: + step_id = str(step.get("id") or "") + result = _run_step( + workflow_manifest, + step, + state=state, + force=args.force, + dry_run=args.dry_run, + timeout_seconds=max(60, int(args.timeout_seconds or 43200)), + ) + state["steps"][step_id] = result + state["updated_at"] = _utcnow() + _write_json(state_file, state) + executed.append(step_id) + if result.get("status") == "FAILED": + overall_rc = int(result.get("returncode") or 1) + break + + summary = { + "runner": "gamma_sbas_runtime_v1", + "operation": broker_manifest.get("operation"), + "workflow_manifest": str(workflow_manifest_path), + "state_path": str(state_file), + "executed_steps": executed, + "returncode": overall_rc, + "dry_run": bool(args.dry_run), + } + print(json.dumps(summary, ensure_ascii=False)) + return overall_rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/INDEX.md b/docs/INDEX.md index c49659d..83159d2 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -55,6 +55,8 @@ - [SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md](SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md) Gamma/LT1 SBAS-InSAR 独立生产页面与统一流水线收口设计。 +- [SBAS_INSAR_GEOJSON_RESULT_MANAGEMENT_DESIGN_20260527.md](SBAS_INSAR_GEOJSON_RESULT_MANAGEMENT_DESIGN_20260527.md) + SBAS-InSAR GeoJSON 覆盖范围、行政区/AOI 按需生产、独立结果管理和产品 catalog 设计。 - [TIMESERIES_LEGACY_DEPRECATION_20260521.md](TIMESERIES_LEGACY_DEPRECATION_20260521.md) 旧 ISCE2/MintPy 时序生产链路停用记录，定义当前入口隐藏、配置默认关闭和后续物理删除条件。 - [GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md](GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md) @@ -137,4 +139,4 @@ - 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*` 最后更新：2026-05-15 -最近修订：2026-05-21 +最近修订：2026-05-27 diff --git a/docs/SBAS_INSAR_GEOJSON_RESULT_MANAGEMENT_DESIGN_20260527.md b/docs/SBAS_INSAR_GEOJSON_RESULT_MANAGEMENT_DESIGN_20260527.md new file mode 100644 index 0000000..c5aec8f --- /dev/null +++ b/docs/SBAS_INSAR_GEOJSON_RESULT_MANAGEMENT_DESIGN_20260527.md @@ -0,0 +1,531 @@ +# SBAS-InSAR GeoJSON Coverage And Result Management Design + +Date: 2026-05-27 + +## 1. Purpose + +Gamma SBAS-InSAR production has completed an end-to-end twelve-node run. The next step is to make production and results usable by geography, not only by time sequence and run status. + +This design defines: + +- how production pages show approximate geographic coverage with GeoJSON/bbox; +- how users select administrative regions or AOI to discover and produce SBAS stacks on demand; +- how completed SBAS runs become result products with searchable geographic extent; +- how the results page should display LOS velocity, LOS sigma, and monitoring-point curves. + +## 2. Current Facts + +The current successful run already contains usable geographic metadata: + +```text +run_id = sbas_7537cc71c998 +stack center bucket = E129.2_N44.1 +stack bbox = 128.7690438245, 43.7486321624, 129.6293024728, 44.3582486206 +monitor point = 129.10207098755, 44.15041727515 +``` + +Available metadata sources: + +```text +stack_manifest.scenes[*].bbox +stack_manifest.scenes[*].center_lon / center_lat +stack_manifest.stack.center_bucket +rdc_dem_summary.dem_source.stack_bbox +monitor_points_summary.monitor_outputs[*].metadata.approx_lonlat +published GeoTIFF bounds from GDAL metadata +``` + +Existing platform capabilities: + +```text +backend/geojson/全国行政区.geojson +backend/geojson/层级映射.json +GET /api/aoi/regions/children +GET /api/aoi/regions/{tree_id}/geometry +backend AOI helpers for region_tree_id / GeoJSON / uploaded AOI parsing +frontend Leaflet map and L.geoJSON support +frontend App.jsx existing source-scene footprint and AOI overlay patterns +``` + +The missing piece is productized SBAS-specific coverage and catalog behavior. + +## 3. Design Principles + +1. SBAS production is temporal, but SBAS result consumption is geographic. +2. Production UI should answer "am I processing the right place?" before a long workflow is submitted. +3. Results UI should answer "where is this product, what time range does it cover, and can I download the main outputs?" +4. One completed SBAS run is one result product bundle, not one product per GeoTIFF. +5. GeoJSON/bbox coverage is enough for the first production UX. Full raster map rendering can come later. +6. Administrative-region filtering should reuse existing AOI infrastructure rather than introduce a parallel region system. + +## 4. Coverage Model + +### 4.1 Stack Coverage + +For stack discovery and production run display: + +```json +{ + "center": {"lon": 129.199, "lat": 44.053}, + "bbox": { + "min_lon": 128.769, + "min_lat": 43.749, + "max_lon": 129.629, + "max_lat": 44.358 + }, + "bbox_geojson": { + "type": "Feature", + "properties": {"role": "sbas_stack_bbox"}, + "geometry": { + "type": "Polygon", + "coordinates": [[ + [128.769, 43.749], + [129.629, 43.749], + [129.629, 44.358], + [128.769, 44.358], + [128.769, 43.749] + ]] + } + }, + "scene_bbox_count": 7, + "scene_footprints_geojson": null +} +``` + +First implementation may use stack bbox only. Per-scene rectangles can be added when the map needs to show coverage stability. + +### 4.2 Product Coverage + +For completed results: + +```json +{ + "coverage_source": "stack_manifest_bbox_union", + "geotiff_bounds_verified": true, + "bbox": {...}, + "center": {...}, + "footprint_geojson": {...}, + "administrative_hint": { + "province": "黑龙江省", + "city": null, + "county": null, + "method": "center_point_lookup" + } +} +``` + +The first administrative hint can be center-point based. Later it should become intersection-based and return all intersected regions with approximate overlap area. + +## 5. Administrative Region And AOI Production + +### 5.1 User Workflow + +Production page should support three AOI sources: + +```text +1. Administrative region selection + province -> city -> county, backed by /aoi/regions endpoints + +2. Map-drawn rectangle + converted to bbox GeoJSON + +3. Uploaded GeoJSON/SHP + reuse existing AOI parsing helpers +``` + +The user flow: + +```text +select AOI / administrative region +discover SBAS stack candidates +show candidate time density + geographic coverage +select candidate +create production Run +submit Gamma SBAS workflow +``` + +### 5.2 Discovery Filter + +Stack discovery should accept: + +```json +{ + "region_tree_id": "230000", + "aoi_geojson": {}, + "aoi_bbox": { + "min_lon": 128.7, + "min_lat": 43.7, + "max_lon": 129.7, + "max_lat": 44.4 + }, + "aoi_overlap_min": 0.0, + "stable_stack_overlap_min": 0.3 +} +``` + +Initial filtering can use bbox intersection: + +```text +candidate stack is valid when union_bbox intersects AOI bbox +``` + +Second-stage filtering should use polygon intersection: + +```text +candidate_score += common_stack_area_intersection_ratio +candidate_score += scene_count / temporal_density +candidate_score -= sparse_time_gap_penalty +``` + +### 5.3 Production Guardrails + +Before running workflow: + +```text +show bbox and administrative hint +show date list and max temporal gap +show DEM coverage status +warn when DEM covers center but not full stack bbox +warn when AOI overlap is low +``` + +The current run demonstrated a real issue: + +```text +DEM source covers stack center = true +DEM source covers full stack bbox = false +``` + +This must be visible in the production page before the user trusts the result. + +## 6. Production Page UI + +Add `Geographic Coverage` block to `SbasInsarProductionPanel`. + +Minimum fields: + +```text +Center: lon / lat +BBox: min_lon, min_lat, max_lon, max_lat +Scene footprints: count +DEM coverage: bbox / center +Administrative hint +Monitor points if generated +Actions: view on map, copy GeoJSON, zoom to footprint +``` + +Map preview options: + +```text +Phase 1: small unframed Leaflet map with rectangle overlay +Phase 2: shared main map overlay using existing App.jsx layer mechanisms +Phase 3: per-scene footprints and monitor points overlay +``` + +The production page should not become the result browser. It only provides enough geographic context to avoid wrong-location production. + +## 7. Result Management Module + +### 7.1 Module Boundary + +Create a separate SBAS-InSAR result management module: + +```text +navigation: Results / SBAS-InSAR Results +backend catalog: catalog_name = sbas_insar +product type: sbas_insar_bundle +source: completed Gamma SBAS production Run +``` + +Do not merge SBAS result semantics into D-InSAR result pages. Reuse the common catalog tables but keep service, API, and frontend module separate. + +### 7.2 Product Row + +One completed run becomes one product row: + +```text +result_products.catalog_name = sbas_insar +result_products.run_key = sbas run_id +result_products.engine_code = gamma +result_products.processor_code = gamma_ipta_sbas +result_products.display_name = platform / orbit / area / date span +``` + +Required product metadata: + +```text +platform +relative_orbit +orbit_direction +polarization +reference_date +start_date +end_date +scene_count +pair_count +bbox_min_lon / bbox_min_lat / bbox_max_lon / bbox_max_lat +center_lon / center_lat +administrative_hint +status +health_status +source_run_id +``` + +### 7.3 Asset Roles + +Primary assets from the expert document: + +```text +primary_velocity_geotiff + current: publish/geotiff/los_rate_toward_m_per_year.tif + expert equivalent: geo_los_def_rate.tif + +primary_velocity_preview + current: publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png + expert equivalent: los_def_rate.bmp / geo_los_def_rate.bmp + +primary_velocity_rgb_geotiff + current: publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif + expert equivalent: geo_los_def_rate_rgb.tif + +quality_sigma_geotiff + current: publish/geotiff/los_sigma_m_per_year.tif + expert equivalent: diff.sigma_ts / geo_diff.sigma_ts + +quality_sigma_preview + current: publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png + expert equivalent: diff.sigma_ts.masked.bmp with cc.cm + +monitor_timeseries_csv + current: publish/monitor_points/*_timeseries.csv + expert equivalent: disp_prt_2d output table + +monitor_timeseries_plot + current: publish/monitor_points/*_timeseries.png + +support_manifest + run_manifest.json, workflow_summary.json, gamma_command_manifest.json +``` + +Default result display should foreground: + +```text +LOS velocity preview +LOS sigma preview +monitor point curve if present +footprint map +``` + +Audit/support files should be grouped separately. + +## 8. Backend Design + +### 8.1 Service + +Add: + +```text +backend/app/services/sbas_insar_catalog_service.py +``` + +Responsibilities: + +```text +scan completed SBAS run directories +validate product bundle readiness +derive bbox/center/footprint GeoJSON +extract or verify GeoTIFF bounds +upsert result_products/result_assets/result_issues +provide list/detail/download APIs +bootstrap self-maintenance on startup +``` + +### 8.2 API + +Add: + +```text +GET /api/sbas-insar-products/catalog-status +POST /api/sbas-insar-products/rebuild-catalog +GET /api/sbas-insar-products +GET /api/sbas-insar-products/{product_id} +GET /api/sbas-insar-products/{product_id}/assets/{asset_id} +``` + +List filters: + +```text +date_from +date_to +reference_date +platform +relative_orbit +orbit_direction +status +health_status +region_tree_id +aoi_bbox +aoi_geojson +intersects_bbox +has_monitor_points +limit / offset +``` + +### 8.3 Run Detail Extension + +Extend production run detail: + +```json +{ + "geographic_coverage": { + "center": {"lon": 129.199, "lat": 44.053}, + "bbox": {"min_lon": 128.769, "min_lat": 43.749, "max_lon": 129.629, "max_lat": 44.358}, + "bbox_geojson": {}, + "scene_bbox_count": 7, + "dem_covers_stack_bbox": false, + "dem_covers_stack_center": true, + "monitor_points": [{"point_id": "auto_low_sigma_high_rate", "lon": 129.102, "lat": 44.150}] + } +} +``` + +### 8.4 Startup Self-Maintenance + +On startup: + +```text +ensure result catalog tables exist through existing maintenance +bootstrap sbas_insar catalog state +scan completed SBAS run publish bundles +upsert missing product rows +record issues for: + missing bbox + missing primary velocity GeoTIFF + missing primary preview + missing sigma GeoTIFF + missing sigma preview + missing monitor files when monitor summary says ready + DEM coverage mismatch +``` + +This matches the current self-maintenance direction used by D-InSAR and PsInSAR catalogs. + +## 9. Frontend Design + +### 9.1 Production Page + +Add: + +```text +GeographicCoveragePanel + bbox text + center text + administrative hint + DEM coverage status + mini map rectangle + copy GeoJSON + zoom/open on main map +``` + +Candidate stack cards should show: + +```text +date span +scene count +center bucket +center lon/lat +bbox short text +AOI overlap indicator when AOI is selected +``` + +### 9.2 Result Page + +Add: + +```text +SbasInsarProductsPanel +``` + +List page: + +```text +filters: date range, administrative region, map AOI, platform, orbit, status +cards/table: product name, date span, bbox/admin hint, scene/pair count, preview thumbnail, health badge +actions: open detail, zoom to map, download primary GeoTIFF +``` + +Detail page: + +```text +footprint map +LOS velocity preview +LOS sigma preview +monitor curve +key metadata table +asset table grouped by role +quality and issue summary +link to production Run +``` + +### 9.3 Map Layer Strategy + +Use GeoJSON for the first implementation: + +```text +bbox polygon for stack/product footprint +administrative region boundary layer from existing AOI endpoints +monitor point markers +optional per-scene footprints +``` + +Do not display RDC BMP as map layer. Only geocoded preview PNG/GeoTIFF-derived bounds should be used for map-oriented display. + +## 10. Implementation Sequence + +Recommended order: + +```text +1. Backend geographic_coverage in SBAS run detail. +2. Production page coverage block and bbox GeoJSON mini-map. +3. Add AOI/region filters to SBAS stack discovery request/response. +4. Add sbas_insar_catalog_service and catalog rebuild API. +5. Register current completed run as first SBAS result product. +6. Build SbasInsarProductsPanel list/detail. +7. Add map AOI filtering and administrative-region filtering to result page. +8. Add startup bootstrap/self-check output. +``` + +The first user-visible win is step 1-2: the operator can immediately see whether a Run covers the intended location. + +## 11. Validation + +Use `sbas_7537cc71c998` as the first validation run. + +Checks: + +```text +geographic_coverage.bbox is present +bbox_geojson draws a rectangle in Leaflet +admin hint is present or explicitly unknown +DEM coverage warning is visible when bbox is not fully covered +result product row exists after catalog rebuild +result detail opens velocity/sigma previews +asset downloads work +map zoom to footprint works +AOI filter returns this product when AOI intersects bbox +AOI filter excludes this product when AOI is far away +``` + +## 12. Open Questions + +1. Administrative-region naming should start with center-point lookup or intersection lookup? + Recommendation: center-point lookup first, intersection later. + +2. Should stack discovery require AOI overlap, or only rank by AOI overlap? + Recommendation: default to rank/filter by bbox intersection, expose minimum overlap later. + +3. Should results use `sbas_insar` or `psinsar` catalog name? + Recommendation: use `sbas_insar`. The current Gamma product is SBAS-InSAR, and old `psinsar` catalog semantics should not be overloaded. + +4. Should GeoTIFF raster be rendered on map immediately? + Recommendation: not in the first slice. Start with footprint GeoJSON and preview images; raster tile rendering can be added after catalog registration is stable. diff --git a/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md b/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md index 977613b..f4d7321 100644 --- a/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md +++ b/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md @@ -10,6 +10,34 @@ The old time-series pairing code may remain temporarily for compatibility and ca The legacy ISCE2/MintPy time-series production chain is disabled by default. The `timeseries-production` backend code and old catalog pages may remain as compatibility code, but they are no longer exposed as production-management subpages. The active SBAS production route is `/api/sbas-insar-production` and the active UI view is `sbas_insar_production`. +Update 2026-05-25: + +- The previous ad-hoc SBAS stage buttons are no longer the primary production design. +- The primary design follows the expert document: workspace directories, a run-level `manifest.json`, step scripts, `state/step_status.json`, and one Gamma SBAS workflow job. +- The already successful experiment is not discarded. Its verified Gamma commands are reused as the first bridge implementation while the scripts are moved toward expert-document templates. +- Old stage endpoints may remain temporarily for compatibility and for reading historical runs, but the UI should favor `Gamma SBAS Workflow`. + +Primary runtime configuration is now read from `.env` through: + +```text +GAMMA_SBAS_ENABLED +GAMMA_SBAS_RUNTIME_ID +GAMMA_SBAS_WSL_DISTRO +GAMMA_SBAS_PYTHON +GAMMA_SBAS_ENV_SCRIPT +GAMMA_SBAS_WORK_ROOT +GAMMA_SBAS_PRODUCT_ROOT +GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT +GAMMA_SBAS_DEFAULT_RLKS +GAMMA_SBAS_DEFAULT_AZLKS +GAMMA_SBAS_DEFAULT_MB_MODE +GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW +GAMMA_SBAS_STEP_TIMEOUT_SECONDS +GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS +``` + +Startup self-check now reports the Gamma SBAS runtime, work root, product root, Python interpreter, and WSL distro. Database self-maintenance still runs through the existing `ensure_database_ready()` startup path; no new SBAS-only database table is required for this slice. + ## Scope Initial production target: @@ -113,6 +141,50 @@ GET /api/sbas-insar-production/trial-runs/{trial_id} GET /api/sbas-insar-production/trial-runs/{trial_id}/artifacts/{relative_path} ``` +Primary Gamma SBAS workflow API: + +```text +POST /api/sbas-insar-production/runs/{run_id}/workflow +POST /api/sbas-insar-production/runs/{run_id}/workflow/jobs +``` + +`workflow` prepares the expert-document workspace and writes: + +```text +runs/{run_id}/workspace.json +runs/{run_id}/manifest.json +runs/{run_id}/state/step_status.json +runs/{run_id}/scripts/01_workspace_data.sh +runs/{run_id}/scripts/02_import_lt1_slc.sh +runs/{run_id}/scripts/03_reference_mli.sh +runs/{run_id}/scripts/04_dem_lookup.sh +runs/{run_id}/scripts/05_coreg_prep.sh +runs/{run_id}/scripts/06_coregister_scenes.sh +runs/{run_id}/scripts/07_rmli_average.sh +runs/{run_id}/scripts/08_diff_network.sh +runs/{run_id}/scripts/09_filter_unwrap.sh +runs/{run_id}/scripts/10_detrend_atm.sh +runs/{run_id}/scripts/11_sbas_inversion.sh +runs/{run_id}/scripts/12_outputs_points.sh +``` + +`workflow/jobs` submits one `SBAS_GAMMA_WORKFLOW` background job. The production workflow is now the twelve-section expert-document workflow. Several sections still reuse the already verified experiment scripts internally, but they are no longer hidden behind an eight-stage production view: + +```text +01_workspace_data -> expert workspace/data-layout check +02_import_lt1_slc -> par_LT1_SLC / orbit correction +03_reference_mli -> multi_look / reference MLI checks +04_dem_lookup -> DEM import, lookup table, RDC height +05_coreg_prep -> common-reference stack preparation +06_coregister_scenes -> scene coregistration +07_rmli_average -> RMLI tab/average intensity +08_diff_network -> baseline network and differential phase +09_filter_unwrap -> adaptive filtering, coherence and unwrap +10_detrend_atm -> quad_fit / atm_mod_2d / sub_phase +11_sbas_inversion -> mb / ts_rate, consuming DIFF_atmsub_tab +12_outputs_points -> geocode, browse products, monitoring curve +``` + Stack discovery and hard-constraint audit API: ```text @@ -403,17 +475,19 @@ backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/scripts/02_coreg_co backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/coregistration_plan.json ``` -Current status before executing the queued job: +The common-reference co-registration job was executed through the backend task/job queue: ```text -status = COREGISTRATION_SCRIPT_READY -next_stage = execute_coregistration -common reference date = 20241007 +task_id = 047092bb-2c57-4639-8e8b-89ce925d3273 +job_id = 15a66c9c-8814-4566-b299-12081f74ec09 +task status = COMPLETED +job status = COMPLETED +last_error = None ``` -The actual `SLC_coreg.py` execution is now wired as a background job endpoint and page action, but has not been production-tested in this pass. It consumes `itab_approved`, not the pre-audit or unapproved pair plan. +Gamma `SLC_coreg.py` completed successfully for all expected secondary scenes. It consumed `itab_approved`, not the pre-audit or unapproved pair plan. -Expected post-job outputs: +Generated post-job outputs: ```text backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/coregistration_summary.json @@ -423,7 +497,18 @@ backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_2 backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_20241007/rmli/*.mli ``` -After the job succeeds: +The coregistration summary reports: + +```text +scene_count = 7 +expected_secondary_count = 6 +ready_secondary_count = 6 +missing_dates = [] +missing_tabs = [] +ready = true +``` + +Current run state: ```text status = COREGISTRATION_READY @@ -437,3 +522,1455 @@ Open product-display decisions: - the first sigma preview should use `los_sigma_mm_per_year.geo_preview.png` - the current monitoring-point curve is a single automatic sample point, not a monitoring network - production monitoring curves need user-selected lon/lat points, imported monitoring points, or a quality-filtered automatic sampler before they can be treated as formal outputs + +## Post-Coregistration Production Design + +Date: 2026-05-21 + +The next production slice should not jump directly from `COREGISTRATION_READY` to final velocity maps. It should keep the same gated pattern used for baseline audit and coregistration: + +```text +generate stage script/plan -> submit background job -> parse summary -> advance manifest status +``` + +The remaining chain is split into four testable stages: + +```text +rdc_dem -> interferograms -> ipta_timeseries -> publish_products +``` + +Each stage gets its own script, summary JSON, task/job type, page action, and artifact entries. This keeps operator testing bounded and avoids hiding a multi-hour Gamma failure inside a single monolithic job. + +### Stage 3: RDC DEM + +Stage id: + +```text +rdc_dem +``` + +Allowed input state: + +```text +run.status = COREGISTRATION_READY +run.next_stage = rdc_dem +``` + +Primary inputs: + +- `work/gamma/common_20241007/SLC_tab` +- `work/gamma/common_20241007/RMLI_tab` +- reference date: `20241007` +- reference MLI parameter file from the common `RMLI_tab` +- project DEM source resolved from the configured DEM strategy + +Do not hardcode the DEM used by the earlier `E131.2/N43.8` trial. That trial script reused: + +```text +backend/runtime/pyint_dem_cache/.../lt1_20230602_20230720...dem +``` + +The active production run is centered near `E129.2/N44.1`, so stage 3 must resolve or build a DEM for this stack explicitly. + +DEM resolution policy: + +1. Prefer an explicitly configured Gamma-compatible or GDAL-readable DEM source. +2. Record the selected DEM source in `rdc_dem_plan.json`. +3. Generate or copy a local run-scoped DEM into: + +```text +work/gamma/dem/ +``` + +4. Fail early if the DEM cannot be read or does not cover the stack/reference geometry. + +Gamma commands: + +- `replace_values` when source DEM nodata cleanup is needed +- `gc_map1` +- `geocode` +- `create_diff_par` +- `init_offsetm` +- `offset_pwrm` +- `offset_fitm` +- `gc_map_fine` +- `geocode` + +Expected files: + +```text +scripts/03_prepare_rdc_dem.sh +rdc_dem_plan.json +rdc_dem_summary.json +logs/20241007_rdc_dem.log +work/gamma/dem/20241007_8rlks.utm.dem +work/gamma/dem/20241007_8rlks.utm.dem.par +work/gamma/dem/20241007_8rlks.UTM_TO_RDC +work/gamma/dem/20241007_8rlks.rdc.dem +work/gamma/dem/20241007_8rlks.diff_par +``` + +Success state: + +```text +status = RDC_DEM_READY +next_stage = interferograms +``` + +Failure state: + +```text +status = RDC_DEM_FAILED +next_stage = fix_rdc_dem +``` + +Summary checks: + +- reference date exists in the common tabs +- reference MLI dimensions are parsed +- DEM source path is recorded +- `rdc.dem`, `UTM_TO_RDC`, `utm.dem.par`, and `diff_par` exist and are non-empty +- Gamma log tail is captured + +User test request after this stage is implemented: + +1. Open the SBAS production page and select `sbas_ab96afabead5`. +2. Confirm it shows `COREGISTRATION_READY` and `next_stage = rdc_dem`. +3. Click `提交 RDC DEM 任务`. +4. Wait for the task to complete. +5. Expected page result: `RDC_DEM_READY`, next stage `interferograms`. +6. Expected artifact: `rdc_dem_summary.json`. +7. If it fails, download or inspect `logs/20241007_rdc_dem.log`. + +### Stage 4: Differential Interferograms + +Stage id: + +```text +interferograms +``` + +Allowed input state: + +```text +run.status = RDC_DEM_READY +run.next_stage = interferograms +``` + +Primary inputs: + +- `work/gamma/common_20241007/SLC_tab` +- `work/gamma/common_20241007/RMLI_tab` +- `work/gamma/common_20241007/itab_approved` +- `work/gamma/dem/20241007_8rlks.rdc.dem` + +Pair source: + +Use the approved Gamma `itab` rows. Do not regenerate pair choices from the old time-series pairing layer at this stage. + +The current approved adjacent network is: + +```text +1 2 1 1 +2 3 2 1 +3 4 3 1 +4 5 4 1 +5 6 5 1 +6 7 6 1 +``` + +Gamma commands per pair: + +- `create_offset` +- `phase_sim_orb` +- `SLC_diff_intf` +- `adf` +- `cc_wave` +- `rasmph_pwr` +- `rasdt_pwr` +- `rascc_mask` +- `mcf` + +Expected files: + +```text +scripts/04_diff_unwrap_common_ref.sh +interferogram_plan.json +interferogram_summary.json +work/gamma/common_20241007/DIFF_tab +work/gamma/common_20241007/itab_ipta +work/gamma/common_20241007/diff/{pair}/{pair}_8rlks.diff +work/gamma/common_20241007/diff/{pair}/{pair}_8rlks.diff_filt +work/gamma/common_20241007/diff/{pair}/{pair}_8rlks.diff_filt.cor +work/gamma/common_20241007/diff/{pair}/{pair}_8rlks.diff_filt.unw +``` + +Success state: + +```text +status = INTERFEROGRAMS_READY +next_stage = ipta_timeseries +``` + +Failure state: + +```text +status = INTERFEROGRAMS_FAILED +next_stage = fix_interferograms +``` + +Summary checks: + +- expected pair count equals approved `itab` row count +- every pair has `diff`, filtered diff, coherence, mask, and unwrapped phase +- `DIFF_tab` line count matches `itab_ipta` row count +- coherence statistics are recorded per pair +- failed pairs are listed explicitly + +User test request after this stage is implemented: + +1. Confirm the run shows `RDC_DEM_READY`. +2. Click `提交差分干涉图任务`. +3. Wait for completion. +4. Expected page result: `INTERFEROGRAMS_READY`, next stage `ipta_timeseries`. +5. Expected summary: `interferogram_summary.json` with `ready_pair_count = 6`. +6. Review any low-coherence warnings before continuing. + +### Stage 5: IPTA Time-Series + +Stage id: + +```text +ipta_timeseries +``` + +Allowed input state: + +```text +run.status = INTERFEROGRAMS_READY +run.next_stage = ipta_timeseries +``` + +Primary inputs: + +- `work/gamma/common_20241007/DIFF_tab` +- `work/gamma/common_20241007/RMLI_tab` +- `work/gamma/common_20241007/itab_ipta` +- reference geometry MLI parameter file + +Gamma commands: + +- `mb` +- `ts_rate` + +The first production implementation should reuse the trial-proven image-based `mb -> ts_rate` invocation shape, but with run-specific paths and the approved `itab`. It should record the exact reference parameter files used by `mb`, because this is a quality-sensitive detail. + +Expected files: + +```text +scripts/05_mb_ts_rate.sh +ipta_timeseries_plan.json +ipta_timeseries_summary.json +work/gamma/common_20241007/timeseries/diff_ts.tab +work/gamma/common_20241007/timeseries/itab_ts +work/gamma/common_20241007/timeseries/sigma_ts +work/gamma/common_20241007/timeseries/hgt_correction +work/gamma/common_20241007/timeseries/ts_rate +work/gamma/common_20241007/timeseries/ts_const +work/gamma/common_20241007/timeseries/sigma_rate +``` + +Success state: + +```text +status = IPTA_TIMESERIES_READY +next_stage = publish_products +``` + +Failure state: + +```text +status = IPTA_TIMESERIES_FAILED +next_stage = fix_ipta_timeseries +``` + +User test request after this stage is implemented: + +1. Confirm the run shows `INTERFEROGRAMS_READY`. +2. Click `提交 IPTA 时序反演任务`. +3. Wait for completion. +4. Expected page result: `IPTA_TIMESERIES_READY`, next stage `publish_products`. +5. Expected summary: `ipta_timeseries_summary.json` showing `ts_rate` and `sigma_rate` exist. + +### Stage 6: Publish Products + +Stage id: + +```text +publish_products +``` + +Allowed input state: + +```text +run.status = IPTA_TIMESERIES_READY +run.next_stage = publish_products +``` + +Primary inputs: + +- `work/gamma/common_20241007/timeseries/ts_rate` +- `work/gamma/common_20241007/timeseries/sigma_rate` +- `work/gamma/common_20241007/timeseries/sigma_ts` +- `work/gamma/common_20241007/timeseries/hgt_correction` +- `work/gamma/dem/20241007_8rlks.UTM_TO_RDC` +- `work/gamma/dem/20241007_8rlks.utm.dem.par` +- reference SLC parameter file for wavelength + +Processing: + +1. Compute wavelength from `radar_frequency`. +2. Convert phase-rate to LOS rate: + +```text +los_rate_away_mm_per_year = phase_rate * wavelength / (4*pi) * 1000 +los_rate_toward_mm_per_year = -phase_rate * wavelength / (4*pi) * 1000 +``` + +3. Use `geocode_back` and `data2geotiff` to export EPSG:4326 GeoTIFFs. +4. Generate web preview PNGs from the geocoded GeoTIFFs, not from RDC BMPs. +5. Write product and quality summaries. + +Expected files: + +```text +scripts/06_publish_products.sh +publish_product_plan.json +publish_product_summary.json +quality_summary.json +publish/geotiff/ts_rate_rad_per_year.tif +publish/geotiff/sigma_rate_rad_per_year.tif +publish/geotiff/los_rate_toward_mm_per_year.tif +publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png +publish/geotiff/los_rate_away_mm_per_year.tif +publish/geotiff/los_sigma_mm_per_year.tif +publish/geotiff/los_sigma_mm_per_year.geo_preview.png +``` + +Success state: + +```text +status = PRODUCTS_READY +next_stage = monitor_points +``` + +Failure state: + +```text +status = PUBLISH_PRODUCTS_FAILED +next_stage = fix_publish_products +``` + +User test request after this stage is implemented: + +1. Confirm the run shows `IPTA_TIMESERIES_READY`. +2. Click `发布 LOS 速率产品`. +3. Wait for completion. +4. Expected page result: `PRODUCTS_READY`. +5. Verify the UI default velocity and sigma previews are geocoded PNGs. +6. Download or open the GeoTIFFs in GIS if needed. +7. Treat RDC BMPs as processing QA only. + +### Stage 7: Monitor Points + +This should not block the first full production result. + +The current automatic sample point is only a debug placeholder. Formal monitoring curves require at least one of: + +- user-provided lon/lat points +- imported monitoring-point layer +- explicitly approved quality-filtered automatic sampler + +Until then, the UI should label monitor curves as sample/debug output, not formal business monitoring points. + +### Implementation Order + +Recommended next coding order: + +1. Add backend `RDC_DEM` stage: plan generation, script generation, queued job, summary parser, manifest update. +2. Add frontend actions and status display for the `rdc_dem` stage. +3. Ask the user to test only `RDC DEM` on `sbas_ab96afabead5`. +4. After that succeeds, implement the interferogram stage. +5. After interferograms succeed, implement IPTA time-series. +6. After IPTA succeeds, implement product publishing and geocoded previews. + +This staged order is intentionally conservative. It keeps each Gamma failure surface small enough to diagnose from one stage log and one summary JSON. + +## 2026-05-22 RDC DEM Implementation Note + +The formal SBAS production workflow now includes a managed `rdc_dem` stage. + +Implemented code paths: + +- Backend service methods: + - `prepare_rdc_dem` + - `execute_rdc_dem` + - `_write_rdc_dem_script` + - `_build_rdc_dem_summary` + - `_refresh_command_manifest_after_rdc_dem` +- FastAPI endpoints: + - `POST /api/sbas-insar-production/runs/{run_id}/rdc-dem` + - `POST /api/sbas-insar-production/runs/{run_id}/rdc-dem/jobs` +- Background job type: + - `SBAS_RDC_DEM` +- Frontend: + - `生成 RDC DEM 脚本` + - `提交 RDC DEM 任务` + - `RDC DEM Plan` status card + +The generated script is intentionally based on the successful trial script: + +```text +gc_map1 +geocode simulated SAR to RDC +create_diff_par +init_offsetm +offset_pwrm +offset_fitm +gc_map_fine +geocode DEM to RDC +``` + +The current `.env` DEM path points to the large SARscape/GDAL raster: + +```text +D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +``` + +That file is not a Gamma `.dem + .dem.par` pair. For this implementation slice, the production stage selects an existing Gamma-format PyINT DEM cache when no explicit Gamma DEM source is configured. On the current machine, the selected LT1 cache covers the stack center and follows the same PyINT/Gamma DEM format that the successful experiment used. + +Known limitation: + +- The existing LT1 PyINT DEM cache covers the stack center but may not fully cover the union of every scene bbox south edge. The plan records both `covers_stack_center` and `covers_stack_bbox`. If the RDC DEM stage fails or creates edge voids, the next fix should generate a fresh Gamma DEM from `D:\DEM\SRTMDEM_RSP_SARscape.wgs84` over the full SBAS stack bbox plus margin, instead of relying on older pair-level DEM caches. + +Local validation completed: + +- Python AST parse passed for: + - `backend/app/services/sbas_insar_production_service.py` + - `backend/app/routers/sbas_insar_production.py` + - `backend/app/services/job_handlers.py` +- `SBAS_RDC_DEM` job handler registration resolves successfully. +- Frontend `npm run build` passed after running with the permission needed for Vite/esbuild subprocess spawn. + +Manual test order for the user: + +1. Restart backend and frontend if they are already running. +2. Open the SBAS-InSAR production page. +3. Select run `sbas_ab96afabead5`. +4. Confirm status is `COREGISTRATION_READY` or `RDC_DEM_SCRIPT_READY`. +5. Click `生成 RDC DEM 脚本`. +6. Confirm `RDC DEM Plan` appears and points to `scripts/03_prepare_rdc_dem.sh`. +7. Click `提交 RDC DEM 任务`. +8. Watch the task until completion. +9. Expected success: + +```text +run.status = RDC_DEM_READY +run.next_stage = interferograms +rdc_dem_summary.ready = true +``` + +Expected output files: + +```text +work/gamma/dem/20241007_8rlks.utm.dem +work/gamma/dem/20241007_8rlks.utm.dem.par +work/gamma/dem/20241007_8rlks.UTM_TO_RDC +work/gamma/dem/20241007_8rlks.rdc.dem +work/gamma/dem/20241007_8rlks.diff_par +logs/20241007_rdc_dem.log +rdc_dem_summary.json +``` + +If the job fails, inspect: + +```text +logs/20241007_rdc_dem.log +rdc_dem_summary.json +``` + +The next production coding stage after `RDC_DEM_READY` is differential interferogram generation. + +## 2026-05-23 Interferogram Stage Implementation Note + +The formal SBAS production workflow now includes a managed `interferograms` stage after `RDC_DEM_READY`. + +Implemented code paths: + +- Backend service methods: + - `prepare_interferograms` + - `execute_interferograms` + - `_write_interferogram_script` + - `_build_interferogram_summary` + - `_build_interferogram_pair_plan` + - `_refresh_command_manifest_after_interferograms` +- FastAPI endpoints: + - `POST /api/sbas-insar-production/runs/{run_id}/interferograms` + - `POST /api/sbas-insar-production/runs/{run_id}/interferograms/jobs` +- Background job type: + - `SBAS_INTERFEROGRAMS` +- Frontend: + - Adds an action to generate the interferogram script. + - Adds an action to submit the interferogram background job. + - Adds an `Interferogram Plan` status card. + +This stage is intentionally derived from the successful Gamma trial script `08_diff_unwrap_common_ref.sh`. The production script keeps the same Gamma command chain: + +```text +create_offset +phase_sim_orb +SLC_diff_intf +adf +cc_wave +rasmph_pwr +rasdt_pwr +rascc_mask +mcf +rasdt_pwr +``` + +The production differences from the trial are: + +- It reads the approved production `itab_approved` instead of hardcoding a trial date list. +- It writes production `DIFF_tab` and `itab_common_ref` for the next `mb` and `ts_rate` stage. +- It records a JSON plan and JSON execution summary under the run directory. +- It keeps all SLCs/RMLIs in the common reference geometry prepared by the coregistration stage. + +Generated files for the current test run: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/interferogram_plan.json +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/scripts/04_diff_unwrap_common_ref.sh +``` + +Current generated plan: + +```text +run_id = sbas_ab96afabead5 +status = INTERFEROGRAMS_SCRIPT_READY +next_stage = execute_interferograms +reference_date = 20241007 +pair_count = 6 +pairs = + 20240422_20240617 + 20240617_20240812 + 20240812_20241007 + 20241007_20250519 + 20250519_20250714 + 20250714_20250908 +``` + +Expected output interface after successful execution: + +```text +work/gamma/common_20241007/DIFF_tab +work/gamma/common_20241007/itab_common_ref +work/gamma/common_20241007/diff//_8rlks.diff_filt.unw +work/gamma/common_20241007/diff//_8rlks.diff_filt.cor +interferogram_summary.json +``` + +Local validation completed: + +- Python AST parse passed for: + - `backend/app/services/sbas_insar_production_service.py` + - `backend/app/routers/sbas_insar_production.py` + - `backend/app/services/job_handlers.py` +- `SBAS_INTERFEROGRAMS` job handler registration resolves successfully. +- The generated `04_diff_unwrap_common_ref.sh` passes `bash -n` in WSL. +- Frontend `npm run build` passed after running with the permission needed for Vite/esbuild subprocess spawn. + +Manual test order for the user: + +1. Restart backend and frontend if they are already running. +2. Open the SBAS-InSAR production page. +3. Select run `sbas_ab96afabead5`. +4. Confirm status is `INTERFEROGRAMS_SCRIPT_READY`. +5. Confirm `Interferogram Plan` shows 6 pairs and `reference_date = 20241007`. +6. Click the interferogram background-job button. +7. Watch the task until completion. +8. Expected success: + +```text +run.status = INTERFEROGRAMS_READY +run.next_stage = ipta_timeseries +interferogram_summary.ready = true +interferogram_summary.ready_pair_count = 6 +``` + +If the job fails, inspect: + +```text +logs/_diff_unwrap_common.log +interferogram_summary.json +work/gamma/common_20241007/DIFF_tab +work/gamma/common_20241007/itab_common_ref +``` + +Known risk to verify in the production test: + +- The trial script proved the command chain with reference-star pairs. The formal production stage uses the approved adjacent `itab` network, including secondary-secondary pairs after common-reference coregistration. This is the right SBAS topology, but the next live test should confirm Gamma accepts the secondary-secondary pair geometry with the current `phase_sim_orb` inputs. If Gamma rejects that geometry, the next correction is to adapt the pair topology or DEM geometry handling while keeping the same managed stage boundary. + +The next production coding stage after `INTERFEROGRAMS_READY` is IPTA time-series inversion with `mb` and `ts_rate`. + +## 2026-05-24 Interferogram Result And IPTA Stage Implementation Note + +The current production run `sbas_ab96afabead5` completed the managed interferogram stage successfully. + +Observed result: + +```text +run.status = INTERFEROGRAMS_READY +run.next_stage = ipta_timeseries +interferogram_summary.ready = true +interferogram_summary.ready_pair_count = 6 +interferogram_summary.pair_count = 6 +DIFF_tab rows = 6 +itab_common_ref rows = 6 +``` + +All six approved SBAS pairs produced the required Gamma outputs: + +```text +_8rlks.off +.sim_unw +_8rlks.diff +_8rlks.diff_filt +_8rlks.diff_filt.cor +_8rlks.diff_filt.cor_mask.bmp +_8rlks.diff_filt.unw +``` + +This confirms that the production adjacent-pair SBAS topology works with the current common-reference geometry, not only the reference-star topology from the earlier experiment. + +The formal SBAS production workflow now includes a managed `ipta_timeseries` stage. + +Implemented code paths: + +- Backend service methods: + - `prepare_ipta_timeseries` + - `execute_ipta_timeseries` + - `_write_ipta_timeseries_script` + - `_build_ipta_timeseries_summary` + - `_select_ipta_mb_reference_mli` + - `_refresh_command_manifest_after_ipta_timeseries` +- FastAPI endpoints: + - `POST /api/sbas-insar-production/runs/{run_id}/ipta-timeseries` + - `POST /api/sbas-insar-production/runs/{run_id}/ipta-timeseries/jobs` +- Background job type: + - `SBAS_IPTA_TIMESERIES` +- Frontend: + - Adds an action to generate the IPTA script. + - Adds an action to submit the IPTA background job. + - Adds an `IPTA Time-Series Plan` status card. + +This stage is derived from the successful Gamma trial script `09_mb_ts_rate.sh`. The production script keeps the same Gamma command chain: + +```text +mb +ts_rate +``` + +The production differences from the trial are: + +- It uses run-specific `DIFF_tab`, `RMLI_tab`, and `itab_common_ref`. +- It records the two `mb` reference parameter files explicitly: + - geometry reference MLI parameter file + - nearest non-reference common-RMLI parameter file used as the `mb` reference parameter +- It writes `ipta_timeseries_plan.json` and `ipta_timeseries_summary.json`. +- It updates the command manifest stage plan and enables the next `publish_products` stage only after `IPTA_TIMESERIES_READY`. + +Generated files for the current test run: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/ipta_timeseries_plan.json +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/scripts/05_mb_ts_rate.sh +``` + +Current generated plan: + +```text +run_id = sbas_ab96afabead5 +status = IPTA_TIMESERIES_SCRIPT_READY +next_stage = execute_ipta_timeseries +reference_date = 20241007 +geometry_reference_mli_par = work/gamma/mli/20241007.mli.par +mb_reference_mli_par = work/gamma/common_20241007/rmli/20240812.mli.par +``` + +Expected output interface after successful execution: + +```text +work/gamma/common_20241007/timeseries/diff_ts.tab +work/gamma/common_20241007/timeseries/itab_ts +work/gamma/common_20241007/timeseries/sigma_ts +work/gamma/common_20241007/timeseries/hgt_correction +work/gamma/common_20241007/timeseries/ts_rate +work/gamma/common_20241007/timeseries/ts_const +work/gamma/common_20241007/timeseries/sigma_rate +ipta_timeseries_summary.json +``` + +Local validation completed: + +- Python AST parse passed for: + - `backend/app/services/sbas_insar_production_service.py` + - `backend/app/routers/sbas_insar_production.py` + - `backend/app/services/job_handlers.py` +- `SBAS_IPTA_TIMESERIES` job handler registration resolves successfully. +- The generated `05_mb_ts_rate.sh` passes `bash -n` in WSL. +- Frontend `npm run build` passed after running with the permission needed for Vite/esbuild subprocess spawn. + +Manual test order for the user: + +1. Restart backend and frontend if they are already running. +2. Open the SBAS-InSAR production page. +3. Select run `sbas_ab96afabead5`. +4. Confirm status is `IPTA_TIMESERIES_SCRIPT_READY`. +5. Confirm `IPTA Time-Series Plan` shows `reference_date = 20241007`. +6. Click the IPTA background-job button. +7. Watch the task until completion. +8. Expected success: + +```text +run.status = IPTA_TIMESERIES_READY +run.next_stage = publish_products +ipta_timeseries_summary.ready = true +``` + +If the job fails, inspect: + +```text +logs/mb_ts_rate.log +ipta_timeseries_summary.json +work/gamma/common_20241007/timeseries/ +``` + +The next production coding stage after `IPTA_TIMESERIES_READY` is product publishing: geocoding `ts_rate` and `sigma_rate`, LOS sign conversion, GeoTIFF generation, web previews, and monitoring-point curve extraction. + +## 2026-05-25 IPTA Failure Triage And Reference-Region Fix + +The first production IPTA run for `sbas_ab96afabead5` failed inside Gamma `mb`. + +Observed result: + +```text +run.status = IPTA_TIMESERIES_FAILED +run.next_stage = fix_ipta_timeseries +execution.returncode = 139 +logs/mb_ts_rate.log = Segmentation fault in mb +``` + +This was not a system-side summary false negative. Gamma `mb` read the input tables successfully: + +```text +DIFF_tab records = 6 +itab_common_ref records = 6 +RMLI_tab entries = 7 +``` + +The failure happened after `mb` printed the reference region and before writing valid `diff_ts.tab` / `itab_ts`. The initial production script reused the experiment's center-pixel reference region: + +```text +range = width / 2 +azimuth = lines / 2 +window = 16 x 16 +``` + +That is fragile for real production stacks. On this run, the center reference window contained zero/invalid unwrapped pixels in multiple interferograms. Gamma `mb` did not emit a clean validation error; it segfaulted. + +Implemented fix: + +- `prepare_ipta_timeseries` now automatically scans the unwrapped interferograms listed in `DIFF_tab`. +- It selects a 16 x 16 reference window with valid nonzero unwrapped values across all pairs and high mean coherence. +- The selected region is written into `ipta_timeseries_plan.json` as `reference_region`. +- `05_mb_ts_rate.sh` now uses the selected `R_REF` / `A_REF` instead of the image center. +- The script removes stale IPTA outputs before running so a retry starts from clean stage outputs. + +Selected reference region for the current run: + +```text +strategy = auto_valid_unwrapped_high_coherence_window +range_pixel = 2152 +azimuth_line = 1512 +window = 16 x 16 +pair_count = 6 +min_valid_pixel_count = 256 +total_valid_pixel_count = 1536 +median_mean_coherence = 0.9953643755 +valid_pixel_count_by_pair = [256, 256, 256, 256, 256, 256] +``` + +Current retry-ready state: + +```text +run.status = IPTA_TIMESERIES_SCRIPT_READY +run.next_stage = execute_ipta_timeseries +scripts/05_mb_ts_rate.sh uses R_REF=2152 and A_REF=1512 +``` + +Validation completed after the fix: + +- Python AST parse passed. +- `SBAS_IPTA_TIMESERIES` job handler resolves successfully. +- Regenerated `05_mb_ts_rate.sh` passes WSL `bash -n`. +- Frontend `npm run build` passed. + +User retry order: + +1. Refresh the SBAS-InSAR production page. +2. Select run `sbas_ab96afabead5`. +3. Confirm status is `IPTA_TIMESERIES_SCRIPT_READY`. +4. Confirm `IPTA Time-Series Plan` contains reference region `2152,1512`. +5. Submit the IPTA task again. + +If the retry still fails, inspect: + +```text +logs/mb_ts_rate.log +ipta_timeseries_summary.json +work/gamma/common_20241007/timeseries/ +``` + +## 2026-05-25 IPTA Mode-1 Failure Closure + +The reference-region fix was necessary but not sufficient. The second production retry still failed inside Gamma `mb` with return code `139`. + +The regenerated plan was correct: + +```text +reference_region = 2152,1512 +window = 16 x 16 +valid_pixel_count_by_pair = [256, 256, 256, 256, 256, 256] +median_mean_coherence = 0.9953643755 +``` + +So the remaining failure was not an invalid reference window. A focused Gamma diagnostic matrix was run against the exact production `DIFF_tab`, `RMLI_tab`, and `itab_common_ref` for run `sbas_ab96afabead5`. + +Diagnostic result: + +```text +full stack, mb mode=1 -> rc=139, segmentation fault +full stack, mb mode=2 -> rc=139, segmentation fault +full stack, mb mode=0 -> rc=0 +reference-star subset, mb mode=1 -> rc=0 +first-3 adjacent-chain subset, mb mode=1 -> rc=0 +single-pair subsets -> non-production diagnostic only, not valid as full stack inversion +``` + +A production-equivalent diagnostic was then run with the same output flags as the experiment script: + +```text +mb sim_flg=1 hgt_flg=1 mode=0 +ts_rate +``` + +That completed successfully and produced: + +```text +diff_ts.tab +itab_ts +sigma_ts +hgt_correction +ts_rate +ts_const +sigma_rate +``` + +The important conclusion is that the original Gamma trial was valid, but the formal production input is not identical to the trial. The trial used a smaller/reference-star style network that Gamma `mb mode=1` accepted. The production run uses the approved full adjacent SBAS chain with 7 dates and 6 interferograms. On the current Gamma 2023 IPTA binary, that full production chain crashes in `mode=1`, while `mode=0` completes. + +Implemented closure: + +- Production default `mb_mode` is now `0`. +- `ipta_timeseries_plan.json` records: + +```text +mb_mode = 0 +mb_mode_description = valid unwrapped phase values required in all layers +``` + +- `05_mb_ts_rate.sh` writes and uses: + +```text +MB_MODE="0" +``` + +- `ipta_timeseries_summary.json` records the mode used by the completed run. +- Router and background job payloads accept `mb_mode`, defaulting to `0`. The UI does not expose this as a normal operator choice yet. +- Script generation now has a fallback path if an existing script file cannot be overwritten because of Windows/WSL ACL drift. + +Current formal service execution result after the fix: + +```text +run_id = sbas_ab96afabead5 +run.status = IPTA_TIMESERIES_READY +run.next_stage = publish_products +execution.returncode = 0 +summary.ready = true +summary.missing_outputs = [] +summary.mb_mode = 0 +diff_ts_row_count = 7 +itab_ts_row_count = 7 +``` + +Output size checks: + +```text +expected_float32_bytes = 45921036 +sigma_ts = 45921036 +hgt_correction = 45921036 +ts_rate = 45921036 +ts_const = 45921036 +sigma_rate = 45921036 +``` + +Operational note: + +During local Codex verification, a direct Windows Python subprocess call to `wsl.exe` returned `Wsl/Service/E_ACCESSDENIED` unless the command was run with external execution permission. That was an execution-context permission issue, not a Gamma processing failure. The same service method completed when WSL execution was allowed. If the production worker reports `Wsl/Service/E_ACCESSDENIED`, fix the worker account/Windows service permissions for WSL access before re-testing Gamma processing. + +One legacy run-directory ACL issue was also fixed manually for `sbas_ab96afabead5`: older files had ACLs that let the current Windows user create new files but not overwrite existing manifest/script files. The current run directory was granted current-user full control so `run_manifest.json`, `ipta_timeseries_plan.json`, and generated scripts can be updated on retries. + +Next stage after this closure is product publishing: + +```text +publish_products +geocode_back ts_rate/sigma_rate +data2geotiff +LOS velocity/sigma products +monitoring point curves +``` + +## 2026-05-26 Product Publishing And Monitoring Point Integration + +The expert-workflow bridge now implements the downstream output work that was previously planned-only. In the current twelve-node workflow these are represented by section 12: + +```text +12_outputs_points +``` + +`10_detrend_atm` is now part of the new development workflow. It is no longer treated as an optional compatibility branch. New production runs should execute section 10 before section 11, and `11_sbas_inversion` should consume `DIFF_atmsub_tab`. + +### 12_outputs_points publish phase + +The stage generates and executes: + +```text +runs/{run_id}/scripts/12_outputs_points.sh +``` + +It follows the experiment-proven path: + +- compute wavelength from `radar_frequency` +- convert `ts_rate` / `sigma_rate` phase rates to LOS +- write both LOS sign conventions: + +```text +los_rate_away_mm_per_year = phase_rate * wavelength / (4*pi) * 1000 +los_rate_toward_mm_per_year = -phase_rate * wavelength / (4*pi) * 1000 +``` + +- geocode with Gamma `geocode_back` +- export GeoTIFFs with Gamma `data2geotiff` +- create RDC QA browse BMPs with `rasdt_pwr` +- create UI map previews from geocoded GeoTIFFs, not from RDC BMPs + +Expected output state: + +```text +run.status = PRODUCTS_READY +run.next_stage = monitor_points +``` + +Expected files: + +```text +publish_product_plan.json +publish_product_summary.json +product_summary.json +quality_summary.json +publish/geotiff/los_rate_toward_mm_per_year.tif +publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png +publish/geotiff/los_rate_away_mm_per_year.tif +publish/geotiff/los_sigma_mm_per_year.tif +publish/geotiff/los_sigma_mm_per_year.geo_preview.png +publish/geotiff/ts_rate_rad_per_year.tif +publish/geotiff/sigma_rate_rad_per_year.tif +``` + +The summary records nonzero finite pixel statistics for LOS velocity and sigma in RDC geometry so the operator can quickly detect blank or extreme outputs. + +### 12_outputs_points monitoring-point phase + +The wrapper also runs the monitoring-point extraction script when product publishing has completed: + +```text +runs/{run_id}/scripts/08_point_timeseries.sh +``` + +It reuses the experiment logic in a parameterized WSL helper: + +```text +deploy/wsl/runners/gamma_sbas_product_tools.py +``` + +Supported point modes in this slice: + +- `auto_low_sigma_high_rate`: automatic non-edge sample point, high absolute LOS velocity and low sigma +- `manual_lonlat`: nearest lookup-table pixel from configured lon/lat points + +The automatic sample remains a diagnostic/sample curve, not a formal monitoring network. + +Expected output state: + +```text +run.status = MONITOR_POINTS_READY +run.next_stage = review_publish_products +``` + +Expected files: + +```text +monitor_points_plan.json +monitor_points_summary.json +publish/monitor_points/{point_id}_timeseries.png +publish/monitor_points/{point_id}_timeseries.csv +publish/monitor_points/{point_id}_metadata.json +``` + +### Workflow Result + +After 10, 11, and 12 complete, the workflow can finish as: + +```text +WORKFLOW_COMPLETED +``` + +instead of `WORKFLOW_PARTIAL`, provided no enabled stage failed. + +Manual test order: + +1. Select run `sbas_ab96afabead5`. +2. Submit Gamma SBAS Workflow with: + +```text +from_step = 12_outputs_points +``` + +3. Expected first completion: + +```text +PRODUCTS_READY +``` + +or full completion: + +```text +MONITOR_POINTS_READY +WORKFLOW_COMPLETED +``` + +4. Confirm the production Run detail shows geocoded LOS velocity and sigma previews. +5. Confirm the monitoring curve is visible and its CSV is downloadable. + +## 2026-05-26 Expert Document Step Index And Color Convention Update + +The user correctly pointed out that the expert document is not an eight-step process. The document has twelve major sections, each with multiple Gamma commands. The production workflow now uses those twelve sections as first-class workflow nodes. The previous eight-stage execution view is retained only as an internal service implementation detail where an already verified experiment script covers more than one expert section. + +The twelve expert sections now appear in `capabilities`, `manifest.json`, `gamma_command_manifest.json`, and the SBAS production page: + +```text +1. Directory and LT1 data preparation +2. Import every LT1 SLC +3. Reference MLI and footprint checks +4. DEM import and lookup table +5. SLC coregistration preparation +6. Coregister every SLC to reference +7. RMLI stack and average intensity +8. Interferogram network and differential phase +9. Adaptive filtering, coherence mask and unwrap +10. Detrend and atmospheric phase removal +11. SBAS inversion +12. Output, geocode and point time-series +``` + +Each section records representative commands from `LT1_GAMMA_SBAS_逐命令处理流程.docx`, mapped workflow stages, and implementation status. Existing successful experiment logic is retained as `implemented_bridge` where it has already proven the same Gamma function, even if the file layout is not yet identical to the document. + +Current acceptance status: + +```text +1-4 implemented_bridge or implemented +5-9 implemented_bridge +10 implemented_bridge: quad_fit/quad_sub/atm_mod_2d/fill_gaps/atm_sim_2d/sub_phase now writes DIFF_atmsub_tab; needs live production validation +11 implemented_bridge: mb/ts_rate now prefers DIFF_atmsub_tab; the full expert multi-pass unw_to_cpx/unw_model refinement is still not fully migrated +12 implemented_bridge: publish and monitor outputs exist; Gamma expert browse color products are now added +``` + +Color and browse products were corrected toward the expert document conventions. Velocity browse products now prefer Gamma `rasdt_pwr` with `hls.cm` and the expert range `-0.08 0.08` m/year. Sigma/quality browse products now use `cc.cm`; because production currently displays LOS sigma-rate rather than `diff.sigma_ts.masked`, the range is adapted to `0.0 0.06` m/year while retaining the expert color table family. Phase, detrend, and atmospheric browse products should use `rmg.cm` with `-6.28 6.28` radians when section 10 is implemented. + +New preferred publish outputs: + +```text +publish/geotiff/los_rate_toward_m_per_year.hls.bmp +publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif +publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png +publish/geotiff/los_sigma_m_per_year.cc.bmp +publish/geotiff/los_sigma_m_per_year.cc.geo_rgb.tif +publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png +publish/geotiff/los_rate_toward_m_per_year.tif +publish/geotiff/los_rate_away_m_per_year.tif +publish/geotiff/los_sigma_m_per_year.tif +``` + +Legacy millimeter-per-year products remain published for comparison with earlier experiments: + +```text +publish/geotiff/los_rate_toward_mm_per_year.tif +publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png +publish/geotiff/los_sigma_mm_per_year.tif +publish/geotiff/los_sigma_mm_per_year.geo_preview.png +``` + +Next implementation target is the rest of section 11. The managed section 10 node now generates `unw.atmsub` products and `DIFF_atmsub_tab`; section 11 consumes that table. The remaining gap is the expert multi-pass `unw_to_cpx` / `unw_model` refinement path. + +## 2026-05-26 Twelve-Node Production Workflow Correction + +The production workflow has been corrected from the temporary `8 coarse stages + 12-section checklist` view to a twelve-node workflow: + +```text +01_workspace_data +02_import_lt1_slc +03_reference_mli +04_dem_lookup +05_coreg_prep +06_coregister_scenes +07_rmli_average +08_diff_network +09_filter_unwrap +10_detrend_atm +11_sbas_inversion +12_outputs_points +``` + +The bridge still reuses verified scripts from the successful experiment where that is safer than rewriting Gamma command chains immediately: + +```text +02-03 reuse the baseline-audit import/multilook/base_calc implementation. +05-07 reuse the common-reference coregistration implementation. +08-09 reuse the differential interferogram/filter/unwrap implementation. +10 uses the expert detrend/atmospheric-correction implementation. +11 uses mb/ts_rate over DIFF_atmsub_tab. +12 runs publish products followed by monitoring-point extraction. +``` + +Dependency handling is no longer a simple linear coarse-stage status check. Section 4 can run after baseline/reference MLI preparation; sections 8-9 require both the DEM stage execution and the coregistration stage execution to be recorded as completed in the current run manifest. + +Old experiment outputs are not accepted as expert-path validation. Development runs may be deleted and regenerated; the twelve-node workflow should be validated from a clean run so that each node has a fresh execution record. Existing summary JSON files may remain as operator evidence, but the workflow runner must not use them to skip expert nodes. + +## 2026-05-26 Expert Section 10 Detrend/ATM Stage + +The formal workflow order is now: + +```text +08_diff_network +09_filter_unwrap +10_detrend_atm +11_sbas_inversion +12_outputs_points +``` + +`10_detrend_atm` writes: + +```text +detrend_atm_plan.json +detrend_atm_summary.json +work/gamma/common_/DIFF_atmsub_tab +work/gamma/common_/itab_atmsub +work/gamma/common_/detrend_atm//_rlks.diff_filt.unw.atmsub +``` + +The stage follows the expert section 10 command family: + +```text +create_diff_par +quad_fit +quad_sub +rasdt_pwr ... rmg.cm +atm_mod_2d +fill_gaps +atm_sim_2d +sub_phase +rasdt_pwr ... rmg.cm +``` + +The `fill_gaps` width for `a0/a1` model grids is inferred from the generated coefficient-file size and the reference MLI aspect ratio. If inference or `fill_gaps` fails, the script falls back to the raw atmospheric coefficients and records the warning in the pair log; this keeps the production test actionable while preserving the expert command path. + +`11_sbas_inversion` now requires `DETREND_ATM_READY` and uses: + +```text +DIFF_TAB = work/gamma/common_/DIFF_atmsub_tab +ITAB = work/gamma/common_/itab_atmsub +``` + +Production test expectation: + +```text +run.status after 10 = DETREND_ATM_READY +run.next_stage after 10 = ipta_timeseries +run.status after 11 = IPTA_TIMESERIES_READY +``` + +## 2026-05-27 Runtime Cleanup And Strict Production Display + +The previous development/test outputs were removed so the next SBAS-InSAR test starts from a clean runtime state: + +```text +backend/runtime/sbas_insar_production/discoveries/* +backend/runtime/sbas_insar_production/runs/* +backend/runtime/sbas_insar_production/stack_manifests/* +backend/runtime/gamma_ipta_trials/* +backend/runtime/gamma_ipta_probe/* +``` + +The frontend SBAS-InSAR production page no longer lists or opens `trial-runs`. It now loads only managed production `runs`, and product links use `/api/sbas-insar-production/runs/{run_id}/artifacts/...`. + +The workflow runner is intentionally strict after this cleanup. Old experiment summaries or sidecar `*_summary.json` files are not accepted as proof that an expert node has completed. A node can be skipped or advanced only when the current run manifest contains a completed execution record for the corresponding stage. + +## 2026-05-27 Result Management And Geographic Coverage Design + +The first clean Gamma SBAS workflow run completed all twelve expert nodes: + +```text +run_id = sbas_7537cc71c998 +status = WORKFLOW_COMPLETED +scene_count = 7 +pair_count = 6 +workflow_summary.completed_count = 12 +workflow_summary.failed_count = 0 +``` + +This means production execution is now viable enough to split the user experience into two modules: + +```text +SBAS-InSAR Production + - discover stack candidates + - create production Run + - submit Gamma SBAS workflow + - inspect 12 expert nodes, scripts, logs, and retry state + +SBAS-InSAR Results + - browse stable products + - inspect geographic footprint and map location + - preview LOS velocity, LOS sigma, and monitoring curves + - download GeoTIFF/PNG/CSV/supporting manifests + - jump back to the source production Run for audit/debug +``` + +### Geographic Coverage Gap + +SBAS is time-series processing, so temporal density is important, but result users primarily ask "where is this product?". The current production page does not make geographic location obvious even though the metadata already exists. + +Available geographic sources in the current run: + +```text +stack_manifest.scenes[*].bbox +stack_manifest.scenes[*].center_lon / center_lat +stack_manifest.stack.center_bucket +rdc_dem_summary.dem_source.stack_bbox +monitor_points_summary.monitor_outputs[*].metadata.approx_lonlat +published GeoTIFF bounds from GDAL metadata +``` + +Example from `sbas_7537cc71c998`: + +```text +stack center bucket = E129.2_N44.1 +stack bbox = 128.7690438245, 43.7486321624, 129.6293024728, 44.3582486206 +monitor point = 129.10207098755, 44.15041727515 +``` + +The production page should add a compact `Geographic Coverage` block near the selected Run summary: + +```text +center lon/lat +stack bbox +scene footprint count +DEM coverage status: covers stack bbox / covers stack center +monitor point lon/lat if generated +open on map / zoom to footprint action +``` + +This block is operational context, not a replacement for a result browser. It helps the operator avoid running or reviewing the wrong location. + +### Result Product Boundary + +One completed SBAS workflow Run should register one result product bundle, not many separate products. This follows the existing D-InSAR/PsInSAR catalog pattern: + +```text +result_products: one row per SBAS bundle +result_assets: multiple files under the bundle +result_issues: missing/invalid/geocoding/quality warnings +catalog_name: sbas_insar +run_key: source SBAS run_id +``` + +The product record should carry first-class query fields: + +```text +platform / satellite +relative_orbit +orbit_direction +polarization +reference_date +start_date +end_date +scene_count +pair_count +bbox_min_lon / bbox_min_lat / bbox_max_lon / bbox_max_lat +center_lon / center_lat +status / health_status +primary_asset_role +quality_asset_role +source_run_id +``` + +Do not treat every GeoTIFF as a separate product row. The user-facing product is the SBAS result for one stack/run over one geographic footprint and time span. + +### Important Assets From The Expert Document + +The expert document makes section 12 outputs the formal product boundary. Important product roles: + +```text +primary_velocity_geotiff + expert source: geo_los_def_rate.tif + current system: publish/geotiff/los_rate_toward_m_per_year.tif + +primary_velocity_rgb_geotiff + expert source: geo_los_def_rate_rgb.tif + current system: publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif + +primary_velocity_preview + expert source: los_def_rate.bmp / geo_los_def_rate.bmp + current system: publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png + +quality_sigma_geotiff + expert source: diff.sigma_ts / geo_diff.sigma_ts + current system: publish/geotiff/los_sigma_m_per_year.tif + +quality_sigma_preview + expert source: diff.sigma_ts.masked.bmp with cc.cm + current system: publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png + +monitor_timeseries_csv + expert source: disp_prt_2d outputs + current system: publish/monitor_points/*_timeseries.csv + +monitor_timeseries_plot + current system: publish/monitor_points/*_timeseries.png + +support_manifest + run_manifest.json, workflow_summary.json, gamma_command_manifest.json, stage summaries +``` + +The Results UI should foreground only the primary and quality products by default. Supporting manifests and stage summaries belong in an "Audit files" section. + +### SBAS-InSAR Results UI + +List view: + +```text +left/top filters: + time range + geographic bbox / map AOI + platform + relative orbit + direction + status / health + has monitor points + +result row/card: + product name + footprint mini-map or bbox text + start/end/reference dates + scene count / pair count + LOS velocity preview thumbnail + sigma health indicator + actions: open details, zoom to map, download primary GeoTIFF +``` + +Detail view: + +```text +map footprint panel +LOS velocity preview +LOS sigma preview +monitoring point curve +key metadata table +asset table grouped by role +quality summary +link back to production Run and 12-node workflow +``` + +Map behavior: + +```text +use stack bbox as initial footprint +prefer GeoTIFF bounds when parsed successfully +show monitor points as point overlays +allow "zoom to result" +allow AOI filter against bbox intersection +``` + +### Backend Work Items + +1. Add `sbas_insar_catalog_service.py`. +2. Register completed SBAS runs into `result_products/result_assets` with `catalog_name = sbas_insar`. +3. Derive `bbox` and `center` from `stack_manifest.scenes[*].bbox`; verify or refine from GeoTIFF metadata when available. +4. Add startup self-maintenance similar to D-InSAR/PsInSAR catalog bootstrapping: + +```text +scan completed SBAS run publish bundles +upsert missing catalog rows +check primary/quality assets exist and are non-empty +record issues for missing bbox, missing GeoTIFF, missing preview, or DEM coverage mismatch +``` + +5. Add API routes: + +```text +GET /api/sbas-insar-products/catalog-status +POST /api/sbas-insar-products/rebuild-catalog +GET /api/sbas-insar-products +GET /api/sbas-insar-products/{product_id} +GET /api/sbas-insar-products/{product_id}/assets/{asset_id} +``` + +6. Extend production run detail to include explicit `geographic_coverage`: + +```json +{ + "center": {"lon": 129.199, "lat": 44.053}, + "bbox": {"min_lon": 128.769, "min_lat": 43.749, "max_lon": 129.629, "max_lat": 44.358}, + "scene_bbox_count": 7, + "dem_covers_stack_bbox": false, + "dem_covers_stack_center": true, + "monitor_points": [{"point_id": "...", "lon": 129.102, "lat": 44.150}] +} +``` + +### Frontend Work Items + +1. Add geographic coverage block to `SbasInsarProductionPanel`. +2. Add new `SbasInsarProductsPanel`. +3. Add API client module for SBAS product catalog. +4. Add navigation entry under result management/result analysis, separate from production management. +5. Reuse existing map overlay patterns from D-InSAR where practical, but keep SBAS asset roles and product semantics independent. diff --git a/frontend/src/SbasInsarProductionPanel.jsx b/frontend/src/SbasInsarProductionPanel.jsx index 5b5d73e..d9c0621 100644 --- a/frontend/src/SbasInsarProductionPanel.jsx +++ b/frontend/src/SbasInsarProductionPanel.jsx @@ -4,17 +4,22 @@ import { auditSbasInsarStack, decideSbasInsarItab, discoverSbasInsarStacks, - getSbasInsarArtifactUrl, getSbasInsarCapabilities, getSbasInsarRun, getSbasInsarRunArtifactUrl, - getSbasInsarTrialRun, listSbasInsarRuns, - listSbasInsarTrialRuns, prepareSbasInsarCoregistration, + prepareSbasInsarInterferograms, + prepareSbasInsarIptaTimeseries, + prepareSbasInsarRdcDem, + prepareSbasInsarWorkflow, runSbasInsarBaselineAudit, submitSbasInsarCoregistrationJob, + submitSbasInsarInterferogramsJob, + submitSbasInsarIptaTimeseriesJob, + submitSbasInsarRdcDemJob, submitSbasInsarRun, + submitSbasInsarWorkflowJob, } from './api/sbasInsarProduction'; const shellStyle = { @@ -90,7 +95,14 @@ function formatBytes(value) { } function StatusBadge({ value }) { - const okValues = new Set(['TRIAL_READY', 'READY', 'READY_FOR_GAMMA_BASELINE_AUDIT']); + const okValues = new Set([ + 'READY', + 'READY_FOR_GAMMA_BASELINE_AUDIT', + 'WORKFLOW_READY', + 'WORKFLOW_COMPLETED', + 'COMPLETED', + 'IPTA_TIMESERIES_READY', + ]); const isOk = okValues.has(value); const color = isOk ? '#0f766e' : '#92400e'; return ( @@ -130,20 +142,6 @@ function Metric({ label, value }) { ); } -function ArtifactLink({ trialId, artifact }) { - const href = getSbasInsarArtifactUrl(trialId, artifact.relative_path); - return ( - - 打开 - - ); -} - function RunArtifactLink({ runId, artifact }) { const href = getSbasInsarRunArtifactUrl(runId, artifact.relative_path); return ( @@ -160,14 +158,10 @@ function RunArtifactLink({ runId, artifact }) { export default function SbasInsarProductionPanel({ readOnly = false }) { const [capabilities, setCapabilities] = useState(null); - const [trials, setTrials] = useState([]); const [runs, setRuns] = useState([]); - const [selectedTrialId, setSelectedTrialId] = useState(''); const [selectedRunId, setSelectedRunId] = useState(''); - const [detail, setDetail] = useState(null); const [runDetail, setRunDetail] = useState(null); const [loading, setLoading] = useState(false); - const [detailLoading, setDetailLoading] = useState(false); const [runDetailLoading, setRunDetailLoading] = useState(false); const [error, setError] = useState(''); const [discovering, setDiscovering] = useState(false); @@ -181,58 +175,43 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { const [coregistrationLoading, setCoregistrationLoading] = useState(false); const [coregistrationJobLoading, setCoregistrationJobLoading] = useState(false); const [coregistrationJob, setCoregistrationJob] = useState(null); + const [rdcDemLoading, setRdcDemLoading] = useState(false); + const [rdcDemJobLoading, setRdcDemJobLoading] = useState(false); + const [rdcDemJob, setRdcDemJob] = useState(null); + const [interferogramLoading, setInterferogramLoading] = useState(false); + const [interferogramJobLoading, setInterferogramJobLoading] = useState(false); + const [interferogramJob, setInterferogramJob] = useState(null); + const [iptaTimeseriesLoading, setIptaTimeseriesLoading] = useState(false); + const [iptaTimeseriesJobLoading, setIptaTimeseriesJobLoading] = useState(false); + const [iptaTimeseriesJob, setIptaTimeseriesJob] = useState(null); + const [workflowLoading, setWorkflowLoading] = useState(false); + const [workflowJobLoading, setWorkflowJobLoading] = useState(false); + const [workflowJob, setWorkflowJob] = useState(null); - const loadTrials = useCallback(async () => { + const loadProductionRuns = useCallback(async () => { setLoading(true); setError(''); try { - const [capabilityData, trialData, runData] = await Promise.all([ + const [capabilityData, runData] = await Promise.all([ getSbasInsarCapabilities(), - listSbasInsarTrialRuns(), listSbasInsarRuns(), ]); - const items = Array.isArray(trialData?.items) ? trialData.items : []; const runItems = Array.isArray(runData?.items) ? runData.items : []; setCapabilities(capabilityData); - setTrials(items); setRuns(runItems); - setSelectedTrialId(current => current || items[0]?.trial_id || ''); setSelectedRunId(current => current || runItems[0]?.run_id || ''); } catch (exc) { setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 列表加载失败'); setCapabilities(null); - setTrials([]); setRuns([]); } finally { setLoading(false); } }, []); - const loadDetail = useCallback(async trialId => { - if (!trialId) { - setDetail(null); - return; - } - setDetailLoading(true); - setError(''); - try { - const data = await getSbasInsarTrialRun(trialId); - setDetail(data); - } catch (exc) { - setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 详情加载失败'); - setDetail(null); - } finally { - setDetailLoading(false); - } - }, []); - useEffect(() => { - loadTrials(); - }, [loadTrials]); - - useEffect(() => { - loadDetail(selectedTrialId); - }, [loadDetail, selectedTrialId]); + loadProductionRuns(); + }, [loadProductionRuns]); const loadRunDetail = useCallback(async runId => { if (!runId) { @@ -308,7 +287,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { : undefined, min_scenes: 3, require_orbits: true, - dry_run: true, + dry_run: false, monitor_point_strategy: 'auto_low_sigma_high_rate', }); const runId = data?.run?.run_id; @@ -326,6 +305,50 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } }, [readOnly, selectedStackId, stackCandidates]); + const workflowPayload = useMemo(() => ({ + force: false, + rlks: 8, + azlks: 8, + reference_window: 16, + mb_mode: 0, + timeout_seconds: 172800, + }), []); + + const handlePrepareWorkflow = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setWorkflowLoading(true); + setError(''); + try { + const data = await prepareSbasInsarWorkflow(selectedRunId, workflowPayload); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'Gamma SBAS workflow 生成失败'); + } finally { + setWorkflowLoading(false); + } + }, [readOnly, selectedRunId, workflowPayload]); + + const handleSubmitWorkflowJob = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setWorkflowJobLoading(true); + setError(''); + try { + const data = await submitSbasInsarWorkflowJob(selectedRunId, workflowPayload); + setWorkflowJob(data); + const detailData = await getSbasInsarRun(selectedRunId); + setRunDetail(detailData); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'Gamma SBAS workflow 任务提交失败'); + setWorkflowJob(null); + } finally { + setWorkflowJobLoading(false); + } + }, [readOnly, selectedRunId, workflowPayload]); + const handleBaselineAudit = useCallback(async (execute = false) => { if (!selectedRunId || readOnly) return; setBaselineAuditLoading(true); @@ -413,31 +436,168 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } }, [readOnly, selectedRunId]); - const artifacts = useMemo(() => detail?.artifacts || [], [detail]); - const primaryPreview = ( - artifacts.find(item => item.key === 'los_rate_toward_mm_per_year_geo_preview_png') - || artifacts.find(item => item.key === 'los_rate_toward_mm_per_year_bmp') - ); - const sigmaPreview = ( - artifacts.find(item => item.key === 'los_sigma_mm_per_year_geo_preview_png') - || artifacts.find(item => item.key === 'los_sigma_mm_per_year_bmp') - ); - const monitorPreview = artifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.png')); - const monitorCsv = artifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.csv')); - const productArtifacts = artifacts.filter(item => item.role !== 'monitor_point'); - const trial = detail?.trial || null; - const stack = trial?.stack || {}; - const summary = detail?.summary || {}; - const radar = summary.radar || {}; - const monitorPoint = Array.isArray(summary.monitor_points) ? summary.monitor_points[0] : null; + const handlePrepareRdcDem = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setRdcDemLoading(true); + setError(''); + try { + const data = await prepareSbasInsarRdcDem(selectedRunId, { + execute: false, + rlks: 8, + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR RDC DEM 计划生成失败'); + } finally { + setRdcDemLoading(false); + } + }, [readOnly, selectedRunId]); + + const handleSubmitRdcDemJob = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setRdcDemJobLoading(true); + setError(''); + try { + const data = await submitSbasInsarRdcDemJob(selectedRunId, { + rlks: 8, + timeout_seconds: 43200, + }); + setRdcDemJob(data); + const detailData = await getSbasInsarRun(selectedRunId); + setRunDetail(detailData); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR RDC DEM 任务提交失败'); + setRdcDemJob(null); + } finally { + setRdcDemJobLoading(false); + } + }, [readOnly, selectedRunId]); + + const handlePrepareInterferograms = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setInterferogramLoading(true); + setError(''); + try { + const data = await prepareSbasInsarInterferograms(selectedRunId, { + execute: false, + rlks: 8, + azlks: 8, + unwrap_threshold: 0.2, + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR interferogram 计划生成失败'); + } finally { + setInterferogramLoading(false); + } + }, [readOnly, selectedRunId]); + + const handleSubmitInterferogramsJob = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setInterferogramJobLoading(true); + setError(''); + try { + const data = await submitSbasInsarInterferogramsJob(selectedRunId, { + rlks: 8, + azlks: 8, + unwrap_threshold: 0.2, + timeout_seconds: 43200, + }); + setInterferogramJob(data); + const detailData = await getSbasInsarRun(selectedRunId); + setRunDetail(detailData); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR interferogram 任务提交失败'); + setInterferogramJob(null); + } finally { + setInterferogramJobLoading(false); + } + }, [readOnly, selectedRunId]); + + const handlePrepareIptaTimeseries = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setIptaTimeseriesLoading(true); + setError(''); + try { + const data = await prepareSbasInsarIptaTimeseries(selectedRunId, { + execute: false, + rlks: 8, + reference_window: 16, + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR IPTA timeseries 计划生成失败'); + } finally { + setIptaTimeseriesLoading(false); + } + }, [readOnly, selectedRunId]); + + const handleSubmitIptaTimeseriesJob = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setIptaTimeseriesJobLoading(true); + setError(''); + try { + const data = await submitSbasInsarIptaTimeseriesJob(selectedRunId, { + rlks: 8, + reference_window: 16, + timeout_seconds: 43200, + }); + setIptaTimeseriesJob(data); + const detailData = await getSbasInsarRun(selectedRunId); + setRunDetail(detailData); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR IPTA timeseries 任务提交失败'); + setIptaTimeseriesJob(null); + } finally { + setIptaTimeseriesJobLoading(false); + } + }, [readOnly, selectedRunId]); + const selectedStack = stackCandidates.find(item => item.stack_id === selectedStackId) || null; const run = runDetail?.run || null; const runManifest = runDetail?.manifest || {}; + const workflowManifest = runDetail?.workflow_manifest || {}; + const workflowState = runDetail?.workflow_state || {}; + const workflowSteps = Array.isArray(workflowManifest.steps) ? workflowManifest.steps : []; + const expertDocumentSteps = Array.isArray(workflowManifest.expert_document?.steps) + ? workflowManifest.expert_document.steps + : []; + const workflowStepState = workflowState.steps || {}; const stagePlan = runDetail?.command_manifest?.stage_plan || []; const runArtifacts = runDetail?.artifacts || []; const baselineSummary = runManifest.baseline_audit?.summary || null; const itabDecision = runManifest.baseline_audit?.itab_decision || null; const coregistrationPlan = runManifest.coregistration || null; + const rdcDemPlan = runManifest.rdc_dem || null; + const interferogramPlan = runManifest.interferograms || null; + const detrendAtmPlan = runManifest.detrend_atm || null; + const iptaTimeseriesPlan = runManifest.ipta_timeseries || null; + const publishProductsPlan = runManifest.publish_products || null; + const monitorProductsPlan = runManifest.monitor_point_products || null; + const runPrimaryPreview = ( + runArtifacts.find(item => item.key === 'los_rate_toward_m_per_year_hls_geo_preview_png') + || runArtifacts.find(item => item.key === 'los_rate_toward_mm_per_year_geo_preview_png') + || runArtifacts.find(item => item.key === 'los_rate_toward_mm_per_year_bmp') + ); + const runSigmaPreview = ( + runArtifacts.find(item => item.key === 'los_sigma_m_per_year_cc_geo_preview_png') + || runArtifacts.find(item => item.key === 'los_sigma_mm_per_year_geo_preview_png') + || runArtifacts.find(item => item.key === 'los_sigma_mm_per_year_bmp') + ); + const runMonitorPreview = runArtifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.png')); + const runMonitorCsv = runArtifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.csv')); const itabApproved = itabDecision?.decision === 'approve' || runManifest.baseline_audit?.approved_for_next_stage === true; const itabRejected = itabDecision?.decision === 'reject'; @@ -453,7 +613,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { + + + {workflowJob && workflowJob.run_id === run.run_id && ( +
+ 已提交后台任务：{workflowJob.task_id}；Job：{workflowJob.job_id} +
+ )} + {workflowSteps.length > 0 && ( +
+ {workflowSteps.map(step => { + const state = workflowStepState[step.id] || {}; + return ( +
+
+
+ {step.id} · {step.name} +
+
+ {(step.expert_tools || []).join(', ') || 'planned'}；{step.enabled ? 'enabled' : 'planned'} +
+
+ +
+ ); + })} +
+ )} + {expertDocumentSteps.length > 0 && ( +
+
Expert document path
+
+ {expertDocumentSteps.length} sections from the LT1 Gamma SBAS expert document. Commands are shown as the acceptance checklist; implementation may be a bridge where the verified experiment already covers the same Gamma function. +
+
+ {expertDocumentSteps.map(item => { + const mappedStatuses = (item.mapped_workflow_steps || []) + .map(mapped => { + const state = workflowStepState[mapped.id] || {}; + return state.status || mapped.status; + }) + .filter(Boolean); + const displayStatus = mappedStatuses[0] || item.implementation_status || 'planned'; + const commandPreview = (item.commands || []).slice(0, 3).join(' | '); + return ( +
+
+
+ {item.order}. {item.title} +
+
+ maps to {(item.workflow_steps || []).join(', ') || '-'}; {item.command_count || 0} commands; {item.implementation_status} +
+ {commandPreview && ( +
+ {commandPreview} +
+ )} +
+ +
+ ); + })} +
+
+ )} + + )} + + {!readOnly && false && (
+ + + + + +
)} @@ -943,6 +1329,296 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { + {rdcDemPlan && ( +
+
RDC DEM Plan
+
+ + + + +
+
+ Script: {rdcDemPlan.script_path || '-'} +
+
+ DEM source: {rdcDemPlan.dem_source?.windows_path || rdcDemPlan.dem_source?.wsl_path || '-'} +
+
+ Output: {rdcDemPlan.outputs?.rdc_dem || '-'} +
+ {rdcDemPlan.execution && ( +
+ Execution: {rdcDemPlan.execution.status || '-'};{' '} + {rdcDemPlan.execution.ended_at || rdcDemPlan.execution.started_at || '-'} +
+ )} + {rdcDemPlan.summary && ( +
+ Ready: {rdcDemPlan.summary.ready ? 'Yes' : 'No'}; missing:{' '} + {(rdcDemPlan.summary.missing_outputs || []).join(', ') || 'none'} +
+ )} + {rdcDemJob && rdcDemJob.run_id === run.run_id && ( +
+ Queued task: {rdcDemJob.task_id}; Job: {rdcDemJob.job_id} +
+ )} +
+ )} + + {interferogramPlan && ( +
+
Interferogram Plan
+
+ + + + +
+
+ Script: {interferogramPlan.script_path || '-'} +
+
+ DIFF_tab: {interferogramPlan.outputs?.diff_tab || '-'} +
+ {interferogramPlan.execution && ( +
+ Execution: {interferogramPlan.execution.status || '-'};{' '} + {interferogramPlan.execution.ended_at || interferogramPlan.execution.started_at || '-'} +
+ )} + {interferogramPlan.summary && ( +
+ Ready pairs: {interferogramPlan.summary.ready_pair_count || 0}/ + {interferogramPlan.summary.pair_count || 0}; missing:{' '} + {(interferogramPlan.summary.missing_pairs || []).join(', ') || 'none'} +
+ )} + {interferogramJob && interferogramJob.run_id === run.run_id && ( +
+ Queued task: {interferogramJob.task_id}; Job: {interferogramJob.job_id} +
+ )} +
+ )} + + {detrendAtmPlan && ( +
+
Detrend / ATM Plan
+
+ + + + +
+
+ Script: {detrendAtmPlan.script_path || '-'} +
+
+ DIFF_atmsub_tab: {detrendAtmPlan.outputs?.diff_atmsub_tab || detrendAtmPlan.summary?.outputs?.diff_atmsub_tab?.path || '-'} +
+ {detrendAtmPlan.execution && ( +
+ Execution: {detrendAtmPlan.execution.status || '-'};{' '} + {detrendAtmPlan.execution.ended_at || detrendAtmPlan.execution.started_at || '-'} +
+ )} + {detrendAtmPlan.summary && ( +
+ Missing pairs: {(detrendAtmPlan.summary.missing_pairs || []).join(', ') || 'none'}; rows:{' '} + {detrendAtmPlan.summary.diff_atmsub_tab_row_count || 0}/ + {detrendAtmPlan.summary.itab_atmsub_row_count || 0} +
+ )} +
+ )} + + {iptaTimeseriesPlan && ( +
+
IPTA Time-Series Plan
+
+ + + + +
+
+ Script: {iptaTimeseriesPlan.script_path || '-'} +
+
+ ts_rate: {iptaTimeseriesPlan.outputs?.ts_rate || iptaTimeseriesPlan.summary?.outputs?.ts_rate?.path || '-'} +
+
+ sigma_rate: {iptaTimeseriesPlan.outputs?.sigma_rate || iptaTimeseriesPlan.summary?.outputs?.sigma_rate?.path || '-'} +
+ {iptaTimeseriesPlan.execution && ( +
+ Execution: {iptaTimeseriesPlan.execution.status || '-'};{' '} + {iptaTimeseriesPlan.execution.ended_at || iptaTimeseriesPlan.execution.started_at || '-'} +
+ )} + {iptaTimeseriesPlan.summary && ( +
+ Missing: {(iptaTimeseriesPlan.summary.missing_outputs || []).join(', ') || 'none'}; rows:{' '} + {iptaTimeseriesPlan.summary.diff_ts_row_count || 0}/ + {iptaTimeseriesPlan.summary.itab_ts_row_count || 0} +
+ )} + {iptaTimeseriesJob && iptaTimeseriesJob.run_id === run.run_id && ( +
+ Queued task: {iptaTimeseriesJob.task_id}; Job: {iptaTimeseriesJob.job_id} +
+ )} +
+ )} + + {publishProductsPlan && ( +
+
Publish Products
+
+ + + + +
+
+ Script: {publishProductsPlan.script_path || '-'} +
+ {publishProductsPlan.summary && ( +
+ Missing: {(publishProductsPlan.summary.missing_outputs || []).join(', ') || 'none'} +
+ )} +
+ )} + + {monitorProductsPlan && ( +
+
Monitoring Point Products
+
+ + + + +
+ {monitorProductsPlan.summary?.monitor_outputs?.length > 0 && ( +
+ Points: {monitorProductsPlan.summary.monitor_outputs.map(item => item.point_id).join(', ')} +
+ )} +
+ )} + + {(runPrimaryPreview || runSigmaPreview || runMonitorPreview) && ( +
+ {runPrimaryPreview && ( +
+
LOS Velocity
+
+ {runPrimaryPreview.key.endsWith('_geo_preview_png') ? 'WGS84 geocoded preview' : 'RDC QA preview'} +
+ Run LOS velocity toward radar positive +
+ )} + {runSigmaPreview && ( +
+
LOS Sigma
+
+ {runSigmaPreview.key.endsWith('_geo_preview_png') ? 'WGS84 geocoded preview' : 'RDC QA preview'} +
+ Run LOS velocity sigma +
+ )} + {runMonitorPreview && ( +
+
Monitoring Curve
+ Run monitoring point LOS displacement time series + {runMonitorCsv && ( +
+ +
+ )} +
+ )} +
+ )} + {runArtifacts.length > 0 && (
@@ -976,192 +1652,6 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { -
-
-
-

试验/生产序列

- {trials.length} 组 -
-
- {trials.map(item => { - const active = item.trial_id === selectedTrialId; - return ( - - ); - })} - {!loading && trials.length === 0 && ( -
- 未发现可读取的 Gamma SBAS/IPTA 试验汇总。 -
- )} -
-
- -
- {detailLoading &&
正在加载详情...
} - {!detailLoading && trial && ( -
-
-
-

{trial.trial_id}

-
- {stack.platform} / relOrbit {stack.relative_orbit} / {stack.direction} / {stack.mode} / {stack.polarization} -
-
- -
- -
- - - - -
- -
- {primaryPreview && ( -
-
LOS 速率图
-
- {primaryPreview.key.endsWith('_geo_preview_png') ? 'WGS84 地理编码预览' : 'RDC 处理几何浏览图'} -
- LOS velocity toward radar positive -
- )} - {monitorPreview && ( -
-
监测点形变曲线
- Monitoring point LOS displacement time series -
- )} - {sigmaPreview && ( -
-
LOS sigma 图
-
- {sigmaPreview.key.endsWith('_geo_preview_png') ? 'WGS84 地理编码预览' : 'RDC 处理几何浏览图'} -
- LOS velocity sigma -
- )} -
- -
-
-
LOS 符号约定
-
- {radar.los_sign_convention || trial.los_sign_convention || '-'} -
-
-
-
监测点
-
- {monitorPoint ? ( - <> - {monitorPoint.point_id}，约 {formatValue(monitorPoint.approx_lonlat?.lon)}E / - {formatValue(monitorPoint.approx_lonlat?.lat)}N，速率 - {formatValue(monitorPoint.los_rate_toward_mm_per_year, ' mm/yr')} - {monitorCsv && ( - <> - {' '} - - - )} - - ) : '-'} -
-
-
-
- 当前曲线是自动选取的单个样例点，用于验证时序曲线能力；正式监测点需要用户点击、导入点位或质量筛选后的点集。 -
- -
-
产品文件
-
-
- - - - - - - - - - {productArtifacts.map(item => ( - - - - - - - ))} - -
产品角色大小操作
{item.label}{item.role} - {formatBytes(item.size_bytes)} - - -
-
- - - )} - {!detailLoading && !trial && ( -
请选择一个 SBAS-InSAR 试验或生产序列。
- )} - - ); } diff --git a/frontend/src/api/sbasInsarProduction.js b/frontend/src/api/sbasInsarProduction.js index 3746de1..8d17cca 100644 --- a/frontend/src/api/sbasInsarProduction.js +++ b/frontend/src/api/sbasInsarProduction.js @@ -24,6 +24,12 @@ export const listSbasInsarRuns = () => export const getSbasInsarRun = runId => apiClient.get(`/sbas-insar-production/runs/${encodeURIComponent(runId)}`).then(r => r.data); +export const prepareSbasInsarWorkflow = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/workflow`, payload).then(r => r.data); + +export const submitSbasInsarWorkflowJob = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/workflow/jobs`, payload).then(r => r.data); + export const runSbasInsarBaselineAudit = (runId, payload = {}) => apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/baseline-audit`, payload).then(r => r.data); @@ -36,14 +42,23 @@ export const prepareSbasInsarCoregistration = (runId, payload = {}) => export const submitSbasInsarCoregistrationJob = (runId, payload = {}) => apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/coregistration/jobs`, payload).then(r => r.data); +export const prepareSbasInsarRdcDem = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/rdc-dem`, payload).then(r => r.data); + +export const submitSbasInsarRdcDemJob = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/rdc-dem/jobs`, payload).then(r => r.data); + +export const prepareSbasInsarInterferograms = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/interferograms`, payload).then(r => r.data); + +export const submitSbasInsarInterferogramsJob = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/interferograms/jobs`, payload).then(r => r.data); + +export const prepareSbasInsarIptaTimeseries = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/ipta-timeseries`, payload).then(r => r.data); + +export const submitSbasInsarIptaTimeseriesJob = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/ipta-timeseries/jobs`, payload).then(r => r.data); + export const getSbasInsarRunArtifactUrl = (runId, relativePath) => `/api/sbas-insar-production/runs/${encodeURIComponent(runId)}/artifacts/${encodeArtifactPath(relativePath)}`; - -export const listSbasInsarTrialRuns = () => - apiClient.get('/sbas-insar-production/trial-runs').then(r => r.data); - -export const getSbasInsarTrialRun = trialId => - apiClient.get(`/sbas-insar-production/trial-runs/${encodeURIComponent(trialId)}`).then(r => r.data); - -export const getSbasInsarArtifactUrl = (trialId, relativePath) => - `/api/sbas-insar-production/trial-runs/${encodeURIComponent(trialId)}/artifacts/${encodeArtifactPath(relativePath)}`;