From 5290b071b3c2ffb177ca62ad62d12a27e57d4239 Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 27 Apr 2026 07:52:20 +0800 Subject: [PATCH] feat(isce2): harden managed production flow - add one-time base DEM preparation utility and prepared-first DEM resolution - support geocode resume/full-geocode and guard large unprepared DEMs - repair missing completion files during in-place ISCE2 publish rebuild - add ISCE2 stabilization update log --- .env.example | 3 + backend/app/dinsar_engines/isce2_engine.py | 35 +- .../app/isce2_pipeline/lt1_input_resolver.py | 2 +- .../isce2_pipeline/prepare_isce2_base_dem.py | 327 ++++++++++++++++++ .../isce2_pipeline/run_lt1_dinsar_pipeline.py | 327 +++++++++++++++--- .../app/services/dinsar_completion_files.py | 242 +++++++++++++ .../services/pyint_input_assets_service.py | 7 +- .../app/services/result_catalog_service.py | 24 ++ deploy/wsl/runners/isce2_runner.py | 4 + .../ISCE2_STABILIZATION_UPDATELOG_20260427.md | 69 ++++ 10 files changed, 995 insertions(+), 45 deletions(-) create mode 100644 backend/app/isce2_pipeline/prepare_isce2_base_dem.py create mode 100644 backend/app/services/dinsar_completion_files.py create mode 100644 docs/ISCE2_STABILIZATION_UPDATELOG_20260427.md diff --git a/.env.example b/.env.example index 8f2ae9c..7e0bdd5 100644 --- a/.env.example +++ b/.env.example @@ -85,6 +85,7 @@ ORBIT_POOL_LANDSAR= # ----------------------------------------------------------------------------- # DEM / 其他数据 # ----------------------------------------------------------------------------- +# Raw D-InSAR DEM source for SARscape/ENVI. Keep the base path without the .wgs84 suffix here. IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape SRTM_DEM_DIR=D:\SRTM30m GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2 @@ -138,6 +139,7 @@ ISCE2_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python ISCE2_PROFILE=lt1_stripmap ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/applications/stripmapApp.py ISCE2_PIPELINE_SCRIPT= +# ISCE2 should point to the prepared WGS84 DEM after the one-time conversion. ISCE2_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84 ISCE2_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\isce2_work ISCE2_OUTPUT_ROOT=D:\production_results\dinsar @@ -159,6 +161,7 @@ PYINT_OUTPUT_ROOT=D:\production_results\dinsar PYINT_DEM_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_dem PYINT_DEM_MODE=local_fabdem PYINT_FABDEM_ROOT= +# When PYINT_DEM_MODE=prepared_file, point this to the same prepared WGS84 DEM. PYINT_PREPARED_DEM_PATH= PYINT_OPENTOPO_DEM_TYPE=SRTMGL1 PYINT_DEM_STRICT=true diff --git a/backend/app/dinsar_engines/isce2_engine.py b/backend/app/dinsar_engines/isce2_engine.py index 3d54bbd..dcc3029 100644 --- a/backend/app/dinsar_engines/isce2_engine.py +++ b/backend/app/dinsar_engines/isce2_engine.py @@ -32,6 +32,7 @@ ORBIT_MARGIN_MAX_SEC = 120.0 TARGET_GRID_SIZE_MIN_M = 5 TARGET_GRID_SIZE_MAX_M = 100 RERUN_MODE_UNFINISHED_ONLY = "unfinished_only" +RESUME_STAGE_CHOICES = {"", "unwrap", "geocode", "export"} def _read_env(name: str, default: str = "") -> str: @@ -70,6 +71,16 @@ def _normalize_optional_path(value: Any) -> str: return os.path.normpath(os.path.abspath(text)) +def _prefer_prepared_dem_variant(path: str) -> str: + normalized = _normalize_optional_path(path) + if not normalized or normalized.lower().endswith(".wgs84"): + return normalized + prepared = normalized + ".wgs84" + if os.path.isfile(prepared) and os.path.isfile(prepared + ".xml"): + return prepared + return normalized + + def _load_json_file(path: str) -> Dict[str, Any]: try: with open(path, "r", encoding="utf-8") as fp: @@ -146,11 +157,11 @@ class Isce2Engine(DinsarEngine): @property def _dem_path(self) -> str: - explicit = _read_env("ISCE2_DEM_PATH", "") + explicit = _prefer_prepared_dem_variant(_read_env("ISCE2_DEM_PATH", "")) if explicit: return explicit base = _read_env("IDL_DINSAR_DEM_BASE_FILE", "") - return f"{base}.wgs84" if base else "" + return _prefer_prepared_dem_variant(base) if base else "" @property def _orbit_pool_isce2(self) -> str: @@ -320,6 +331,18 @@ class Isce2Engine(DinsarEngine): ) normalized["orbit_margin_sec"] = orbit_margin + if "full_geocode" in normalized: + normalized["full_geocode"] = bool(normalized["full_geocode"]) + + if "resume_from" in normalized and normalized["resume_from"] is not None: + resume_from = str(normalized["resume_from"]).strip().lower() + if resume_from not in RESUME_STAGE_CHOICES: + raise ValueError("resume_from must be one of: unwrap, geocode, export") + normalized["resume_from"] = resume_from + + if bool(normalized.get("force")) and str(normalized.get("resume_from") or "").strip(): + raise ValueError("force cannot be used together with resume_from") + return normalized # ------------------------------------------------------------------ @@ -507,6 +530,8 @@ class Isce2Engine(DinsarEngine): bbox_margin: Any, wavelength: Any, orbit_margin_sec: Any, + full_geocode: bool, + resume_from: str, pair_meta: Dict[str, Any], ) -> Dict[str, Any]: return { @@ -541,6 +566,8 @@ class Isce2Engine(DinsarEngine): "bbox_margin": bbox_margin, "wavelength": wavelength, "orbit_margin_sec": orbit_margin_sec, + "full_geocode": bool(full_geocode), + "resume_from": str(resume_from or "").strip(), }, "pair_meta": dict(pair_meta or {}), } @@ -650,6 +677,8 @@ class Isce2Engine(DinsarEngine): bbox_margin = extra.get("bbox_margin") wavelength = LT1_FIXED_WAVELENGTH orbit_margin_sec = extra.get("orbit_margin_sec") + full_geocode = bool(extra.get("full_geocode")) + resume_from = str(extra.get("resume_from") or "").strip().lower() wsl_isce2_pool = "" if self._orbit_pool_isce2: @@ -792,6 +821,8 @@ class Isce2Engine(DinsarEngine): bbox_margin=bbox_margin, wavelength=wavelength, orbit_margin_sec=orbit_margin_sec, + full_geocode=full_geocode, + resume_from=resume_from, pair_meta=pair_meta, ) broker_result = wsl_broker.run_manifest( diff --git a/backend/app/isce2_pipeline/lt1_input_resolver.py b/backend/app/isce2_pipeline/lt1_input_resolver.py index 90b3a4b..39b9df5 100644 --- a/backend/app/isce2_pipeline/lt1_input_resolver.py +++ b/backend/app/isce2_pipeline/lt1_input_resolver.py @@ -250,4 +250,4 @@ def _prepared_dem_variants(value: str | Path) -> tuple[str | Path, ...]: return () if text.lower().endswith(".wgs84"): return (value,) - return (value, f"{text}.wgs84") + return (f"{text}.wgs84", value) diff --git a/backend/app/isce2_pipeline/prepare_isce2_base_dem.py b/backend/app/isce2_pipeline/prepare_isce2_base_dem.py new file mode 100644 index 0000000..bfe84a4 --- /dev/null +++ b/backend/app/isce2_pipeline/prepare_isce2_base_dem.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +try: + from .lt1_input_resolver import load_env_file +except ImportError: + SCRIPT_DIR = Path(__file__).resolve().parent + if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + from lt1_input_resolver import load_env_file # type: ignore + + +@dataclass(frozen=True) +class DemResolution: + source_label: str + configured_value: str + source_path: Path + prepared_path: Path + + +def _configure_proj_environment() -> None: + if os.environ.get("PROJ_DATA") and os.environ.get("PROJ_LIB"): + return + + candidates: list[Path] = [] + + isce2_python = str(os.environ.get("ISCE2_PYTHON") or "").strip() + if isce2_python: + candidates.append(Path(isce2_python).resolve().parents[1] / "share" / "proj") + + candidates.append(Path(sys.executable).resolve().parents[1] / "share" / "proj") + + for candidate in candidates: + if candidate.exists(): + proj_path = candidate.as_posix() + os.environ.setdefault("PROJ_DATA", proj_path) + os.environ.setdefault("PROJ_LIB", proj_path) + return + + +def normalize_linux_path(value: str | Path) -> Path: + text = str(value or "").strip().strip('"').strip("'") + if not text: + return Path("") + if text.startswith("\\\\"): + raise ValueError("UNC paths are not supported directly. Mount them in WSL first.") + + match = re.match(r"^([A-Za-z]):[\\/](.*)$", text) + if match: + drive = match.group(1).lower() + rest = match.group(2).replace("\\", "/") + return Path(f"/mnt/{drive}/{rest}") + return Path(text) + + +def linux_path_to_windows(path: Path) -> str: + text = str(path) + match = re.match(r"^/mnt/([a-zA-Z])/(.*)$", text) + if not match: + return text + drive = match.group(1).upper() + rest = match.group(2).replace("/", "\\") + return f"{drive}:\\{rest}" + + +def parse_args() -> argparse.Namespace: + script_dir = Path(__file__).resolve().parent + repo_root = script_dir.parent.parent.parent + parser = argparse.ArgumentParser( + description="Prepare a reusable WGS84 '.wgs84' DEM once for ISCE2 and PyINT." + ) + parser.add_argument( + "--source-dem", + default=None, + help="Optional source DEM base path. Accepts either a raw DEM base path or an existing .wgs84 path.", + ) + parser.add_argument( + "--env-file", + default=str(repo_root / ".env"), + help="Project .env file used to resolve default DEM paths.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Rebuild the prepared .wgs84 outputs even if they already exist.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Resolve paths and print the planned output without modifying files.", + ) + return parser.parse_args() + + +def _remove_prepare_outputs(base_path: Path) -> None: + for suffix in ("", ".xml", ".vrt", ".hdr", ".aux.xml"): + candidate = Path(str(base_path) + suffix) + if candidate.exists(): + candidate.unlink() + + +def _ensure_source_xml(path: Path) -> None: + if not Path(str(path) + ".xml").exists(): + raise FileNotFoundError(f"Missing DEM XML sidecar: {path}.xml") + + +def _maybe_prepared_path(path: Path) -> Path: + return path if str(path).lower().endswith(".wgs84") else Path(str(path) + ".wgs84") + + +def _existing_path(path: Path) -> bool: + return path.exists() and Path(str(path) + ".xml").exists() + + +def _resolve_from_value(label: str, value: str) -> DemResolution | None: + normalized = normalize_linux_path(value) + if not str(normalized): + return None + + if _existing_path(normalized): + if str(normalized).lower().endswith(".wgs84"): + raw_candidate = normalized.with_suffix("") + if _existing_path(raw_candidate): + return DemResolution(label, value, raw_candidate, normalized) + return DemResolution(label, value, normalized, normalized) + return DemResolution(label, value, normalized, _maybe_prepared_path(normalized)) + + if str(normalized).lower().endswith(".wgs84"): + raw_candidate = normalized.with_suffix("") + if _existing_path(raw_candidate): + return DemResolution(label, value, raw_candidate, normalized) + else: + prepared_candidate = Path(str(normalized) + ".wgs84") + if _existing_path(prepared_candidate): + return DemResolution(label, value, normalized, prepared_candidate) + if Path(str(normalized) + ".xml").exists(): + return DemResolution(label, value, normalized, prepared_candidate) + return None + + +def resolve_dem_from_env(explicit_source: str | None, env_file: Path) -> DemResolution: + env_values = load_env_file(env_file) + candidates: list[tuple[str, str]] = [] + if explicit_source: + candidates.append(("explicit", explicit_source)) + + # Keep the raw source path explicit in .env via IDL_DINSAR_DEM_BASE_FILE. + for key in ( + "IDL_DINSAR_DEM_BASE_FILE", + "ISCE2_DEM_PATH", + "PYINT_PREPARED_DEM_PATH", + ): + value = str(env_values.get(key) or "").strip() + if value: + candidates.append((key, value)) + + for label, value in candidates: + resolved = _resolve_from_value(label, value) + if resolved is not None: + return resolved + + raise FileNotFoundError( + "Unable to resolve a DEM source from --source-dem, IDL_DINSAR_DEM_BASE_FILE, " + "ISCE2_DEM_PATH, or PYINT_PREPARED_DEM_PATH." + ) + + +def inspect_dem(path: Path) -> dict[str, Any]: + _configure_proj_environment() + import isce # noqa: F401 + import isceobj + + _ensure_source_xml(path) + dem = isceobj.createDemImage() + dem.load(str(path) + ".xml") + return { + "path": str(path), + "reference": str(dem.reference or "").strip(), + "width": int(dem.width or 0), + "length": int(dem.length or 0), + "first_lon": float(dem.coord1.coordStart or 0.0), + "first_lat": float(dem.coord2.coordStart or 0.0), + "delta_lon": float(dem.coord1.coordDelta or 0.0), + "delta_lat": float(dem.coord2.coordDelta or 0.0), + } + + +def ensure_prepared_dem(source_path: Path, prepared_path: Path, *, force: bool) -> dict[str, Any]: + _configure_proj_environment() + import isce # noqa: F401 + import isceobj + from iscesys.DataManager import createManager + + if source_path != prepared_path: + _ensure_source_xml(source_path) + + if _existing_path(prepared_path) and not force: + prepared_meta = inspect_dem(prepared_path) + if prepared_meta["reference"].upper() != "WGS84": + raise RuntimeError( + f"Prepared DEM exists but is not WGS84: {prepared_path} ({prepared_meta['reference']})" + ) + return { + "action": "validated_existing", + "source": inspect_dem(source_path), + "prepared": prepared_meta, + } + + if source_path == prepared_path: + prepared_meta = inspect_dem(prepared_path) + if prepared_meta["reference"].upper() != "WGS84": + raise RuntimeError( + f"Provided DEM is not WGS84: {prepared_path} ({prepared_meta['reference']})" + ) + return { + "action": "already_wgs84", + "source": prepared_meta, + "prepared": prepared_meta, + } + + if force: + _remove_prepare_outputs(prepared_path) + + source_dem = isceobj.createDemImage() + source_dem.load(str(source_path) + ".xml") + # Some DEM XML sidecars store only the basename. Force an absolute filename + # so ISCE2 writes the generated ".wgs84" next to the source DEM, not in cwd. + source_dem.filename = str(source_path) + if not Path(str(source_path) + ".vrt").exists(): + source_dem.renderVRT() + + source_reference = str(source_dem.reference or "").strip().upper() + if source_reference != "EGM96": + raise RuntimeError( + f"Expected an EGM96 raw DEM before preparation, got: {source_dem.reference or ''}" + ) + + dem_stitcher = createManager("dem1", "iscestitcher") + dem_stitcher.noFilling = False + prepared_dem = dem_stitcher.correct(source_dem) + prepared_dem.metadatalocation = str(prepared_path) + ".xml" + prepared_dem._extraFilename = str(prepared_path) + ".vrt" + if not Path(prepared_dem.metadatalocation).exists(): + prepared_dem.dump(prepared_dem.metadatalocation) + if not Path(prepared_dem._extraFilename).exists(): + prepared_dem.renderVRT() + + prepared_meta = inspect_dem(prepared_path) + if prepared_meta["reference"].upper() != "WGS84": + raise RuntimeError( + f"Generated prepared DEM is not WGS84: {prepared_path} ({prepared_meta['reference']})" + ) + + return { + "action": "converted", + "source": inspect_dem(source_path), + "prepared": prepared_meta, + } + + +def build_report(resolution: DemResolution, outcome: dict[str, Any]) -> dict[str, Any]: + source_windows = linux_path_to_windows(resolution.source_path) + prepared_windows = linux_path_to_windows(resolution.prepared_path) + return { + "source_label": resolution.source_label, + "configured_value": resolution.configured_value, + "source_dem_wsl": str(resolution.source_path), + "source_dem_windows": source_windows, + "prepared_dem_wsl": str(resolution.prepared_path), + "prepared_dem_windows": prepared_windows, + "action": outcome["action"], + "source_dem": outcome["source"], + "prepared_dem": outcome["prepared"], + "suggested_env": { + "IDL_DINSAR_DEM_BASE_FILE": source_windows, + "ISCE2_DEM_PATH": prepared_windows, + "PYINT_PREPARED_DEM_PATH": prepared_windows, + }, + } + + +def main() -> int: + args = parse_args() + env_file = normalize_linux_path(args.env_file) + resolution = resolve_dem_from_env(args.source_dem, env_file) + + if args.dry_run: + outcome = { + "action": "dry_run", + "source": inspect_dem(resolution.source_path), + "prepared": inspect_dem(resolution.prepared_path) + if _existing_path(resolution.prepared_path) + else {"path": str(resolution.prepared_path), "reference": "", "width": 0, "length": 0}, + } + else: + outcome = ensure_prepared_dem( + resolution.source_path, + resolution.prepared_path, + force=bool(args.force), + ) + + report = build_report(resolution, outcome) + report_path = Path(str(resolution.prepared_path) + ".prepare_report.json") + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + + print(f"Action: {report['action']}") + print(f"Source label: {report['source_label']}") + print(f"Source DEM: {report['source_dem_windows']}") + print(f"Prepared DEM: {report['prepared_dem_windows']}") + print(f"Report: {linux_path_to_windows(report_path)}") + print("Suggested .env values:") + for key, value in report["suggested_env"].items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py b/backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py index 6da3c6c..f737d47 100644 --- a/backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py +++ b/backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py @@ -3,10 +3,12 @@ from __future__ import annotations import argparse import ast +import os import re import shutil import subprocess import sys +import time import xml.etree.ElementTree as ET from dataclasses import dataclass from pathlib import Path @@ -21,6 +23,13 @@ from lt1_input_resolver import ( DEFAULT_TARGET_GRID_SIZE_M = 10 METERS_PER_DEGREE = 111320.0 +LARGE_BASE_DEM_PIXEL_THRESHOLD = 200_000_000 +PIPELINE_STAGE_ORDER = ("filter", "unwrap", "geocode", "export") +RESUME_STAGE_CHOICES = PIPELINE_STAGE_ORDER[1:] +DEFAULT_EXPORT_GEOCODE_PRODUCTS = [ + "interferogram/filt_topophase.unw", + "interferogram/topophase.cor", +] @dataclass @@ -43,6 +52,7 @@ class PipelineConfig: bbox: list[float] | None target_grid_size_m: int geo_posting_deg: float + geocode_products: list[str] | None def parse_args() -> argparse.Namespace: @@ -147,6 +157,17 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Also export the unmasked displacement GeoTIFF for debugging", ) + parser.add_argument( + "--full-geocode", + action="store_true", + help="Let ISCE2 geocode its full default product list instead of the reduced export-only list.", + ) + parser.add_argument( + "--resume-from", + choices=RESUME_STAGE_CHOICES, + default=None, + help="Resume from an existing work directory starting at the given stage.", + ) parser.add_argument( "--wavelength", type=float, @@ -178,6 +199,8 @@ def parse_args() -> argparse.Namespace: raise ValueError("--orbit-margin-sec must be between 60 and 120 seconds") if args.target_grid_size_m <= 0: raise ValueError("--target-grid-size-m must be greater than 0") + if args.force and args.resume_from: + raise ValueError("--force cannot be used together with --resume-from") return args @@ -340,6 +363,55 @@ def resolve_dem(dem_value: str | None) -> Path: ) +def _read_xml_property_value(root: ET.Element, name: str) -> str: + for prop in root.findall("property"): + if str(prop.get("name") or "").strip() != name: + continue + return str(prop.findtext("value") or "").strip() + return "" + + +def read_dem_dimensions(dem_path: Path) -> tuple[int, int] | None: + xml_path = Path(str(dem_path) + ".xml") + if not xml_path.exists(): + return None + root = ET.fromstring(xml_path.read_text(encoding="utf-8", errors="ignore")) + width_text = _read_xml_property_value(root, "width") + length_text = _read_xml_property_value(root, "length") + if not width_text or not length_text: + return None + try: + return int(float(width_text)), int(float(length_text)) + except ValueError: + return None + + +def has_prepared_dem_sibling(dem_path: Path) -> bool: + if dem_path.as_posix().lower().endswith(".wgs84"): + return True + sibling = Path(str(dem_path) + ".wgs84") + return sibling.exists() and Path(str(sibling) + ".xml").exists() + + +def guard_large_unprepared_base_dem(dem_path: Path) -> None: + if has_prepared_dem_sibling(dem_path): + return + dimensions = read_dem_dimensions(dem_path) + if dimensions is None: + return + width, length = dimensions + pixel_count = width * length + if pixel_count < LARGE_BASE_DEM_PIXEL_THRESHOLD: + return + raise RuntimeError( + "Configured DEM resolves to a large base raster without a prepared '.wgs84' sibling. " + f"Selected DEM: {dem_path} ({width}x{length}, {pixel_count} pixels). " + "A fresh ISCE2 run would spend a very long time rebuilding the geoid-corrected DEM during " + "verifyDEM/topo. Prepare '.wgs84' once, or point ISCE2_DEM_PATH directly to the " + "prepared file before starting a fresh run." + ) + + def resolve_task( task_dir: Path, orbit_root: Path, @@ -399,12 +471,20 @@ def render_bbox(bbox: list[float] | None) -> str: return f' [{values}]\n' +def render_string_list(name: str, values: list[str] | None) -> str: + if not values: + return "" + rendered = ", ".join(repr(str(value)) for value in values if str(value).strip()) + return f' [{rendered}]\n' if rendered else "" + + def meters_to_geoposting_degrees(target_grid_size_m: int) -> float: return float(target_grid_size_m) / METERS_PER_DEGREE def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None: bbox_xml = render_bbox(config.bbox) + geocode_list_xml = render_string_list("geocode list", config.geocode_products) text = ( "\n" " \n" @@ -417,6 +497,7 @@ def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None: f" {config.target_grid_size_m}\n" f" {config.geo_posting_deg:.12f}\n" f"{bbox_xml}" + f"{geocode_list_xml}" f" {config.dem_path.as_posix()}\n" "\n" " \n" @@ -436,13 +517,24 @@ def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None: xml_path.write_text(text, encoding="utf-8") -def run_logged(cmd: list[str], cwd: Path, log_path: Path) -> None: +def run_logged(stage_name: str, cmd: list[str], cwd: Path, log_path: Path) -> None: + started_monotonic = time.monotonic() + started_text = time.strftime("%Y-%m-%d %H:%M:%S") log_path.parent.mkdir(parents=True, exist_ok=True) - print("Running:") - print(" " + " ".join(cmd)) - print(f"Log: {log_path}") + print(f"[{stage_name}] Starting at {started_text}", flush=True) + print("Running:", flush=True) + print(" " + " ".join(cmd), flush=True) + print(f"Log: {log_path}", flush=True) with log_path.open("w", encoding="utf-8") as handle: + handle.write(f"[{stage_name}] Starting at {started_text}\n") + handle.write("Running:\n") + handle.write(" " + " ".join(cmd) + "\n") + handle.write(f"Log: {log_path}\n") + handle.flush() + + child_env = os.environ.copy() + child_env["PYTHONUNBUFFERED"] = "1" proc = subprocess.Popen( cmd, cwd=cwd, @@ -450,14 +542,25 @@ def run_logged(cmd: list[str], cwd: Path, log_path: Path) -> None: stderr=subprocess.STDOUT, text=True, bufsize=1, + env=child_env, ) assert proc.stdout is not None for line in proc.stdout: sys.stdout.write(line) + sys.stdout.flush() handle.write(line) + handle.flush() status = proc.wait() + elapsed_seconds = time.monotonic() - started_monotonic + handle.write(f"[{stage_name}] Finished with exit code {status} after {elapsed_seconds:.1f}s\n") + handle.flush() + + print( + f"[{stage_name}] Finished with exit code {status} after {elapsed_seconds:.1f}s", + flush=True, + ) if status != 0: raise RuntimeError(f"Command failed with exit code {status}: {' '.join(cmd)}") @@ -494,6 +597,119 @@ def expand_bbox(bbox: list[float], margin: float) -> list[float]: ] +def resolve_auto_geocode_bbox(work_dir: Path, bbox_margin: float) -> tuple[list[float], list[float]]: + topo_xml = work_dir / "PICKLE" / "topo.xml" + if not topo_xml.exists(): + raise FileNotFoundError("topo step output is missing; cannot resolve the geocode bounding box.") + estimated_bbox = load_estimated_bbox(topo_xml) + return estimated_bbox, expand_bbox(estimated_bbox, bbox_margin) + + +def ensure_geocode_bbox(work_dir: Path, config: PipelineConfig, bbox_margin: float) -> None: + if config.bbox is not None: + return + estimated_bbox, expanded_bbox = resolve_auto_geocode_bbox(work_dir, bbox_margin) + config.bbox = expanded_bbox + print(f"Auto bbox from topo: {estimated_bbox}") + print(f"Expanded bbox used for geocode: {config.bbox}") + + +def cleanup_geocode_outputs(work_dir: Path, geocode_products: list[str] | None) -> None: + if not geocode_products: + return + + removed: list[Path] = [] + for product in geocode_products: + base_path = work_dir / product + for suffix in (".geo", ".geo.xml", ".geo.vrt", ".geo.aux.xml"): + candidate = Path(str(base_path) + suffix) + if candidate.exists(): + candidate.unlink() + removed.append(candidate) + + if removed: + print(f"Removed {len(removed)} stale geocode output file(s).") + + +def cleanup_dem_subset_outputs(base_path: Path) -> None: + for suffix in ("", ".hdr", ".xml", ".vrt", ".aux.xml"): + candidate = Path(str(base_path) + suffix) + if candidate.exists(): + candidate.unlink() + + +def prepare_geocode_dem_subset(work_dir: Path, source_dem_path: Path, bbox: list[float]) -> Path: + import isce # noqa: F401 # Ensures the bundled ISCE packages are initialized on sys.path. + import isceobj + from osgeo import gdal + + gdal.UseExceptions() + + source_xml = Path(str(source_dem_path) + ".xml") + if not source_xml.exists(): + raise FileNotFoundError(f"Missing DEM XML sidecar: {source_xml}") + + source_vrt = Path(str(source_dem_path) + ".vrt") + source_open_path = source_vrt if source_vrt.exists() else source_dem_path + if not source_open_path.exists(): + raise FileNotFoundError(f"Missing DEM source for geocode subset: {source_open_path}") + + subset_base = work_dir / "geocode_dem" + cleanup_dem_subset_outputs(subset_base) + + south, north, west, east = bbox + ds = gdal.Translate( + subset_base.as_posix(), + source_open_path.as_posix(), + format="ENVI", + projWin=[west, north, east, south], + ) + if ds is None: + raise RuntimeError(f"Failed to crop DEM subset from {source_open_path}") + + width = int(ds.RasterXSize or 0) + length = int(ds.RasterYSize or 0) + geotransform = ds.GetGeoTransform(can_return_null=True) + ds = None + + if width <= 0 or length <= 0 or geotransform is None: + raise RuntimeError("Cropped DEM subset is empty or missing georeferencing metadata.") + + source_dem = isceobj.createDemImage() + source_dem.load(source_xml.as_posix()) + dem_reference = str(source_dem.reference or "").strip() or "UNKNOWN" + + source_dem.filename = subset_base.as_posix() + source_dem.width = width + source_dem.length = length + source_dem.coord1.coordStart = geotransform[0] + source_dem.coord1.coordDelta = geotransform[1] + source_dem.coord1.coordSize = width + source_dem.coord2.coordStart = geotransform[3] + source_dem.coord2.coordDelta = geotransform[5] + source_dem.coord2.coordSize = length + source_dem.dump(subset_base.as_posix() + ".xml") + source_dem.renderVRT() + + print( + "Prepared geocode DEM subset: " + f"{subset_base} ({width}x{length}, reference={dem_reference})" + ) + return subset_base + + +def prepare_geocode_dem(work_dir: Path, config: PipelineConfig) -> None: + if config.bbox is None: + raise ValueError("Cannot prepare a geocode DEM subset without a resolved bbox.") + config.dem_path = prepare_geocode_dem_subset(work_dir, config.dem_path, config.bbox) + + +def should_run_stage(start_stage: str, stage_name: str) -> bool: + start_index = PIPELINE_STAGE_ORDER.index(start_stage) + stage_index = PIPELINE_STAGE_ORDER.index(stage_name) + return stage_index >= start_index + + def prepare_snaphu_resume(work_dir: Path, bbox: list[float] | None) -> None: pickle_dir = work_dir / "PICKLE" src = pickle_dir / "filter" @@ -568,10 +784,20 @@ def print_summary( print(f"BBox: {config.bbox if config.bbox is not None else 'auto'}") print(f"Target grid: {config.target_grid_size_m} m") print(f"Geo posting: {config.geo_posting_deg:.12f} deg") + print( + "Geocode list: " + + ( + ", ".join(config.geocode_products) + if config.geocode_products + else "ISCE2 default" + ) + ) def main() -> int: args = parse_args() + resume_from = str(args.resume_from or "").strip().lower() + start_stage = resume_from or PIPELINE_STAGE_ORDER[0] task_dir = normalize_linux_path(args.task_dir).resolve() if not task_dir.exists(): raise FileNotFoundError(f"Task directory not found: {task_dir}") @@ -589,9 +815,16 @@ def main() -> int: ) if work_dir.exists(): - if not args.force: + if resume_from: + pass + elif args.force: + shutil.rmtree(work_dir) + else: raise FileExistsError(f"Work directory already exists: {work_dir}. Use --force to recreate it.") - shutil.rmtree(work_dir) + elif resume_from: + raise FileNotFoundError( + f"Resume requested from {resume_from}, but work directory does not exist: {work_dir}" + ) work_dir.mkdir(parents=True, exist_ok=True) @@ -620,7 +853,16 @@ def main() -> int: bbox=bbox, target_grid_size_m=args.target_grid_size_m, geo_posting_deg=geo_posting_deg, + geocode_products=None if args.full_geocode else list(DEFAULT_EXPORT_GEOCODE_PRODUCTS), ) + if start_stage == PIPELINE_STAGE_ORDER[0]: + guard_large_unprepared_base_dem(config.dem_path) + + if resume_from in {"unwrap", "geocode", "export"}: + ensure_geocode_bbox(work_dir, config, args.bbox_margin) + if should_run_stage(start_stage, "geocode"): + prepare_geocode_dem(work_dir, config) + print_summary(task_dir=task_dir, work_dir=work_dir, output_dir=output_dir, config=config) xml_path = work_dir / f"{task_name}_stripmap.xml" @@ -632,45 +874,52 @@ def main() -> int: app_py = locate_stripmap_app() - run_logged( - [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--end=filter"], - cwd=work_dir, - log_path=work_dir / "01_to_filter.log", - ) + if should_run_stage(start_stage, "filter"): + run_logged( + "01_to_filter", + [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--end=filter"], + cwd=work_dir, + log_path=work_dir / "01_to_filter.log", + ) - if config.bbox is None: - estimated_bbox = load_estimated_bbox(work_dir / "PICKLE" / "topo.xml") - config.bbox = expand_bbox(estimated_bbox, args.bbox_margin) + if should_run_stage(start_stage, "geocode") and config.bbox is None: + ensure_geocode_bbox(work_dir, config, args.bbox_margin) + prepare_geocode_dem(work_dir, config) write_stripmap_xml(xml_path, config) - print(f"Auto bbox from topo: {estimated_bbox}") - print(f"Expanded bbox used for geocode: {config.bbox}") - prepare_snaphu_resume(work_dir, config.bbox) - run_logged( - [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=unwrap", "--end=unwrap"], - cwd=work_dir, - log_path=work_dir / "02_unwrap_snaphu.log", - ) + if should_run_stage(start_stage, "unwrap"): + prepare_snaphu_resume(work_dir, config.bbox) + run_logged( + "02_unwrap_snaphu", + [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=unwrap", "--end=unwrap"], + cwd=work_dir, + log_path=work_dir / "02_unwrap_snaphu.log", + ) - prepare_geocode_resume(work_dir) - run_logged( - [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=geocode", "--end=geocode"], - cwd=work_dir, - log_path=work_dir / "03_geocode.log", - ) + if should_run_stage(start_stage, "geocode"): + prepare_geocode_resume(work_dir) + cleanup_geocode_outputs(work_dir, config.geocode_products) + run_logged( + "03_geocode", + [sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=geocode", "--end=geocode"], + cwd=work_dir, + log_path=work_dir / "03_geocode.log", + ) - outputs = export_products( - work_dir=work_dir, - output_dir=output_dir, - prefix=output_prefix, - wavelength=args.wavelength, - coh_threshold=args.coh_threshold, - include_disp_full=args.include_disp_full, - ) + outputs: dict[str, Path] = {} + if should_run_stage(start_stage, "export"): + outputs = export_products( + work_dir=work_dir, + output_dir=output_dir, + prefix=output_prefix, + wavelength=args.wavelength, + coh_threshold=args.coh_threshold, + include_disp_full=args.include_disp_full, + ) - print("Pipeline finished.") - for key, path in outputs.items(): - print(f"{key}: {path}") + print("Pipeline finished.") + for key, path in outputs.items(): + print(f"{key}: {path}") return 0 diff --git a/backend/app/services/dinsar_completion_files.py b/backend/app/services/dinsar_completion_files.py new file mode 100644 index 0000000..a70a69a --- /dev/null +++ b/backend/app/services/dinsar_completion_files.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import json +import os +import re +from datetime import datetime +from typing import Any, Dict, List, Optional + +from ..config import settings +from .dinsar_naming import RUN_META_FILENAME + + +CURRENT_POINTER_DIRNAME = "current" +EXECUTION_MANIFEST_FILENAME = "execution_manifest.json" +EXECUTION_STATUS_COMPLETED = "COMPLETED" +_SAFE_POINTER_RE = re.compile(r"[^0-9A-Za-z._-]+") + + +def _normalize_path(path: Any) -> str: + text = str(path or "").strip() + if not text: + return "" + return os.path.normpath(os.path.abspath(text)) + + +def _load_json(path: str) -> Optional[Dict[str, Any]]: + try: + with open(path, "r", encoding="utf-8") as fp: + payload = json.load(fp) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + +def _write_json(path: str, payload: Dict[str, Any]) -> str: + target = _normalize_path(path) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as fp: + json.dump(payload, fp, ensure_ascii=False, indent=2) + return target + + +def _first_text(*values: Any) -> str: + for value in values: + text = str(value or "").strip() + if text: + return text + return "" + + +def _utc_text(value: Optional[str]) -> str: + text = str(value or "").strip() + if text: + return text + return datetime.utcnow().isoformat(timespec="seconds") + "Z" + + +def _sanitize_pointer_fragment(value: str, default: str) -> str: + text = _SAFE_POINTER_RE.sub("_", str(value or "").strip()).strip("._") + return text or default + + +def _current_pointer_path(results_root_dir: str, *, engine_code: str, profile_code: str) -> str: + pointer_name = ( + f"{_sanitize_pointer_fragment(engine_code or 'engine', 'engine')}__" + f"{_sanitize_pointer_fragment(profile_code or 'profile', 'profile')}.json" + ) + return os.path.join(_normalize_path(results_root_dir), CURRENT_POINTER_DIRNAME, pointer_name) + + +def _runtime_id_for_engine(engine_code: str) -> Optional[str]: + normalized = str(engine_code or "").strip().lower() + if normalized == "isce2": + return settings.ISCE2_RUNTIME_ID or None + if normalized in {"pyint", "gamma"}: + return settings.PYINT_RUNTIME_ID or None + return None + + +def _normalize_source_files(primary_file: str, source_files: List[str] | None) -> List[str]: + normalized_primary = _normalize_path(primary_file) + normalized: List[str] = [] + seen: set[str] = set() + for raw_path in [normalized_primary, *(source_files or [])]: + path = _normalize_path(raw_path) + if not path or path in seen or not os.path.isfile(path): + continue + seen.add(path) + normalized.append(path) + return normalized + + +def _infer_results_root_dir(run_dir: str, pair_key: str) -> str: + normalized_run_dir = _normalize_path(run_dir) + runs_dir = os.path.dirname(normalized_run_dir) + if os.path.basename(runs_dir).lower() == "runs": + return os.path.dirname(runs_dir) + if pair_key: + return os.path.join(os.path.dirname(normalized_run_dir), pair_key) + return os.path.dirname(normalized_run_dir) + + +def repair_managed_completion_files( + run_dir: str, + *, + primary_file: str, + source_files: List[str] | None = None, + run_meta: Optional[Dict[str, Any]] = None, + update_run_metadata: bool = True, +) -> Dict[str, Any]: + normalized_run_dir = _normalize_path(run_dir) + if not os.path.isdir(normalized_run_dir): + raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}") + + run_meta_path = os.path.join(normalized_run_dir, RUN_META_FILENAME) + payload = dict(run_meta or _load_json(run_meta_path) or {}) + if not payload: + raise FileNotFoundError(f"Run metadata not found: {run_meta_path}") + + normalized_primary = _normalize_path(primary_file or payload.get("primary_file")) + normalized_sources = _normalize_source_files( + normalized_primary, + source_files or payload.get("source_files"), + ) + if not normalized_primary or not os.path.isfile(normalized_primary): + raise FileNotFoundError(f"Primary output file not found: {normalized_primary or ''}") + if not normalized_sources: + normalized_sources = [normalized_primary] + + run_key = _first_text(payload.get("run_key"), os.path.basename(normalized_run_dir)) + pair_key = _first_text( + payload.get("pair_key"), + os.path.basename(os.path.dirname(os.path.dirname(normalized_run_dir))), + ) + engine_code = _first_text(payload.get("engine_code"), "isce2") + profile_code = _first_text(payload.get("profile_code"), "unknown") + task_name = _first_text(payload.get("task_name"), payload.get("task_alias"), run_key) + task_alias = _first_text(payload.get("task_alias"), payload.get("task_name"), task_name) + output_dir = _normalize_path(payload.get("output_dir") or normalized_run_dir) + native_output_dir = _normalize_path( + payload.get("native_output_dir") or os.path.join(normalized_run_dir, "native") + ) + results_root_dir = _infer_results_root_dir(normalized_run_dir, pair_key) + publish_root_dir = _normalize_path(os.path.dirname(results_root_dir)) + + metrics: Dict[str, Any] = {} + acceptance = payload.get("acceptance") if isinstance(payload.get("acceptance"), dict) else {} + acceptance_metrics = acceptance.get("metrics") if isinstance(acceptance.get("metrics"), dict) else {} + run_metrics = payload.get("metrics") if isinstance(payload.get("metrics"), dict) else {} + if acceptance_metrics: + metrics["acceptance"] = acceptance_metrics + if run_metrics: + metrics["run_metrics"] = run_metrics + if payload.get("manual_recovery") or payload.get("recovery_mode"): + metrics["recovery"] = { + "manual_recovery": bool(payload.get("manual_recovery")), + "recovery_mode": _first_text(payload.get("recovery_mode")), + "recovered_at": _first_text(payload.get("recovered_at")), + } + + execution_payload = { + "format_version": 1, + "run_id": _first_text(payload.get("production_run_id"), payload.get("run_id")), + "product_family": "dinsar", + "run_key": run_key, + "task_id": _first_text(payload.get("task_id")), + "engine_code": engine_code, + "profile_code": profile_code, + "runtime_id": _first_text(payload.get("runtime_id")) or _runtime_id_for_engine(engine_code), + "mode": _first_text(payload.get("mode"), "managed"), + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "pair_uid": _first_text(payload.get("pair_uid"), payload.get("scene_pair_uid")), + "network_run_id": _first_text(payload.get("network_run_id")), + "network_edge_id": payload.get("network_edge_id"), + "policy_version": _first_text(payload.get("policy_version")), + "selection_strategy": _first_text(payload.get("selection_strategy")), + "source_root": _first_text(payload.get("source_root")), + "source_task_dir": _first_text(payload.get("task_dir")), + "results_root_dir": results_root_dir, + "publish_root_dir": publish_root_dir, + "output_dir": output_dir, + "native_output_dir": native_output_dir, + "primary_file": normalized_primary, + "source_files": normalized_sources, + "status": EXECUTION_STATUS_COMPLETED, + "metrics": metrics, + "created_at": _utc_text(payload.get("started_at")), + "finished_at": _utc_text(payload.get("finished_at") or payload.get("recovered_at")), + "recovered_from_run_meta": True, + } + execution_manifest_path = _write_json( + os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME), + execution_payload, + ) + + pointer_payload = { + "format_version": 1, + "product_family": "dinsar", + "engine_code": engine_code, + "profile_code": profile_code, + "runtime_id": execution_payload.get("runtime_id"), + "run_key": run_key, + "execution_id": None, + "status": EXECUTION_STATUS_COMPLETED, + "output_dir": output_dir, + "native_output_dir": native_output_dir, + "manifest_path": execution_manifest_path, + "primary_file": normalized_primary, + "source_files": normalized_sources, + "updated_at": _utc_text(payload.get("finished_at") or payload.get("recovered_at")), + "recovered_from_run_meta": True, + } + pointer_path = _write_json( + _current_pointer_path( + results_root_dir, + engine_code=engine_code, + profile_code=profile_code, + ), + pointer_payload, + ) + + if update_run_metadata: + payload["execution_manifest_path"] = execution_manifest_path + payload["current_pointer_path"] = pointer_path + payload["output_dir"] = output_dir + payload["native_output_dir"] = native_output_dir + payload["primary_file"] = normalized_primary + payload["source_files"] = normalized_sources + _write_json(run_meta_path, payload) + + return { + "run_dir": normalized_run_dir, + "results_root_dir": results_root_dir, + "execution_manifest_path": execution_manifest_path, + "current_pointer_path": pointer_path, + "engine_code": engine_code, + "profile_code": profile_code, + "run_key": run_key, + "pair_key": pair_key, + } diff --git a/backend/app/services/pyint_input_assets_service.py b/backend/app/services/pyint_input_assets_service.py index 2dbe603..9d1c2bf 100644 --- a/backend/app/services/pyint_input_assets_service.py +++ b/backend/app/services/pyint_input_assets_service.py @@ -106,12 +106,13 @@ def _prepared_dem_variants(value: Any) -> List[str]: if not normalized: return [] - candidates = [normalized] root, ext = os.path.splitext(normalized) if ext.lower() == ".wgs84": - candidates.append(root) + candidates = [normalized, root] elif not ext: - candidates.append(normalized + ".wgs84") + candidates = [normalized + ".wgs84", normalized] + else: + candidates = [normalized] unique: List[str] = [] seen: set[str] = set() diff --git a/backend/app/services/result_catalog_service.py b/backend/app/services/result_catalog_service.py index a04a4f3..35958ec 100644 --- a/backend/app/services/result_catalog_service.py +++ b/backend/app/services/result_catalog_service.py @@ -32,6 +32,7 @@ from .manifest_snapshot_service import ( iter_manifest_paths, ) from .image_service import image_service +from .dinsar_completion_files import repair_managed_completion_files from .dinsar_naming import ( PAIR_META_FILENAME, RUN_META_FILENAME, @@ -809,6 +810,7 @@ class ResultCatalogService: disp_dir = os.path.join(package_dir, "assets", "disp") preview_dir = _ensure_directory(os.path.join(package_dir, "preview")) asset_rows: List[Dict[str, Any]] = [] + completion_files_result: Optional[Dict[str, Any]] = None if candidate["engine_code"] == "envi": source_primary = _normalize_path(candidate["source_files"][0]) @@ -866,6 +868,7 @@ class ResultCatalogService: } ) else: + target_source_files: List[str] = [] if in_place_source: target_primary = _normalize_path(primary_file) else: @@ -878,6 +881,7 @@ class ResultCatalogService: overwritten += 1 else: skipped += 1 + target_source_files.append(target_primary) asset_rows.append( { "role": "disp", @@ -904,6 +908,7 @@ class ResultCatalogService: overwritten += 1 else: skipped += 1 + target_source_files.append(target_coh) asset_rows.append( { "role": "coh", @@ -953,6 +958,15 @@ class ResultCatalogService: manifest_path = os.path.join(package_dir, "manifest.json") with open(manifest_path, "w", encoding="utf-8") as fp: json.dump(manifest, fp, ensure_ascii=False, indent=2) + if candidate["engine_code"] == "isce2" and in_place_source: + try: + completion_files_result = repair_managed_completion_files( + package_dir, + primary_file=target_primary, + source_files=target_source_files, + ) + except FileNotFoundError: + completion_files_result = None details.append( { @@ -965,6 +979,16 @@ class ResultCatalogService: "package_dir": package_dir, "in_place": in_place_source, "thumb_created": thumb_ok, + "execution_manifest_path": ( + completion_files_result.get("execution_manifest_path") + if completion_files_result + else None + ), + "current_pointer_path": ( + completion_files_result.get("current_pointer_path") + if completion_files_result + else None + ), "status": "ok", } ) diff --git a/deploy/wsl/runners/isce2_runner.py b/deploy/wsl/runners/isce2_runner.py index aa21dc7..a36c9e5 100644 --- a/deploy/wsl/runners/isce2_runner.py +++ b/deploy/wsl/runners/isce2_runner.py @@ -68,8 +68,11 @@ def _build_pipeline_argv(payload: Mapping[str, Any], *, dry_run: bool = False) - _append_optional(argv, "--bbox-margin", params.get("bbox_margin")) _append_optional(argv, "--wavelength", params.get("wavelength")) _append_optional(argv, "--orbit-margin-sec", params.get("orbit_margin_sec")) + _append_optional(argv, "--resume-from", params.get("resume_from")) _append_optional(argv, "--reference-satellite", pair_meta.get("master_satellite")) _append_optional(argv, "--secondary-satellite", pair_meta.get("slave_satellite")) + if bool(params.get("full_geocode")): + argv.append("--full-geocode") if dry_run: argv.append("--dry-run") return argv @@ -81,6 +84,7 @@ def _build_child_env() -> Dict[str, str]: proj_data = env_root / "share" / "proj" if proj_data.exists(): env["PROJ_DATA"] = proj_data.as_posix() + env["PROJ_LIB"] = proj_data.as_posix() return env diff --git a/docs/ISCE2_STABILIZATION_UPDATELOG_20260427.md b/docs/ISCE2_STABILIZATION_UPDATELOG_20260427.md new file mode 100644 index 0000000..c36d1b4 --- /dev/null +++ b/docs/ISCE2_STABILIZATION_UPDATELOG_20260427.md @@ -0,0 +1,69 @@ +# ISCE2 Stabilization Update Log + +Date: `2026-04-27` + +## Scope + +This update hardens the managed `ISCE2` LT-1 stripmap D-InSAR production path against +large raw DEM reuse, long geocode stalls, and incomplete recovered-run metadata. + +## Delivered Changes + +### 1. Managed ISCE2 pipeline hardening + +- Added `--resume-from unwrap|geocode|export` support to the LT-1 pipeline. +- Added `--full-geocode` support. The default path now geocodes only the export-critical + products instead of ISCE2's full default list. +- Added stage-aware logging with stdout flush to improve long-run observability. +- Added geocode DEM subset preparation so resumed geocode/export runs do not reprocess the + full base DEM. +- Added a guard that blocks fresh runs when the selected DEM resolves to a very large raw + base raster without a prepared `.wgs84` sibling. + +### 2. Engine and runtime configuration fixes + +- `ISCE2` now prefers an existing prepared `.wgs84` DEM over the raw base path. +- `PyINT` DEM resolution now follows the same prepared-first preference. +- The WSL ISCE2 runner now passes `resume_from` and `full_geocode` through to the pipeline. +- The WSL runner now sets both `PROJ_DATA` and `PROJ_LIB`. + +### 3. Recovery and catalog self-healing + +- Added a completion-file repair helper for managed ISCE2 runs. +- In-place ISCE2 publish/rebuild now repairs missing: + - `execution_manifest.json` + - `current/isce2__.json` +- This allows recovered runs to be reintroduced into the managed result catalog without + hand-editing completion markers. + +### 4. One-time DEM preparation tooling + +- Added `backend/app/isce2_pipeline/prepare_isce2_base_dem.py`. +- The script resolves DEM paths from `.env`, validates existing prepared outputs, and can + generate a reusable `WGS84` `.wgs84` DEM from an `EGM96` base DEM. +- The script also auto-configures `PROJ` paths for standalone execution. + +### 5. Configuration guidance + +- Updated `.env.example` comments to distinguish: + - raw SARscape/ENVI DEM source path + - prepared ISCE2/PyINT `.wgs84` path + +## Validation + +- Python syntax validation was run for the modified ISCE2 pipeline, engine, runtime, and + result-catalog modules. +- The one-time full DEM preparation completed successfully and produced: + - `.wgs84` + - `.wgs84.xml` + - `.wgs84.vrt` +- The prepared DEM metadata was verified to report `WGS84`. + +## Required Local Follow-Up + +These operational steps are intentionally not committed: + +- Point local `.env` `ISCE2_DEM_PATH` to the prepared `.wgs84` file. +- Point local `.env` `PYINT_PREPARED_DEM_PATH` to the same prepared `.wgs84` file. +- Restart the backend so the running process reloads the updated `.env`. +