feat(isce2): harden managed production flow
- add one-time base DEM preparation utility and prepared-first DEM resolution - support geocode resume/full-geocode and guard large unprepared DEMs - repair missing completion files during in-place ISCE2 publish rebuild - add ISCE2 stabilization update log
This commit is contained in:
@@ -85,6 +85,7 @@ ORBIT_POOL_LANDSAR=
|
||||
# -----------------------------------------------------------------------------
|
||||
# DEM / 其他数据
|
||||
# -----------------------------------------------------------------------------
|
||||
# Raw D-InSAR DEM source for SARscape/ENVI. Keep the base path without the .wgs84 suffix here.
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape
|
||||
SRTM_DEM_DIR=D:\SRTM30m
|
||||
GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2
|
||||
@@ -138,6 +139,7 @@ ISCE2_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
ISCE2_PROFILE=lt1_stripmap
|
||||
ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/applications/stripmapApp.py
|
||||
ISCE2_PIPELINE_SCRIPT=
|
||||
# ISCE2 should point to the prepared WGS84 DEM after the one-time conversion.
|
||||
ISCE2_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84
|
||||
ISCE2_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\isce2_work
|
||||
ISCE2_OUTPUT_ROOT=D:\production_results\dinsar
|
||||
@@ -159,6 +161,7 @@ PYINT_OUTPUT_ROOT=D:\production_results\dinsar
|
||||
PYINT_DEM_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_dem
|
||||
PYINT_DEM_MODE=local_fabdem
|
||||
PYINT_FABDEM_ROOT=
|
||||
# When PYINT_DEM_MODE=prepared_file, point this to the same prepared WGS84 DEM.
|
||||
PYINT_PREPARED_DEM_PATH=
|
||||
PYINT_OPENTOPO_DEM_TYPE=SRTMGL1
|
||||
PYINT_DEM_STRICT=true
|
||||
|
||||
@@ -32,6 +32,7 @@ ORBIT_MARGIN_MAX_SEC = 120.0
|
||||
TARGET_GRID_SIZE_MIN_M = 5
|
||||
TARGET_GRID_SIZE_MAX_M = 100
|
||||
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
|
||||
RESUME_STAGE_CHOICES = {"", "unwrap", "geocode", "export"}
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
@@ -70,6 +71,16 @@ def _normalize_optional_path(value: Any) -> str:
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _prefer_prepared_dem_variant(path: str) -> str:
|
||||
normalized = _normalize_optional_path(path)
|
||||
if not normalized or normalized.lower().endswith(".wgs84"):
|
||||
return normalized
|
||||
prepared = normalized + ".wgs84"
|
||||
if os.path.isfile(prepared) and os.path.isfile(prepared + ".xml"):
|
||||
return prepared
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_json_file(path: str) -> Dict[str, Any]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fp:
|
||||
@@ -146,11 +157,11 @@ class Isce2Engine(DinsarEngine):
|
||||
|
||||
@property
|
||||
def _dem_path(self) -> str:
|
||||
explicit = _read_env("ISCE2_DEM_PATH", "")
|
||||
explicit = _prefer_prepared_dem_variant(_read_env("ISCE2_DEM_PATH", ""))
|
||||
if explicit:
|
||||
return explicit
|
||||
base = _read_env("IDL_DINSAR_DEM_BASE_FILE", "")
|
||||
return f"{base}.wgs84" if base else ""
|
||||
return _prefer_prepared_dem_variant(base) if base else ""
|
||||
|
||||
@property
|
||||
def _orbit_pool_isce2(self) -> str:
|
||||
@@ -320,6 +331,18 @@ class Isce2Engine(DinsarEngine):
|
||||
)
|
||||
normalized["orbit_margin_sec"] = orbit_margin
|
||||
|
||||
if "full_geocode" in normalized:
|
||||
normalized["full_geocode"] = bool(normalized["full_geocode"])
|
||||
|
||||
if "resume_from" in normalized and normalized["resume_from"] is not None:
|
||||
resume_from = str(normalized["resume_from"]).strip().lower()
|
||||
if resume_from not in RESUME_STAGE_CHOICES:
|
||||
raise ValueError("resume_from must be one of: unwrap, geocode, export")
|
||||
normalized["resume_from"] = resume_from
|
||||
|
||||
if bool(normalized.get("force")) and str(normalized.get("resume_from") or "").strip():
|
||||
raise ValueError("force cannot be used together with resume_from")
|
||||
|
||||
return normalized
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -507,6 +530,8 @@ class Isce2Engine(DinsarEngine):
|
||||
bbox_margin: Any,
|
||||
wavelength: Any,
|
||||
orbit_margin_sec: Any,
|
||||
full_geocode: bool,
|
||||
resume_from: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
@@ -541,6 +566,8 @@ class Isce2Engine(DinsarEngine):
|
||||
"bbox_margin": bbox_margin,
|
||||
"wavelength": wavelength,
|
||||
"orbit_margin_sec": orbit_margin_sec,
|
||||
"full_geocode": bool(full_geocode),
|
||||
"resume_from": str(resume_from or "").strip(),
|
||||
},
|
||||
"pair_meta": dict(pair_meta or {}),
|
||||
}
|
||||
@@ -650,6 +677,8 @@ class Isce2Engine(DinsarEngine):
|
||||
bbox_margin = extra.get("bbox_margin")
|
||||
wavelength = LT1_FIXED_WAVELENGTH
|
||||
orbit_margin_sec = extra.get("orbit_margin_sec")
|
||||
full_geocode = bool(extra.get("full_geocode"))
|
||||
resume_from = str(extra.get("resume_from") or "").strip().lower()
|
||||
|
||||
wsl_isce2_pool = ""
|
||||
if self._orbit_pool_isce2:
|
||||
@@ -792,6 +821,8 @@ class Isce2Engine(DinsarEngine):
|
||||
bbox_margin=bbox_margin,
|
||||
wavelength=wavelength,
|
||||
orbit_margin_sec=orbit_margin_sec,
|
||||
full_geocode=full_geocode,
|
||||
resume_from=resume_from,
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
broker_result = wsl_broker.run_manifest(
|
||||
|
||||
@@ -250,4 +250,4 @@ def _prepared_dem_variants(value: str | Path) -> tuple[str | Path, ...]:
|
||||
return ()
|
||||
if text.lower().endswith(".wgs84"):
|
||||
return (value,)
|
||||
return (value, f"{text}.wgs84")
|
||||
return (f"{text}.wgs84", value)
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from .lt1_input_resolver import load_env_file
|
||||
except ImportError:
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
from lt1_input_resolver import load_env_file # type: ignore
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DemResolution:
|
||||
source_label: str
|
||||
configured_value: str
|
||||
source_path: Path
|
||||
prepared_path: Path
|
||||
|
||||
|
||||
def _configure_proj_environment() -> None:
|
||||
if os.environ.get("PROJ_DATA") and os.environ.get("PROJ_LIB"):
|
||||
return
|
||||
|
||||
candidates: list[Path] = []
|
||||
|
||||
isce2_python = str(os.environ.get("ISCE2_PYTHON") or "").strip()
|
||||
if isce2_python:
|
||||
candidates.append(Path(isce2_python).resolve().parents[1] / "share" / "proj")
|
||||
|
||||
candidates.append(Path(sys.executable).resolve().parents[1] / "share" / "proj")
|
||||
|
||||
for candidate in candidates:
|
||||
if candidate.exists():
|
||||
proj_path = candidate.as_posix()
|
||||
os.environ.setdefault("PROJ_DATA", proj_path)
|
||||
os.environ.setdefault("PROJ_LIB", proj_path)
|
||||
return
|
||||
|
||||
|
||||
def normalize_linux_path(value: str | Path) -> Path:
|
||||
text = str(value or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return Path("")
|
||||
if text.startswith("\\\\"):
|
||||
raise ValueError("UNC paths are not supported directly. Mount them in WSL first.")
|
||||
|
||||
match = re.match(r"^([A-Za-z]):[\\/](.*)$", text)
|
||||
if match:
|
||||
drive = match.group(1).lower()
|
||||
rest = match.group(2).replace("\\", "/")
|
||||
return Path(f"/mnt/{drive}/{rest}")
|
||||
return Path(text)
|
||||
|
||||
|
||||
def linux_path_to_windows(path: Path) -> str:
|
||||
text = str(path)
|
||||
match = re.match(r"^/mnt/([a-zA-Z])/(.*)$", text)
|
||||
if not match:
|
||||
return text
|
||||
drive = match.group(1).upper()
|
||||
rest = match.group(2).replace("/", "\\")
|
||||
return f"{drive}:\\{rest}"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
repo_root = script_dir.parent.parent.parent
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Prepare a reusable WGS84 '.wgs84' DEM once for ISCE2 and PyINT."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-dem",
|
||||
default=None,
|
||||
help="Optional source DEM base path. Accepts either a raw DEM base path or an existing .wgs84 path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env-file",
|
||||
default=str(repo_root / ".env"),
|
||||
help="Project .env file used to resolve default DEM paths.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Rebuild the prepared .wgs84 outputs even if they already exist.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Resolve paths and print the planned output without modifying files.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _remove_prepare_outputs(base_path: Path) -> None:
|
||||
for suffix in ("", ".xml", ".vrt", ".hdr", ".aux.xml"):
|
||||
candidate = Path(str(base_path) + suffix)
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
|
||||
|
||||
def _ensure_source_xml(path: Path) -> None:
|
||||
if not Path(str(path) + ".xml").exists():
|
||||
raise FileNotFoundError(f"Missing DEM XML sidecar: {path}.xml")
|
||||
|
||||
|
||||
def _maybe_prepared_path(path: Path) -> Path:
|
||||
return path if str(path).lower().endswith(".wgs84") else Path(str(path) + ".wgs84")
|
||||
|
||||
|
||||
def _existing_path(path: Path) -> bool:
|
||||
return path.exists() and Path(str(path) + ".xml").exists()
|
||||
|
||||
|
||||
def _resolve_from_value(label: str, value: str) -> DemResolution | None:
|
||||
normalized = normalize_linux_path(value)
|
||||
if not str(normalized):
|
||||
return None
|
||||
|
||||
if _existing_path(normalized):
|
||||
if str(normalized).lower().endswith(".wgs84"):
|
||||
raw_candidate = normalized.with_suffix("")
|
||||
if _existing_path(raw_candidate):
|
||||
return DemResolution(label, value, raw_candidate, normalized)
|
||||
return DemResolution(label, value, normalized, normalized)
|
||||
return DemResolution(label, value, normalized, _maybe_prepared_path(normalized))
|
||||
|
||||
if str(normalized).lower().endswith(".wgs84"):
|
||||
raw_candidate = normalized.with_suffix("")
|
||||
if _existing_path(raw_candidate):
|
||||
return DemResolution(label, value, raw_candidate, normalized)
|
||||
else:
|
||||
prepared_candidate = Path(str(normalized) + ".wgs84")
|
||||
if _existing_path(prepared_candidate):
|
||||
return DemResolution(label, value, normalized, prepared_candidate)
|
||||
if Path(str(normalized) + ".xml").exists():
|
||||
return DemResolution(label, value, normalized, prepared_candidate)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_dem_from_env(explicit_source: str | None, env_file: Path) -> DemResolution:
|
||||
env_values = load_env_file(env_file)
|
||||
candidates: list[tuple[str, str]] = []
|
||||
if explicit_source:
|
||||
candidates.append(("explicit", explicit_source))
|
||||
|
||||
# Keep the raw source path explicit in .env via IDL_DINSAR_DEM_BASE_FILE.
|
||||
for key in (
|
||||
"IDL_DINSAR_DEM_BASE_FILE",
|
||||
"ISCE2_DEM_PATH",
|
||||
"PYINT_PREPARED_DEM_PATH",
|
||||
):
|
||||
value = str(env_values.get(key) or "").strip()
|
||||
if value:
|
||||
candidates.append((key, value))
|
||||
|
||||
for label, value in candidates:
|
||||
resolved = _resolve_from_value(label, value)
|
||||
if resolved is not None:
|
||||
return resolved
|
||||
|
||||
raise FileNotFoundError(
|
||||
"Unable to resolve a DEM source from --source-dem, IDL_DINSAR_DEM_BASE_FILE, "
|
||||
"ISCE2_DEM_PATH, or PYINT_PREPARED_DEM_PATH."
|
||||
)
|
||||
|
||||
|
||||
def inspect_dem(path: Path) -> dict[str, Any]:
|
||||
_configure_proj_environment()
|
||||
import isce # noqa: F401
|
||||
import isceobj
|
||||
|
||||
_ensure_source_xml(path)
|
||||
dem = isceobj.createDemImage()
|
||||
dem.load(str(path) + ".xml")
|
||||
return {
|
||||
"path": str(path),
|
||||
"reference": str(dem.reference or "").strip(),
|
||||
"width": int(dem.width or 0),
|
||||
"length": int(dem.length or 0),
|
||||
"first_lon": float(dem.coord1.coordStart or 0.0),
|
||||
"first_lat": float(dem.coord2.coordStart or 0.0),
|
||||
"delta_lon": float(dem.coord1.coordDelta or 0.0),
|
||||
"delta_lat": float(dem.coord2.coordDelta or 0.0),
|
||||
}
|
||||
|
||||
|
||||
def ensure_prepared_dem(source_path: Path, prepared_path: Path, *, force: bool) -> dict[str, Any]:
|
||||
_configure_proj_environment()
|
||||
import isce # noqa: F401
|
||||
import isceobj
|
||||
from iscesys.DataManager import createManager
|
||||
|
||||
if source_path != prepared_path:
|
||||
_ensure_source_xml(source_path)
|
||||
|
||||
if _existing_path(prepared_path) and not force:
|
||||
prepared_meta = inspect_dem(prepared_path)
|
||||
if prepared_meta["reference"].upper() != "WGS84":
|
||||
raise RuntimeError(
|
||||
f"Prepared DEM exists but is not WGS84: {prepared_path} ({prepared_meta['reference']})"
|
||||
)
|
||||
return {
|
||||
"action": "validated_existing",
|
||||
"source": inspect_dem(source_path),
|
||||
"prepared": prepared_meta,
|
||||
}
|
||||
|
||||
if source_path == prepared_path:
|
||||
prepared_meta = inspect_dem(prepared_path)
|
||||
if prepared_meta["reference"].upper() != "WGS84":
|
||||
raise RuntimeError(
|
||||
f"Provided DEM is not WGS84: {prepared_path} ({prepared_meta['reference']})"
|
||||
)
|
||||
return {
|
||||
"action": "already_wgs84",
|
||||
"source": prepared_meta,
|
||||
"prepared": prepared_meta,
|
||||
}
|
||||
|
||||
if force:
|
||||
_remove_prepare_outputs(prepared_path)
|
||||
|
||||
source_dem = isceobj.createDemImage()
|
||||
source_dem.load(str(source_path) + ".xml")
|
||||
# Some DEM XML sidecars store only the basename. Force an absolute filename
|
||||
# so ISCE2 writes the generated ".wgs84" next to the source DEM, not in cwd.
|
||||
source_dem.filename = str(source_path)
|
||||
if not Path(str(source_path) + ".vrt").exists():
|
||||
source_dem.renderVRT()
|
||||
|
||||
source_reference = str(source_dem.reference or "").strip().upper()
|
||||
if source_reference != "EGM96":
|
||||
raise RuntimeError(
|
||||
f"Expected an EGM96 raw DEM before preparation, got: {source_dem.reference or '<empty>'}"
|
||||
)
|
||||
|
||||
dem_stitcher = createManager("dem1", "iscestitcher")
|
||||
dem_stitcher.noFilling = False
|
||||
prepared_dem = dem_stitcher.correct(source_dem)
|
||||
prepared_dem.metadatalocation = str(prepared_path) + ".xml"
|
||||
prepared_dem._extraFilename = str(prepared_path) + ".vrt"
|
||||
if not Path(prepared_dem.metadatalocation).exists():
|
||||
prepared_dem.dump(prepared_dem.metadatalocation)
|
||||
if not Path(prepared_dem._extraFilename).exists():
|
||||
prepared_dem.renderVRT()
|
||||
|
||||
prepared_meta = inspect_dem(prepared_path)
|
||||
if prepared_meta["reference"].upper() != "WGS84":
|
||||
raise RuntimeError(
|
||||
f"Generated prepared DEM is not WGS84: {prepared_path} ({prepared_meta['reference']})"
|
||||
)
|
||||
|
||||
return {
|
||||
"action": "converted",
|
||||
"source": inspect_dem(source_path),
|
||||
"prepared": prepared_meta,
|
||||
}
|
||||
|
||||
|
||||
def build_report(resolution: DemResolution, outcome: dict[str, Any]) -> dict[str, Any]:
|
||||
source_windows = linux_path_to_windows(resolution.source_path)
|
||||
prepared_windows = linux_path_to_windows(resolution.prepared_path)
|
||||
return {
|
||||
"source_label": resolution.source_label,
|
||||
"configured_value": resolution.configured_value,
|
||||
"source_dem_wsl": str(resolution.source_path),
|
||||
"source_dem_windows": source_windows,
|
||||
"prepared_dem_wsl": str(resolution.prepared_path),
|
||||
"prepared_dem_windows": prepared_windows,
|
||||
"action": outcome["action"],
|
||||
"source_dem": outcome["source"],
|
||||
"prepared_dem": outcome["prepared"],
|
||||
"suggested_env": {
|
||||
"IDL_DINSAR_DEM_BASE_FILE": source_windows,
|
||||
"ISCE2_DEM_PATH": prepared_windows,
|
||||
"PYINT_PREPARED_DEM_PATH": prepared_windows,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
env_file = normalize_linux_path(args.env_file)
|
||||
resolution = resolve_dem_from_env(args.source_dem, env_file)
|
||||
|
||||
if args.dry_run:
|
||||
outcome = {
|
||||
"action": "dry_run",
|
||||
"source": inspect_dem(resolution.source_path),
|
||||
"prepared": inspect_dem(resolution.prepared_path)
|
||||
if _existing_path(resolution.prepared_path)
|
||||
else {"path": str(resolution.prepared_path), "reference": "", "width": 0, "length": 0},
|
||||
}
|
||||
else:
|
||||
outcome = ensure_prepared_dem(
|
||||
resolution.source_path,
|
||||
resolution.prepared_path,
|
||||
force=bool(args.force),
|
||||
)
|
||||
|
||||
report = build_report(resolution, outcome)
|
||||
report_path = Path(str(resolution.prepared_path) + ".prepare_report.json")
|
||||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
print(f"Action: {report['action']}")
|
||||
print(f"Source label: {report['source_label']}")
|
||||
print(f"Source DEM: {report['source_dem_windows']}")
|
||||
print(f"Prepared DEM: {report['prepared_dem_windows']}")
|
||||
print(f"Report: {linux_path_to_windows(report_path)}")
|
||||
print("Suggested .env values:")
|
||||
for key, value in report["suggested_env"].items():
|
||||
print(f"{key}={value}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -3,10 +3,12 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -21,6 +23,13 @@ from lt1_input_resolver import (
|
||||
|
||||
DEFAULT_TARGET_GRID_SIZE_M = 10
|
||||
METERS_PER_DEGREE = 111320.0
|
||||
LARGE_BASE_DEM_PIXEL_THRESHOLD = 200_000_000
|
||||
PIPELINE_STAGE_ORDER = ("filter", "unwrap", "geocode", "export")
|
||||
RESUME_STAGE_CHOICES = PIPELINE_STAGE_ORDER[1:]
|
||||
DEFAULT_EXPORT_GEOCODE_PRODUCTS = [
|
||||
"interferogram/filt_topophase.unw",
|
||||
"interferogram/topophase.cor",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -43,6 +52,7 @@ class PipelineConfig:
|
||||
bbox: list[float] | None
|
||||
target_grid_size_m: int
|
||||
geo_posting_deg: float
|
||||
geocode_products: list[str] | None
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -147,6 +157,17 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Also export the unmasked displacement GeoTIFF for debugging",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--full-geocode",
|
||||
action="store_true",
|
||||
help="Let ISCE2 geocode its full default product list instead of the reduced export-only list.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resume-from",
|
||||
choices=RESUME_STAGE_CHOICES,
|
||||
default=None,
|
||||
help="Resume from an existing work directory starting at the given stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--wavelength",
|
||||
type=float,
|
||||
@@ -178,6 +199,8 @@ def parse_args() -> argparse.Namespace:
|
||||
raise ValueError("--orbit-margin-sec must be between 60 and 120 seconds")
|
||||
if args.target_grid_size_m <= 0:
|
||||
raise ValueError("--target-grid-size-m must be greater than 0")
|
||||
if args.force and args.resume_from:
|
||||
raise ValueError("--force cannot be used together with --resume-from")
|
||||
return args
|
||||
|
||||
|
||||
@@ -340,6 +363,55 @@ def resolve_dem(dem_value: str | None) -> Path:
|
||||
)
|
||||
|
||||
|
||||
def _read_xml_property_value(root: ET.Element, name: str) -> str:
|
||||
for prop in root.findall("property"):
|
||||
if str(prop.get("name") or "").strip() != name:
|
||||
continue
|
||||
return str(prop.findtext("value") or "").strip()
|
||||
return ""
|
||||
|
||||
|
||||
def read_dem_dimensions(dem_path: Path) -> tuple[int, int] | None:
|
||||
xml_path = Path(str(dem_path) + ".xml")
|
||||
if not xml_path.exists():
|
||||
return None
|
||||
root = ET.fromstring(xml_path.read_text(encoding="utf-8", errors="ignore"))
|
||||
width_text = _read_xml_property_value(root, "width")
|
||||
length_text = _read_xml_property_value(root, "length")
|
||||
if not width_text or not length_text:
|
||||
return None
|
||||
try:
|
||||
return int(float(width_text)), int(float(length_text))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def has_prepared_dem_sibling(dem_path: Path) -> bool:
|
||||
if dem_path.as_posix().lower().endswith(".wgs84"):
|
||||
return True
|
||||
sibling = Path(str(dem_path) + ".wgs84")
|
||||
return sibling.exists() and Path(str(sibling) + ".xml").exists()
|
||||
|
||||
|
||||
def guard_large_unprepared_base_dem(dem_path: Path) -> None:
|
||||
if has_prepared_dem_sibling(dem_path):
|
||||
return
|
||||
dimensions = read_dem_dimensions(dem_path)
|
||||
if dimensions is None:
|
||||
return
|
||||
width, length = dimensions
|
||||
pixel_count = width * length
|
||||
if pixel_count < LARGE_BASE_DEM_PIXEL_THRESHOLD:
|
||||
return
|
||||
raise RuntimeError(
|
||||
"Configured DEM resolves to a large base raster without a prepared '.wgs84' sibling. "
|
||||
f"Selected DEM: {dem_path} ({width}x{length}, {pixel_count} pixels). "
|
||||
"A fresh ISCE2 run would spend a very long time rebuilding the geoid-corrected DEM during "
|
||||
"verifyDEM/topo. Prepare '<dem>.wgs84' once, or point ISCE2_DEM_PATH directly to the "
|
||||
"prepared file before starting a fresh run."
|
||||
)
|
||||
|
||||
|
||||
def resolve_task(
|
||||
task_dir: Path,
|
||||
orbit_root: Path,
|
||||
@@ -399,12 +471,20 @@ def render_bbox(bbox: list[float] | None) -> str:
|
||||
return f' <property name="geocode bounding box">[{values}]</property>\n'
|
||||
|
||||
|
||||
def render_string_list(name: str, values: list[str] | None) -> str:
|
||||
if not values:
|
||||
return ""
|
||||
rendered = ", ".join(repr(str(value)) for value in values if str(value).strip())
|
||||
return f' <property name="{name}">[{rendered}]</property>\n' if rendered else ""
|
||||
|
||||
|
||||
def meters_to_geoposting_degrees(target_grid_size_m: int) -> float:
|
||||
return float(target_grid_size_m) / METERS_PER_DEGREE
|
||||
|
||||
|
||||
def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None:
|
||||
bbox_xml = render_bbox(config.bbox)
|
||||
geocode_list_xml = render_string_list("geocode list", config.geocode_products)
|
||||
text = (
|
||||
"<stripmapApp>\n"
|
||||
" <component name=\"stripmapApp\">\n"
|
||||
@@ -417,6 +497,7 @@ def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None:
|
||||
f" <property name=\"posting\">{config.target_grid_size_m}</property>\n"
|
||||
f" <property name=\"geoPosting\">{config.geo_posting_deg:.12f}</property>\n"
|
||||
f"{bbox_xml}"
|
||||
f"{geocode_list_xml}"
|
||||
f" <property name=\"demFilename\">{config.dem_path.as_posix()}</property>\n"
|
||||
"\n"
|
||||
" <component name=\"Reference\">\n"
|
||||
@@ -436,13 +517,24 @@ def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None:
|
||||
xml_path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def run_logged(cmd: list[str], cwd: Path, log_path: Path) -> None:
|
||||
def run_logged(stage_name: str, cmd: list[str], cwd: Path, log_path: Path) -> None:
|
||||
started_monotonic = time.monotonic()
|
||||
started_text = time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
print("Running:")
|
||||
print(" " + " ".join(cmd))
|
||||
print(f"Log: {log_path}")
|
||||
print(f"[{stage_name}] Starting at {started_text}", flush=True)
|
||||
print("Running:", flush=True)
|
||||
print(" " + " ".join(cmd), flush=True)
|
||||
print(f"Log: {log_path}", flush=True)
|
||||
|
||||
with log_path.open("w", encoding="utf-8") as handle:
|
||||
handle.write(f"[{stage_name}] Starting at {started_text}\n")
|
||||
handle.write("Running:\n")
|
||||
handle.write(" " + " ".join(cmd) + "\n")
|
||||
handle.write(f"Log: {log_path}\n")
|
||||
handle.flush()
|
||||
|
||||
child_env = os.environ.copy()
|
||||
child_env["PYTHONUNBUFFERED"] = "1"
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
@@ -450,14 +542,25 @@ def run_logged(cmd: list[str], cwd: Path, log_path: Path) -> None:
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
env=child_env,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
for line in proc.stdout:
|
||||
sys.stdout.write(line)
|
||||
sys.stdout.flush()
|
||||
handle.write(line)
|
||||
handle.flush()
|
||||
|
||||
status = proc.wait()
|
||||
elapsed_seconds = time.monotonic() - started_monotonic
|
||||
handle.write(f"[{stage_name}] Finished with exit code {status} after {elapsed_seconds:.1f}s\n")
|
||||
handle.flush()
|
||||
|
||||
print(
|
||||
f"[{stage_name}] Finished with exit code {status} after {elapsed_seconds:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if status != 0:
|
||||
raise RuntimeError(f"Command failed with exit code {status}: {' '.join(cmd)}")
|
||||
@@ -494,6 +597,119 @@ def expand_bbox(bbox: list[float], margin: float) -> list[float]:
|
||||
]
|
||||
|
||||
|
||||
def resolve_auto_geocode_bbox(work_dir: Path, bbox_margin: float) -> tuple[list[float], list[float]]:
|
||||
topo_xml = work_dir / "PICKLE" / "topo.xml"
|
||||
if not topo_xml.exists():
|
||||
raise FileNotFoundError("topo step output is missing; cannot resolve the geocode bounding box.")
|
||||
estimated_bbox = load_estimated_bbox(topo_xml)
|
||||
return estimated_bbox, expand_bbox(estimated_bbox, bbox_margin)
|
||||
|
||||
|
||||
def ensure_geocode_bbox(work_dir: Path, config: PipelineConfig, bbox_margin: float) -> None:
|
||||
if config.bbox is not None:
|
||||
return
|
||||
estimated_bbox, expanded_bbox = resolve_auto_geocode_bbox(work_dir, bbox_margin)
|
||||
config.bbox = expanded_bbox
|
||||
print(f"Auto bbox from topo: {estimated_bbox}")
|
||||
print(f"Expanded bbox used for geocode: {config.bbox}")
|
||||
|
||||
|
||||
def cleanup_geocode_outputs(work_dir: Path, geocode_products: list[str] | None) -> None:
|
||||
if not geocode_products:
|
||||
return
|
||||
|
||||
removed: list[Path] = []
|
||||
for product in geocode_products:
|
||||
base_path = work_dir / product
|
||||
for suffix in (".geo", ".geo.xml", ".geo.vrt", ".geo.aux.xml"):
|
||||
candidate = Path(str(base_path) + suffix)
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
removed.append(candidate)
|
||||
|
||||
if removed:
|
||||
print(f"Removed {len(removed)} stale geocode output file(s).")
|
||||
|
||||
|
||||
def cleanup_dem_subset_outputs(base_path: Path) -> None:
|
||||
for suffix in ("", ".hdr", ".xml", ".vrt", ".aux.xml"):
|
||||
candidate = Path(str(base_path) + suffix)
|
||||
if candidate.exists():
|
||||
candidate.unlink()
|
||||
|
||||
|
||||
def prepare_geocode_dem_subset(work_dir: Path, source_dem_path: Path, bbox: list[float]) -> Path:
|
||||
import isce # noqa: F401 # Ensures the bundled ISCE packages are initialized on sys.path.
|
||||
import isceobj
|
||||
from osgeo import gdal
|
||||
|
||||
gdal.UseExceptions()
|
||||
|
||||
source_xml = Path(str(source_dem_path) + ".xml")
|
||||
if not source_xml.exists():
|
||||
raise FileNotFoundError(f"Missing DEM XML sidecar: {source_xml}")
|
||||
|
||||
source_vrt = Path(str(source_dem_path) + ".vrt")
|
||||
source_open_path = source_vrt if source_vrt.exists() else source_dem_path
|
||||
if not source_open_path.exists():
|
||||
raise FileNotFoundError(f"Missing DEM source for geocode subset: {source_open_path}")
|
||||
|
||||
subset_base = work_dir / "geocode_dem"
|
||||
cleanup_dem_subset_outputs(subset_base)
|
||||
|
||||
south, north, west, east = bbox
|
||||
ds = gdal.Translate(
|
||||
subset_base.as_posix(),
|
||||
source_open_path.as_posix(),
|
||||
format="ENVI",
|
||||
projWin=[west, north, east, south],
|
||||
)
|
||||
if ds is None:
|
||||
raise RuntimeError(f"Failed to crop DEM subset from {source_open_path}")
|
||||
|
||||
width = int(ds.RasterXSize or 0)
|
||||
length = int(ds.RasterYSize or 0)
|
||||
geotransform = ds.GetGeoTransform(can_return_null=True)
|
||||
ds = None
|
||||
|
||||
if width <= 0 or length <= 0 or geotransform is None:
|
||||
raise RuntimeError("Cropped DEM subset is empty or missing georeferencing metadata.")
|
||||
|
||||
source_dem = isceobj.createDemImage()
|
||||
source_dem.load(source_xml.as_posix())
|
||||
dem_reference = str(source_dem.reference or "").strip() or "UNKNOWN"
|
||||
|
||||
source_dem.filename = subset_base.as_posix()
|
||||
source_dem.width = width
|
||||
source_dem.length = length
|
||||
source_dem.coord1.coordStart = geotransform[0]
|
||||
source_dem.coord1.coordDelta = geotransform[1]
|
||||
source_dem.coord1.coordSize = width
|
||||
source_dem.coord2.coordStart = geotransform[3]
|
||||
source_dem.coord2.coordDelta = geotransform[5]
|
||||
source_dem.coord2.coordSize = length
|
||||
source_dem.dump(subset_base.as_posix() + ".xml")
|
||||
source_dem.renderVRT()
|
||||
|
||||
print(
|
||||
"Prepared geocode DEM subset: "
|
||||
f"{subset_base} ({width}x{length}, reference={dem_reference})"
|
||||
)
|
||||
return subset_base
|
||||
|
||||
|
||||
def prepare_geocode_dem(work_dir: Path, config: PipelineConfig) -> None:
|
||||
if config.bbox is None:
|
||||
raise ValueError("Cannot prepare a geocode DEM subset without a resolved bbox.")
|
||||
config.dem_path = prepare_geocode_dem_subset(work_dir, config.dem_path, config.bbox)
|
||||
|
||||
|
||||
def should_run_stage(start_stage: str, stage_name: str) -> bool:
|
||||
start_index = PIPELINE_STAGE_ORDER.index(start_stage)
|
||||
stage_index = PIPELINE_STAGE_ORDER.index(stage_name)
|
||||
return stage_index >= start_index
|
||||
|
||||
|
||||
def prepare_snaphu_resume(work_dir: Path, bbox: list[float] | None) -> None:
|
||||
pickle_dir = work_dir / "PICKLE"
|
||||
src = pickle_dir / "filter"
|
||||
@@ -568,10 +784,20 @@ def print_summary(
|
||||
print(f"BBox: {config.bbox if config.bbox is not None else 'auto'}")
|
||||
print(f"Target grid: {config.target_grid_size_m} m")
|
||||
print(f"Geo posting: {config.geo_posting_deg:.12f} deg")
|
||||
print(
|
||||
"Geocode list: "
|
||||
+ (
|
||||
", ".join(config.geocode_products)
|
||||
if config.geocode_products
|
||||
else "ISCE2 default"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
resume_from = str(args.resume_from or "").strip().lower()
|
||||
start_stage = resume_from or PIPELINE_STAGE_ORDER[0]
|
||||
task_dir = normalize_linux_path(args.task_dir).resolve()
|
||||
if not task_dir.exists():
|
||||
raise FileNotFoundError(f"Task directory not found: {task_dir}")
|
||||
@@ -589,9 +815,16 @@ def main() -> int:
|
||||
)
|
||||
|
||||
if work_dir.exists():
|
||||
if not args.force:
|
||||
raise FileExistsError(f"Work directory already exists: {work_dir}. Use --force to recreate it.")
|
||||
if resume_from:
|
||||
pass
|
||||
elif args.force:
|
||||
shutil.rmtree(work_dir)
|
||||
else:
|
||||
raise FileExistsError(f"Work directory already exists: {work_dir}. Use --force to recreate it.")
|
||||
elif resume_from:
|
||||
raise FileNotFoundError(
|
||||
f"Resume requested from {resume_from}, but work directory does not exist: {work_dir}"
|
||||
)
|
||||
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -620,7 +853,16 @@ def main() -> int:
|
||||
bbox=bbox,
|
||||
target_grid_size_m=args.target_grid_size_m,
|
||||
geo_posting_deg=geo_posting_deg,
|
||||
geocode_products=None if args.full_geocode else list(DEFAULT_EXPORT_GEOCODE_PRODUCTS),
|
||||
)
|
||||
if start_stage == PIPELINE_STAGE_ORDER[0]:
|
||||
guard_large_unprepared_base_dem(config.dem_path)
|
||||
|
||||
if resume_from in {"unwrap", "geocode", "export"}:
|
||||
ensure_geocode_bbox(work_dir, config, args.bbox_margin)
|
||||
if should_run_stage(start_stage, "geocode"):
|
||||
prepare_geocode_dem(work_dir, config)
|
||||
|
||||
print_summary(task_dir=task_dir, work_dir=work_dir, output_dir=output_dir, config=config)
|
||||
|
||||
xml_path = work_dir / f"{task_name}_stripmap.xml"
|
||||
@@ -632,33 +874,40 @@ def main() -> int:
|
||||
|
||||
app_py = locate_stripmap_app()
|
||||
|
||||
if should_run_stage(start_stage, "filter"):
|
||||
run_logged(
|
||||
"01_to_filter",
|
||||
[sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--end=filter"],
|
||||
cwd=work_dir,
|
||||
log_path=work_dir / "01_to_filter.log",
|
||||
)
|
||||
|
||||
if config.bbox is None:
|
||||
estimated_bbox = load_estimated_bbox(work_dir / "PICKLE" / "topo.xml")
|
||||
config.bbox = expand_bbox(estimated_bbox, args.bbox_margin)
|
||||
if should_run_stage(start_stage, "geocode") and config.bbox is None:
|
||||
ensure_geocode_bbox(work_dir, config, args.bbox_margin)
|
||||
prepare_geocode_dem(work_dir, config)
|
||||
write_stripmap_xml(xml_path, config)
|
||||
print(f"Auto bbox from topo: {estimated_bbox}")
|
||||
print(f"Expanded bbox used for geocode: {config.bbox}")
|
||||
|
||||
if should_run_stage(start_stage, "unwrap"):
|
||||
prepare_snaphu_resume(work_dir, config.bbox)
|
||||
run_logged(
|
||||
"02_unwrap_snaphu",
|
||||
[sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=unwrap", "--end=unwrap"],
|
||||
cwd=work_dir,
|
||||
log_path=work_dir / "02_unwrap_snaphu.log",
|
||||
)
|
||||
|
||||
if should_run_stage(start_stage, "geocode"):
|
||||
prepare_geocode_resume(work_dir)
|
||||
cleanup_geocode_outputs(work_dir, config.geocode_products)
|
||||
run_logged(
|
||||
"03_geocode",
|
||||
[sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=geocode", "--end=geocode"],
|
||||
cwd=work_dir,
|
||||
log_path=work_dir / "03_geocode.log",
|
||||
)
|
||||
|
||||
outputs: dict[str, Path] = {}
|
||||
if should_run_stage(start_stage, "export"):
|
||||
outputs = export_products(
|
||||
work_dir=work_dir,
|
||||
output_dir=output_dir,
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..config import settings
|
||||
from .dinsar_naming import RUN_META_FILENAME
|
||||
|
||||
|
||||
CURRENT_POINTER_DIRNAME = "current"
|
||||
EXECUTION_MANIFEST_FILENAME = "execution_manifest.json"
|
||||
EXECUTION_STATUS_COMPLETED = "COMPLETED"
|
||||
_SAFE_POINTER_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
|
||||
|
||||
def _normalize_path(path: Any) -> str:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _load_json(path: str) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
return payload if isinstance(payload, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _write_json(path: str, payload: Dict[str, Any]) -> str:
|
||||
target = _normalize_path(path)
|
||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||
with open(target, "w", encoding="utf-8") as fp:
|
||||
json.dump(payload, fp, ensure_ascii=False, indent=2)
|
||||
return target
|
||||
|
||||
|
||||
def _first_text(*values: Any) -> str:
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _utc_text(value: Optional[str]) -> str:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
|
||||
def _sanitize_pointer_fragment(value: str, default: str) -> str:
|
||||
text = _SAFE_POINTER_RE.sub("_", str(value or "").strip()).strip("._")
|
||||
return text or default
|
||||
|
||||
|
||||
def _current_pointer_path(results_root_dir: str, *, engine_code: str, profile_code: str) -> str:
|
||||
pointer_name = (
|
||||
f"{_sanitize_pointer_fragment(engine_code or 'engine', 'engine')}__"
|
||||
f"{_sanitize_pointer_fragment(profile_code or 'profile', 'profile')}.json"
|
||||
)
|
||||
return os.path.join(_normalize_path(results_root_dir), CURRENT_POINTER_DIRNAME, pointer_name)
|
||||
|
||||
|
||||
def _runtime_id_for_engine(engine_code: str) -> Optional[str]:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "isce2":
|
||||
return settings.ISCE2_RUNTIME_ID or None
|
||||
if normalized in {"pyint", "gamma"}:
|
||||
return settings.PYINT_RUNTIME_ID or None
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_source_files(primary_file: str, source_files: List[str] | None) -> List[str]:
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw_path in [normalized_primary, *(source_files or [])]:
|
||||
path = _normalize_path(raw_path)
|
||||
if not path or path in seen or not os.path.isfile(path):
|
||||
continue
|
||||
seen.add(path)
|
||||
normalized.append(path)
|
||||
return normalized
|
||||
|
||||
|
||||
def _infer_results_root_dir(run_dir: str, pair_key: str) -> str:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
runs_dir = os.path.dirname(normalized_run_dir)
|
||||
if os.path.basename(runs_dir).lower() == "runs":
|
||||
return os.path.dirname(runs_dir)
|
||||
if pair_key:
|
||||
return os.path.join(os.path.dirname(normalized_run_dir), pair_key)
|
||||
return os.path.dirname(normalized_run_dir)
|
||||
|
||||
|
||||
def repair_managed_completion_files(
|
||||
run_dir: str,
|
||||
*,
|
||||
primary_file: str,
|
||||
source_files: List[str] | None = None,
|
||||
run_meta: Optional[Dict[str, Any]] = None,
|
||||
update_run_metadata: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
if not os.path.isdir(normalized_run_dir):
|
||||
raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}")
|
||||
|
||||
run_meta_path = os.path.join(normalized_run_dir, RUN_META_FILENAME)
|
||||
payload = dict(run_meta or _load_json(run_meta_path) or {})
|
||||
if not payload:
|
||||
raise FileNotFoundError(f"Run metadata not found: {run_meta_path}")
|
||||
|
||||
normalized_primary = _normalize_path(primary_file or payload.get("primary_file"))
|
||||
normalized_sources = _normalize_source_files(
|
||||
normalized_primary,
|
||||
source_files or payload.get("source_files"),
|
||||
)
|
||||
if not normalized_primary or not os.path.isfile(normalized_primary):
|
||||
raise FileNotFoundError(f"Primary output file not found: {normalized_primary or '<empty>'}")
|
||||
if not normalized_sources:
|
||||
normalized_sources = [normalized_primary]
|
||||
|
||||
run_key = _first_text(payload.get("run_key"), os.path.basename(normalized_run_dir))
|
||||
pair_key = _first_text(
|
||||
payload.get("pair_key"),
|
||||
os.path.basename(os.path.dirname(os.path.dirname(normalized_run_dir))),
|
||||
)
|
||||
engine_code = _first_text(payload.get("engine_code"), "isce2")
|
||||
profile_code = _first_text(payload.get("profile_code"), "unknown")
|
||||
task_name = _first_text(payload.get("task_name"), payload.get("task_alias"), run_key)
|
||||
task_alias = _first_text(payload.get("task_alias"), payload.get("task_name"), task_name)
|
||||
output_dir = _normalize_path(payload.get("output_dir") or normalized_run_dir)
|
||||
native_output_dir = _normalize_path(
|
||||
payload.get("native_output_dir") or os.path.join(normalized_run_dir, "native")
|
||||
)
|
||||
results_root_dir = _infer_results_root_dir(normalized_run_dir, pair_key)
|
||||
publish_root_dir = _normalize_path(os.path.dirname(results_root_dir))
|
||||
|
||||
metrics: Dict[str, Any] = {}
|
||||
acceptance = payload.get("acceptance") if isinstance(payload.get("acceptance"), dict) else {}
|
||||
acceptance_metrics = acceptance.get("metrics") if isinstance(acceptance.get("metrics"), dict) else {}
|
||||
run_metrics = payload.get("metrics") if isinstance(payload.get("metrics"), dict) else {}
|
||||
if acceptance_metrics:
|
||||
metrics["acceptance"] = acceptance_metrics
|
||||
if run_metrics:
|
||||
metrics["run_metrics"] = run_metrics
|
||||
if payload.get("manual_recovery") or payload.get("recovery_mode"):
|
||||
metrics["recovery"] = {
|
||||
"manual_recovery": bool(payload.get("manual_recovery")),
|
||||
"recovery_mode": _first_text(payload.get("recovery_mode")),
|
||||
"recovered_at": _first_text(payload.get("recovered_at")),
|
||||
}
|
||||
|
||||
execution_payload = {
|
||||
"format_version": 1,
|
||||
"run_id": _first_text(payload.get("production_run_id"), payload.get("run_id")),
|
||||
"product_family": "dinsar",
|
||||
"run_key": run_key,
|
||||
"task_id": _first_text(payload.get("task_id")),
|
||||
"engine_code": engine_code,
|
||||
"profile_code": profile_code,
|
||||
"runtime_id": _first_text(payload.get("runtime_id")) or _runtime_id_for_engine(engine_code),
|
||||
"mode": _first_text(payload.get("mode"), "managed"),
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"pair_uid": _first_text(payload.get("pair_uid"), payload.get("scene_pair_uid")),
|
||||
"network_run_id": _first_text(payload.get("network_run_id")),
|
||||
"network_edge_id": payload.get("network_edge_id"),
|
||||
"policy_version": _first_text(payload.get("policy_version")),
|
||||
"selection_strategy": _first_text(payload.get("selection_strategy")),
|
||||
"source_root": _first_text(payload.get("source_root")),
|
||||
"source_task_dir": _first_text(payload.get("task_dir")),
|
||||
"results_root_dir": results_root_dir,
|
||||
"publish_root_dir": publish_root_dir,
|
||||
"output_dir": output_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"primary_file": normalized_primary,
|
||||
"source_files": normalized_sources,
|
||||
"status": EXECUTION_STATUS_COMPLETED,
|
||||
"metrics": metrics,
|
||||
"created_at": _utc_text(payload.get("started_at")),
|
||||
"finished_at": _utc_text(payload.get("finished_at") or payload.get("recovered_at")),
|
||||
"recovered_from_run_meta": True,
|
||||
}
|
||||
execution_manifest_path = _write_json(
|
||||
os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME),
|
||||
execution_payload,
|
||||
)
|
||||
|
||||
pointer_payload = {
|
||||
"format_version": 1,
|
||||
"product_family": "dinsar",
|
||||
"engine_code": engine_code,
|
||||
"profile_code": profile_code,
|
||||
"runtime_id": execution_payload.get("runtime_id"),
|
||||
"run_key": run_key,
|
||||
"execution_id": None,
|
||||
"status": EXECUTION_STATUS_COMPLETED,
|
||||
"output_dir": output_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"manifest_path": execution_manifest_path,
|
||||
"primary_file": normalized_primary,
|
||||
"source_files": normalized_sources,
|
||||
"updated_at": _utc_text(payload.get("finished_at") or payload.get("recovered_at")),
|
||||
"recovered_from_run_meta": True,
|
||||
}
|
||||
pointer_path = _write_json(
|
||||
_current_pointer_path(
|
||||
results_root_dir,
|
||||
engine_code=engine_code,
|
||||
profile_code=profile_code,
|
||||
),
|
||||
pointer_payload,
|
||||
)
|
||||
|
||||
if update_run_metadata:
|
||||
payload["execution_manifest_path"] = execution_manifest_path
|
||||
payload["current_pointer_path"] = pointer_path
|
||||
payload["output_dir"] = output_dir
|
||||
payload["native_output_dir"] = native_output_dir
|
||||
payload["primary_file"] = normalized_primary
|
||||
payload["source_files"] = normalized_sources
|
||||
_write_json(run_meta_path, payload)
|
||||
|
||||
return {
|
||||
"run_dir": normalized_run_dir,
|
||||
"results_root_dir": results_root_dir,
|
||||
"execution_manifest_path": execution_manifest_path,
|
||||
"current_pointer_path": pointer_path,
|
||||
"engine_code": engine_code,
|
||||
"profile_code": profile_code,
|
||||
"run_key": run_key,
|
||||
"pair_key": pair_key,
|
||||
}
|
||||
@@ -106,12 +106,13 @@ def _prepared_dem_variants(value: Any) -> List[str]:
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
candidates = [normalized]
|
||||
root, ext = os.path.splitext(normalized)
|
||||
if ext.lower() == ".wgs84":
|
||||
candidates.append(root)
|
||||
candidates = [normalized, root]
|
||||
elif not ext:
|
||||
candidates.append(normalized + ".wgs84")
|
||||
candidates = [normalized + ".wgs84", normalized]
|
||||
else:
|
||||
candidates = [normalized]
|
||||
|
||||
unique: List[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
@@ -32,6 +32,7 @@ from .manifest_snapshot_service import (
|
||||
iter_manifest_paths,
|
||||
)
|
||||
from .image_service import image_service
|
||||
from .dinsar_completion_files import repair_managed_completion_files
|
||||
from .dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
RUN_META_FILENAME,
|
||||
@@ -809,6 +810,7 @@ class ResultCatalogService:
|
||||
disp_dir = os.path.join(package_dir, "assets", "disp")
|
||||
preview_dir = _ensure_directory(os.path.join(package_dir, "preview"))
|
||||
asset_rows: List[Dict[str, Any]] = []
|
||||
completion_files_result: Optional[Dict[str, Any]] = None
|
||||
|
||||
if candidate["engine_code"] == "envi":
|
||||
source_primary = _normalize_path(candidate["source_files"][0])
|
||||
@@ -866,6 +868,7 @@ class ResultCatalogService:
|
||||
}
|
||||
)
|
||||
else:
|
||||
target_source_files: List[str] = []
|
||||
if in_place_source:
|
||||
target_primary = _normalize_path(primary_file)
|
||||
else:
|
||||
@@ -878,6 +881,7 @@ class ResultCatalogService:
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
target_source_files.append(target_primary)
|
||||
asset_rows.append(
|
||||
{
|
||||
"role": "disp",
|
||||
@@ -904,6 +908,7 @@ class ResultCatalogService:
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
target_source_files.append(target_coh)
|
||||
asset_rows.append(
|
||||
{
|
||||
"role": "coh",
|
||||
@@ -953,6 +958,15 @@ class ResultCatalogService:
|
||||
manifest_path = os.path.join(package_dir, "manifest.json")
|
||||
with open(manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
if candidate["engine_code"] == "isce2" and in_place_source:
|
||||
try:
|
||||
completion_files_result = repair_managed_completion_files(
|
||||
package_dir,
|
||||
primary_file=target_primary,
|
||||
source_files=target_source_files,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
completion_files_result = None
|
||||
|
||||
details.append(
|
||||
{
|
||||
@@ -965,6 +979,16 @@ class ResultCatalogService:
|
||||
"package_dir": package_dir,
|
||||
"in_place": in_place_source,
|
||||
"thumb_created": thumb_ok,
|
||||
"execution_manifest_path": (
|
||||
completion_files_result.get("execution_manifest_path")
|
||||
if completion_files_result
|
||||
else None
|
||||
),
|
||||
"current_pointer_path": (
|
||||
completion_files_result.get("current_pointer_path")
|
||||
if completion_files_result
|
||||
else None
|
||||
),
|
||||
"status": "ok",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -68,8 +68,11 @@ def _build_pipeline_argv(payload: Mapping[str, Any], *, dry_run: bool = False) -
|
||||
_append_optional(argv, "--bbox-margin", params.get("bbox_margin"))
|
||||
_append_optional(argv, "--wavelength", params.get("wavelength"))
|
||||
_append_optional(argv, "--orbit-margin-sec", params.get("orbit_margin_sec"))
|
||||
_append_optional(argv, "--resume-from", params.get("resume_from"))
|
||||
_append_optional(argv, "--reference-satellite", pair_meta.get("master_satellite"))
|
||||
_append_optional(argv, "--secondary-satellite", pair_meta.get("slave_satellite"))
|
||||
if bool(params.get("full_geocode")):
|
||||
argv.append("--full-geocode")
|
||||
if dry_run:
|
||||
argv.append("--dry-run")
|
||||
return argv
|
||||
@@ -81,6 +84,7 @@ def _build_child_env() -> Dict[str, str]:
|
||||
proj_data = env_root / "share" / "proj"
|
||||
if proj_data.exists():
|
||||
env["PROJ_DATA"] = proj_data.as_posix()
|
||||
env["PROJ_LIB"] = proj_data.as_posix()
|
||||
return env
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# ISCE2 Stabilization Update Log
|
||||
|
||||
Date: `2026-04-27`
|
||||
|
||||
## Scope
|
||||
|
||||
This update hardens the managed `ISCE2` LT-1 stripmap D-InSAR production path against
|
||||
large raw DEM reuse, long geocode stalls, and incomplete recovered-run metadata.
|
||||
|
||||
## Delivered Changes
|
||||
|
||||
### 1. Managed ISCE2 pipeline hardening
|
||||
|
||||
- Added `--resume-from unwrap|geocode|export` support to the LT-1 pipeline.
|
||||
- Added `--full-geocode` support. The default path now geocodes only the export-critical
|
||||
products instead of ISCE2's full default list.
|
||||
- Added stage-aware logging with stdout flush to improve long-run observability.
|
||||
- Added geocode DEM subset preparation so resumed geocode/export runs do not reprocess the
|
||||
full base DEM.
|
||||
- Added a guard that blocks fresh runs when the selected DEM resolves to a very large raw
|
||||
base raster without a prepared `.wgs84` sibling.
|
||||
|
||||
### 2. Engine and runtime configuration fixes
|
||||
|
||||
- `ISCE2` now prefers an existing prepared `.wgs84` DEM over the raw base path.
|
||||
- `PyINT` DEM resolution now follows the same prepared-first preference.
|
||||
- The WSL ISCE2 runner now passes `resume_from` and `full_geocode` through to the pipeline.
|
||||
- The WSL runner now sets both `PROJ_DATA` and `PROJ_LIB`.
|
||||
|
||||
### 3. Recovery and catalog self-healing
|
||||
|
||||
- Added a completion-file repair helper for managed ISCE2 runs.
|
||||
- In-place ISCE2 publish/rebuild now repairs missing:
|
||||
- `execution_manifest.json`
|
||||
- `current/isce2__<profile>.json`
|
||||
- This allows recovered runs to be reintroduced into the managed result catalog without
|
||||
hand-editing completion markers.
|
||||
|
||||
### 4. One-time DEM preparation tooling
|
||||
|
||||
- Added `backend/app/isce2_pipeline/prepare_isce2_base_dem.py`.
|
||||
- The script resolves DEM paths from `.env`, validates existing prepared outputs, and can
|
||||
generate a reusable `WGS84` `.wgs84` DEM from an `EGM96` base DEM.
|
||||
- The script also auto-configures `PROJ` paths for standalone execution.
|
||||
|
||||
### 5. Configuration guidance
|
||||
|
||||
- Updated `.env.example` comments to distinguish:
|
||||
- raw SARscape/ENVI DEM source path
|
||||
- prepared ISCE2/PyINT `.wgs84` path
|
||||
|
||||
## Validation
|
||||
|
||||
- Python syntax validation was run for the modified ISCE2 pipeline, engine, runtime, and
|
||||
result-catalog modules.
|
||||
- The one-time full DEM preparation completed successfully and produced:
|
||||
- `.wgs84`
|
||||
- `.wgs84.xml`
|
||||
- `.wgs84.vrt`
|
||||
- The prepared DEM metadata was verified to report `WGS84`.
|
||||
|
||||
## Required Local Follow-Up
|
||||
|
||||
These operational steps are intentionally not committed:
|
||||
|
||||
- Point local `.env` `ISCE2_DEM_PATH` to the prepared `.wgs84` file.
|
||||
- Point local `.env` `PYINT_PREPARED_DEM_PATH` to the same prepared `.wgs84` file.
|
||||
- Restart the backend so the running process reloads the updated `.env`.
|
||||
|
||||
Reference in New Issue
Block a user