Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initialize insar management system v2

Browse Source
This commit is contained in:
Harmon
2026-04-14 13:16:01 +08:00
commit ecc72ec9cd
361 changed files with 2142522 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/isce2_sbas_timeseries/scripts/.gitkeep
+1
View File
@@ -0,0 +1 @@
experiments/isce2_sbas_timeseries/scripts/build_lt1_stack_prep.py
+604
View File
@@ -0,0 +1,604 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import importlib.util
import json
import re
import sys
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
REPO_ROOT = Path(__file__).resolve().parents[3]
DEFAULT_MANIFEST_PATH = (
REPO_ROOT
/ "experiments"
/ "isce2_sbas_timeseries"
/ "configs"
/ "sample_stack_e123p3_n46p1.json"
)
DEFAULT_STACK_SCRIPT_WSL = (
"/home/administrator/miniconda3/envs/isce2/share/isce2/stripmapStack/stackStripMap.py"
)
DEFAULT_CONDA_WSL = "/home/administrator/miniconda3/bin/conda"
DEFAULT_ISCE2_SHARE_WSL = "/home/administrator/miniconda3/envs/isce2/share/isce2"
DEFAULT_STRIPMAP_STACK_DIR_WSL = f"{DEFAULT_ISCE2_SHARE_WSL}/stripmapStack"
SUPPORTED_STACK_WORKFLOWS = ("slc", "interferogram", "ionosphere")
DEFAULT_STACK_TEXT_CMD = (
f"export PATH={DEFAULT_STRIPMAP_STACK_DIR_WSL}:$PATH; "
f"export PYTHONPATH={DEFAULT_STRIPMAP_STACK_DIR_WSL}:{DEFAULT_ISCE2_SHARE_WSL}${{PYTHONPATH:+:$PYTHONPATH}}; "
)
def windows_to_wsl(path: str | Path) -> str:
text = str(path)
match = re.match(r"^([A-Za-z]):[\\/](.*)$", text)
if not match:
return text.replace("\\", "/")
drive = match.group(1).lower()
tail = match.group(2).replace("\\", "/").lstrip("/")
return f"/mnt/{drive}/{tail}"
def load_isce2_input_helper_module():
helper_path = REPO_ROOT / "backend" / "app" / "isce2_pipeline" / "lt1_input_resolver.py"
spec = importlib.util.spec_from_file_location("lt1_orbit_helper", helper_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load orbit helper module: {helper_path}")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
ISCE2_INPUT_HELPER = load_isce2_input_helper_module()
def require_file(path: Path, label: str) -> None:
if not path.exists():
raise FileNotFoundError(f"Missing {label}: {path}")
def shelve_stem_exists(stem: Path) -> bool:
for suffix in ("", ".db", ".dat", ".dir", ".bak"):
if Path(str(stem) + suffix).exists():
return True
return False
@dataclass
class ScenePlan:
date: str
target_dir_windows: str
target_dir_wsl: str
source_scene_json_windows: str
source_scene_json_wsl: str
source_tiff_windows: str
source_tiff_wsl: str
source_meta_windows: str
source_meta_wsl: str
orbit_xml_windows: Optional[str]
orbit_xml_wsl: Optional[str]
orbit_xml_exists: bool
orbit_resolution_mode: Optional[str]
orbit_resolution_error: Optional[str]
source_exists: bool
scene_start_utc: str
scene_stop_utc: str
orbit_window_start_utc: str
orbit_window_stop_utc: str
expected_slc_windows: str
expected_slc_wsl: str
expected_slc_xml_windows: str
expected_slc_xml_wsl: str
expected_data_shelve_windows: str
expected_data_shelve_wsl: str
materialized_slc_exists: bool
materialized_data_exists: bool
stack_ready: bool
status: str
def build_scene_plan(
scene: Dict[str, Any],
slc_root: Path,
orbit_pool: Optional[Path],
orbit_stage_dir: Path,
margin_sec: float,
) -> ScenePlan:
date = str(scene["imaging_date"])
satellite = str(scene["satellite"])
source_tiff = Path(scene["tiff_path"])
source_meta = Path(scene["meta_path"])
require_file(source_tiff, f"scene TIFF for {date}")
require_file(source_meta, f"scene meta XML for {date}")
scene_start_dt, scene_stop_dt = ISCE2_INPUT_HELPER.parse_scene_window(source_meta, margin_sec=0.0)
orbit_window_start_dt, orbit_window_stop_dt = ISCE2_INPUT_HELPER.parse_scene_window(
source_meta,
margin_sec=margin_sec,
)
scene_start_utc = scene_start_dt.isoformat()
scene_stop_utc = scene_stop_dt.isoformat()
orbit_window_start_utc = orbit_window_start_dt.isoformat()
orbit_window_stop_utc = orbit_window_stop_dt.isoformat()
target_dir = slc_root / date
expected_slc = target_dir / f"{date}.slc"
expected_slc_xml = target_dir / f"{date}.slc.xml"
expected_data = target_dir / "data"
source_scene_json = target_dir / "source_scene.json"
orbit_xml: Optional[Path] = None
orbit_resolution_mode: Optional[str] = None
orbit_resolution_error: Optional[str] = None
if orbit_pool is not None:
try:
orbit_resolution = ISCE2_INPUT_HELPER.ensure_lt1_orbit_xml(
date_yyyymmdd=date,
satellite=satellite,
annotation_xml=source_meta,
orbit_root=orbit_pool,
orbit_output_dir=orbit_stage_dir,
margin_sec=margin_sec,
)
orbit_xml = orbit_resolution.path
orbit_resolution_mode = orbit_resolution.source
except Exception as exc:
orbit_resolution_error = str(exc)
materialized_slc_exists = expected_slc.exists() and expected_slc_xml.exists()
materialized_data_exists = shelve_stem_exists(expected_data)
stack_ready = bool(orbit_xml and materialized_slc_exists and materialized_data_exists)
if not orbit_xml:
status = "missing_orbit_xml"
elif not materialized_slc_exists and not materialized_data_exists:
status = "waiting_for_scene_materializer"
elif not materialized_slc_exists:
status = "missing_slc"
elif not materialized_data_exists:
status = "missing_data_shelve"
else:
status = "ready"
return ScenePlan(
date=date,
target_dir_windows=str(target_dir),
target_dir_wsl=windows_to_wsl(target_dir),
source_scene_json_windows=str(source_scene_json),
source_scene_json_wsl=windows_to_wsl(source_scene_json),
source_tiff_windows=str(source_tiff),
source_tiff_wsl=windows_to_wsl(source_tiff),
source_meta_windows=str(source_meta),
source_meta_wsl=windows_to_wsl(source_meta),
orbit_xml_windows=str(orbit_xml) if orbit_xml else None,
orbit_xml_wsl=windows_to_wsl(orbit_xml) if orbit_xml else None,
orbit_xml_exists=bool(orbit_xml),
orbit_resolution_mode=orbit_resolution_mode,
orbit_resolution_error=orbit_resolution_error,
source_exists=True,
scene_start_utc=scene_start_utc,
scene_stop_utc=scene_stop_utc,
orbit_window_start_utc=orbit_window_start_utc,
orbit_window_stop_utc=orbit_window_stop_utc,
expected_slc_windows=str(expected_slc),
expected_slc_wsl=windows_to_wsl(expected_slc),
expected_slc_xml_windows=str(expected_slc_xml),
expected_slc_xml_wsl=windows_to_wsl(expected_slc_xml),
expected_data_shelve_windows=str(expected_data),
expected_data_shelve_wsl=windows_to_wsl(expected_data),
materialized_slc_exists=materialized_slc_exists,
materialized_data_exists=materialized_data_exists,
stack_ready=stack_ready,
status=status,
)
def render_stack_command(
slc_dir_wsl: str,
dem_wsl: str,
work_dir_wsl: str,
reference_date: str,
workflow: str,
) -> List[str]:
return [
DEFAULT_CONDA_WSL,
"run",
"-n",
"isce2",
"python",
DEFAULT_STACK_SCRIPT_WSL,
"-s",
slc_dir_wsl,
"-d",
dem_wsl,
"-w",
work_dir_wsl,
"-m",
reference_date,
"--nofocus",
"-W",
workflow,
"-u",
"snaphu",
"-c",
DEFAULT_STACK_TEXT_CMD,
]
def shell_quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"
def render_shell_command(argv: List[str]) -> str:
return " ".join(shell_quote(item) for item in argv)
def build_blockers(scene_plans: List[ScenePlan], orbit_pool: Optional[Path], dem_path: Optional[Path]) -> List[str]:
blockers: List[str] = []
if orbit_pool is None:
blockers.append("ORBIT_POOL_ISCE2 was not resolved.")
if dem_path is None:
blockers.append("Prepared DEM with .xml sidecar was not resolved.")
missing_orbit = [item.date for item in scene_plans if not item.orbit_xml_exists]
if missing_orbit:
blockers.append("Missing orbit XML for dates: " + ", ".join(missing_orbit))
orbit_errors = [f"{item.date}: {item.orbit_resolution_error}" for item in scene_plans if item.orbit_resolution_error]
if orbit_errors:
blockers.append("Orbit resolution errors: " + "; ".join(orbit_errors))
missing_slc = [item.date for item in scene_plans if not item.materialized_slc_exists]
if missing_slc:
blockers.append("Materialized .slc/.slc.xml are missing for dates: " + ", ".join(missing_slc))
missing_data = [item.date for item in scene_plans if not item.materialized_data_exists]
if missing_data:
blockers.append("ISCE data shelve is missing for dates: " + ", ".join(missing_data))
return blockers
def render_contract_markdown(report: Dict[str, Any]) -> str:
lines: List[str] = []
ready = bool(report["readiness"]["ready_for_stackStripMap_nofocus"])
lines.append("# LT-1 Stack Prep Contract")
lines.append("")
lines.append(f"Generated: {report['generated_at_utc']}")
lines.append("")
lines.append("## Selected Stack")
lines.append("")
lines.append(f"- Group key: `{report['group_key']}`")
lines.append(f"- Reference date: `{report['reference_date']}`")
lines.append(f"- Workflow: `{report['processing_workflow']}`")
lines.append(f"- Scene count: `{report['scene_count']}`")
lines.append("")
lines.append("## Resolved Runtime Inputs")
lines.append("")
lines.append(f"- Orbit pool (Windows): `{report['resolved_dependencies']['orbit_pool_windows'] or 'UNRESOLVED'}`")
lines.append(f"- Orbit pool (WSL): `{report['resolved_dependencies']['orbit_pool_wsl'] or 'UNRESOLVED'}`")
lines.append(f"- DEM (Windows): `{report['resolved_dependencies']['dem_path_windows'] or 'UNRESOLVED'}`")
lines.append(f"- DEM (WSL): `{report['resolved_dependencies']['dem_path_wsl'] or 'UNRESOLVED'}`")
lines.append("")
lines.append("## Confirmed stripmapStack Contract")
lines.append("")
lines.append("- `stackStripMap.py --nofocus` discovers dates from `SLC/YYYYMMDD/YYYYMMDD.slc`.")
lines.append("- `topo.py` opens `SLC/YYYYMMDD/data` for the reference acquisition.")
lines.append("- `geo2rdr.py` opens `SLC/YYYYMMDD/data` for each secondary acquisition.")
lines.append("- Therefore each acquisition directory must contain at least:")
lines.append(" - `YYYYMMDD.slc`")
lines.append(" - `YYYYMMDD.slc.xml`")
lines.append(" - `data` shelve with `frame` and optional `doppler`")
lines.append("")
lines.append("## Scene Status")
lines.append("")
lines.append("| Date | Orbit XML | SLC | Data | Status |")
lines.append("| --- | --- | --- | --- | --- |")
for scene in report["scenes"]:
orbit_ok = "yes" if scene["orbit_xml_exists"] else "no"
slc_ok = "yes" if scene["materialized_slc_exists"] else "no"
data_ok = "yes" if scene["materialized_data_exists"] else "no"
lines.append(f"| {scene['date']} | {orbit_ok} | {slc_ok} | {data_ok} | `{scene['status']}` |")
lines.append("")
lines.append("## Draft stackStripMap Command")
lines.append("")
lines.append("```bash")
lines.append(report["stack_command"]["shell"])
lines.append("```")
lines.append("")
lines.append("## Current Blockers")
lines.append("")
blockers = report["readiness"]["blocking_reasons"]
if blockers:
for blocker in blockers:
lines.append(f"- {blocker}")
else:
lines.append("- none")
lines.append("")
lines.append("## Next Tasks")
lines.append("")
if ready:
lines.append("- Execute `run_01_reference` and confirm geometry generation succeeds.")
lines.append("- Execute `run_02` to `run_07` step by step and record any LT-1-specific failures.")
lines.append("- Inspect `baselines/`, `configs/`, and the first coarse coregistration outputs.")
lines.append("- Install MintPy only after the stack run outputs are stable.")
else:
lines.append("- Use the LT-1 scene materializer to finish the remaining acquisitions under the generated `SLC/` root.")
lines.append("- Re-run the generated preflight script, then execute `stackStripMap.py --nofocus`.")
lines.append("- Install MintPy only after the stack materializer contract is working end to end.")
lines.append("")
return "\n".join(lines)
def render_run_script(report: Dict[str, Any]) -> str:
slc_dir = report["workspace"]["slc_dir_wsl"]
work_dir = report["workspace"]["stack_work_dir_wsl"]
dem_path = report["resolved_dependencies"]["dem_path_wsl"] or "__MISSING_DEM__"
reference_date = report["reference_date"]
dates = " ".join(scene["date"] for scene in report["scenes"])
command = report["stack_command"]["shell"]
return f"""#!/usr/bin/env bash
set -euo pipefail
SLC_DIR={shell_quote(slc_dir)}
WORK_DIR={shell_quote(work_dir)}
DEM={shell_quote(dem_path)}
REFERENCE_DATE={shell_quote(reference_date)}
ISCE2_SHARE={shell_quote(DEFAULT_ISCE2_SHARE_WSL)}
STRIPMAP_STACK_DIR={shell_quote(DEFAULT_STRIPMAP_STACK_DIR_WSL)}
DATES=({dates})
export PYTHONPATH="$STRIPMAP_STACK_DIR:$ISCE2_SHARE${{PYTHONPATH:+:$PYTHONPATH}}"
export PATH="$STRIPMAP_STACK_DIR:$PATH"
echo "LT-1 stripmap stack dry-run preflight"
echo "SLC root: $SLC_DIR"
echo "Work dir: $WORK_DIR"
echo "DEM: $DEM"
echo "Reference date: $REFERENCE_DATE"
echo "PYTHONPATH: $PYTHONPATH"
echo "PATH prefix: $STRIPMAP_STACK_DIR"
missing=0
for d in "${{DATES[@]}}"; do
if [[ ! -f "$SLC_DIR/$d/$d.slc" ]]; then
echo "MISSING: $SLC_DIR/$d/$d.slc"
missing=1
fi
if [[ ! -f "$SLC_DIR/$d/$d.slc.xml" ]]; then
echo "MISSING: $SLC_DIR/$d/$d.slc.xml"
missing=1
fi
if [[ ! -e "$SLC_DIR/$d/data" && ! -e "$SLC_DIR/$d/data.db" && ! -e "$SLC_DIR/$d/data.dat" && ! -e "$SLC_DIR/$d/data.dir" && ! -e "$SLC_DIR/$d/data.bak" ]]; then
echo "MISSING: $SLC_DIR/$d/data"
missing=1
fi
done
if [[ "$missing" -ne 0 ]]; then
echo "Dry-run only. LT-1 scene materialization is still missing."
exit 2
fi
echo "Preflight passed. Running stackStripMap."
{command}
"""
def write_json(path: Path, payload: Dict[str, Any]) -> None:
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Build a dry-run LT-1 SBAS stack-prep workspace for ISCE2 stripmapStack."
)
parser.add_argument(
"--manifest-path",
default=str(DEFAULT_MANIFEST_PATH),
help="Path to the selected stack manifest JSON.",
)
parser.add_argument(
"--scratch-root",
default=None,
help="Override the stack scratch root directory. Defaults to proposed_scratch_windows in the manifest.",
)
parser.add_argument(
"--orbit-pool",
default=None,
help="Override ORBIT_POOL_ISCE2 (Windows path containing LT1A_GpsData_GAS_C_YYYYMMDD.xml).",
)
parser.add_argument(
"--dem-path",
default=None,
help="Override the prepared DEM base path (must have a .xml sidecar).",
)
parser.add_argument(
"--orbit-margin-sec",
type=float,
default=60.0,
help="Margin used when reporting the recommended orbit clip window.",
)
parser.add_argument(
"--workflow",
default="slc",
choices=SUPPORTED_STACK_WORKFLOWS,
help="stripmapStack workflow to generate: slc, interferogram, or ionosphere.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
env_values = ISCE2_INPUT_HELPER.load_env_file(REPO_ROOT / ".env")
manifest_path = Path(args.manifest_path).resolve()
require_file(manifest_path, "stack manifest")
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
scratch_root = Path(args.scratch_root or manifest["proposed_scratch_windows"]).resolve()
slc_root = scratch_root / "SLC"
orbits_dir = scratch_root / "orbits"
logs_dir = scratch_root / "logs"
notes_dir = scratch_root / "notes"
inputs_dir = scratch_root / "inputs"
stack_work_dir = scratch_root / "stack_work"
for path in (scratch_root, slc_root, orbits_dir, logs_dir, notes_dir, inputs_dir, stack_work_dir):
path.mkdir(parents=True, exist_ok=True)
orbit_pool = ISCE2_INPUT_HELPER.resolve_orbit_pool_path(
explicit_path=args.orbit_pool,
env_values=env_values,
default_candidates=ISCE2_INPUT_HELPER.DEFAULT_WINDOWS_ORBIT_POOL_CANDIDATES,
)
local_dem_candidate = inputs_dir / "dem" / "stack_dem_window.wgs84"
dem_path = ISCE2_INPUT_HELPER.resolve_prepared_dem_path(
explicit_path=args.dem_path,
env_values=env_values,
extra_candidates=[local_dem_candidate],
default_candidates=ISCE2_INPUT_HELPER.DEFAULT_WINDOWS_DEM_CANDIDATES,
)
scene_plans = [
build_scene_plan(
scene,
slc_root=slc_root,
orbit_pool=orbit_pool,
orbit_stage_dir=orbits_dir,
margin_sec=args.orbit_margin_sec,
)
for scene in manifest["scenes"]
]
scene_plans.sort(key=lambda item: item.date)
for plan, source_scene in zip(scene_plans, sorted(manifest["scenes"], key=lambda item: item["imaging_date"])):
target_dir = Path(plan.target_dir_windows)
target_dir.mkdir(parents=True, exist_ok=True)
scene_payload = dict(source_scene)
scene_payload["stack_prep"] = {
"date": plan.date,
"target_dir_windows": plan.target_dir_windows,
"target_dir_wsl": plan.target_dir_wsl,
"orbit_xml_windows": plan.orbit_xml_windows,
"orbit_xml_wsl": plan.orbit_xml_wsl,
"orbit_resolution_mode": plan.orbit_resolution_mode,
"orbit_resolution_error": plan.orbit_resolution_error,
"scene_start_utc": plan.scene_start_utc,
"scene_stop_utc": plan.scene_stop_utc,
"orbit_window_start_utc": plan.orbit_window_start_utc,
"orbit_window_stop_utc": plan.orbit_window_stop_utc,
"expected_slc_windows": plan.expected_slc_windows,
"expected_slc_xml_windows": plan.expected_slc_xml_windows,
"expected_data_shelve_windows": plan.expected_data_shelve_windows,
"status": plan.status,
}
write_json(target_dir / "source_scene.json", scene_payload)
stack_command_argv = render_stack_command(
slc_dir_wsl=windows_to_wsl(slc_root),
dem_wsl=windows_to_wsl(dem_path) if dem_path else "__MISSING_DEM__",
work_dir_wsl=windows_to_wsl(stack_work_dir),
reference_date=manifest["reference_date"],
workflow=args.workflow,
)
blockers = build_blockers(scene_plans, orbit_pool=orbit_pool, dem_path=dem_path)
readiness = {
"all_orbits_resolved": all(item.orbit_xml_exists for item in scene_plans),
"all_materialized_slc_present": all(item.materialized_slc_exists for item in scene_plans),
"all_data_shelves_present": all(item.materialized_data_exists for item in scene_plans),
"ready_for_stackStripMap_nofocus": not blockers,
"blocking_reasons": blockers,
}
report: Dict[str, Any] = {
"manifest_version": 1,
"generated_at_utc": datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
"source_manifest_windows": str(manifest_path),
"source_manifest_wsl": windows_to_wsl(manifest_path),
"group_key": manifest["group_key"],
"tile_key": manifest["tile_key"],
"scene_count": manifest["scene_count"],
"reference_date": manifest["reference_date"],
"reference_strategy": manifest["reference_strategy"],
"processing_workflow": args.workflow,
"sensor_name": "LUTAN1",
"stack_driver": "isce2.stripmapStack.stackStripMap",
"workspace": {
"root_windows": str(scratch_root),
"root_wsl": windows_to_wsl(scratch_root),
"slc_dir_windows": str(slc_root),
"slc_dir_wsl": windows_to_wsl(slc_root),
"orbits_dir_windows": str(orbits_dir),
"orbits_dir_wsl": windows_to_wsl(orbits_dir),
"logs_dir_windows": str(logs_dir),
"logs_dir_wsl": windows_to_wsl(logs_dir),
"notes_dir_windows": str(notes_dir),
"notes_dir_wsl": windows_to_wsl(notes_dir),
"inputs_dir_windows": str(inputs_dir),
"inputs_dir_wsl": windows_to_wsl(inputs_dir),
"stack_work_dir_windows": str(stack_work_dir),
"stack_work_dir_wsl": windows_to_wsl(stack_work_dir),
},
"resolved_dependencies": {
"orbit_pool_windows": str(orbit_pool) if orbit_pool else None,
"orbit_pool_wsl": windows_to_wsl(orbit_pool) if orbit_pool else None,
"dem_path_windows": str(dem_path) if dem_path else None,
"dem_path_wsl": windows_to_wsl(dem_path) if dem_path else None,
},
"stack_contract": {
"mode": "nofocus",
"workflow": args.workflow,
"required_per_acquisition_files": [
"YYYYMMDD.slc",
"YYYYMMDD.slc.xml",
"data shelve",
],
"current_source_layout": "per_scene_folder_with_tiff_meta_rpc",
"adapter_needed": True,
"adapter_goal": "materialize a stripmapStack-ready date directory from LT-1 TIFF/meta/orbit inputs",
},
"stack_command": {
"argv": stack_command_argv,
"shell": render_shell_command(stack_command_argv),
},
"readiness": readiness,
"scenes": [plan.__dict__ for plan in scene_plans],
"next_tasks": [
"Use the LT-1 scene materializer to build YYYYMMDD.slc and data shelve for the remaining acquisitions.",
"Keep raw scene data external and store only lightweight source manifests plus generated ISCE products under scratch/SLC/YYYYMMDD.",
f"Run stripmapStack in --nofocus mode with workflow={args.workflow} once every date directory is materialized.",
"Install MintPy only after stackStripMap produces stable interferogram outputs.",
],
}
report_path = scratch_root / "stack_input_manifest.json"
contract_path = scratch_root / "stack_prep_contract.md"
run_script_path = scratch_root / "run_stripmap_stack_dryrun.sh"
write_json(report_path, report)
contract_path.write_text(render_contract_markdown(report), encoding="utf-8")
run_script_path.write_text(render_run_script(report), encoding="utf-8", newline="\n")
print(f"Manifest: {report_path}")
print(f"Contract: {contract_path}")
print(f"Run script: {run_script_path}")
print(f"Scratch root: {scratch_root}")
print(f"Orbit pool: {orbit_pool if orbit_pool else 'UNRESOLVED'}")
print(f"DEM: {dem_path if dem_path else 'UNRESOLVED'}")
print(f"Ready: {readiness['ready_for_stackStripMap_nofocus']}")
if blockers:
print("Blockers:")
for blocker in blockers:
print(f" - {blocker}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/build_mintpy_publish_bundle.py
+202
View File
@@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""Build a publish-style manifest and preview bundle from MintPy outputs."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
import h5py
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
def _decode_date(value):
return value.decode() if isinstance(value, (bytes, np.bytes_)) else str(value)
def _read_h5_summary(h5_path: Path) -> dict:
with h5py.File(h5_path, "r") as f:
datasets = sorted(f.keys())
attrs = {k: (v.item() if hasattr(v, "item") else v) for k, v in f.attrs.items()}
serializable_attrs = {}
for key, value in attrs.items():
if isinstance(value, bytes):
serializable_attrs[key] = value.decode()
elif isinstance(value, np.ndarray):
serializable_attrs[key] = value.tolist()
else:
serializable_attrs[key] = value
summary = {
"path": h5_path.name,
"datasets": datasets,
"attrs": serializable_attrs,
}
if "date" in f:
summary["dates"] = [_decode_date(x) for x in f["date"][:]]
return summary
def _write_velocity_preview(geo_velocity_h5: Path, output_png: Path) -> dict:
with h5py.File(geo_velocity_h5, "r") as f:
velocity = f["velocity"][:]
finite = np.isfinite(velocity)
valid = velocity[finite]
if valid.size == 0:
raise RuntimeError(f"No finite velocity values found in {geo_velocity_h5}")
vmax = float(np.nanpercentile(np.abs(valid), 98))
vmax = max(vmax, 1e-6)
vmin = -vmax
fig = plt.figure(figsize=(10, 7), dpi=150)
ax = fig.add_subplot(111)
im = ax.imshow(velocity, cmap="RdBu_r", vmin=vmin, vmax=vmax)
ax.set_title("Velocity Preview (m/year)")
ax.set_xticks([])
ax.set_yticks([])
cbar = fig.colorbar(im, ax=ax, shrink=0.82)
cbar.set_label("m/year")
fig.tight_layout()
output_png.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(output_png, bbox_inches="tight")
plt.close(fig)
return {
"vmin": vmin,
"vmax": vmax,
"valid_pixels": int(valid.size),
}
def _count_mask_pixels(mask_h5: Path) -> dict:
with h5py.File(mask_h5, "r") as f:
dataset_name = "mask" if "mask" in f else "waterMask"
data = f[dataset_name][:]
total = int(data.size)
valid = int(np.count_nonzero(data))
return {
"dataset": dataset_name,
"valid_pixels": valid,
"total_pixels": total,
"valid_ratio": valid / total if total else 0.0,
}
def build_bundle(mintpy_work_dir: Path, publish_dir: Path, group_key: str | None) -> None:
assets_dir = publish_dir / "assets"
preview_dir = publish_dir / "preview"
metadata_dir = publish_dir / "metadata"
geo_velocity_h5 = assets_dir / "geo_velocity.h5"
geo_timeseries_h5 = assets_dir / "geo_timeseries.h5"
geo_temporal_coh_h5 = assets_dir / "geo_temporalCoherence.h5"
geo_mask_temp_coh_h5 = assets_dir / "geo_maskTempCoh.h5"
preview_stats = _write_velocity_preview(
geo_velocity_h5=geo_velocity_h5,
output_png=preview_dir / "velocity_preview.png",
)
with h5py.File(mintpy_work_dir / "timeseries.h5", "r") as ts_file:
ref_date = ts_file.attrs.get("REF_DATE")
ref_x = ts_file.attrs.get("REF_X")
ref_y = ts_file.attrs.get("REF_Y")
stack_dates = [_decode_date(x) for x in ts_file["date"][:]]
manifest = {
"schema_version": "psinsar.publish.v1",
"catalog_name": "psinsar",
"mode": "sbas",
"engine_code": "isce2",
"processor_code": "isce2_stack_mintpy",
"group_key": group_key,
"mintpy_work_dir": str(mintpy_work_dir),
"publish_dir": str(publish_dir),
"reference_date": _decode_date(ref_date) if ref_date is not None else None,
"reference_point": {
"x": int(ref_x) if ref_x is not None else None,
"y": int(ref_y) if ref_y is not None else None,
},
"stack_dates": stack_dates,
"artifacts": [
{"product_type": "timeseries_cube", "path": "assets/geo_timeseries.h5"},
{"product_type": "velocity_map", "path": "assets/geo_velocity.h5"},
{"product_type": "velocity_geotiff", "path": "assets/velocity.tif"},
{"product_type": "temporal_coherence", "path": "assets/geo_temporalCoherence.h5"},
{"product_type": "temporal_coherence_geotiff", "path": "assets/temporalCoherence.tif"},
{"product_type": "quality_mask", "path": "assets/geo_maskTempCoh.h5"},
{"product_type": "quality_mask_geotiff", "path": "assets/maskTempCoh.tif"},
{"product_type": "preview_png", "path": "preview/velocity_preview.png"},
{"product_type": "diagnostic_png", "path": "preview/numTriNonzeroIntAmbiguity.png"},
],
"quality": {
"mask_all_valid": _count_mask_pixels(mintpy_work_dir / "maskAllValid.h5"),
"mask_temp_coh": _count_mask_pixels(mintpy_work_dir / "maskTempCoh.h5"),
"velocity_preview": preview_stats,
},
"summaries": {
"geo_velocity": _read_h5_summary(geo_velocity_h5),
"geo_timeseries": _read_h5_summary(geo_timeseries_h5),
"geo_temporal_coherence": _read_h5_summary(geo_temporal_coh_h5),
"geo_mask_temp_coh": _read_h5_summary(geo_mask_temp_coh_h5),
},
"metadata_files": [
"metadata/smallbaselineApp.cfg",
"metadata/source_quality_summary.json",
],
}
summary_json = {
"maskAllValid": manifest["quality"]["mask_all_valid"],
"maskTempCoh": manifest["quality"]["mask_temp_coh"],
"preview": manifest["quality"]["velocity_preview"],
}
publish_dir.mkdir(parents=True, exist_ok=True)
metadata_dir.mkdir(parents=True, exist_ok=True)
(publish_dir / "manifest.json").write_text(
json.dumps(manifest, indent=2, ensure_ascii=False),
encoding="utf-8",
)
(metadata_dir / "source_quality_summary.json").write_text(
json.dumps(summary_json, indent=2, ensure_ascii=False),
encoding="utf-8",
)
print(f"Wrote manifest: {publish_dir / 'manifest.json'}")
print(f"Wrote quality summary: {metadata_dir / 'source_quality_summary.json'}")
print(f"Wrote preview: {preview_dir / 'velocity_preview.png'}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Build publish-style artifacts for MintPy SBAS outputs.")
parser.add_argument("--mintpy-work-dir", required=True, help="MintPy work directory containing timeseries.h5, velocity.h5, etc.")
parser.add_argument("--publish-dir", required=True, help="Publish output directory.")
parser.add_argument("--group-key", default=None, help="Optional stack group key to embed in manifest.")
return parser.parse_args()
def main() -> int:
args = parse_args()
build_bundle(
mintpy_work_dir=Path(args.mintpy_work_dir),
publish_dir=Path(args.publish_dir),
group_key=args.group_key,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/check_env_ubuntu2404.sh
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euo pipefail
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
REPO_ROOT="${REPO_ROOT:-/mnt/z/Code/Insar_management_system_v2}"
EXP_ROOT="${EXP_ROOT:-$REPO_ROOT/experiments/isce2_sbas_timeseries}"
echo "== repo =="
echo "$REPO_ROOT"
test -d "$REPO_ROOT"
echo "== experiment root =="
echo "$EXP_ROOT"
test -d "$EXP_ROOT"
echo "== python3 =="
python3 --version
echo "== conda env list =="
"$CONDA_BIN" env list
echo "== isce2 runtime =="
"$CONDA_BIN" run -n isce2 python -c "import sys; import isce; print(sys.executable); print(isce.__file__)"
echo "== Lutan1 sensor module =="
"$CONDA_BIN" run -n isce2 python -c "from isce.components.isceobj.Sensor import Lutan1; print(Lutan1.__file__)"
echo "== mintpy import check =="
"$CONDA_BIN" run -n isce2 python -c "import importlib.util; print('mintpy:present' if importlib.util.find_spec('mintpy') else 'mintpy:missing')"
echo "== candidate ISCE stack directories =="
find /home/administrator/miniconda3/envs/isce2 -maxdepth 6 \
\( -iname 'stripmapStack' -o -iname 'topsStack' -o -iname 'stack' \) 2>/dev/null || true
experiments/isce2_sbas_timeseries/scripts/create_mintpy_all_ifgram_mask.py
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env python3
"""Create a strict MintPy mask containing only pixels valid in all interferograms."""
from __future__ import annotations
import argparse
from pathlib import Path
import h5py
import numpy as np
def build_mask(ifgram_stack: Path, output_path: Path, block_rows: int) -> None:
with h5py.File(ifgram_stack, "r") as src:
unwrap = src["unwrapPhase"]
conn = src.get("connectComponent")
num_ifg, length, width = unwrap.shape
mask = np.ones((length, width), dtype=np.bool_)
print(f"Input stack: {ifgram_stack}")
print(f"Interferograms: {num_ifg}")
print(f"Shape: {length} x {width}")
print(f"Block rows: {block_rows}")
for row0 in range(0, length, block_rows):
row1 = min(row0 + block_rows, length)
block = unwrap[:, row0:row1, :]
block_mask = np.all(np.isfinite(block) & (block != 0.0), axis=0)
if conn is not None:
conn_block = conn[:, row0:row1, :]
block_mask &= np.all(conn_block != 0, axis=0)
mask[row0:row1, :] = block_mask
print(f"Processed rows {row0}:{row1}")
attrs = dict(src.attrs)
output_path.parent.mkdir(parents=True, exist_ok=True)
with h5py.File(output_path, "w") as dst:
dst.create_dataset("mask", data=mask, dtype=np.bool_)
for key, value in attrs.items():
dst.attrs[key] = value
dst.attrs["FILE_TYPE"] = "mask"
dst.attrs["DATASET_NAME"] = "mask"
dst.attrs["SOURCE_FILE"] = str(ifgram_stack)
dst.attrs["MASK_RULE"] = "all_ifgrams_finite_nonzero_and_conncomp_nonzero"
valid_pixels = int(mask.sum())
total_pixels = int(mask.size)
print(f"Output mask: {output_path}")
print(f"Valid pixels: {valid_pixels}/{total_pixels} ({valid_pixels / total_pixels * 100:.2f}%)")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Create a strict mask of pixels valid in all MintPy interferograms."
)
parser.add_argument("--ifgram-stack", required=True, help="Path to MintPy inputs/ifgramStack.h5")
parser.add_argument("--output", required=True, help="Output HDF5 path, e.g. maskAllValid.h5")
parser.add_argument(
"--block-rows",
type=int,
default=256,
help="Number of image rows processed per block.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
build_mask(
ifgram_stack=Path(args.ifgram_stack),
output_path=Path(args.output),
block_rows=args.block_rows,
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/create_synthetic_watermask.py
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from datetime import datetime
from pathlib import Path
import xml.etree.ElementTree as ET
import numpy as np
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Create a synthetic stripmapStack water mask in radar coordinates. "
"The default fill value 1 means all-land, which preserves downstream pixels."
)
)
parser.add_argument(
"--like-image",
required=True,
help="Existing ISCE image base path or .xml path used only for shape/metadata, for example shadowMask.rdr",
)
parser.add_argument(
"--output",
required=True,
help="Output water-mask base path, for example .../geom_reference/waterMask.rdr",
)
parser.add_argument(
"--fill-value",
type=int,
default=1,
choices=(0, 1),
help="Pixel value to write. 1 keeps all pixels, 0 masks all pixels.",
)
parser.add_argument(
"--force",
action="store_true",
help="Overwrite an existing output mask.",
)
parser.add_argument(
"--report",
default=None,
help="Optional JSON report path.",
)
return parser.parse_args()
def resolve_like_paths(value: str) -> tuple[Path, Path]:
candidate = Path(value)
if candidate.suffix == ".xml":
xml_path = candidate
image_path = Path(str(candidate)[:-4])
else:
image_path = candidate
xml_path = Path(str(candidate) + ".xml")
if not xml_path.exists():
raise FileNotFoundError(f"Template image XML not found: {xml_path}")
return image_path, xml_path
def maybe_unlink(path: Path) -> None:
if path.exists():
path.unlink()
def require_xml_value(root: ET.Element, property_name: str) -> str:
value_node = root.find(f"./property[@name='{property_name}']/value")
if value_node is None or value_node.text is None:
raise ValueError(f"Missing XML property '{property_name}'")
return value_node.text.strip()
def write_template_metadata(template_image: Path, template_xml: Path, output: Path) -> tuple[int, int]:
root = ET.parse(template_xml).getroot()
width = int(require_xml_value(root, "width"))
length = int(require_xml_value(root, "length"))
file_name_node = root.find("./property[@name='file_name']/value")
if file_name_node is None:
raise ValueError(f"Missing XML file_name entry: {template_xml}")
file_name_node.text = str(output)
xml_output = Path(str(output) + ".xml")
ET.indent(root, space=" ")
ET.ElementTree(root).write(xml_output, encoding="utf-8")
hdr_template = template_image.with_suffix(".hdr")
hdr_output = output.with_suffix(".hdr")
if hdr_template.exists():
hdr_text = hdr_template.read_text(encoding="utf-8", errors="ignore")
hdr_output.write_text(hdr_text.replace(str(template_image), str(output)), encoding="utf-8")
vrt_template = Path(str(template_image) + ".vrt")
vrt_output = Path(str(output) + ".vrt")
if vrt_template.exists():
vrt_text = vrt_template.read_text(encoding="utf-8", errors="ignore")
vrt_text = vrt_text.replace(template_image.name, output.name)
vrt_output.write_text(vrt_text, encoding="utf-8")
return width, length
def main() -> int:
args = parse_args()
template_image, template_xml = resolve_like_paths(args.like_image)
output = Path(args.output)
if output.exists() and not args.force:
raise FileExistsError(f"Output already exists, use --force to overwrite: {output}")
output.parent.mkdir(parents=True, exist_ok=True)
width, length = write_template_metadata(template_image=template_image, template_xml=template_xml, output=output)
mask = np.full((length, width), args.fill_value, dtype=np.uint8)
mask.tofile(output)
maybe_unlink(output.with_suffix(".rdr.aux.xml"))
report = {
"generated_at_utc": datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
"template_xml": str(template_xml),
"output": str(output),
"width": width,
"length": length,
"fill_value": args.fill_value,
"data_type": "BYTE",
"note": "Synthetic all-land water mask for local stripmapStack experiments without Earthdata SWBD access.",
}
report_path = Path(args.report) if args.report else output.parent / "synthetic_watermask_report.json"
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Template: {template_xml}")
print(f"Output: {output}")
print(f"Shape: {length} x {width}")
print(f"Value: {args.fill_value}")
print(f"Report: {report_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/export_conda_env_snapshot_ubuntu2404.sh
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "Usage: $0 <env-name> <output-dir-wsl>" >&2
exit 1
fi
ENV_NAME="$1"
OUTPUT_DIR="$2"
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
mkdir -p "$OUTPUT_DIR"
SAFE_NAME="${ENV_NAME//[^A-Za-z0-9._-]/_}"
YAML_PATH="$OUTPUT_DIR/${SAFE_NAME}.no_builds.yml"
EXPLICIT_PATH="$OUTPUT_DIR/${SAFE_NAME}.explicit.txt"
LIST_PATH="$OUTPUT_DIR/${SAFE_NAME}.conda_list.txt"
RUNTIME_PATH="$OUTPUT_DIR/${SAFE_NAME}.runtime_versions.txt"
echo "Exporting conda environment snapshot"
echo "Env: $ENV_NAME"
echo "Output dir: $OUTPUT_DIR"
"$CONDA_BIN" env export -n "$ENV_NAME" --no-builds > "$YAML_PATH"
"$CONDA_BIN" list -n "$ENV_NAME" --explicit > "$EXPLICIT_PATH"
"$CONDA_BIN" list -n "$ENV_NAME" > "$LIST_PATH"
"$CONDA_BIN" run -n "$ENV_NAME" python -c "
import importlib.util
import logging
import platform
import sys
logging.getLogger().setLevel(logging.WARNING)
def version_of(name):
try:
mod = __import__(name)
return getattr(mod, '__version__', '<missing>')
except Exception as exc:
return f'<import failed: {exc}>'
for line in [
f'python_executable={sys.executable}',
f'python_version={platform.python_version()}',
f'isce_present={importlib.util.find_spec(\"isce\") is not None}',
f'mintpy_present={importlib.util.find_spec(\"mintpy\") is not None}',
f'h5py_present={importlib.util.find_spec(\"h5py\") is not None}',
]:
print(line)
if importlib.util.find_spec('isce') is not None:
import isce
print(f'isce_file={isce.__file__}')
print(f'isce_version={getattr(isce, \"__version__\", \"<missing>\")}')
if importlib.util.find_spec('mintpy') is not None:
import mintpy
print(f'mintpy_file={mintpy.__file__}')
print(f'mintpy_version={getattr(mintpy, \"__version__\", \"<missing>\")}')
if importlib.util.find_spec('h5py') is not None:
import h5py
print(f'h5py_version={h5py.__version__}')
" > "$RUNTIME_PATH"
echo "Wrote: $YAML_PATH"
echo "Wrote: $EXPLICIT_PATH"
echo "Wrote: $LIST_PATH"
echo "Wrote: $RUNTIME_PATH"
experiments/isce2_sbas_timeseries/scripts/export_mintpy_publish_products_ubuntu2404.sh
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 2 || $# -gt 3 ]]; then
echo "Usage: $0 <mintpy-work-dir-wsl> <publish-dir-wsl> [group-key]" >&2
exit 1
fi
MINTPY_WORK_DIR="$1"
PUBLISH_DIR="$2"
GROUP_KEY="${3:-}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MINTPY_RUNNER="${MINTPY_RUNNER:-$SCRIPT_DIR/run_mintpy_with_isce_ubuntu2404.sh}"
PUBLISH_BUILDER="$SCRIPT_DIR/build_mintpy_publish_bundle.py"
GEO_LAT_STEP="${GEO_LAT_STEP:--0.000185185}"
GEO_LON_STEP="${GEO_LON_STEP:-0.000185185}"
GEO_INTERP_METHOD="${GEO_INTERP_METHOD:-nearest}"
ASSETS_DIR="$PUBLISH_DIR/assets"
PREVIEW_DIR="$PUBLISH_DIR/preview"
METADATA_DIR="$PUBLISH_DIR/metadata"
mkdir -p "$ASSETS_DIR" "$PREVIEW_DIR" "$METADATA_DIR"
LOOKUP_FILE="$MINTPY_WORK_DIR/inputs/geometryRadar.h5"
echo "MintPy publish export"
echo "Work dir: $MINTPY_WORK_DIR"
echo "Publish dir: $PUBLISH_DIR"
echo "Lookup file: $LOOKUP_FILE"
echo "Runner: $MINTPY_RUNNER"
echo "Geo step: $GEO_LAT_STEP, $GEO_LON_STEP"
echo "Interp: $GEO_INTERP_METHOD"
for src in velocity.h5 temporalCoherence.h5 maskTempCoh.h5 timeseries.h5; do
bash "$MINTPY_RUNNER" geocode.py \
"$MINTPY_WORK_DIR/$src" \
-l "$LOOKUP_FILE" \
--lalo "$GEO_LAT_STEP" "$GEO_LON_STEP" \
-i "$GEO_INTERP_METHOD" \
--outdir "$ASSETS_DIR" \
--update
done
bash "$MINTPY_RUNNER" save_gdal.py "$ASSETS_DIR/geo_velocity.h5" -d velocity -o "$ASSETS_DIR/velocity.tif"
bash "$MINTPY_RUNNER" save_gdal.py "$ASSETS_DIR/geo_temporalCoherence.h5" -d temporalCoherence -o "$ASSETS_DIR/temporalCoherence.tif"
bash "$MINTPY_RUNNER" save_gdal.py "$ASSETS_DIR/geo_maskTempCoh.h5" -d mask -o "$ASSETS_DIR/maskTempCoh.tif"
cp "$MINTPY_WORK_DIR/smallbaselineApp.cfg" "$METADATA_DIR/smallbaselineApp.cfg"
cp "$MINTPY_WORK_DIR/numTriNonzeroIntAmbiguity.png" "$PREVIEW_DIR/numTriNonzeroIntAmbiguity.png"
if [[ -n "$GROUP_KEY" ]]; then
bash "$MINTPY_RUNNER" python "$PUBLISH_BUILDER" \
--mintpy-work-dir "$MINTPY_WORK_DIR" \
--publish-dir "$PUBLISH_DIR" \
--group-key "$GROUP_KEY"
else
bash "$MINTPY_RUNNER" python "$PUBLISH_BUILDER" \
--mintpy-work-dir "$MINTPY_WORK_DIR" \
--publish-dir "$PUBLISH_DIR"
fi
experiments/isce2_sbas_timeseries/scripts/export_mintpy_publish_products_unified_env_ubuntu2404.sh
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GENERIC_EXPORTER="$SCRIPT_DIR/export_mintpy_publish_products_ubuntu2404.sh"
export MINTPY_RUNNER="${MINTPY_RUNNER:-$SCRIPT_DIR/run_mintpy_unified_env_ubuntu2404.sh}"
bash "$GENERIC_EXPORTER" "$@"
experiments/isce2_sbas_timeseries/scripts/export_phase4_env_snapshots_ubuntu2404.sh
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 1 ]]; then
echo "Usage: $0 <output-dir-wsl>" >&2
exit 1
fi
OUTPUT_DIR="$1"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPORTER="$SCRIPT_DIR/export_conda_env_snapshot_ubuntu2404.sh"
bash "$EXPORTER" isce2 "$OUTPUT_DIR"
bash "$EXPORTER" isce2_mintpy_v1 "$OUTPUT_DIR"
experiments/isce2_sbas_timeseries/scripts/install_isce2_stack_runtime_ubuntu2404.sh
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
CONDA_ENV="${CONDA_ENV:-isce2}"
PIP_INDEX_URL="${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple}"
PACKAGES=(
matplotlib
)
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
echo "ISCE2 stack runtime bootstrap"
echo "Conda: $CONDA_BIN"
echo "Env: $CONDA_ENV"
echo "Index: $PIP_INDEX_URL"
for pkg in "${PACKAGES[@]}"; do
echo "Installing $pkg into $CONDA_ENV"
"$CONDA_BIN" run -n "$CONDA_ENV" python -m pip install -i "$PIP_INDEX_URL" "$pkg"
done
echo "Runtime bootstrap complete"
experiments/isce2_sbas_timeseries/scripts/install_mintpy_into_cloned_isce2_env_ubuntu2404.sh
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env bash
set -euo pipefail
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
SOURCE_ENV="${SOURCE_ENV:-isce2}"
TARGET_ENV="${TARGET_ENV:-isce2_mintpy}"
PYTHON_VERSION="${PYTHON_VERSION:-3.11}"
BOOTSTRAP_MODE="${BOOTSTRAP_MODE:-clone}"
USE_TUNA_MIRROR="${USE_TUNA_MIRROR:-1}"
CHANNEL_CONDA_FORGE="${CHANNEL_CONDA_FORGE:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge}"
CHANNEL_MAIN="${CHANNEL_MAIN:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main}"
CHANNEL_R="${CHANNEL_R:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r}"
MINTPY_SPEC="${MINTPY_SPEC:-mintpy}"
CLONE_OFFLINE="${CLONE_OFFLINE:-1}"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
channel_args=()
if [[ "$USE_TUNA_MIRROR" == "1" ]]; then
channel_args=(
--override-channels
-c "$CHANNEL_CONDA_FORGE"
-c "$CHANNEL_MAIN"
-c "$CHANNEL_R"
)
fi
env_exists() {
"$CONDA_BIN" env list | awk '{print $1}' | grep -Fxq "$1"
}
echo "Unified ISCE2 + MintPy runtime bootstrap"
echo "Conda: $CONDA_BIN"
echo "Source env: $SOURCE_ENV"
echo "Target env: $TARGET_ENV"
echo "Python: $PYTHON_VERSION"
echo "Mode: $BOOTSTRAP_MODE"
echo "MintPy spec: $MINTPY_SPEC"
echo "Use mirror: $USE_TUNA_MIRROR"
echo "Clone offline:$CLONE_OFFLINE"
if ! env_exists "$SOURCE_ENV"; then
echo "Missing source environment: $SOURCE_ENV" >&2
exit 1
fi
if [[ "$BOOTSTRAP_MODE" != "clone" && "$BOOTSTRAP_MODE" != "recreate" ]]; then
echo "Unsupported BOOTSTRAP_MODE: $BOOTSTRAP_MODE" >&2
exit 1
fi
if env_exists "$TARGET_ENV"; then
echo "Environment $TARGET_ENV already exists. Reusing it."
else
if [[ "$BOOTSTRAP_MODE" == "clone" ]]; then
echo "Cloning $SOURCE_ENV into $TARGET_ENV"
clone_args=("${channel_args[@]}" -y -n "$TARGET_ENV" --clone "$SOURCE_ENV")
if [[ "$CLONE_OFFLINE" == "1" ]]; then
clone_args+=(--offline)
fi
"$CONDA_BIN" create "${clone_args[@]}"
else
tmp_export="$(mktemp)"
tmp_conda_specs="$(mktemp)"
tmp_pip_specs="$(mktemp)"
trap 'rm -f "$tmp_export" "$tmp_conda_specs" "$tmp_pip_specs"' EXIT
echo "Exporting $SOURCE_ENV into a recreate spec"
"$CONDA_BIN" env export -n "$SOURCE_ENV" --no-builds > "$tmp_export"
awk \
'
/^dependencies:/ {
in_dependencies = 1
next
}
/^prefix:/ {
exit
}
in_dependencies == 1 && /^ - pip:$/ {
exit
}
in_dependencies == 1 && /^ - / {
print substr($0, 5)
}
' "$tmp_export" > "$tmp_conda_specs"
awk \
'
/^ - pip:$/ {
in_pip = 1
next
}
/^prefix:/ {
exit
}
in_pip == 1 && /^ - / {
print substr($0, 7)
}
' "$tmp_export" > "$tmp_pip_specs"
mapfile -t conda_specs < "$tmp_conda_specs"
if [[ ${#conda_specs[@]} -eq 0 ]]; then
echo "Failed to extract conda dependency specs from $SOURCE_ENV export" >&2
exit 1
fi
echo "Recreating $TARGET_ENV from exported dependency list"
"$CONDA_BIN" create -y -n "$TARGET_ENV" "${channel_args[@]}" "${conda_specs[@]}"
if [[ -s "$tmp_pip_specs" ]]; then
mapfile -t pip_specs < "$tmp_pip_specs"
echo "Reinstalling exported pip packages into $TARGET_ENV"
"$CONDA_BIN" run -n "$TARGET_ENV" python -m pip install "${pip_specs[@]}"
fi
fi
fi
echo "Installing MintPy into $TARGET_ENV"
"$CONDA_BIN" install -y -n "$TARGET_ENV" "${channel_args[@]}" "$MINTPY_SPEC"
echo "Verifying unified runtime imports"
"$CONDA_BIN" run -n "$TARGET_ENV" python -c "import sys; import isce; import mintpy; import h5py; print(sys.executable); print(isce.__file__); print(mintpy.__file__); print('h5py=' + h5py.__version__)"
echo "Unified runtime bootstrap complete"
experiments/isce2_sbas_timeseries/scripts/install_mintpy_runtime_ubuntu2404.sh
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
TARGET_ENV="${TARGET_ENV:-mintpy}"
PYTHON_VERSION="${PYTHON_VERSION:-3.11}"
USE_TUNA_MIRROR="${USE_TUNA_MIRROR:-1}"
CHANNEL_CONDA_FORGE="${CHANNEL_CONDA_FORGE:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge}"
CHANNEL_MAIN="${CHANNEL_MAIN:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main}"
CHANNEL_R="${CHANNEL_R:-https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r}"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
channel_args=()
if [[ "$USE_TUNA_MIRROR" == "1" ]]; then
channel_args=(
--override-channels
-c "$CHANNEL_CONDA_FORGE"
-c "$CHANNEL_MAIN"
-c "$CHANNEL_R"
)
fi
env_exists() {
"$CONDA_BIN" env list | awk '{print $1}' | grep -Fxq "$TARGET_ENV"
}
echo "MintPy runtime bootstrap"
echo "Conda: $CONDA_BIN"
echo "Target env: $TARGET_ENV"
echo "Python: $PYTHON_VERSION"
echo "Use mirror: $USE_TUNA_MIRROR"
if env_exists; then
echo "Environment $TARGET_ENV already exists. Installing or updating MintPy."
"$CONDA_BIN" install -y -n "$TARGET_ENV" "${channel_args[@]}" mintpy
else
echo "Creating environment $TARGET_ENV with MintPy."
"$CONDA_BIN" create -y -n "$TARGET_ENV" "${channel_args[@]}" "python=$PYTHON_VERSION" mintpy
fi
echo "Verifying MintPy import"
"$CONDA_BIN" run -n "$TARGET_ENV" python -c "import mintpy; print(mintpy.__file__)"
echo "MintPy runtime bootstrap complete"
experiments/isce2_sbas_timeseries/scripts/materialize_lt1_stack_scenes.py
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import os
import shelve
import shutil
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
def choose_path(scene: Dict[str, Any], primary_key: str, fallback_key: str) -> str:
primary = scene.get(primary_key)
fallback = scene.get(fallback_key)
for candidate in (primary, fallback):
if candidate and Path(candidate).exists():
return str(Path(candidate))
raise FileNotFoundError(
f"Neither {primary_key} nor {fallback_key} exists for scene {scene.get('date') or scene.get('imaging_date')}"
)
def load_manifest(path: Path) -> Dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def remove_existing_shelve(stem: Path) -> None:
for suffix in ("", ".db", ".dat", ".dir", ".bak"):
candidate = Path(str(stem) + suffix)
if candidate.exists():
if candidate.is_dir():
shutil.rmtree(candidate)
else:
candidate.unlink()
def shelve_stem_exists(stem: Path) -> bool:
for suffix in ("", ".db", ".dat", ".dir", ".bak"):
if Path(str(stem) + suffix).exists():
return True
return False
@dataclass
class SceneResult:
date: str
output_dir: str
slc_path: str
data_shelve: str
status: str
bytes_written: Optional[int]
started_at_utc: str
ended_at_utc: str
def materialize_one_scene(
scene: Dict[str, Any],
force: bool,
) -> SceneResult:
import isce
from isceobj.Sensor import createSensor
date = str(scene["date"])
output_dir = Path(scene["target_dir_wsl"])
output_dir.mkdir(parents=True, exist_ok=True)
slc_path = Path(scene["expected_slc_wsl"])
slc_xml_path = Path(scene["expected_slc_xml_wsl"])
data_shelve = Path(scene["expected_data_shelve_wsl"])
tiff_path = choose_path(scene, "source_tiff_wsl", "source_tiff_windows")
orbit_xml = choose_path(scene, "orbit_xml_wsl", "orbit_xml_windows")
if force:
for path in (slc_path, slc_xml_path, Path(str(slc_path) + ".vrt")):
if path.exists():
path.unlink()
remove_existing_shelve(data_shelve)
if slc_path.exists() and slc_xml_path.exists() and shelve_stem_exists(data_shelve):
now = datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
return SceneResult(
date=date,
output_dir=str(output_dir),
slc_path=str(slc_path),
data_shelve=str(data_shelve),
status="skipped_existing",
bytes_written=slc_path.stat().st_size,
started_at_utc=now,
ended_at_utc=now,
)
started_at = datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
sensor = createSensor("LUTAN1")
sensor.configure()
sensor.tiff = tiff_path
sensor.orbitFile = orbit_xml
sensor.output = str(slc_path)
sensor.extractImage()
sensor.extractDoppler()
sensor.frame.getImage().renderHdr()
remove_existing_shelve(data_shelve)
with shelve.open(str(data_shelve)) as db:
db["frame"] = sensor.frame
ended_at = datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
report = {
"date": date,
"source_tiff": tiff_path,
"orbit_xml": orbit_xml,
"output_slc": str(slc_path),
"output_slc_xml": str(slc_xml_path),
"data_shelve": str(data_shelve),
"frame_lines": sensor.frame.getNumberOfLines(),
"frame_samples": sensor.frame.getNumberOfSamples(),
"started_at_utc": started_at,
"ended_at_utc": ended_at,
}
(output_dir / "materialization_report.json").write_text(
json.dumps(report, indent=2, ensure_ascii=False),
encoding="utf-8",
)
return SceneResult(
date=date,
output_dir=str(output_dir),
slc_path=str(slc_path),
data_shelve=str(data_shelve),
status="materialized",
bytes_written=slc_path.stat().st_size if slc_path.exists() else None,
started_at_utc=started_at,
ended_at_utc=ended_at,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Materialize LT-1 stack acquisitions into stripmapStack-ready SLC/date directories."
)
parser.add_argument(
"--stack-manifest",
required=True,
help="Path to stack_input_manifest.json generated by build_lt1_stack_prep.py",
)
parser.add_argument(
"--dates",
nargs="+",
default=None,
help="Optional subset of acquisition dates to materialize, for example 20250510 20250705",
)
parser.add_argument(
"--force",
action="store_true",
help="Overwrite existing .slc/.xml/data outputs for the selected dates.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
manifest_path = Path(args.stack_manifest)
if not manifest_path.exists():
raise FileNotFoundError(f"Stack manifest not found: {manifest_path}")
manifest = load_manifest(manifest_path)
scenes = list(manifest.get("scenes", []))
if not scenes:
raise ValueError(f"No scenes found in stack manifest: {manifest_path}")
selected_dates = set(args.dates or [])
if selected_dates:
scenes = [scene for scene in scenes if str(scene["date"]) in selected_dates]
if not scenes:
raise ValueError(f"No matching dates found in manifest for selection: {sorted(selected_dates)}")
results: List[SceneResult] = []
for scene in scenes:
print(f"Materializing {scene['date']} -> {scene['target_dir_wsl']}")
result = materialize_one_scene(scene, force=args.force)
results.append(result)
print(f" status={result.status} slc={result.slc_path}")
report = {
"generated_at_utc": datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
"stack_manifest": str(manifest_path),
"results": [result.__dict__ for result in results],
}
report_path = manifest_path.parent / "materialization_summary.json"
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Summary: {report_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/phase0_todo.md
+59
View File
@@ -0,0 +1,59 @@
# Phase 0 Practical TODO
## Immediate
- [x] Run `scripts/check_env_ubuntu2404.sh` inside `Ubuntu-24.04`.
- [x] Use `scripts/scan_lt1_stack_candidates.py` to keep one baseline sample stack manifest current.
- [x] Treat `E123.3_N46.1` as the first tile-level smoke-test sample unless a better sample appears.
- [x] Confirm ISCE2 stack-processing scripts are present.
- [x] Confirm the official helper scripts do not advertise LT-1/LUTAN1 stack prep.
- [x] Record required orbit, DEM, and metadata adaptations.
- [x] Run `scripts/build_lt1_stack_prep.py` to keep the dry-run stack workspace current.
## Before first end-to-end run
- [x] Implement an LT-1 scene materializer that creates `YYYYMMDD.slc`, `YYYYMMDD.slc.xml`, and `data`.
- [x] Materialize the remaining acquisitions for `E123.3_N46.1` under `scratch/.../SLC/`.
- [x] Smoke-test the materializer on the reference date `20250510`.
- [x] Run the generated `run_stripmap_stack_dryrun.sh` preflight and then `stackStripMap.py --nofocus`.
- [x] Inspect the produced `baseline/`, `configs/`, and `run_files/` outputs.
- [x] Prepare a stack-local DEM to avoid global-DEM bbox behavior during `createWaterMask`.
- [x] Add a reproducible synthetic `waterMask` fallback for `run_01_reference` when Earthdata credentials are unavailable.
Working rule: DEM is already local and sufficient; do not download `SWBD` during this experiment stage.
- [x] Extract shared LT-1 input preparation helper for DEM/orbit resolution.
Compatibility rule: original D-InSAR entry logic remains in place; only the duplicated input-prep internals were consolidated.
- [x] Decide MintPy installation strategy after stack generation is stable.
Decision: default to a dedicated WSL conda env named `mintpy` so the working `isce2` processing env stays unchanged on the development machine.
- [x] Freeze the first smoke-test command chain.
Frozen chain: `run_01_reference -> run_02_focus_split -> run_03_geo2rdr_coarseResamp -> run_04_refineSecondaryTiming -> run_05_invertMisreg -> run_06_fineResamp -> run_07_grid_baseline`
- [x] Execute `run_01_reference` through the WSL wrapper and verify the fallback-recovered geometry outputs.
- [x] Execute `run_02` to `run_07` and record LT-1-specific failures if they appear.
Result: all stages exited `0` in `Ubuntu-24.04`. `run_04_refineSecondaryTiming` logs still contain `Bad match at level 1` and `correlation error`, but pair-level `misreg`, date-level `misreg`, merged SLC, and merged baseline products were all generated successfully.
## Next Focus
- [x] Run `scripts/install_mintpy_runtime_ubuntu2404.sh` in `Ubuntu-24.04` and verify the new env.
Result: dedicated WSL env `mintpy` was created successfully and `smallbaselineApp.py` / `prep_isce.py` are available.
- [x] Validate MintPy ingestion against the current `stack_work/merged/` outputs.
Result: `build_lt1_stack_prep.py --workflow interferogram` plus `run_08_igram` produced `Igrams/*/filt_*_snaphu.unw`, and `prep_isce.py` completed successfully after bridging the working `isce2` Python package into the `mintpy` env.
- [x] Draft the first `smallbaselineApp.cfg` for the LT-1 sample stack.
Result: `configs/sample_smallbaseline_lt1_e123p3_n46p1.cfg` now records the first runnable LT-1 stripmapStack -> MintPy SBAS contract.
- [x] Execute the first MintPy workflow steps after `prep_isce.py`.
Result: the repo-local smoke-test chain now reaches radar-coordinate `timeseries.h5` and `velocity.h5` in `stack_work/mintpy_sbas_v5/`.
Current helper chain:
- `scripts/run_mintpy_with_isce_ubuntu2404.sh`
- `scripts/create_mintpy_all_ifgram_mask.py`
- `scripts/run_smallbaselineApp_patched.py`
- `scripts/run_mintpy_sbas_smoketest_ubuntu2404.sh`
- [x] Draft the first production-side SBAS artifact manifest and publish contract.
Result:
- `configs/sample_psinsar_manifest_lt1_e123p3_n46p1.json`
- `docs/ISCE2_SBAS_TIMESERIES_DESIGN.md`
## New Follow-up
- [ ] Decide whether production should keep the repo-local patched MintPy launcher or pin an upstream-fixed MintPy version.
- [x] Add the geocode/export stage needed for publishable SBAS rasters and previews.
Result: experiment-layer publish export now succeeds into `publish/mintpy_sbas_v5/` with geocoded HDF5, GeoTIFF, preview PNG, and `manifest.json`.
- [ ] Wire the validated SBAS runtime chain into backend workflow submission and artifact publishing.
- [ ] Run a separate unified-environment experiment by cloning the current WSL `isce2` env and installing MintPy directly inside it.
experiments/isce2_sbas_timeseries/scripts/prepare_lt1_stack_dem.py
+152
View File
@@ -0,0 +1,152 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any, Dict, List
def load_manifest(path: Path) -> Dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def resolve_dem_source(arg_value: str | None, manifest: Dict[str, Any]) -> Path:
candidates: List[str] = []
if arg_value:
candidates.append(arg_value)
resolved = manifest.get("resolved_dependencies", {})
for key in ("dem_path_wsl", "dem_path_windows"):
value = resolved.get(key)
if value:
candidates.append(value)
for candidate in candidates:
path = Path(candidate)
if path.exists() and Path(str(path) + ".xml").exists():
return path
raise FileNotFoundError("Unable to resolve source DEM from arguments or stack manifest")
def compute_bbox(manifest: Dict[str, Any], margin_deg: float) -> List[float]:
lons: List[float] = []
lats: List[float] = []
for scene in manifest["scenes"]:
lon = scene.get("scene_center_lon")
lat = scene.get("scene_center_lat")
if lon is None or lat is None:
source_scene_json = scene.get("source_scene_json_wsl") or scene.get("source_scene_json_windows")
if source_scene_json and Path(source_scene_json).exists():
source_payload = json.loads(Path(source_scene_json).read_text(encoding="utf-8"))
lon = source_payload.get("scene_center_lon")
lat = source_payload.get("scene_center_lat")
if lon is not None and lat is not None:
lons.append(float(lon))
lats.append(float(lat))
if not lons or not lats:
raise ValueError("Stack manifest does not include usable scene center coordinates")
west = min(lons) - margin_deg
east = max(lons) + margin_deg
south = min(lats) - margin_deg
north = max(lats) + margin_deg
return [south, north, west, east]
def prepare_dem(source_dem: Path, output_dem: Path, bbox: List[float]) -> None:
from osgeo import gdal
from isce.applications.gdal2isce_xml import gdal2isce_xml
south, north, west, east = bbox
src_open_path = Path(str(source_dem) + ".vrt")
if not src_open_path.exists():
src_open_path = source_dem
output_dem.parent.mkdir(parents=True, exist_ok=True)
output_vrt = Path(str(output_dem) + ".vrt")
output_xml = Path(str(output_dem) + ".xml")
output_hdr = Path(str(output_dem) + ".hdr")
fallback_hdr = output_dem.with_suffix(".hdr")
src_ds = gdal.Open(str(src_open_path), gdal.GA_ReadOnly)
if src_ds is None:
raise RuntimeError(f"Unable to open DEM source: {src_open_path}")
translate_options = gdal.TranslateOptions(
format="ENVI",
projWin=[west, north, east, south],
)
out_ds = gdal.Translate(str(output_dem), src_ds, options=translate_options)
if out_ds is None:
raise RuntimeError("gdal.Translate failed while clipping the DEM")
out_ds = None
src_ds = None
vrt_ds = gdal.Open(str(output_dem), gdal.GA_ReadOnly)
if vrt_ds is None:
raise RuntimeError(f"Unable to reopen clipped DEM: {output_dem}")
gdal.Translate(str(output_vrt), vrt_ds, options=gdal.TranslateOptions(format="VRT"))
vrt_ds = None
gdal2isce_xml(str(output_vrt))
if not output_xml.exists():
raise RuntimeError(f"Expected ISCE XML was not created: {output_xml}")
if not output_hdr.exists() and not fallback_hdr.exists():
raise RuntimeError(f"Expected ENVI header was not created: {output_hdr} or {fallback_hdr}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Clip a local DEM window for one LT-1 stack workspace."
)
parser.add_argument(
"--stack-manifest",
required=True,
help="Path to stack_input_manifest.json generated by build_lt1_stack_prep.py",
)
parser.add_argument(
"--source-dem",
default=None,
help="Override source DEM base path.",
)
parser.add_argument(
"--margin-deg",
type=float,
default=1.0,
help="Margin around stack scene-center extents in degrees.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
manifest_path = Path(args.stack_manifest)
if not manifest_path.exists():
raise FileNotFoundError(f"Stack manifest not found: {manifest_path}")
manifest = load_manifest(manifest_path)
source_dem = resolve_dem_source(args.source_dem, manifest)
bbox = compute_bbox(manifest, margin_deg=args.margin_deg)
workspace = manifest["workspace"]
dem_dir = Path(workspace["inputs_dir_wsl"]) / "dem"
output_dem = dem_dir / "stack_dem_window.wgs84"
prepare_dem(source_dem=source_dem, output_dem=output_dem, bbox=bbox)
report = {
"stack_manifest": str(manifest_path),
"source_dem": str(source_dem),
"output_dem": str(output_dem),
"bbox_south_north_west_east": bbox,
}
report_path = dem_dir / "stack_dem_window_report.json"
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Source DEM: {source_dem}")
print(f"Output DEM: {output_dem}")
print(f"BBox: {bbox}")
print(f"Report: {report_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
experiments/isce2_sbas_timeseries/scripts/run_generated_stack_runfile_ubuntu2404.sh
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 2 ]]; then
echo "Usage: $0 <scratch_root_wsl> <run_file_name>" >&2
echo "Example: $0 /mnt/z/Code/Insar_management_system_v2/experiments/isce2_sbas_timeseries/scratch/lt1a_strip1_hh_descending_e123p3_n46p1 run_01_reference" >&2
exit 1
fi
SCRATCH_ROOT="$1"
RUN_FILE_NAME="$2"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
CONDA_ENV="${CONDA_ENV:-isce2}"
ISCE2_SHARE="${ISCE2_SHARE:-/home/administrator/miniconda3/envs/isce2/share/isce2}"
STRIPMAP_STACK_DIR="${STRIPMAP_STACK_DIR:-$ISCE2_SHARE/stripmapStack}"
SYNTHETIC_WATERMASK_SCRIPT="${SYNTHETIC_WATERMASK_SCRIPT:-$SCRIPT_DIR/create_synthetic_watermask.py}"
ALLOW_SYNTHETIC_WATERMASK="${ALLOW_SYNTHETIC_WATERMASK:-1}"
STACK_WORK="$SCRATCH_ROOT/stack_work"
RUN_FILE="$STACK_WORK/run_files/$RUN_FILE_NAME"
LOG_DIR="$STACK_WORK/logs"
LOG_FILE="$LOG_DIR/$RUN_FILE_NAME.log"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
if [[ ! -f "$RUN_FILE" ]]; then
echo "Run file not found: $RUN_FILE" >&2
exit 1
fi
mkdir -p "$LOG_DIR"
export PYTHONPATH="$STRIPMAP_STACK_DIR:$ISCE2_SHARE${PYTHONPATH:+:$PYTHONPATH}"
export PATH="$STRIPMAP_STACK_DIR:$PATH"
recover_reference_watermask() {
local like_image="$STACK_WORK/geom_reference/shadowMask.rdr"
local output_mask="$STACK_WORK/geom_reference/waterMask.rdr"
local report_path="$LOG_DIR/$RUN_FILE_NAME.synthetic_watermask.json"
local watermask_failure_pattern='Please create a \.netrc file|Running: createWaterMask|DataRetriever - ERROR|There was a problem in retrieving the file|SRTMSWBD\.003|SWBD'
if [[ "$RUN_FILE_NAME" != "run_01_reference" ]]; then
return 1
fi
if [[ "$ALLOW_SYNTHETIC_WATERMASK" != "1" ]]; then
return 1
fi
if [[ ! -f "$LOG_FILE" ]]; then
return 1
fi
# Recover only the known offline water-mask failure modes observed in this
# experiment: missing Earthdata credentials or SWBD retrieval failure.
if ! grep -Eq "$watermask_failure_pattern" "$LOG_FILE"; then
return 1
fi
if [[ ! -f "$like_image" || ! -f "$like_image.xml" ]]; then
echo "Synthetic water-mask fallback could not find template image: $like_image" >&2
return 1
fi
if [[ ! -f "$SYNTHETIC_WATERMASK_SCRIPT" ]]; then
echo "Synthetic water-mask helper script not found: $SYNTHETIC_WATERMASK_SCRIPT" >&2
return 1
fi
echo "Earthdata credentials are unavailable. Creating a synthetic all-land water mask."
"$CONDA_BIN" run -n "$CONDA_ENV" python "$SYNTHETIC_WATERMASK_SCRIPT" \
--like-image "$like_image" \
--output "$output_mask" \
--fill-value 1 \
--force \
--report "$report_path"
}
echo "Executing stripmap stack run file"
echo "Scratch root: $SCRATCH_ROOT"
echo "Run file: $RUN_FILE"
echo "Log file: $LOG_FILE"
echo "Conda env: $CONDA_ENV"
echo "PYTHONPATH: $PYTHONPATH"
echo "PATH prefix: $STRIPMAP_STACK_DIR"
set -o pipefail
"$CONDA_BIN" run -n "$CONDA_ENV" bash "$RUN_FILE" 2>&1 | tee "$LOG_FILE"
RUN_STATUS=${PIPESTATUS[0]}
if [[ "$RUN_STATUS" -eq 0 ]]; then
exit 0
fi
if recover_reference_watermask; then
echo "Recovered $RUN_FILE_NAME with a synthetic all-land water mask."
exit 0
fi
exit "$RUN_STATUS"
experiments/isce2_sbas_timeseries/scripts/run_mintpy_sbas_smoketest_ubuntu2404.sh
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "Usage: $0 <smallbaseline-config-wsl> <mintpy-work-dir-wsl>" >&2
exit 1
fi
CFG_PATH="$1"
WORK_DIR="$2"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BRIDGE_RUNNER="$SCRIPT_DIR/run_mintpy_with_isce_ubuntu2404.sh"
PATCHED_APP="$SCRIPT_DIR/run_smallbaselineApp_patched.py"
STRICT_MASK_BUILDER="$SCRIPT_DIR/create_mintpy_all_ifgram_mask.py"
echo "MintPy SBAS smoketest"
echo "Config: $CFG_PATH"
echo "Work dir: $WORK_DIR"
bash "$BRIDGE_RUNNER" python "$PATCHED_APP" "$CFG_PATH" --dir "$WORK_DIR" --dostep load_data
bash "$BRIDGE_RUNNER" python "$STRICT_MASK_BUILDER" \
--ifgram-stack "$WORK_DIR/inputs/ifgramStack.h5" \
--output "$WORK_DIR/maskAllValid.h5"
bash "$BRIDGE_RUNNER" python "$PATCHED_APP" "$CFG_PATH" --dir "$WORK_DIR" --start modify_network --end velocity
experiments/isce2_sbas_timeseries/scripts/run_mintpy_sbas_unified_env_smoketest_ubuntu2404.sh
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "Usage: $0 <smallbaseline-config-wsl> <mintpy-work-dir-wsl>" >&2
exit 1
fi
CFG_PATH="$1"
WORK_DIR="$2"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
UNIFIED_RUNNER="$SCRIPT_DIR/run_mintpy_unified_env_ubuntu2404.sh"
PATCHED_APP="$SCRIPT_DIR/run_smallbaselineApp_patched.py"
STRICT_MASK_BUILDER="$SCRIPT_DIR/create_mintpy_all_ifgram_mask.py"
echo "MintPy SBAS unified-env smoketest"
echo "Config: $CFG_PATH"
echo "Work dir: $WORK_DIR"
bash "$UNIFIED_RUNNER" python "$PATCHED_APP" "$CFG_PATH" --dir "$WORK_DIR" --dostep load_data
bash "$UNIFIED_RUNNER" python "$STRICT_MASK_BUILDER" \
--ifgram-stack "$WORK_DIR/inputs/ifgramStack.h5" \
--output "$WORK_DIR/maskAllValid.h5"
bash "$UNIFIED_RUNNER" python "$PATCHED_APP" "$CFG_PATH" --dir "$WORK_DIR" --start modify_network --end velocity
experiments/isce2_sbas_timeseries/scripts/run_mintpy_unified_env_ubuntu2404.sh
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <mintpy-command> [args...]" >&2
echo "Example: $0 prep_isce.py -h" >&2
exit 1
fi
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
MINTPY_ENV="${MINTPY_ENV:-isce2_mintpy}"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
echo "MintPy command in unified env"
echo "Conda: $CONDA_BIN"
echo "Target env: $MINTPY_ENV"
echo "Command: $*"
"$CONDA_BIN" run -n "$MINTPY_ENV" "$@"
experiments/isce2_sbas_timeseries/scripts/run_mintpy_with_isce_ubuntu2404.sh
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 1 ]]; then
echo "Usage: $0 <mintpy-command> [args...]" >&2
echo "Example: $0 prep_isce.py -h" >&2
exit 1
fi
CONDA_BIN="${CONDA_BIN:-/home/administrator/miniconda3/bin/conda}"
MINTPY_ENV="${MINTPY_ENV:-mintpy}"
ISCE_SITE_PACKAGES="${ISCE_SITE_PACKAGES:-/home/administrator/miniconda3/envs/isce2/lib/python3.11/site-packages}"
ISCE_PACKAGE_DIR="${ISCE_PACKAGE_DIR:-$ISCE_SITE_PACKAGES/isce}"
ISCE_BRIDGE_DIR="${ISCE_BRIDGE_DIR:-$HOME/.cache/mintpy_isce_bridge}"
if [[ ! -x "$CONDA_BIN" ]]; then
echo "Missing conda binary: $CONDA_BIN" >&2
exit 1
fi
if [[ ! -d "$ISCE_SITE_PACKAGES" ]]; then
echo "Missing ISCE site-packages directory: $ISCE_SITE_PACKAGES" >&2
exit 1
fi
if [[ ! -d "$ISCE_PACKAGE_DIR" ]]; then
echo "Missing ISCE package directory: $ISCE_PACKAGE_DIR" >&2
exit 1
fi
mkdir -p "$ISCE_BRIDGE_DIR"
ln -sfn "$ISCE_PACKAGE_DIR" "$ISCE_BRIDGE_DIR/isce"
# Bridge only the top-level ISCE package into the MintPy env.
# The package itself extends sys.path to its internal components on import,
# which avoids shadowing MintPy's own numpy/h5py stack with the isce2 env.
export PYTHONPATH="$ISCE_BRIDGE_DIR${PYTHONPATH:+:$PYTHONPATH}"
echo "MintPy command bridge"
echo "Conda: $CONDA_BIN"
echo "MintPy env: $MINTPY_ENV"
echo "ISCE bridge: $ISCE_BRIDGE_DIR -> $ISCE_PACKAGE_DIR"
echo "Command: $*"
"$CONDA_BIN" run -n "$MINTPY_ENV" "$@"
experiments/isce2_sbas_timeseries/scripts/run_smallbaselineApp_patched.py
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Run MintPy smallbaselineApp with a local workaround for a single-pixel inversion bug."""
from __future__ import annotations
import sys
import numpy as np
import mintpy.ifgram_inversion as ifgram_inversion
from mintpy.cli.smallbaselineApp import main as mintpy_smallbaseline_main
_ORIGINAL_ESTIMATE_TIMESERIES = ifgram_inversion.estimate_timeseries
def _patched_estimate_timeseries(*args, **kwargs):
ts, inv_quality, num_inv_obs = _ORIGINAL_ESTIMATE_TIMESERIES(*args, **kwargs)
# MintPy 1.6.2 may return a shape-(1,) inversion quality array for the
# single-pixel partial-network branch, while the caller expects a scalar.
if isinstance(inv_quality, np.ndarray) and inv_quality.size == 1:
inv_quality = np.asarray(inv_quality).reshape(-1)[0].item()
if isinstance(num_inv_obs, np.ndarray) and num_inv_obs.size == 1:
num_inv_obs = int(np.asarray(num_inv_obs).reshape(-1)[0])
return ts, inv_quality, num_inv_obs
def main(argv: list[str] | None = None) -> int:
ifgram_inversion.estimate_timeseries = _patched_estimate_timeseries
print("Applied local MintPy estimate_timeseries single-pixel fix.")
return mintpy_smallbaseline_main(argv)
if __name__ == "__main__":
raise SystemExit(main(sys.argv[1:]))
experiments/isce2_sbas_timeseries/scripts/scan_lt1_stack_candidates.py
+373
View File
@@ -0,0 +1,373 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import importlib.util
import json
import os
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
def _repo_root() -> Path:
return Path(__file__).resolve().parents[3]
def _load_utils_module():
utils_path = _repo_root() / "backend" / "app" / "utils.py"
spec = importlib.util.spec_from_file_location("repo_utils", utils_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load repo utils module: {utils_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
UTILS = _load_utils_module()
LT1_NAME_RE = re.compile(
r"^(?P<satellite>LT1[AB])_"
r"(?P<satellite_mode>[^_]+)_"
r"(?P<receiving_station>[^_]+)_"
r"(?P<imaging_mode>[^_]+)_"
r"(?P<abs_orbit>\d+)_"
r"(?P<lon>E\d+\.\d+)_"
r"(?P<lat>N\d+\.\d+)_"
r"(?P<date>\d{8})_"
r"(?P<product_type>[^_]+)_"
r"(?P<polarization>[^_]+)_"
r"(?P<product_level>[^_]+)_"
r"(?P<product_unique_id>\d+)$"
)
@dataclass
class SceneRecord:
folder_name: str
folder_path: str
folder_path_wsl: str
tiff_path: str
tiff_path_wsl: str
meta_path: str
meta_path_wsl: str
file_size_bytes: int
satellite: str
imaging_date: str
imaging_mode: Optional[str]
polarization: Optional[str]
orbit_direction: Optional[str]
satellite_mode: Optional[str]
receiving_station: Optional[str]
orbit_circle: Optional[str]
scene_center_lon: Optional[float]
scene_center_lat: Optional[float]
acquisition_time_utc: Optional[str]
product_type: Optional[str]
product_level: Optional[str]
product_unique_id: Optional[str]
tile_key: str
group_key: str
orbit_txt_expected_name: str
def windows_to_wsl(path: str | Path) -> str:
text = str(path)
match = re.match(r"^([A-Za-z]):[\\/](.*)$", os.path.normpath(text))
if not match:
return text.replace("\\", "/")
drive = match.group(1).lower()
normalized_tail = match.group(2).replace("\\", "/").lstrip("/")
return f"/mnt/{drive}/{normalized_tail}"
def choose_tiff(folder: Path) -> Optional[Path]:
candidates = sorted(folder.glob("*.tiff"))
if not candidates:
return None
slc_candidates = [path for path in candidates if "_SLC_" in path.name]
if len(slc_candidates) == 1:
return slc_candidates[0]
if len(candidates) == 1:
return candidates[0]
return candidates[0]
def merge_metadata(name_meta: Dict[str, Any], xml_meta: Dict[str, Any]) -> Dict[str, Any]:
merged = dict(name_meta or {})
prefer_name_keys = {"product_unique_id"}
for key, value in (xml_meta or {}).items():
if value in (None, ""):
continue
if key in prefer_name_keys and merged.get(key):
continue
merged[key] = value
return merged
def parse_scene(folder: Path) -> Optional[SceneRecord]:
match = LT1_NAME_RE.match(folder.name)
if not match:
return None
name_meta = UTILS.get_parser(folder.name, UTILS.RADAR_PARSERS)
if not name_meta:
return None
xml_file_path = UTILS.find_xml_file(str(folder))
if not xml_file_path:
return None
coverage_polygon, xml_meta = UTILS.parse_xml_metadata(xml_file_path)
if not coverage_polygon:
return None
tiff_path = choose_tiff(folder)
if tiff_path is None:
return None
merged = merge_metadata(name_meta, xml_meta or {})
tile_key = f"{match.group('lon')}_{match.group('lat')}"
orbit_direction = str(merged.get("orbit_direction") or "").upper() or None
group_key = "|".join(
[
str(merged.get("satellite") or ""),
str(merged.get("imaging_mode") or ""),
str(merged.get("polarization") or ""),
str(orbit_direction or ""),
tile_key,
]
)
satellite = str(merged.get("satellite") or "")
imaging_date = str(merged.get("imaging_date") or "")
return SceneRecord(
folder_name=folder.name,
folder_path=str(folder),
folder_path_wsl=windows_to_wsl(folder),
tiff_path=str(tiff_path),
tiff_path_wsl=windows_to_wsl(tiff_path),
meta_path=str(xml_file_path),
meta_path_wsl=windows_to_wsl(xml_file_path),
file_size_bytes=tiff_path.stat().st_size,
satellite=satellite,
imaging_date=imaging_date,
imaging_mode=merged.get("imaging_mode"),
polarization=merged.get("polarization"),
orbit_direction=orbit_direction,
satellite_mode=merged.get("satellite_mode"),
receiving_station=merged.get("receiving_station"),
orbit_circle=merged.get("orbit_circle"),
scene_center_lon=merged.get("scene_center_lon"),
scene_center_lat=merged.get("scene_center_lat"),
acquisition_time_utc=merged.get("acquisition_time_utc"),
product_type=merged.get("product_type"),
product_level=merged.get("product_level"),
product_unique_id=merged.get("product_unique_id"),
tile_key=tile_key,
group_key=group_key,
orbit_txt_expected_name=f"{satellite}_GpsData_GAS_C_{imaging_date}.txt",
)
def scan_scenes(root_dir: Path) -> List[SceneRecord]:
scenes: List[SceneRecord] = []
for entry in sorted(root_dir.iterdir()):
if not entry.is_dir():
continue
scene = parse_scene(entry)
if scene:
scenes.append(scene)
return scenes
def build_group_summary(scenes: List[SceneRecord]) -> List[Dict[str, Any]]:
groups: Dict[str, List[SceneRecord]] = {}
for scene in scenes:
groups.setdefault(scene.group_key, []).append(scene)
summary: List[Dict[str, Any]] = []
for key, items in groups.items():
items.sort(key=lambda item: item.imaging_date)
first = items[0]
summary.append(
{
"group_key": key,
"count": len(items),
"satellite": first.satellite,
"imaging_mode": first.imaging_mode,
"polarization": first.polarization,
"orbit_direction": first.orbit_direction,
"tile_key": first.tile_key,
"dates": [item.imaging_date for item in items],
"receiving_stations": sorted({item.receiving_station for item in items if item.receiving_station}),
}
)
summary.sort(key=lambda item: (-item["count"], item["group_key"]))
return summary
def select_group(
summary: List[Dict[str, Any]],
tile_key: Optional[str],
group_key: Optional[str],
min_scenes: int,
) -> Optional[str]:
if group_key:
return group_key
if tile_key:
for item in summary:
if item["tile_key"] == tile_key and item["count"] >= min_scenes:
return item["group_key"]
return None
for item in summary:
if item["count"] >= min_scenes:
return item["group_key"]
return None
def build_manifest(root_dir: Path, group_key: str, scenes: List[SceneRecord]) -> Dict[str, Any]:
group_scenes = [scene for scene in scenes if scene.group_key == group_key]
if not group_scenes:
raise ValueError(f"Group not found: {group_key}")
group_scenes.sort(key=lambda item: item.imaging_date)
first = group_scenes[0]
reference_index = len(group_scenes) // 2
reference_scene = group_scenes[reference_index]
slug = (
f"{first.satellite.lower()}_"
f"{(first.imaging_mode or 'unknown').lower()}_"
f"{(first.polarization or 'unknown').lower()}_"
f"{(first.orbit_direction or 'unknown').lower()}_"
f"{first.tile_key.lower().replace('.', 'p')}"
)
scratch_root = _repo_root() / "experiments" / "isce2_sbas_timeseries" / "scratch" / slug
scratch_root_wsl = windows_to_wsl(scratch_root)
return {
"source_root_windows": str(root_dir),
"source_root_wsl": windows_to_wsl(root_dir),
"group_key": group_key,
"tile_key": first.tile_key,
"scene_count": len(group_scenes),
"reference_strategy": "middle_by_date",
"reference_date": reference_scene.imaging_date,
"stack_group": {
"satellite": first.satellite,
"imaging_mode": first.imaging_mode,
"polarization": first.polarization,
"orbit_direction": first.orbit_direction,
"receiving_stations": sorted({item.receiving_station for item in group_scenes if item.receiving_station}),
},
"proposed_scratch_windows": str(scratch_root),
"proposed_scratch_wsl": scratch_root_wsl,
"proposed_layout": {
"stack_input_manifest": f"{scratch_root_wsl}/stack_input_manifest.json",
"slc_dir": f"{scratch_root_wsl}/SLC",
"orbits_dir": f"{scratch_root_wsl}/orbits",
"logs_dir": f"{scratch_root_wsl}/logs",
},
"stack_prep_assessment": {
"current_scene_layout": "per_scene_folder_with_tiff_meta_rpc",
"official_stripmapStack_expected_layout": "SLC/YYYYMMDD/YYYYMMDD.raw or YYYYMMDD.slc",
"direct_compatibility": "unproven",
"lt1_adapter_required_likely": True,
"notes": [
"Current repo can read these scene folders as RadarData assets.",
"Official stripmapStack helper scripts do not advertise LT-1/LUTAN1 preparation hooks.",
"A custom LT-1 stack preparation layer is likely needed before official stack execution.",
],
},
"scenes": [asdict(scene) for scene in group_scenes],
}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Scan LT-1 scene folders and build a dry-run SBAS stack-prep manifest."
)
parser.add_argument(
"--root-dir",
default=r"F:\Insar_data_pool_1",
help="Windows root directory containing LT-1 scene folders.",
)
parser.add_argument(
"--min-scenes",
type=int,
default=4,
help="Minimum scenes required for candidate groups.",
)
parser.add_argument(
"--top-n",
type=int,
default=20,
help="How many candidate groups to print.",
)
parser.add_argument(
"--tile-key",
default=None,
help="Pick one candidate by tile key, for example E123.3_N46.1.",
)
parser.add_argument(
"--group-key",
default=None,
help="Pick one candidate by full group key.",
)
parser.add_argument(
"--manifest-path",
default=None,
help="Optional JSON output path for the selected group's dry-run manifest.",
)
return parser
def main() -> int:
args = build_parser().parse_args()
root_dir = Path(args.root_dir)
if not root_dir.exists():
raise FileNotFoundError(f"Root directory does not exist: {root_dir}")
scenes = scan_scenes(root_dir)
summary = build_group_summary(scenes)
print(f"scanned_scenes={len(scenes)}")
print(f"candidate_groups={len(summary)}")
print("top_candidates:")
for item in summary[: args.top_n]:
print(
json.dumps(
{
"count": item["count"],
"tile_key": item["tile_key"],
"group_key": item["group_key"],
"dates": item["dates"],
"receiving_stations": item["receiving_stations"],
},
ensure_ascii=False,
)
)
selected_group = select_group(summary, args.tile_key, args.group_key, args.min_scenes)
if not selected_group:
print("selected_group=None")
return 0
manifest = build_manifest(root_dir, selected_group, scenes)
print(f"selected_group={selected_group}")
print(f"reference_date={manifest['reference_date']}")
if args.manifest_path:
manifest_path = Path(args.manifest_path)
manifest_path.parent.mkdir(parents=True, exist_ok=True)
manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"manifest_written={manifest_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())