Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: engineer SBAS timeseries production workflow

Browse Source
This commit is contained in:
Harmon
2026-04-29 14:43:31 +08:00
parent dace8b20f6
commit 4c0d1f2c2b
54 changed files with 5843 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/config.py
+16 -5
View File
@@ -506,14 +506,25 @@ class Settings(BaseSettings):
if not self.TIMESERIES_WSL_DISTRO:
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO)
if not self.TIMESERIES_ENV_NAME:
object.__setattr__(self, "TIMESERIES_ENV_NAME", "isce2_mintpy_v1")
if not self.TIMESERIES_PYTHON:
env_name = str(self.TIMESERIES_ENV_NAME or "isce2_mintpy_v1").strip() or "isce2_mintpy_v1"
object.__setattr__(
self,
"TIMESERIES_PYTHON",
f"/home/administrator/miniconda3/envs/{env_name}/bin/python",
"TIMESERIES_ENV_NAME",
str(self.WSL_SHARED_CONDA_ENV or "insar_wsl_v1").strip() or "insar_wsl_v1",
)
if not self.TIMESERIES_PYTHON:
shared_python = str(self.WSL_SHARED_PYTHON or "").strip()
if shared_python:
object.__setattr__(self, "TIMESERIES_PYTHON", shared_python)
else:
env_name = (
str(self.TIMESERIES_ENV_NAME or "insar_wsl_v1").strip()
or "insar_wsl_v1"
)
object.__setattr__(
self,
"TIMESERIES_PYTHON",
f"/home/administrator/miniconda3/envs/{env_name}/bin/python",
)
if not self.TIMESERIES_WORK_ROOT:
object.__setattr__(
self,
backend/app/db_maintenance.py
+1
View File
@@ -33,6 +33,7 @@ MIGRATION_FILES = [
"004_pairing_refactor.sql",
"005_pairing_task_trace.sql",
"006_result_pairing_trace.sql",
"007_timeseries_stack_plan_trace.sql",
]
backend/app/dinsar_engines/isce2_engine.py
+406 -2
View File
@@ -27,8 +27,22 @@ LT1_FIXED_WAVELENGTH = 0.23793052222222222
DEFAULT_TARGET_GRID_SIZE_M = 10
DEFAULT_BBOX_MARGIN = 0.05
DEFAULT_COH_THRESHOLD = 0.05
DEFAULT_REFERENCE_MODE = "none"
DEFAULT_REFERENCE_MODE = "coh_median"
DEFAULT_REFERENCE_COH_THRESHOLD = 0.30
DEFAULT_DERAMP_MODE = "plane"
DEFAULT_DERAMP_COH_THRESHOLD = 0.30
DEFAULT_DENSE_OFFSETS = True
DEFAULT_RUBBERSHEET_RANGE = True
DEFAULT_RUBBERSHEET_AZIMUTH = True
DEFAULT_IONOSPHERE_CORRECTION = True
DEFAULT_RUBBER_SHEET_SNR_THRESHOLD = 5.0
DEFAULT_RUBBER_SHEET_FILTER_SIZE = 9
DEFAULT_DENSE_WINDOW_WIDTH = 64
DEFAULT_DENSE_WINDOW_HEIGHT = 64
DEFAULT_DENSE_SEARCH_WIDTH = 20
DEFAULT_DENSE_SEARCH_HEIGHT = 20
DEFAULT_DENSE_SKIP_WIDTH = 32
DEFAULT_DENSE_SKIP_HEIGHT = 32
ORBIT_MARGIN_MIN_SEC = 60.0
ORBIT_MARGIN_MAX_SEC = 120.0
TARGET_GRID_SIZE_MIN_M = 5
@@ -36,6 +50,7 @@ TARGET_GRID_SIZE_MAX_M = 100
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
RESUME_STAGE_CHOICES = {"", "unwrap", "geocode", "export"}
REFERENCE_MODE_CHOICES = {"none", "coh_median"}
DERAMP_MODE_CHOICES = {"none", "plane"}
def _read_env(name: str, default: str = "") -> str:
@@ -206,7 +221,7 @@ class Isce2Engine(DinsarEngine):
# Profiles
# ------------------------------------------------------------------
def get_profiles(self) -> List[EngineProfile]:
def _legacy_get_profiles(self) -> List[EngineProfile]:
return [
EngineProfile(
code="lt1_stripmap",
@@ -281,6 +296,243 @@ class Isce2Engine(DinsarEngine):
),
]
def get_profiles(self) -> List[EngineProfile]:
return [
EngineProfile(
code="lt1_stripmap",
label="LT-1 Stripmap",
description=(
"Managed LT-1 stripmap D-InSAR production in WSL with the standard "
"ISCE2 enhancement steps and split-spectrum ionosphere correction enabled by default."
),
params_schema={
"force": {
"label": "Rebuild Work Dir",
"type": "boolean",
"default": False,
"section": "Execution",
"description": "Delete the existing work directory before rerunning the task.",
"recommendation": "Use only when the previous work directory can be discarded.",
},
"target_grid_size_m": {
"label": "Target Grid Size (m)",
"type": "number",
"default": DEFAULT_TARGET_GRID_SIZE_M,
"step": 1,
"min": TARGET_GRID_SIZE_MIN_M,
"max": TARGET_GRID_SIZE_MAX_M,
"section": "Execution",
"description": "Controls multilook scale and geocoded output spacing.",
"recommendation": "Use 10 by default; try 5 for more detail or 15-20 for more stability.",
},
"bbox": {
"label": "Geocode BBox",
"type": "string",
"default": "",
"placeholder": "south,north,west,east",
"section": "Execution",
"description": "Optional manual geocode bounding box.",
"recommendation": "Leave empty unless you need to constrain the output area.",
},
"coh_threshold": {
"label": "Coherence Threshold",
"type": "number",
"default": DEFAULT_COH_THRESHOLD,
"step": 0.01,
"min": 0,
"max": 1,
"section": "Delivery",
"description": "Masks displacement pixels below this coherence threshold in the exported product.",
"recommendation": "Use 0.05 for broad inspection and 0.10+ for stricter delivery.",
},
"reference_mode": {
"label": "Reference Mode",
"type": "string",
"default": DEFAULT_REFERENCE_MODE,
"enum": sorted(REFERENCE_MODE_CHOICES),
"section": "Delivery",
"description": "Normalizes the final displacement field before delivery.",
"recommendation": "Use coh_median for production so the result is centered on stable high-coherence pixels.",
},
"reference_coh_threshold": {
"label": "Reference Coh Threshold",
"type": "number",
"default": DEFAULT_REFERENCE_COH_THRESHOLD,
"step": 0.01,
"min": 0,
"max": 1,
"section": "Delivery",
"description": "Minimum coherence used when selecting pixels for displacement referencing.",
"recommendation": "Use 0.30 by default; raise it only when you have enough high-quality support pixels.",
},
"deramp_mode": {
"label": "Deramp Mode",
"type": "string",
"default": DEFAULT_DERAMP_MODE,
"enum": sorted(DERAMP_MODE_CHOICES),
"section": "Delivery",
"description": "Removes long-wavelength ramp residuals after referencing.",
"recommendation": "Use plane for LT-1 production unless you are explicitly debugging raw ISCE2 output.",
},
"deramp_coh_threshold": {
"label": "Deramp Coh Threshold",
"type": "number",
"default": DEFAULT_DERAMP_COH_THRESHOLD,
"step": 0.01,
"min": 0,
"max": 1,
"section": "Delivery",
"description": "Minimum coherence used when selecting pixels for deramp fitting.",
"recommendation": "Use 0.30 by default so the ramp is fitted on cleaner support pixels.",
},
"bbox_margin": {
"label": "BBox Margin (deg)",
"type": "number",
"default": DEFAULT_BBOX_MARGIN,
"step": 0.01,
"min": 0,
"section": "Execution",
"description": "Extra degree margin added to the auto-estimated geocode bounding box.",
"recommendation": "Use 0.05 by default; increase only if edges are clipped.",
},
"ionosphere_correction": {
"label": "Enable Split-Spectrum Ionosphere Correction",
"type": "boolean",
"default": DEFAULT_IONOSPHERE_CORRECTION,
"section": "Enhancement",
"description": "Runs the split-spectrum dispersive correction branch before geocode and export.",
"recommendation": "Keep enabled by default; disable it when the correction itself is suspected to degrade a scene.",
},
"dense_offsets": {
"label": "Enable Dense Offsets",
"type": "boolean",
"default": DEFAULT_DENSE_OFFSETS,
"section": "Enhancement",
"description": "Run ISCE2 dense offset estimation before fine resampling.",
"recommendation": "Keep enabled for LT-1 stripmap production.",
},
"rubbersheet_range": {
"label": "Enable Range Rubbersheeting",
"type": "boolean",
"default": DEFAULT_RUBBERSHEET_RANGE,
"section": "Enhancement",
"description": "Update range offsets with dense offsets before fine resampling.",
"recommendation": "Keep enabled for LT-1 stripmap production.",
},
"rubbersheet_azimuth": {
"label": "Enable Azimuth Rubbersheeting",
"type": "boolean",
"default": DEFAULT_RUBBERSHEET_AZIMUTH,
"section": "Enhancement",
"description": "Update azimuth offsets with dense offsets before fine resampling.",
"recommendation": "Keep enabled for LT-1 stripmap production.",
},
"rubber_sheet_snr_threshold": {
"label": "Rubbersheet SNR Threshold",
"type": "number",
"default": DEFAULT_RUBBER_SHEET_SNR_THRESHOLD,
"step": 0.5,
"min": 0,
"section": "Enhancement",
"description": "SNR threshold used when masking dense offsets for rubbersheeting.",
"recommendation": "Start with 5.0 unless a scene-specific diagnosis suggests otherwise.",
},
"rubber_sheet_filter_size": {
"label": "Rubbersheet Filter Size",
"type": "number",
"default": DEFAULT_RUBBER_SHEET_FILTER_SIZE,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Median filter size used when smoothing masked dense offsets.",
"recommendation": "Start with 9.",
},
"dense_window_width": {
"label": "Dense Window Width",
"type": "number",
"default": DEFAULT_DENSE_WINDOW_WIDTH,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset correlation window width.",
"recommendation": "Start with 64.",
},
"dense_window_height": {
"label": "Dense Window Height",
"type": "number",
"default": DEFAULT_DENSE_WINDOW_HEIGHT,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset correlation window height.",
"recommendation": "Start with 64.",
},
"dense_search_width": {
"label": "Dense Search Width",
"type": "number",
"default": DEFAULT_DENSE_SEARCH_WIDTH,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset search width.",
"recommendation": "Start with 20.",
},
"dense_search_height": {
"label": "Dense Search Height",
"type": "number",
"default": DEFAULT_DENSE_SEARCH_HEIGHT,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset search height.",
"recommendation": "Start with 20.",
},
"dense_skip_width": {
"label": "Dense Skip Width",
"type": "number",
"default": DEFAULT_DENSE_SKIP_WIDTH,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset sampling stride in range direction.",
"recommendation": "Start with 32.",
},
"dense_skip_height": {
"label": "Dense Skip Height",
"type": "number",
"default": DEFAULT_DENSE_SKIP_HEIGHT,
"step": 1,
"min": 1,
"section": "Enhancement",
"description": "Dense offset sampling stride in azimuth direction.",
"recommendation": "Start with 32.",
},
"wavelength": {
"label": "Radar Wavelength (m)",
"type": "number",
"default": LT1_FIXED_WAVELENGTH,
"step": 0.000001,
"readonly": True,
"include_in_payload": False,
"section": "Execution",
"description": "Fixed LT-1 radar wavelength used for displacement conversion.",
"recommendation": "This value is locked by the system.",
},
"orbit_margin_sec": {
"label": "Orbit Margin (sec)",
"type": "number",
"default": ORBIT_MARGIN_MIN_SEC,
"step": 1,
"min": ORBIT_MARGIN_MIN_SEC,
"max": ORBIT_MARGIN_MAX_SEC,
"section": "Execution",
"description": "Extra time margin preserved when clipping the precise orbit XML.",
"recommendation": "Use 60 by default; raise to 90-120 only if scene timing is tight.",
},
},
),
]
def normalize_extra(self, extra: Dict[str, Any] | None) -> Dict[str, Any]:
normalized: Dict[str, Any] = dict(extra or {})
normalized.pop("wavelength", None)
@@ -330,6 +582,22 @@ class Isce2Engine(DinsarEngine):
raise ValueError("reference_coh_threshold must be between 0 and 1")
normalized["reference_coh_threshold"] = reference_coh_threshold
if "deramp_mode" in normalized and normalized["deramp_mode"] is not None:
deramp_mode = str(normalized["deramp_mode"]).strip().lower()
if deramp_mode not in DERAMP_MODE_CHOICES:
supported_modes = ", ".join(sorted(DERAMP_MODE_CHOICES))
raise ValueError(f"deramp_mode must be one of: {supported_modes}")
normalized["deramp_mode"] = deramp_mode
if "deramp_coh_threshold" in normalized and normalized["deramp_coh_threshold"] is not None:
try:
deramp_coh_threshold = float(normalized["deramp_coh_threshold"])
except (TypeError, ValueError) as exc:
raise ValueError("deramp_coh_threshold must be numeric") from exc
if deramp_coh_threshold < 0 or deramp_coh_threshold > 1:
raise ValueError("deramp_coh_threshold must be between 0 and 1")
normalized["deramp_coh_threshold"] = deramp_coh_threshold
if "bbox_margin" in normalized and normalized["bbox_margin"] is not None:
try:
bbox_margin = float(normalized["bbox_margin"])
@@ -339,6 +607,40 @@ class Isce2Engine(DinsarEngine):
raise ValueError("范围外扩量不能小于 0。")
normalized["bbox_margin"] = bbox_margin
for bool_key in ("ionosphere_correction", "dense_offsets", "rubbersheet_range", "rubbersheet_azimuth"):
if bool_key in normalized:
normalized[bool_key] = bool(normalized[bool_key])
if "rubber_sheet_snr_threshold" in normalized and normalized["rubber_sheet_snr_threshold"] is not None:
try:
snr_threshold = float(normalized["rubber_sheet_snr_threshold"])
except (TypeError, ValueError) as exc:
raise ValueError("rubber_sheet_snr_threshold must be numeric") from exc
if snr_threshold < 0:
raise ValueError("rubber_sheet_snr_threshold must be non-negative")
normalized["rubber_sheet_snr_threshold"] = snr_threshold
for int_key in (
"rubber_sheet_filter_size",
"dense_window_width",
"dense_window_height",
"dense_search_width",
"dense_search_height",
"dense_skip_width",
"dense_skip_height",
):
if int_key not in normalized or normalized[int_key] is None:
continue
try:
numeric_value = float(normalized[int_key])
except (TypeError, ValueError) as exc:
raise ValueError(f"{int_key} must be numeric") from exc
if int(numeric_value) != numeric_value:
raise ValueError(f"{int_key} must be an integer")
if int(numeric_value) <= 0:
raise ValueError(f"{int_key} must be greater than 0")
normalized[int_key] = int(numeric_value)
if "orbit_margin_sec" in normalized and normalized["orbit_margin_sec"] is not None:
try:
orbit_margin = float(normalized["orbit_margin_sec"])
@@ -548,7 +850,21 @@ class Isce2Engine(DinsarEngine):
coh_threshold: Any,
reference_mode: str,
reference_coh_threshold: Any,
deramp_mode: str,
deramp_coh_threshold: Any,
bbox_margin: Any,
dense_offsets: bool,
rubbersheet_range: bool,
rubbersheet_azimuth: bool,
ionosphere_correction: bool,
rubber_sheet_snr_threshold: Any,
rubber_sheet_filter_size: Any,
dense_window_width: Any,
dense_window_height: Any,
dense_search_width: Any,
dense_search_height: Any,
dense_skip_width: Any,
dense_skip_height: Any,
wavelength: Any,
orbit_margin_sec: Any,
full_geocode: bool,
@@ -586,11 +902,26 @@ class Isce2Engine(DinsarEngine):
"coh_threshold": coh_threshold,
"reference_mode": str(reference_mode or "").strip(),
"reference_coh_threshold": reference_coh_threshold,
"deramp_mode": str(deramp_mode or "").strip(),
"deramp_coh_threshold": deramp_coh_threshold,
"bbox_margin": bbox_margin,
"dense_offsets": bool(dense_offsets),
"rubbersheet_range": bool(rubbersheet_range),
"rubbersheet_azimuth": bool(rubbersheet_azimuth),
"ionosphere_correction": bool(ionosphere_correction),
"rubber_sheet_snr_threshold": rubber_sheet_snr_threshold,
"rubber_sheet_filter_size": rubber_sheet_filter_size,
"dense_window_width": dense_window_width,
"dense_window_height": dense_window_height,
"dense_search_width": dense_search_width,
"dense_search_height": dense_search_height,
"dense_skip_width": dense_skip_width,
"dense_skip_height": dense_skip_height,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
"full_geocode": bool(full_geocode),
"resume_from": str(resume_from or "").strip(),
"split_spectrum": bool(ionosphere_correction),
},
"pair_meta": dict(pair_meta or {}),
}
@@ -704,7 +1035,34 @@ class Isce2Engine(DinsarEngine):
"reference_coh_threshold",
DEFAULT_REFERENCE_COH_THRESHOLD,
)
deramp_mode = str(
extra.get("deramp_mode", DEFAULT_DERAMP_MODE) or DEFAULT_DERAMP_MODE
).strip().lower()
deramp_coh_threshold = extra.get(
"deramp_coh_threshold",
DEFAULT_DERAMP_COH_THRESHOLD,
)
bbox_margin = extra.get("bbox_margin", DEFAULT_BBOX_MARGIN)
ionosphere_correction = bool(
extra.get("ionosphere_correction", DEFAULT_IONOSPHERE_CORRECTION)
)
dense_offsets = bool(extra.get("dense_offsets", DEFAULT_DENSE_OFFSETS))
rubbersheet_range = bool(extra.get("rubbersheet_range", DEFAULT_RUBBERSHEET_RANGE))
rubbersheet_azimuth = bool(extra.get("rubbersheet_azimuth", DEFAULT_RUBBERSHEET_AZIMUTH))
rubber_sheet_snr_threshold = extra.get(
"rubber_sheet_snr_threshold",
DEFAULT_RUBBER_SHEET_SNR_THRESHOLD,
)
rubber_sheet_filter_size = extra.get(
"rubber_sheet_filter_size",
DEFAULT_RUBBER_SHEET_FILTER_SIZE,
)
dense_window_width = extra.get("dense_window_width", DEFAULT_DENSE_WINDOW_WIDTH)
dense_window_height = extra.get("dense_window_height", DEFAULT_DENSE_WINDOW_HEIGHT)
dense_search_width = extra.get("dense_search_width", DEFAULT_DENSE_SEARCH_WIDTH)
dense_search_height = extra.get("dense_search_height", DEFAULT_DENSE_SEARCH_HEIGHT)
dense_skip_width = extra.get("dense_skip_width", DEFAULT_DENSE_SKIP_WIDTH)
dense_skip_height = extra.get("dense_skip_height", DEFAULT_DENSE_SKIP_HEIGHT)
wavelength = LT1_FIXED_WAVELENGTH
orbit_margin_sec = extra.get("orbit_margin_sec", ORBIT_MARGIN_MIN_SEC)
full_geocode = bool(extra.get("full_geocode"))
@@ -850,7 +1208,21 @@ class Isce2Engine(DinsarEngine):
coh_threshold=coh_threshold,
reference_mode=reference_mode,
reference_coh_threshold=reference_coh_threshold,
deramp_mode=deramp_mode,
deramp_coh_threshold=deramp_coh_threshold,
bbox_margin=bbox_margin,
dense_offsets=dense_offsets,
rubbersheet_range=rubbersheet_range,
rubbersheet_azimuth=rubbersheet_azimuth,
ionosphere_correction=ionosphere_correction,
rubber_sheet_snr_threshold=rubber_sheet_snr_threshold,
rubber_sheet_filter_size=rubber_sheet_filter_size,
dense_window_width=dense_window_width,
dense_window_height=dense_window_height,
dense_search_width=dense_search_width,
dense_search_height=dense_search_height,
dense_skip_width=dense_skip_width,
dense_skip_height=dense_skip_height,
wavelength=wavelength,
orbit_margin_sec=orbit_margin_sec,
full_geocode=full_geocode,
@@ -930,9 +1302,25 @@ class Isce2Engine(DinsarEngine):
"coh_threshold": coh_threshold,
"reference_mode": reference_mode,
"reference_coh_threshold": reference_coh_threshold,
"deramp_mode": deramp_mode,
"deramp_coh_threshold": deramp_coh_threshold,
"bbox_margin": bbox_margin,
"ionosphere_correction": ionosphere_correction,
"dense_offsets": dense_offsets,
"rubbersheet_range": rubbersheet_range,
"rubbersheet_azimuth": rubbersheet_azimuth,
"rubber_sheet_snr_threshold": rubber_sheet_snr_threshold,
"rubber_sheet_filter_size": rubber_sheet_filter_size,
"dense_window_width": dense_window_width,
"dense_window_height": dense_window_height,
"dense_search_width": dense_search_width,
"dense_search_height": dense_search_height,
"dense_skip_width": dense_skip_width,
"dense_skip_height": dense_skip_height,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
"split_spectrum": ionosphere_correction,
"ionosphere_correction": ionosphere_correction,
},
"master_path": pair_meta.get("master_path"),
"slave_path": pair_meta.get("slave_path"),
@@ -1053,9 +1441,25 @@ class Isce2Engine(DinsarEngine):
"coh_threshold": coh_threshold,
"reference_mode": reference_mode,
"reference_coh_threshold": reference_coh_threshold,
"deramp_mode": deramp_mode,
"deramp_coh_threshold": deramp_coh_threshold,
"bbox_margin": bbox_margin,
"ionosphere_correction": ionosphere_correction,
"dense_offsets": dense_offsets,
"rubbersheet_range": rubbersheet_range,
"rubbersheet_azimuth": rubbersheet_azimuth,
"rubber_sheet_snr_threshold": rubber_sheet_snr_threshold,
"rubber_sheet_filter_size": rubber_sheet_filter_size,
"dense_window_width": dense_window_width,
"dense_window_height": dense_window_height,
"dense_search_width": dense_search_width,
"dense_search_height": dense_search_height,
"dense_skip_width": dense_skip_width,
"dense_skip_height": dense_skip_height,
"wavelength": wavelength,
"orbit_margin_sec": orbit_margin_sec,
"split_spectrum": ionosphere_correction,
"ionosphere_correction": ionosphere_correction,
"runtime_id": runtime.runtime_id,
"command": last_task_result.get("command", ""),
"runner_argv": last_task_result.get("runner_argv", []),
backend/app/isce2_pipeline/export_isce_geotiff.py
+279 -22
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
import numpy as np
@@ -11,9 +12,12 @@ gdal.UseExceptions()
DEFAULT_WAVELENGTH = 0.23793052222222222
DEFAULT_NODATA = -9999.0
DEFAULT_REFERENCE_MODE = "none"
DEFAULT_REFERENCE_MODE = "coh_median"
DEFAULT_REFERENCE_COH_THRESHOLD = 0.30
DEFAULT_DERAMP_MODE = "plane"
DEFAULT_DERAMP_COH_THRESHOLD = 0.30
REFERENCE_MODE_CHOICES = ("none", "coh_median")
DERAMP_MODE_CHOICES = ("none", "plane")
def parse_args() -> argparse.Namespace:
@@ -54,7 +58,7 @@ def parse_args() -> argparse.Namespace:
type=str,
choices=REFERENCE_MODE_CHOICES,
default=DEFAULT_REFERENCE_MODE,
help="Optional reference normalization mode for debug exports",
help="Reference normalization mode applied before final displacement export",
)
parser.add_argument(
"--reference-coh-threshold",
@@ -62,10 +66,23 @@ def parse_args() -> argparse.Namespace:
default=DEFAULT_REFERENCE_COH_THRESHOLD,
help="Minimum coherence used to select reference pixels for normalization",
)
parser.add_argument(
"--deramp-mode",
type=str,
choices=DERAMP_MODE_CHOICES,
default=DEFAULT_DERAMP_MODE,
help="Optional long-wavelength ramp removal applied after reference normalization",
)
parser.add_argument(
"--deramp-coh-threshold",
type=float,
default=DEFAULT_DERAMP_COH_THRESHOLD,
help="Minimum coherence used when selecting pixels for deramp fitting",
)
parser.add_argument(
"--include-disp-full",
action="store_true",
help="Also export the coherence-unmasked displacement GeoTIFF for debugging",
help="Also export the coherence-unmasked final displacement GeoTIFF",
)
return parser.parse_args()
@@ -93,6 +110,51 @@ def write_geotiff(array: np.ndarray, ref_ds: gdal.Dataset, out_path: Path, nodat
ds = None
def _resolve_phase_source(work_dir: Path) -> dict[str, str | bool]:
ionosphere_phase = work_dir / "ionosphere" / "nondispersive.bil.unwCor.filt.geo.vrt"
ionosphere_mask = work_dir / "ionosphere" / "mask.bil.geo.vrt"
full_unwrap = work_dir / "interferogram" / "filt_topophase.unw.geo.vrt"
if ionosphere_phase.exists():
return {
"phase_path": str(ionosphere_phase),
"phase_source": "ionosphere_nondispersive",
"mask_path": str(ionosphere_mask) if ionosphere_mask.exists() else "",
"ionosphere_corrected": True,
}
return {
"phase_path": str(full_unwrap),
"phase_source": "interferogram_unwrapped",
"mask_path": "",
"ionosphere_corrected": False,
}
def _select_support_mask(
*,
base_mask: np.ndarray,
amp_valid: np.ndarray,
disp_valid: np.ndarray,
coh: np.ndarray,
selection_threshold: float,
) -> tuple[np.ndarray, dict[str, float | int | str]]:
fallback = ""
support_mask = base_mask & (coh >= selection_threshold)
if not support_mask.any():
support_mask = base_mask & (coh > 0)
fallback = "coh>0"
if not support_mask.any():
support_mask = amp_valid & disp_valid
fallback = "amp_only"
stats: dict[str, float | int | str] = {
"selection_threshold": float(selection_threshold),
"fallback": fallback,
"support_ratio": float(base_mask.mean()),
"support_count": int(support_mask.sum()),
"support_mask_ratio": float(support_mask.mean()),
}
return support_mask, stats
def compute_reference_offset(
disp_m_raw: np.ndarray,
amp: np.ndarray,
@@ -100,7 +162,7 @@ def compute_reference_offset(
coh_threshold: float,
reference_mode: str,
reference_coh_threshold: float,
) -> tuple[float, dict[str, float | int | str]]:
) -> tuple[float, np.ndarray, dict[str, float | int | str]]:
amp_valid = np.isfinite(amp) & (amp != 0)
coh_finite = np.isfinite(coh)
disp_valid = np.isfinite(disp_m_raw)
@@ -121,17 +183,16 @@ def compute_reference_offset(
"fallback": "",
}
if normalized_mode == "none":
return 0.0, stats
return 0.0, base_mask, stats
selection_threshold = min(1.0, max(0.0, max(float(coh_threshold), float(reference_coh_threshold))))
reference_mask = base_mask & (coh >= selection_threshold)
fallback = ""
if not reference_mask.any():
reference_mask = base_mask & (coh > 0)
fallback = "coh>0"
if not reference_mask.any():
reference_mask = amp_valid & disp_valid
fallback = "amp_only"
reference_mask, mask_stats = _select_support_mask(
base_mask=base_mask,
amp_valid=amp_valid,
disp_valid=disp_valid,
coh=coh,
selection_threshold=selection_threshold,
)
reference_count = int(reference_mask.sum())
if reference_count <= 0:
@@ -141,11 +202,101 @@ def compute_reference_offset(
{
"reference_count": reference_count,
"reference_ratio": float(reference_mask.mean()),
"selection_threshold": float(selection_threshold),
"fallback": fallback,
"selection_threshold": float(mask_stats["selection_threshold"]),
"fallback": str(mask_stats["fallback"]),
}
)
return float(np.median(disp_m_raw[reference_mask])), stats
return float(np.median(disp_m_raw[reference_mask])), reference_mask, stats
def compute_deramp_surface(
disp_m: np.ndarray,
amp: np.ndarray,
coh: np.ndarray,
coh_threshold: float,
deramp_mode: str,
deramp_coh_threshold: float,
) -> tuple[np.ndarray, np.ndarray, dict[str, float | int | str | bool]]:
amp_valid = np.isfinite(amp) & (amp != 0)
coh_finite = np.isfinite(coh)
disp_valid = np.isfinite(disp_m)
base_mask = amp_valid & coh_finite & disp_valid
normalized_mode = str(deramp_mode or DEFAULT_DERAMP_MODE).strip().lower()
if normalized_mode not in DERAMP_MODE_CHOICES:
raise ValueError(f"Unsupported deramp mode: {deramp_mode}")
empty_surface = np.zeros_like(disp_m, dtype=np.float32)
stats: dict[str, float | int | str | bool] = {
"mode": normalized_mode,
"applied": False,
"fit_count": 0,
"fit_ratio": 0.0,
"selection_threshold": 0.0,
"fallback": "",
"sample_step": 0,
"sample_count": 0,
}
if normalized_mode == "none":
return empty_surface, base_mask, stats
if not base_mask.any():
return empty_surface, base_mask, stats
selection_threshold = min(1.0, max(0.0, max(float(coh_threshold), float(deramp_coh_threshold))))
fit_mask, mask_stats = _select_support_mask(
base_mask=base_mask,
amp_valid=amp_valid,
disp_valid=disp_valid,
coh=coh,
selection_threshold=selection_threshold,
)
fit_count = int(fit_mask.sum())
stats.update(
{
"fit_count": fit_count,
"fit_ratio": float(fit_mask.mean()),
"selection_threshold": float(mask_stats["selection_threshold"]),
"fallback": str(mask_stats["fallback"]),
}
)
if fit_count < 3:
stats["fallback"] = "insufficient_support"
return empty_surface, fit_mask, stats
yy, xx = np.indices(disp_m.shape, dtype=np.float64)
xs = xx[fit_mask]
ys = yy[fit_mask]
zs = disp_m[fit_mask].astype(np.float64)
sample_step = max(1, fit_count // 250_000)
if sample_step > 1:
xs = xs[::sample_step]
ys = ys[::sample_step]
zs = zs[::sample_step]
sample_count = int(zs.size)
stats["sample_step"] = int(sample_step)
stats["sample_count"] = sample_count
if sample_count < 3:
stats["fallback"] = "insufficient_sample"
return empty_surface, fit_mask, stats
design = np.column_stack([xs, ys, np.ones_like(xs)])
coeffs, _, _, _ = np.linalg.lstsq(design, zs, rcond=None)
plane = (
coeffs[0] * xx
+ coeffs[1] * yy
+ coeffs[2]
).astype(np.float32)
stats.update(
{
"applied": True,
"coef_x_per_pixel": float(coeffs[0]),
"coef_y_per_pixel": float(coeffs[1]),
"intercept_m": float(coeffs[2]),
"left_right_delta_m": float(coeffs[0] * max(disp_m.shape[1] - 1, 0)),
"top_bottom_delta_m": float(coeffs[1] * max(disp_m.shape[0] - 1, 0)),
}
)
return plane, fit_mask, stats
def export_products(
@@ -156,32 +307,48 @@ def export_products(
coh_threshold: float,
reference_mode: str = DEFAULT_REFERENCE_MODE,
reference_coh_threshold: float = DEFAULT_REFERENCE_COH_THRESHOLD,
deramp_mode: str = DEFAULT_DERAMP_MODE,
deramp_coh_threshold: float = DEFAULT_DERAMP_COH_THRESHOLD,
include_disp_full: bool = False,
nodata: float = DEFAULT_NODATA,
) -> dict[str, Path]:
unw_path = work_dir / "interferogram" / "filt_topophase.unw.geo.vrt"
cor_path = work_dir / "interferogram" / "topophase.cor.geo.vrt"
phase_source = _resolve_phase_source(work_dir)
phase_path = Path(str(phase_source["phase_path"]))
mask_path = Path(str(phase_source["mask_path"])) if str(phase_source["mask_path"]) else None
if not unw_path.exists():
raise FileNotFoundError(f"Missing unwrapped product: {unw_path}")
if not cor_path.exists():
raise FileNotFoundError(f"Missing coherence product: {cor_path}")
if not phase_path.exists():
raise FileNotFoundError(f"Missing phase source product: {phase_path}")
unw_ds = gdal.Open(str(unw_path))
cor_ds = gdal.Open(str(cor_path))
if unw_ds is None or cor_ds is None:
phase_ds = gdal.Open(str(phase_path))
mask_ds = gdal.Open(str(mask_path)) if mask_path is not None else None
if unw_ds is None or cor_ds is None or phase_ds is None:
raise RuntimeError("Failed to open ISCE2 geo products with GDAL.")
amp = unw_ds.GetRasterBand(1).ReadAsArray().astype(np.float32)
phase = unw_ds.GetRasterBand(2).ReadAsArray().astype(np.float32)
if bool(phase_source["ionosphere_corrected"]):
phase = phase_ds.GetRasterBand(1).ReadAsArray().astype(np.float32)
else:
phase = unw_ds.GetRasterBand(2).ReadAsArray().astype(np.float32)
coh_band = 2 if cor_ds.RasterCount >= 2 else 1
coh = cor_ds.GetRasterBand(coh_band).ReadAsArray().astype(np.float32)
coh_valid = np.isfinite(coh) & (coh > 0)
amp_valid = np.isfinite(amp) & (amp != 0)
ionosphere_mask_valid = None
if mask_ds is not None:
ionosphere_mask = mask_ds.GetRasterBand(1).ReadAsArray().astype(np.float32)
ionosphere_mask_valid = np.isfinite(ionosphere_mask) & (ionosphere_mask > 0)
disp_m_raw = phase * wavelength / (4.0 * np.pi)
reference_offset_m, reference_stats = compute_reference_offset(
reference_offset_m, reference_mask, reference_stats = compute_reference_offset(
disp_m_raw=disp_m_raw,
amp=amp,
coh=coh,
@@ -189,10 +356,25 @@ def export_products(
reference_mode=reference_mode,
reference_coh_threshold=reference_coh_threshold,
)
disp_m = disp_m_raw - reference_offset_m
disp_m_ref = disp_m_raw - reference_offset_m
deramp_surface_m, deramp_mask, deramp_stats = compute_deramp_surface(
disp_m=disp_m_ref,
amp=amp,
coh=coh,
coh_threshold=coh_threshold,
deramp_mode=deramp_mode,
deramp_coh_threshold=deramp_coh_threshold,
)
disp_m = disp_m_ref - deramp_surface_m
disp_m_full = disp_m.copy()
mask = (~amp_valid) | (~np.isfinite(disp_m)) | (~np.isfinite(coh)) | (coh < coh_threshold)
if ionosphere_mask_valid is not None:
mask |= ~ionosphere_mask_valid
disp_m_raw_masked = disp_m_raw.copy()
disp_m_raw_masked[mask] = nodata
disp_m_ref_masked = disp_m_ref.copy()
disp_m_ref_masked[mask] = nodata
disp_m_masked = disp_m.copy()
disp_m_masked[mask] = nodata
disp_m_full[(~amp_valid) | (~np.isfinite(disp_m_full))] = nodata
@@ -202,8 +384,13 @@ def export_products(
output_dir.mkdir(parents=True, exist_ok=True)
out_disp = output_dir / f"{prefix}_disp.tif"
out_disp_raw = output_dir / f"{prefix}_disp_raw.tif"
out_disp_ref = output_dir / f"{prefix}_disp_ref.tif"
out_coh = output_dir / f"{prefix}_coh.tif"
out_meta = output_dir / f"{prefix}_disp_meta.json"
write_geotiff(disp_m_raw_masked, unw_ds, out_disp_raw, nodata)
write_geotiff(disp_m_ref_masked, unw_ds, out_disp_ref, nodata)
write_geotiff(disp_m_masked, unw_ds, out_disp, nodata)
write_geotiff(coh_out, cor_ds, out_coh, nodata)
out_disp_full = None
@@ -211,14 +398,50 @@ def export_products(
out_disp_full = output_dir / f"{prefix}_disp_full.tif"
write_geotiff(disp_m_full, unw_ds, out_disp_full, nodata)
valid_raw_masked = disp_m_raw_masked[disp_m_raw_masked != nodata]
valid_ref_masked = disp_m_ref_masked[disp_m_ref_masked != nodata]
valid_disp = disp_m_masked[disp_m_masked != nodata]
valid_coh = coh_out[coh_out != nodata]
valid_full = disp_m_full[disp_m_full != nodata] if include_disp_full else np.array([], dtype=np.float32)
valid_raw = disp_m_raw[amp_valid & np.isfinite(disp_m_raw)]
using_reference = str(reference_stats["mode"]) != "none"
using_deramp = bool(deramp_stats["applied"])
meta_payload = {
"work_dir": str(work_dir),
"output_dir": str(output_dir),
"prefix": prefix,
"coh_threshold": float(coh_threshold),
"phase_source": {
"kind": str(phase_source["phase_source"]),
"path": str(phase_path),
"ionosphere_corrected": bool(phase_source["ionosphere_corrected"]),
"mask_path": str(mask_path) if mask_path is not None else "",
"mask_applied": bool(ionosphere_mask_valid is not None),
"mask_valid_ratio": float(ionosphere_mask_valid.mean()) if ionosphere_mask_valid is not None else None,
},
"reference": {
**reference_stats,
"offset_m": float(reference_offset_m),
"support_count": int(reference_mask.sum()),
},
"deramp": {
**deramp_stats,
"support_count": int(deramp_mask.sum()),
},
"ranges_m": {
"raw_valid": [float(valid_raw.min()), float(valid_raw.max())] if valid_raw.size else [],
"raw_masked": [float(valid_raw_masked.min()), float(valid_raw_masked.max())] if valid_raw_masked.size else [],
"ref_masked": [float(valid_ref_masked.min()), float(valid_ref_masked.max())] if valid_ref_masked.size else [],
"final_masked": [float(valid_disp.min()), float(valid_disp.max())] if valid_disp.size else [],
"final_full": [float(valid_full.min()), float(valid_full.max())] if valid_full.size else [],
},
}
out_meta.write_text(json.dumps(meta_payload, indent=2, ensure_ascii=False), encoding="utf-8")
print(f"Work dir: {work_dir}")
print(f"Output prefix: {prefix}")
print(f"Phase source: {phase_source['phase_source']}")
print(f"Coherence threshold: {coh_threshold}")
print(f"Reference mode: {reference_stats['mode']}")
if using_reference:
@@ -233,29 +456,61 @@ def export_products(
print(f"Reference offset: {reference_offset_m:.4f} m")
if reference_stats["fallback"]:
print(f"Reference fallback: {reference_stats['fallback']}")
print(f"Deramp mode: {deramp_stats['mode']}")
if using_deramp:
print(
"Deramp coh floor: "
f"{float(deramp_stats['selection_threshold']):.2f}"
)
print(
"Deramp pixel ratio: "
f"{float(deramp_stats['fit_ratio'])*100:.2f}%"
)
print(
"Deramp plane delta: "
f"dx={float(deramp_stats['left_right_delta_m']):.4f} m, "
f"dy={float(deramp_stats['top_bottom_delta_m']):.4f} m"
)
if deramp_stats["fallback"]:
print(f"Deramp fallback: {deramp_stats['fallback']}")
elif deramp_stats["fallback"]:
print(f"Deramp fallback: {deramp_stats['fallback']}")
print(f"Unwrap support ratio: {amp_valid.mean()*100:.2f}%")
print(f"Coherence support ratio: {coh_valid.mean()*100:.2f}%")
print(f"Masked disp ratio: {(disp_m_masked != nodata).mean()*100:.2f}%")
if valid_raw.size:
print(f"Raw disp range: [{valid_raw.min():.4f}, {valid_raw.max():.4f}] m")
if valid_raw_masked.size:
print(f"Raw masked range: [{valid_raw_masked.min():.4f}, {valid_raw_masked.max():.4f}] m")
if valid_ref_masked.size:
label = "Ref disp range" if using_reference else "Ref disp range"
print(f"{label + ':':24}[{valid_ref_masked.min():.4f}, {valid_ref_masked.max():.4f}] m")
if valid_disp.size:
label = "Norm disp range" if using_reference else "Disp range"
label = "Final disp range" if using_reference or using_deramp else "Disp range"
print(f"{label + ':':24}[{valid_disp.min():.4f}, {valid_disp.max():.4f}] m")
if include_disp_full and valid_full.size:
label = "Norm full disp range" if using_reference else "Full disp range"
label = "Final full disp range" if using_reference or using_deramp else "Full disp range"
print(f"{label + ':':24}[{valid_full.min():.4f}, {valid_full.max():.4f}] m")
if valid_coh.size:
print(f"Coherence range: [{valid_coh.min():.4f}, {valid_coh.max():.4f}]")
print(f"Wrote: {out_disp_raw}")
print(f"Wrote: {out_disp_ref}")
print(f"Wrote: {out_disp}")
if out_disp_full is not None:
print(f"Wrote: {out_disp_full}")
print(f"Wrote: {out_coh}")
print(f"Wrote: {out_meta}")
unw_ds = None
cor_ds = None
phase_ds = None
mask_ds = None
outputs: dict[str, Path] = {
"disp_raw": out_disp_raw,
"disp_ref": out_disp_ref,
"disp": out_disp,
"coh": out_coh,
"meta": out_meta,
}
if out_disp_full is not None:
outputs["disp_full"] = out_disp_full
@@ -276,6 +531,8 @@ def main() -> int:
coh_threshold=args.coh_threshold,
reference_mode=args.reference_mode,
reference_coh_threshold=args.reference_coh_threshold,
deramp_mode=args.deramp_mode,
deramp_coh_threshold=args.deramp_coh_threshold,
include_disp_full=args.include_disp_full,
)
return 0
backend/app/isce2_pipeline/lt1_input_resolver.py
+100 -1
View File
@@ -2,9 +2,11 @@
from __future__ import annotations
import sys
import re
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Iterable, Mapping, Optional
from typing import Any, Callable, Iterable, Mapping, Optional
try:
from .convert_lt1_orbit_to_isce_xml import (
@@ -37,6 +39,7 @@ DEFAULT_WSL_DEM_CANDIDATES = (
"/mnt/d/SRTM30m/SRTMDEM_RSP_SARscape",
)
DEFAULT_WINDOWS_ORBIT_POOL_CANDIDATES = (r"D:\orbit_pools\isce2",)
DEM_SIDECAR_PROPERTY_NAMES = ("file_name", "metadata_location", "extra_file_name")
@dataclass(frozen=True)
@@ -126,6 +129,24 @@ def resolve_prepared_dem_path(
return resolve_existing_prepared_file(candidates, path_transform=path_transform)
def repair_related_dem_sidecars(
dem_path: Path,
*,
write_changes: bool = True,
) -> list[dict[str, Any]]:
reports: list[dict[str, Any]] = []
seen: set[str] = set()
for candidate in _related_dem_sidecar_candidates(dem_path):
key = str(candidate)
if key in seen:
continue
seen.add(key)
report = repair_dem_sidecar_paths(candidate, write_changes=write_changes)
if report.get("exists"):
reports.append(report)
return reports
def resolve_existing_directory(
candidates: Iterable[str | Path],
path_transform: PathTransform = identity_path_transform,
@@ -162,6 +183,63 @@ def resolve_existing_prepared_file(
return None
def repair_dem_sidecar_paths(
dem_path: Path,
*,
write_changes: bool = True,
) -> dict[str, Any]:
normalized_path = Path(str(dem_path))
xml_path = Path(str(normalized_path) + ".xml")
vrt_path = Path(str(normalized_path) + ".vrt")
report: dict[str, Any] = {
"dem_path": str(normalized_path),
"xml_path": str(xml_path),
"vrt_path": str(vrt_path),
"exists": xml_path.exists(),
"changed": False,
"updated_fields": [],
"expected": {},
"current": {},
}
if not xml_path.exists():
return report
expected_values = {
"file_name": _to_isce_sidecar_path(normalized_path),
"metadata_location": _to_isce_sidecar_path(xml_path),
"extra_file_name": _to_isce_sidecar_path(vrt_path) if vrt_path.exists() else "",
}
tree = ET.parse(xml_path)
root = tree.getroot()
updates: list[str] = []
for prop in root.findall("property"):
name = str(prop.get("name") or "").strip()
if name not in DEM_SIDECAR_PROPERTY_NAMES:
continue
value_node = prop.find("value")
if value_node is None:
value_node = ET.SubElement(prop, "value")
current_value = str(value_node.text or "").strip()
expected_value = expected_values.get(name, "")
report["current"][name] = current_value
report["expected"][name] = expected_value
if not expected_value:
continue
if current_value == expected_value:
continue
value_node.text = expected_value
updates.append(name)
if updates and write_changes:
ET.indent(tree, space=" ")
tree.write(xml_path, encoding="utf-8")
report["changed"] = bool(updates)
report["updated_fields"] = updates
return report
def ensure_lt1_orbit_xml(
date_yyyymmdd: str,
satellite: str,
@@ -251,3 +329,24 @@ def _prepared_dem_variants(value: str | Path) -> tuple[str | Path, ...]:
if text.lower().endswith(".wgs84"):
return (value,)
return (f"{text}.wgs84", value)
def _related_dem_sidecar_candidates(dem_path: Path) -> tuple[Path, ...]:
text = str(dem_path).strip()
if not text:
return ()
if text.lower().endswith(".wgs84"):
raw_path = Path(text[:-6])
return (dem_path, raw_path)
prepared_path = Path(text + ".wgs84")
return (dem_path, prepared_path)
def _to_isce_sidecar_path(path: Path) -> str:
text = str(path).strip()
match = re.match(r"^([A-Za-z]):[\\/](.*)$", text)
if match:
drive = match.group(1).lower()
rest = match.group(2).replace("\\", "/")
return f"/mnt/{drive}/{rest}"
return Path(text).as_posix()
backend/app/isce2_pipeline/repair_dem_sidecars.py
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import sys
from pathlib import Path
try:
from .lt1_input_resolver import repair_dem_sidecar_paths
except ImportError:
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from lt1_input_resolver import repair_dem_sidecar_paths # type: ignore
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Audit and optionally repair moved ISCE DEM XML sidecars."
)
parser.add_argument(
"--root",
type=Path,
required=True,
help="Directory containing DEM files and sidecars",
)
parser.add_argument(
"--repair",
action="store_true",
help="Write repaired file_name / metadata_location / extra_file_name values back to XML",
)
return parser.parse_args()
def iter_dem_sidecars(root: Path) -> list[Path]:
sidecars: list[Path] = []
for xml_path in sorted(root.rglob("*.xml")):
if xml_path.name.lower().endswith(".aux.xml"):
continue
dem_path = Path(str(xml_path)[:-4])
if dem_path.exists():
sidecars.append(dem_path)
return sidecars
def main() -> int:
args = parse_args()
root = args.root.resolve()
if not root.exists() or not root.is_dir():
raise FileNotFoundError(f"DEM root directory not found: {root}")
sidecars = iter_dem_sidecars(root)
changed_count = 0
mismatch_count = 0
print(f"DEM root: {root}")
print(f"Sidecars: {len(sidecars)}")
for dem_path in sidecars:
report = repair_dem_sidecar_paths(dem_path, write_changes=bool(args.repair))
updated_fields = list(report.get("updated_fields") or [])
if updated_fields:
changed_count += 1
mismatch_count += 1
print(
f"[fixed] {report['xml_path']} -> {', '.join(updated_fields)}"
if args.repair
else f"[mismatch] {report['xml_path']} -> {', '.join(updated_fields)}"
)
continue
current = report.get("current") or {}
expected = report.get("expected") or {}
mismatched = [
key
for key, expected_value in expected.items()
if expected_value and str(current.get(key) or "").strip() != str(expected_value).strip()
]
if mismatched:
mismatch_count += 1
print(f"[mismatch] {report['xml_path']} -> {', '.join(mismatched)}")
else:
print(f"[ok] {report['xml_path']}")
print(f"Mismatched: {mismatch_count}")
if args.repair:
print(f"Repaired: {changed_count}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
backend/app/isce2_pipeline/run_lt1_dinsar_pipeline.py
+356 -59
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import ast
import importlib.util
import os
import re
import shutil
@@ -14,15 +15,19 @@ from dataclasses import dataclass
from pathlib import Path
from export_isce_geotiff import (
DEFAULT_DERAMP_COH_THRESHOLD,
DEFAULT_DERAMP_MODE,
DEFAULT_REFERENCE_COH_THRESHOLD,
DEFAULT_REFERENCE_MODE,
DEFAULT_WAVELENGTH,
DERAMP_MODE_CHOICES,
REFERENCE_MODE_CHOICES,
export_products,
)
from lt1_input_resolver import (
DEFAULT_WSL_DEM_CANDIDATES,
ensure_lt1_orbit_xml,
repair_related_dem_sidecars,
resolve_prepared_dem_path,
)
@@ -35,7 +40,22 @@ RESUME_STAGE_CHOICES = PIPELINE_STAGE_ORDER[1:]
DEFAULT_EXPORT_GEOCODE_PRODUCTS = [
"interferogram/filt_topophase.unw",
"interferogram/topophase.cor",
"ionosphere/dispersive.bil.unwCor.filt",
"ionosphere/nondispersive.bil.unwCor.filt",
"ionosphere/mask.bil",
]
DEFAULT_EXPORT_GEOCODE_PRODUCTS_NO_IONO = [
"interferogram/filt_topophase.unw",
"interferogram/topophase.cor",
]
DEFAULT_RUBBER_SHEET_SNR_THRESHOLD = 5.0
DEFAULT_RUBBER_SHEET_FILTER_SIZE = 9
DEFAULT_DENSE_WINDOW_WIDTH = 64
DEFAULT_DENSE_WINDOW_HEIGHT = 64
DEFAULT_DENSE_SEARCH_WIDTH = 20
DEFAULT_DENSE_SEARCH_HEIGHT = 20
DEFAULT_DENSE_SKIP_WIDTH = 32
DEFAULT_DENSE_SKIP_HEIGHT = 32
@dataclass
@@ -59,6 +79,18 @@ class PipelineConfig:
target_grid_size_m: int
geo_posting_deg: float
geocode_products: list[str] | None
ionosphere_correction: bool
dense_offsets: bool
rubbersheet_range: bool
rubbersheet_azimuth: bool
rubber_sheet_snr_threshold: float
rubber_sheet_filter_size: int
dense_window_width: int
dense_window_height: int
dense_search_width: int
dense_search_height: int
dense_skip_width: int
dense_skip_height: int
def parse_args() -> argparse.Namespace:
@@ -66,7 +98,7 @@ def parse_args() -> argparse.Namespace:
repo_root = script_dir.parent
parser = argparse.ArgumentParser(
description="Run an LT-1 ISCE2 DInSAR production pipeline with SNAPHU."
description="Run an LT-1 ISCE2 DInSAR production pipeline with the standard stripmap workflow."
)
parser.add_argument("task_dir", help="Task directory, for example Task_20250112_20250309")
parser.add_argument(
@@ -156,7 +188,7 @@ def parse_args() -> argparse.Namespace:
"--reference-mode",
choices=REFERENCE_MODE_CHOICES,
default=DEFAULT_REFERENCE_MODE,
help="Optional reference normalization mode used only for debug exports",
help="Reference normalization mode applied during final displacement export",
)
parser.add_argument(
"--reference-coh-threshold",
@@ -164,6 +196,18 @@ def parse_args() -> argparse.Namespace:
default=DEFAULT_REFERENCE_COH_THRESHOLD,
help="Minimum coherence used when selecting reference pixels for export normalization",
)
parser.add_argument(
"--deramp-mode",
choices=DERAMP_MODE_CHOICES,
default=DEFAULT_DERAMP_MODE,
help="Optional ramp-removal mode applied after reference normalization",
)
parser.add_argument(
"--deramp-coh-threshold",
type=float,
default=DEFAULT_DERAMP_COH_THRESHOLD,
help="Minimum coherence used when selecting pixels for deramp fitting",
)
parser.add_argument(
"--target-grid-size-m",
type=int,
@@ -180,6 +224,76 @@ def parse_args() -> argparse.Namespace:
action="store_true",
help="Let ISCE2 geocode its full default product list instead of the reduced export-only list.",
)
parser.add_argument(
"--no-ionosphere-correction",
action="store_false",
dest="ionosphere_correction",
help="Disable split-spectrum dispersive correction and export the standard unwrapped interferogram.",
)
parser.set_defaults(ionosphere_correction=True)
parser.add_argument(
"--dense-offsets",
action="store_true",
help="Enable ISCE2 dense offset estimation before fine resampling.",
)
parser.add_argument(
"--rubbersheet-range",
action="store_true",
help="Enable ISCE2 range rubbersheeting using dense offsets.",
)
parser.add_argument(
"--rubbersheet-azimuth",
action="store_true",
help="Enable ISCE2 azimuth rubbersheeting using dense offsets.",
)
parser.add_argument(
"--rubber-sheet-snr-threshold",
type=float,
default=DEFAULT_RUBBER_SHEET_SNR_THRESHOLD,
help="SNR threshold used by ISCE2 rubbersheet offset masking.",
)
parser.add_argument(
"--rubber-sheet-filter-size",
type=int,
default=DEFAULT_RUBBER_SHEET_FILTER_SIZE,
help="Median filter size used by ISCE2 rubbersheet offset masking.",
)
parser.add_argument(
"--dense-window-width",
type=int,
default=DEFAULT_DENSE_WINDOW_WIDTH,
help="Dense offset correlation window width.",
)
parser.add_argument(
"--dense-window-height",
type=int,
default=DEFAULT_DENSE_WINDOW_HEIGHT,
help="Dense offset correlation window height.",
)
parser.add_argument(
"--dense-search-width",
type=int,
default=DEFAULT_DENSE_SEARCH_WIDTH,
help="Dense offset search window width.",
)
parser.add_argument(
"--dense-search-height",
type=int,
default=DEFAULT_DENSE_SEARCH_HEIGHT,
help="Dense offset search window height.",
)
parser.add_argument(
"--dense-skip-width",
type=int,
default=DEFAULT_DENSE_SKIP_WIDTH,
help="Dense offset sampling stride in range direction.",
)
parser.add_argument(
"--dense-skip-height",
type=int,
default=DEFAULT_DENSE_SKIP_HEIGHT,
help="Dense offset sampling stride in azimuth direction.",
)
parser.add_argument(
"--resume-from",
choices=RESUME_STAGE_CHOICES,
@@ -219,11 +333,75 @@ def parse_args() -> argparse.Namespace:
raise ValueError("--target-grid-size-m must be greater than 0")
if args.reference_coh_threshold < 0 or args.reference_coh_threshold > 1:
raise ValueError("--reference-coh-threshold must be between 0 and 1")
if args.deramp_coh_threshold < 0 or args.deramp_coh_threshold > 1:
raise ValueError("--deramp-coh-threshold must be between 0 and 1")
if args.rubber_sheet_snr_threshold < 0:
raise ValueError("--rubber-sheet-snr-threshold must be non-negative")
if args.rubber_sheet_filter_size <= 0:
raise ValueError("--rubber-sheet-filter-size must be greater than 0")
for field_name in (
"dense_window_width",
"dense_window_height",
"dense_search_width",
"dense_search_height",
"dense_skip_width",
"dense_skip_height",
):
if int(getattr(args, field_name)) <= 0:
raise ValueError(f"--{field_name.replace('_', '-')} must be greater than 0")
if args.force and args.resume_from:
raise ValueError("--force cannot be used together with --resume-from")
return args
def _find_python_module(module_name: str) -> bool:
try:
return importlib.util.find_spec(module_name) is not None
except ModuleNotFoundError:
return False
def validate_runtime_dependencies(args: argparse.Namespace) -> str:
errors: list[str] = []
env_for_cli = build_process_env()
ionosphere_correction = bool(getattr(args, "ionosphere_correction", True))
if ionosphere_correction:
missing_ionosphere_modules: list[str] = []
if not _find_python_module("cv2"):
missing_ionosphere_modules.append("cv2")
if not _find_python_module("scipy"):
missing_ionosphere_modules.append("scipy")
if missing_ionosphere_modules:
errors.append(
"Missing Python dependencies for the ISCE2 stripmap ionosphere step: "
+ ", ".join(missing_ionosphere_modules)
+ ". The managed LT-1 workflow enables split-spectrum dispersive correction "
"before geocode. Install the missing packages in the WSL runtime, for example: "
"conda install -n insar_wsl_v1 -c conda-forge opencv scipy."
)
if (args.rubbersheet_range or args.rubbersheet_azimuth) and not _find_python_module(
"astropy.convolution"
):
errors.append(
"Missing Python dependency 'astropy.convolution'. "
"ISCE2 stripmap rubbersheeting imports astropy.convolution in "
"runRubbersheetRange.py. Install astropy in the WSL runtime, for example: "
"conda install -n insar_wsl_v1 -c conda-forge astropy."
)
if ionosphere_correction and not shutil.which("imageMath.py", path=str(env_for_cli.get("PATH") or "")):
errors.append(
"Missing CLI dependency 'imageMath.py' on PATH. "
"ISCE2 stripmap shells out to imageMath.py in the ionosphere step, so a missing PATH entry "
"will only surface late in the run. Export the active conda env bin directory into PATH "
"before launching production."
)
return "\n".join(errors)
def locate_stripmap_app() -> Path:
import isce
@@ -233,6 +411,29 @@ def locate_stripmap_app() -> Path:
return app_path
def locate_isce_applications_dir() -> Path | None:
spec = importlib.util.find_spec("isce")
if not spec or not spec.origin:
return None
app_dir = Path(spec.origin).resolve().parent / "applications"
if app_dir.exists():
return app_dir
return None
def build_process_env(base_env: dict[str, str] | None = None) -> dict[str, str]:
env = dict(base_env or os.environ.copy())
path_prefixes = [Path(sys.executable).resolve().parent.as_posix()]
app_dir = locate_isce_applications_dir()
if app_dir:
path_prefixes.append(app_dir.as_posix())
current_path = str(env.get("PATH") or "")
env["PATH"] = ":".join(path_prefixes + ([current_path] if current_path else []))
return env
def normalize_linux_path(value: str | Path) -> Path:
text = str(value).strip()
if text.startswith("\\\\"):
@@ -374,6 +575,14 @@ def resolve_dem(dem_value: str | None) -> Path:
path_transform=normalize_linux_path,
)
if dem_path is not None:
repair_reports = repair_related_dem_sidecars(dem_path)
for report in repair_reports:
if not report.get("changed"):
continue
print(
"Repaired DEM sidecar paths: "
f"{report['xml_path']} -> {', '.join(report['updated_fields'])}"
)
return dem_path
searched = ", ".join(str(path) for path in DEFAULT_WSL_DEM_CANDIDATES)
@@ -502,9 +711,30 @@ def meters_to_geoposting_degrees(target_grid_size_m: int) -> float:
return float(target_grid_size_m) / METERS_PER_DEGREE
def build_default_geocode_products(*, ionosphere_correction: bool) -> list[str]:
return list(
DEFAULT_EXPORT_GEOCODE_PRODUCTS
if ionosphere_correction
else DEFAULT_EXPORT_GEOCODE_PRODUCTS_NO_IONO
)
def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None:
bbox_xml = render_bbox(config.bbox)
geocode_list_xml = render_string_list("geocode list", config.geocode_products)
enhancement_props = (
f" <property name=\"do denseoffsets\">{str(config.dense_offsets)}</property>\n"
f" <property name=\"do rubbersheetingRange\">{str(config.rubbersheet_range)}</property>\n"
f" <property name=\"do rubbersheetingAzimuth\">{str(config.rubbersheet_azimuth)}</property>\n"
f" <property name=\"rubber sheet SNR Threshold\">{config.rubber_sheet_snr_threshold}</property>\n"
f" <property name=\"rubber sheet filter size\">{config.rubber_sheet_filter_size}</property>\n"
f" <property name=\"dense window width\">{config.dense_window_width}</property>\n"
f" <property name=\"dense window height\">{config.dense_window_height}</property>\n"
f" <property name=\"dense search width\">{config.dense_search_width}</property>\n"
f" <property name=\"dense search height\">{config.dense_search_height}</property>\n"
f" <property name=\"dense skip width\">{config.dense_skip_width}</property>\n"
f" <property name=\"dense skip height\">{config.dense_skip_height}</property>\n"
)
text = (
"<stripmapApp>\n"
" <component name=\"stripmapApp\">\n"
@@ -514,10 +744,13 @@ def write_stripmap_xml(xml_path: Path, config: PipelineConfig) -> None:
" <property name=\"renderer\">xml</property>\n"
" <property name=\"do unwrap\">True</property>\n"
" <property name=\"unwrapper name\">snaphu</property>\n"
f" <property name=\"do split spectrum\">{str(config.ionosphere_correction)}</property>\n"
f" <property name=\"do dispersive\">{str(config.ionosphere_correction)}</property>\n"
f" <property name=\"posting\">{config.target_grid_size_m}</property>\n"
f" <property name=\"geoPosting\">{config.geo_posting_deg:.12f}</property>\n"
f"{bbox_xml}"
f"{geocode_list_xml}"
f"{enhancement_props}"
f" <property name=\"demFilename\">{config.dem_path.as_posix()}</property>\n"
"\n"
" <component name=\"Reference\">\n"
@@ -553,7 +786,7 @@ def run_logged(stage_name: str, cmd: list[str], cwd: Path, log_path: Path) -> No
handle.write(f"Log: {log_path}\n")
handle.flush()
child_env = os.environ.copy()
child_env = build_process_env()
child_env["PYTHONUNBUFFERED"] = "1"
proc = subprocess.Popen(
cmd,
@@ -730,59 +963,57 @@ def should_run_stage(start_stage: str, stage_name: str) -> bool:
return stage_index >= start_index
def prepare_snaphu_resume(work_dir: Path, bbox: list[float] | None) -> None:
def has_pickle_state(work_dir: Path, state_name: str) -> bool:
pickle_dir = work_dir / "PICKLE"
src = pickle_dir / "filter"
src_xml = pickle_dir / "filter.xml"
dst = pickle_dir / "filter_high_band"
dst_xml = pickle_dir / "filter_high_band.xml"
if not src.exists() or not src_xml.exists():
raise FileNotFoundError("filter step output is missing; cannot prepare SNAPHU resume state.")
shutil.copy2(src, dst)
shutil.copy2(src_xml, dst_xml)
root = ET.fromstring(dst_xml.read_text(encoding="utf-8"))
props = {prop.attrib.get("name"): prop for prop in root.findall("property")}
required = {
"referenceslccroppedproduct": "reference_slc.xml",
"secondaryslccroppedproduct": "secondary_slc.xml",
"referenceslcproduct": "reference_slc.xml",
"secondaryslcproduct": "secondary_slc.xml",
"referencegeometrysystem": "Zero Doppler",
"secondarygeometrysystem": "Zero Doppler",
}
if bbox is not None:
required["estimatedboundingbox"] = str(bbox)
for name, value in required.items():
if name in props:
node = props[name].find("value")
if node is None:
node = ET.SubElement(props[name], "value")
node.text = value
continue
prop = ET.SubElement(root, "property", {"name": name})
ET.SubElement(prop, "value").text = value
dst_xml.write_text(ET.tostring(root, encoding="unicode"), encoding="utf-8")
return (pickle_dir / state_name).exists() and (pickle_dir / f"{state_name}.xml").exists()
def prepare_geocode_resume(work_dir: Path) -> None:
pickle_dir = work_dir / "PICKLE"
unwrap = pickle_dir / "unwrap"
unwrap_xml = pickle_dir / "unwrap.xml"
ionosphere = pickle_dir / "ionosphere"
ionosphere_xml = pickle_dir / "ionosphere.xml"
def resolve_unwrap_start_step(work_dir: Path, *, ionosphere_correction: bool) -> str:
if ionosphere_correction:
if has_pickle_state(work_dir, "ionosphere"):
return "ionosphere"
if has_pickle_state(work_dir, "unwrap_low_band") and has_pickle_state(
work_dir, "unwrap_high_band"
):
return "ionosphere"
if has_pickle_state(work_dir, "filter_low_band") and has_pickle_state(
work_dir, "filter_high_band"
):
return "unwrap"
if has_pickle_state(work_dir, "filter"):
return "filter_low_band"
raise FileNotFoundError(
"Unable to resume the ISCE2 unwrap/ionosphere stage. Missing PICKLE state for "
"filter, filter_low_band/filter_high_band, unwrap_low_band/unwrap_high_band, or ionosphere."
)
if not unwrap.exists() or not unwrap_xml.exists():
raise FileNotFoundError("unwrap step output is missing; cannot prepare geocode resume state.")
if has_pickle_state(work_dir, "unwrap"):
return "unwrap"
if has_pickle_state(work_dir, "filter"):
return "unwrap"
raise FileNotFoundError(
"Unable to resume the ISCE2 unwrap stage. Missing PICKLE state for filter or unwrap."
)
shutil.copy2(unwrap, ionosphere)
shutil.copy2(unwrap_xml, ionosphere_xml)
def resolve_geocode_start_step(work_dir: Path, *, ionosphere_correction: bool) -> str:
if ionosphere_correction:
if has_pickle_state(work_dir, "ionosphere"):
return "geocode"
if has_pickle_state(work_dir, "unwrap_low_band") and has_pickle_state(
work_dir, "unwrap_high_band"
):
return "ionosphere"
raise FileNotFoundError(
"Unable to resume the ISCE2 geocode stage. Missing PICKLE state for ionosphere or "
"unwrap_low_band/unwrap_high_band."
)
if has_pickle_state(work_dir, "unwrap"):
return "geocode"
raise FileNotFoundError(
"Unable to resume the ISCE2 geocode stage. Missing PICKLE state for unwrap."
)
def print_summary(
@@ -804,6 +1035,25 @@ def print_summary(
print(f"BBox: {config.bbox if config.bbox is not None else 'auto'}")
print(f"Target grid: {config.target_grid_size_m} m")
print(f"Geo posting: {config.geo_posting_deg:.12f} deg")
print(
"Enhancement: "
f"split_spectrum={config.ionosphere_correction}, "
f"ionosphere={config.ionosphere_correction}, "
f"dense_offsets={config.dense_offsets}, "
f"rubbersheet_range={config.rubbersheet_range}, "
f"rubbersheet_azimuth={config.rubbersheet_azimuth}"
)
print(
"Dense params: "
f"window={config.dense_window_width}x{config.dense_window_height}, "
f"search={config.dense_search_width}x{config.dense_search_height}, "
f"skip={config.dense_skip_width}x{config.dense_skip_height}"
)
print(
"Rubber mask: "
f"snr_threshold={config.rubber_sheet_snr_threshold}, "
f"filter_size={config.rubber_sheet_filter_size}"
)
print(
"Geocode list: "
+ (
@@ -816,6 +1066,11 @@ def print_summary(
def main() -> int:
args = parse_args()
if not args.dry_run:
dependency_error = validate_runtime_dependencies(args)
if dependency_error:
print(dependency_error, file=sys.stderr)
return 2
resume_from = str(args.resume_from or "").strip().lower()
start_stage = resume_from or PIPELINE_STAGE_ORDER[0]
task_dir = normalize_linux_path(args.task_dir).resolve()
@@ -873,12 +1128,30 @@ def main() -> int:
bbox=bbox,
target_grid_size_m=args.target_grid_size_m,
geo_posting_deg=geo_posting_deg,
geocode_products=None if args.full_geocode else list(DEFAULT_EXPORT_GEOCODE_PRODUCTS),
geocode_products=(
None
if args.full_geocode
else build_default_geocode_products(
ionosphere_correction=bool(args.ionosphere_correction)
)
),
ionosphere_correction=bool(args.ionosphere_correction),
dense_offsets=bool(args.dense_offsets),
rubbersheet_range=bool(args.rubbersheet_range),
rubbersheet_azimuth=bool(args.rubbersheet_azimuth),
rubber_sheet_snr_threshold=float(args.rubber_sheet_snr_threshold),
rubber_sheet_filter_size=int(args.rubber_sheet_filter_size),
dense_window_width=int(args.dense_window_width),
dense_window_height=int(args.dense_window_height),
dense_search_width=int(args.dense_search_width),
dense_search_height=int(args.dense_search_height),
dense_skip_width=int(args.dense_skip_width),
dense_skip_height=int(args.dense_skip_height),
)
if start_stage == PIPELINE_STAGE_ORDER[0]:
guard_large_unprepared_base_dem(config.dem_path)
if resume_from in {"unwrap", "geocode", "export"}:
if resume_from in {"unwrap", "geocode"}:
ensure_geocode_bbox(work_dir, config, args.bbox_margin)
if should_run_stage(start_stage, "geocode"):
prepare_geocode_dem(work_dir, config)
@@ -908,20 +1181,42 @@ def main() -> int:
write_stripmap_xml(xml_path, config)
if should_run_stage(start_stage, "unwrap"):
prepare_snaphu_resume(work_dir, config.bbox)
unwrap_start_step = resolve_unwrap_start_step(
work_dir,
ionosphere_correction=config.ionosphere_correction,
)
unwrap_end_step = "ionosphere" if config.ionosphere_correction else "unwrap"
unwrap_stage_name = "02_to_ionosphere" if config.ionosphere_correction else "02_to_unwrap"
run_logged(
"02_unwrap_snaphu",
[sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=unwrap", "--end=unwrap"],
unwrap_stage_name,
[
sys.executable,
app_py.as_posix(),
xml_path.as_posix(),
"--steps",
f"--start={unwrap_start_step}",
f"--end={unwrap_end_step}",
],
cwd=work_dir,
log_path=work_dir / "02_unwrap_snaphu.log",
log_path=work_dir / f"{unwrap_stage_name}.log",
)
if should_run_stage(start_stage, "geocode"):
prepare_geocode_resume(work_dir)
geocode_start_step = resolve_geocode_start_step(
work_dir,
ionosphere_correction=config.ionosphere_correction,
)
cleanup_geocode_outputs(work_dir, config.geocode_products)
run_logged(
"03_geocode",
[sys.executable, app_py.as_posix(), xml_path.as_posix(), "--steps", "--start=geocode", "--end=geocode"],
[
sys.executable,
app_py.as_posix(),
xml_path.as_posix(),
"--steps",
f"--start={geocode_start_step}",
"--end=geocode",
],
cwd=work_dir,
log_path=work_dir / "03_geocode.log",
)
@@ -936,6 +1231,8 @@ def main() -> int:
coh_threshold=args.coh_threshold,
reference_mode=args.reference_mode,
reference_coh_threshold=args.reference_coh_threshold,
deramp_mode=args.deramp_mode,
deramp_coh_threshold=args.deramp_coh_threshold,
include_disp_full=args.include_disp_full,
)
backend/app/models/__init__.py
+7 -1
View File
@@ -15,6 +15,8 @@ from .orm import (
PairingMetricCacheORM,
PairingNetworkRunORM,
PairingNetworkEdgeORM,
TimeseriesStackPlanORM,
TimeseriesStackPlanItemORM,
HazardPointORM,
SystemTaskORM,
TaskLogORM,
@@ -59,6 +61,9 @@ from .schemas import (
RadarPair,
PairingResponse,
PsRequest,
TimeseriesStackPlan,
TimeseriesStackPlanItem,
TimeseriesStackPlanDetail,
TaskInfo,
AuthUserInfo,
AuthAuditLogInfo,
@@ -84,6 +89,7 @@ __all__ = [
"ResultIssueORM", "ResultCatalogStateORM",
"PairingCacheStateORM", "PairingDirtySceneORM", "PairingMetricCacheORM",
"PairingNetworkRunORM", "PairingNetworkEdgeORM",
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM",
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
@@ -99,7 +105,7 @@ __all__ = [
"HazardPoint", "DinsarResult", "ScanRequest", "ManagedRootInfo", "ScanCursorInfo",
"RadarData", "RadarDataPage", "DinsarResultPage",
"PairingRequest", "RadarPair", "PairingResponse",
"PsRequest", "TaskInfo",
"PsRequest", "TimeseriesStackPlan", "TimeseriesStackPlanItem", "TimeseriesStackPlanDetail", "TaskInfo",
"AuthUserInfo", "AuthAuditLogInfo", "RadarPreviewStatusInfo",
"DinsarTaskBatch", "DinsarTaskItem", "PsTaskBatch", "PsTaskItem", "PsTimeseriesRun",
"WaterDetectRequest", "WaterDetectResponse",
backend/app/models/orm.py
+71
View File
@@ -442,6 +442,72 @@ class PairingNetworkEdgeORM(Base):
)
class TimeseriesStackPlanORM(Base):
__tablename__ = "timeseries_stack_plans"
id = Column(Integer, primary_key=True, autoincrement=True)
plan_id = Column(String(64), unique=True, index=True, nullable=False)
strategy = Column(String(32), index=True, nullable=False, default="sbas_stack")
request_hash = Column(String(64), index=True, nullable=True)
request_params_json = Column(JSON, nullable=True)
aoi_source = Column(String(32), nullable=True)
aoi_hash = Column(String(64), index=True, nullable=True)
aoi_summary_json = Column(JSON, nullable=True)
direction = Column(String(32), index=True, nullable=True)
scene_count = Column(Integer, nullable=False, default=0)
stack_key = Column(String(128), index=True, nullable=True)
group_key = Column(String(128), index=True, nullable=True)
status = Column(String(16), index=True, nullable=False, default="READY")
created_by = Column(String(64), nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
items = relationship(
"TimeseriesStackPlanItemORM",
back_populates="plan",
cascade="all, delete-orphan",
)
__table_args__ = (
Index("idx_timeseries_stack_plans_direction_created", "direction", "created_at"),
)
class TimeseriesStackPlanItemORM(Base):
__tablename__ = "timeseries_stack_plan_items"
id = Column(Integer, primary_key=True, autoincrement=True)
plan_ref_id = Column(
Integer,
ForeignKey("timeseries_stack_plans.id", ondelete="CASCADE"),
index=True,
nullable=False,
)
radar_data_ref_id = Column(
Integer,
ForeignKey("radar_data.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
scene_rank = Column(Integer, nullable=False, default=0)
file_path = Column(String, nullable=False)
satellite = Column(String, nullable=True)
imaging_date = Column(String, nullable=True)
imaging_mode = Column(String, nullable=True)
polarization = Column(String, nullable=True)
has_orbit_data = Column(Boolean, nullable=False, default=False)
selection_meta_json = Column(JSON, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
plan = relationship("TimeseriesStackPlanORM", back_populates="items")
radar_data = relationship("RadarDataORM")
__table_args__ = (
UniqueConstraint("plan_ref_id", "scene_rank", name="uq_timeseries_plan_items_plan_rank"),
Index("idx_timeseries_plan_items_plan_date", "plan_ref_id", "imaging_date"),
)
class HazardPointORM(Base):
__tablename__ = 'hazard_points'
@@ -901,6 +967,8 @@ class PsTaskBatchORM(Base):
batch_id = Column(String, unique=True, index=True, nullable=False)
name = Column(String, nullable=True)
direction = Column(String, nullable=True)
plan_id = Column(String(64), index=True, nullable=True)
plan_strategy = Column(String(32), nullable=True)
status = Column(String, index=True, nullable=False, default="PENDING")
total_items = Column(Integer, default=0)
completed_items = Column(Integer, default=0)
@@ -915,6 +983,7 @@ class PsTaskItemORM(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
batch_id = Column(String, ForeignKey("ps_task_batches.batch_id"), index=True, nullable=False)
plan_item_ref_id = Column(Integer, index=True, nullable=True)
file_path = Column(String, nullable=False)
satellite = Column(String, nullable=True)
@@ -937,6 +1006,8 @@ class PsTimeseriesRunORM(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
run_id = Column(String(64), unique=True, index=True, nullable=False)
batch_id = Column(String, ForeignKey("ps_task_batches.batch_id"), index=True, nullable=False)
plan_id = Column(String(64), index=True, nullable=True)
plan_strategy = Column(String(32), nullable=True)
product_family = Column(String(32), index=True, nullable=True)
run_name = Column(String(255), nullable=False)
backend/app/models/schemas.py
+58 -2
View File
@@ -194,6 +194,15 @@ class RadarData(BaseModel):
preview_cache_version: Optional[str] = None
preview_cache_updated_at: Optional[datetime] = None
preview_cache_error: Optional[str] = None
stack_plan_id: Optional[str] = None
stack_plan_item_id: Optional[int] = None
stack_scene_rank: Optional[int] = None
stack_group_key: Optional[str] = None
stack_key: Optional[str] = None
stack_common_aoi_coverage_ratio: Optional[float] = None
stack_coverage_consistency_ratio: Optional[float] = None
stack_threshold_satisfied: Optional[bool] = None
stack_selection_mode: Optional[str] = None
model_config = ConfigDict(from_attributes=True)
@@ -342,8 +351,50 @@ class PairingResponse(BaseModel):
class PsRequest(BaseModel):
"""PS-InSAR 时序分析数据准备的请求模型。"""
initial_overlap_threshold: float = 0.3
final_overlap_threshold: float = 0.95
initial_overlap_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
final_overlap_threshold: float = Field(default=0.95, ge=0.0, le=1.0)
class TimeseriesStackPlanItem(BaseModel):
id: int
plan_ref_id: int
radar_data_ref_id: Optional[int] = None
scene_rank: int
file_path: str
satellite: Optional[str] = None
imaging_date: Optional[str] = None
imaging_mode: Optional[str] = None
polarization: Optional[str] = None
has_orbit_data: bool
selection_meta_json: Optional[Dict[str, Any]] = None
created_at: datetime
model_config = ConfigDict(from_attributes=True)
class TimeseriesStackPlan(BaseModel):
id: int
plan_id: str
strategy: str
request_hash: Optional[str] = None
request_params_json: Optional[Dict[str, Any]] = None
aoi_source: Optional[str] = None
aoi_hash: Optional[str] = None
aoi_summary_json: Optional[Dict[str, Any]] = None
direction: Optional[str] = None
scene_count: int
stack_key: Optional[str] = None
group_key: Optional[str] = None
status: str
created_by: Optional[str] = None
created_at: datetime
updated_at: Optional[datetime] = None
model_config = ConfigDict(from_attributes=True)
class TimeseriesStackPlanDetail(TimeseriesStackPlan):
items: List[TimeseriesStackPlanItem] = Field(default_factory=list)
class TaskInfo(BaseModel):
@@ -446,6 +497,8 @@ class PsTaskBatch(BaseModel):
batch_id: str
name: Optional[str] = None
direction: Optional[str] = None
plan_id: Optional[str] = None
plan_strategy: Optional[str] = None
status: str
total_items: int
completed_items: int
@@ -458,6 +511,7 @@ class PsTaskBatch(BaseModel):
class PsTaskItem(BaseModel):
id: int
batch_id: str
plan_item_ref_id: Optional[int] = None
file_path: str
satellite: Optional[str] = None
imaging_date: Optional[str] = None
@@ -474,6 +528,8 @@ class PsTaskItem(BaseModel):
class PsTimeseriesRun(BaseModel):
run_id: str
batch_id: str
plan_id: Optional[str] = None
plan_strategy: Optional[str] = None
product_family: Optional[str] = None
run_name: str
catalog_name: str
backend/app/routers/dependencies.py
+26 -3
View File
@@ -158,6 +158,7 @@ AOI_UPLOAD_MAX_TOTAL_BYTES = max(
AOI_UPLOAD_MAX_SINGLE_FILE_BYTES,
)
AOI_UPLOAD_STREAM_CHUNK_BYTES = 1024 * 1024
_SHAPEFILE_READ_LOCK = asyncio.Lock()
# ---------------------------------------------------------------------------
# Region index caches
@@ -805,6 +806,20 @@ def _parse_aoi_geojson_form_value(aoi_geojson: Optional[str]) -> Optional[Tuple[
return merged_geometry.wkt, feature_collection
def _read_aoi_shapefile_with_restore_shx(shp_path: str):
import geopandas as gpd
previous_restore_shx = os.environ.get("SHAPE_RESTORE_SHX")
os.environ["SHAPE_RESTORE_SHX"] = "YES"
try:
return gpd.read_file(shp_path, engine="pyogrio")
finally:
if previous_restore_shx is None:
os.environ.pop("SHAPE_RESTORE_SHX", None)
else:
os.environ["SHAPE_RESTORE_SHX"] = previous_restore_shx
async def _parse_aoi_from_files(files: Optional[List[UploadFile]]) -> Optional[Tuple[str, Dict[str, Any]]]:
if not files:
return None
@@ -869,9 +884,17 @@ async def _parse_aoi_from_files(files: Optional[List[UploadFile]]) -> Optional[T
geojson_payload = json.loads(Path(dest_path).read_text(encoding="gbk"))
if shp_path:
import geopandas as gpd
gdf = await asyncio.to_thread(gpd.read_file, shp_path, engine="pyogrio")
try:
async with _SHAPEFILE_READ_LOCK:
gdf = await asyncio.to_thread(_read_aoi_shapefile_with_restore_shx, shp_path)
except Exception as exc:
raise HTTPException(
status_code=400,
detail=(
"AOI Shapefile 读取失败。系统已尝试自动恢复缺失的 .shx 索引；"
f"请确认已上传 .shp/.dbf/.prj/.shx 或可恢复的标准 Shapefile。原始错误: {exc}"
),
) from exc
if gdf.crs and gdf.crs.to_epsg() != 4326:
gdf = gdf.to_crs(epsg=4326)
feature_collection = json.loads(gdf.to_json())
backend/app/routers/pairing.py
+36
View File
@@ -16,6 +16,11 @@ from ..models import (
PairingResponse,
PsRequest,
RadarData,
TimeseriesStackPlan,
TimeseriesStackPlanDetail,
TimeseriesStackPlanItem,
TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM,
)
from ..services.pairing_cache_service import pairing_cache_service
from ..services.spatial_service import spatial_service
@@ -177,6 +182,37 @@ async def get_pairing_network_run_endpoint(
}
@router.get("/timeseries-plans/{plan_id}", response_model=TimeseriesStackPlanDetail)
async def get_timeseries_stack_plan_endpoint(
plan_id: str,
db: AsyncSession = Depends(get_db),
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
normalized_plan_id = str(plan_id or "").strip()
if not normalized_plan_id:
raise HTTPException(status_code=400, detail="plan_id is required.")
plan_result = await db.execute(
select(TimeseriesStackPlanORM).where(TimeseriesStackPlanORM.plan_id == normalized_plan_id)
)
plan = plan_result.scalar_one_or_none()
if plan is None:
raise HTTPException(status_code=404, detail="Timeseries stack plan not found.")
items_result = await db.execute(
select(TimeseriesStackPlanItemORM)
.where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id)
.order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
)
payload = TimeseriesStackPlan.model_validate(plan).model_dump()
payload["items"] = [
TimeseriesStackPlanItem.model_validate(item)
for item in items_result.scalars().all()
]
return TimeseriesStackPlanDetail.model_validate(payload)
@router.post("/find-pairs", response_model=PairingResponse)
async def find_pairs_endpoint(
params: PairingRequest = Depends(get_pairing_request_from_form),
backend/app/routers/task_batches.py
+176 -2
View File
@@ -1,8 +1,10 @@
from __future__ import annotations
import json
import os
import uuid
from datetime import datetime
from typing import List, Optional
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field, field_validator
@@ -23,6 +25,8 @@ from ..models import (
PsTaskItemORM,
RadarData,
RadarPair,
TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM,
)
from .dependencies import (
_add_operation_audit_log,
@@ -109,7 +113,9 @@ class DinsarBatchCreateRequest(BaseModel):
class PsBatchCreateRequest(BaseModel):
name: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
plan_id: Optional[str] = Field(default=None, max_length=64)
stack: List[RadarData]
planning_context: Optional[Dict[str, Any]] = None
@field_validator("stack")
@classmethod
@@ -118,6 +124,8 @@ class PsBatchCreateRequest(BaseModel):
raise ValueError(
f"stack exceeds max item count ({TASK_BATCH_MAX_ITEMS})."
)
if len(value) < 3:
raise ValueError("SBAS timeseries batch requires at least 3 scenes.")
return value
@@ -126,6 +134,57 @@ class BatchItemUpdateRequest(BaseModel):
remark: Optional[str] = Field(default=None, max_length=BATCH_REMARK_MAX_LENGTH)
def _normalize_lookup_key(value: Optional[str]) -> str:
text = str(value or "").strip()
if not text:
return ""
return os.path.normcase(os.path.normpath(text))
def _build_plan_context(
plan: TimeseriesStackPlanORM,
plan_items: List[TimeseriesStackPlanItemORM],
) -> Dict[str, Any]:
request_params = plan.request_params_json if isinstance(plan.request_params_json, dict) else {}
ordered_items = sorted(
plan_items,
key=lambda item: (int(item.scene_rank or 0), int(item.id or 0)),
)
scenes = [
{
"plan_item_id": item.id,
"scene_id": item.radar_data_ref_id,
"scene_rank": item.scene_rank,
"scene_file_path": item.file_path,
"scene_imaging_date": item.imaging_date,
"scene_satellite": item.satellite,
"scene_imaging_mode": item.imaging_mode,
"scene_polarization": item.polarization,
"selection_meta": item.selection_meta_json if isinstance(item.selection_meta_json, dict) else None,
}
for item in ordered_items
]
return {
"source": "timeseries_stack_plan",
"plan_id": plan.plan_id,
"strategy": plan.strategy,
"direction": plan.direction,
"scene_count": int(plan.scene_count or len(scenes)),
"stack_key": plan.stack_key,
"group_key": plan.group_key,
"request_hash": plan.request_hash,
"aoi_summary": plan.aoi_summary_json if isinstance(plan.aoi_summary_json, dict) else None,
"initial_overlap_threshold": request_params.get("initial_overlap_threshold"),
"final_overlap_threshold": request_params.get("final_overlap_threshold"),
"stack_dates": [
str(item.imaging_date).strip()
for item in ordered_items
if str(item.imaging_date or "").strip()
],
"scenes": scenes,
}
@router.post("/task-batches/dinsar", response_model=DinsarTaskBatch)
async def create_dinsar_batch_endpoint(
request: DinsarBatchCreateRequest,
@@ -303,12 +362,85 @@ async def create_ps_batch_endpoint(
if not request.stack:
raise HTTPException(status_code=400, detail="No PS items provided.")
request_plan_id = (
request.planning_context.get("plan_id")
if isinstance(request.planning_context, dict)
else None
)
explicit_plan_id = str(request.plan_id or request_plan_id or "").strip() or None
inferred_plan_ids = sorted(
{
str(item.stack_plan_id or "").strip()
for item in request.stack
if str(item.stack_plan_id or "").strip()
}
)
if len(inferred_plan_ids) > 1:
raise HTTPException(status_code=400, detail="PS stack items belong to multiple stack plans.")
if explicit_plan_id and inferred_plan_ids and explicit_plan_id != inferred_plan_ids[0]:
raise HTTPException(status_code=400, detail="request.plan_id does not match stack scene plan metadata.")
effective_plan_id = explicit_plan_id or (inferred_plan_ids[0] if inferred_plan_ids else None)
plan: Optional[TimeseriesStackPlanORM] = None
plan_items: List[TimeseriesStackPlanItemORM] = []
plan_item_by_id: Dict[int, TimeseriesStackPlanItemORM] = {}
plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
plan_item_by_path: Dict[str, TimeseriesStackPlanItemORM] = {}
planning_context = request.planning_context if isinstance(request.planning_context, dict) else None
if effective_plan_id:
plan_result = await db.execute(
select(TimeseriesStackPlanORM).where(TimeseriesStackPlanORM.plan_id == effective_plan_id)
)
plan = plan_result.scalar_one_or_none()
if plan is None:
raise HTTPException(status_code=404, detail=f"Timeseries stack plan not found: {effective_plan_id}")
if (
str(request.direction or "").strip()
and str(plan.direction or "").strip()
and str(request.direction).strip().upper() != str(plan.direction).strip().upper()
):
raise HTTPException(status_code=400, detail="request.direction does not match the referenced stack plan.")
items_result = await db.execute(
select(TimeseriesStackPlanItemORM)
.where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id)
.order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
)
plan_items = items_result.scalars().all()
plan_item_by_id = {int(item.id): item for item in plan_items if item.id is not None}
plan_item_by_scene_id = {
int(item.radar_data_ref_id): item
for item in plan_items
if item.radar_data_ref_id is not None
}
plan_item_by_path = {
_normalize_lookup_key(item.file_path): item
for item in plan_items
if _normalize_lookup_key(item.file_path)
}
if not planning_context:
planning_context = _build_plan_context(plan, plan_items)
else:
merged_context = {
**_build_plan_context(plan, plan_items),
**planning_context,
}
if "scenes" not in planning_context:
merged_context["scenes"] = _build_plan_context(plan, plan_items).get("scenes") or []
planning_context = merged_context
batch_id = str(uuid.uuid4())
batch_name = request.name or f"PS_{(request.direction or 'STACK')}_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
batch = PsTaskBatchORM(
batch_id=batch_id,
name=batch_name,
direction=request.direction,
plan_id=plan.plan_id if plan is not None else effective_plan_id,
plan_strategy=(
(plan.strategy if plan is not None else None)
or (planning_context or {}).get("strategy")
),
status="PENDING",
total_items=len(request.stack),
completed_items=0,
@@ -316,14 +448,49 @@ async def create_ps_batch_endpoint(
db.add(batch)
for img in request.stack:
matched_plan_item: Optional[TimeseriesStackPlanItemORM] = None
if img.stack_plan_item_id is not None and int(img.stack_plan_item_id) in plan_item_by_id:
matched_plan_item = plan_item_by_id[int(img.stack_plan_item_id)]
elif img.id is not None and int(img.id) in plan_item_by_scene_id:
matched_plan_item = plan_item_by_scene_id[int(img.id)]
else:
matched_plan_item = plan_item_by_path.get(_normalize_lookup_key(img.file_path))
if batch.plan_id and matched_plan_item is None:
raise HTTPException(
status_code=400,
detail=f"PS stack scene is not part of referenced stack plan: {img.file_path}",
)
remark_payload = None
if planning_context:
planning_summary = {
key: value
for key, value in planning_context.items()
if key != "scenes"
}
remark_payload = {
**planning_summary,
"plan_id": batch.plan_id,
"plan_item_id": int(matched_plan_item.id) if matched_plan_item and matched_plan_item.id is not None else None,
"scene_id": img.id,
"scene_file_path": img.file_path,
"scene_imaging_date": img.imaging_date,
"scene_satellite": img.satellite,
}
item = PsTaskItemORM(
batch_id=batch_id,
plan_item_ref_id=(
int(matched_plan_item.id)
if matched_plan_item is not None and matched_plan_item.id is not None
else None
),
file_path=img.file_path,
satellite=img.satellite,
imaging_date=img.imaging_date,
polarization=img.polarization,
has_orbit_data=bool(img.has_orbit_data),
status="PENDING",
remark=json.dumps(remark_payload, ensure_ascii=False) if remark_payload else None,
)
db.add(item)
@@ -332,7 +499,14 @@ async def create_ps_batch_endpoint(
request=http_request,
action="batch_created",
resource=f"task-batches/ps/{batch_id}",
detail={"batch_name": batch_name, "items": len(request.stack), "direction": request.direction},
detail={
"batch_name": batch_name,
"items": len(request.stack),
"direction": request.direction,
"plan_id": batch.plan_id,
"plan_strategy": batch.plan_strategy,
"planning_context": planning_context,
},
)
await db.commit()
await db.refresh(batch)
backend/app/routers/timeseries_production.py
+94
View File
@@ -31,6 +31,45 @@ class TimeseriesRunCreateRequest(BaseModel):
return text
class TimeseriesWslCheckRequest(BaseModel):
distro: Optional[str] = Field(default=None, max_length=128)
smoke_test: bool = Field(default=False)
@field_validator("distro", mode="before")
@classmethod
def _normalize_distro(cls, value: Optional[str]) -> Optional[str]:
if value is None:
return None
text = str(value).strip()
return text or None
class TimeseriesPreflightRequest(BaseModel):
batch_id: str = Field(..., description="PS batch id")
reference_date: Optional[str] = Field(default=None, pattern=r"^\d{8}$|^$")
water_mask_mode: str = Field(default="synthetic_fallback", max_length=64)
@field_validator("batch_id", mode="before")
@classmethod
def _validate_batch_id(cls, value: str) -> str:
text = str(value or "").strip()
if not text:
raise ValueError("batch_id is required")
return text
class TimeseriesRetryStepRequest(BaseModel):
step_id: str = Field(..., max_length=128)
@field_validator("step_id", mode="before")
@classmethod
def _validate_step_id(cls, value: str) -> str:
text = str(value or "").strip()
if not text:
raise ValueError("step_id is required")
return text
@router.post("/runs", status_code=202)
async def create_timeseries_run(
request: TimeseriesRunCreateRequest,
@@ -53,6 +92,39 @@ async def create_timeseries_run(
raise HTTPException(status_code=status_code, detail=message) from exc
@router.post("/wsl-check")
async def run_timeseries_wsl_check(
request: TimeseriesWslCheckRequest,
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
try:
return await timeseries_service.get_runtime_report(
distro=request.distro,
smoke_test=request.smoke_test,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/preflight")
async def run_timeseries_preflight(
request: TimeseriesPreflightRequest,
current_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = current_user
try:
return await timeseries_service.get_preflight_report(
batch_id=request.batch_id,
reference_date=request.reference_date,
water_mask_mode=request.water_mask_mode,
db=db,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/runs")
async def list_timeseries_runs(
limit: int = 50,
@@ -75,3 +147,25 @@ async def get_timeseries_run_detail(
if detail is None:
raise HTTPException(status_code=404, detail="Timeseries run not found")
return detail
@router.post("/runs/{run_id}/retry-step", status_code=202)
async def retry_timeseries_run_step(
run_id: str,
request: TimeseriesRetryStepRequest,
current_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = current_user
try:
return await timeseries_service.retry_step(
run_id,
step_id=request.step_id,
db=db,
)
except ValueError as exc:
message = str(exc)
status_code = 409 if "cannot be retried" in message or "running steps" in message else 400
if "not found" in message:
status_code = 404
raise HTTPException(status_code=status_code, detail=message) from exc
backend/app/services/product_packaging.py
+3
View File
@@ -143,6 +143,8 @@ def upgrade_timeseries_package_manifest(
"run_id": run_context.get("run_id"),
"run_name": run_context.get("run_name"),
"batch_id": run_context.get("batch_id"),
"plan_id": run_context.get("plan_id"),
"plan_strategy": run_context.get("plan_strategy"),
"task_id": run_context.get("task_id"),
"workflow_run_id": run_context.get("workflow_run_id"),
"mode": run_context.get("mode"),
@@ -173,6 +175,7 @@ def upgrade_timeseries_package_manifest(
**_clean_dict(document.get("identity")),
"stack_key": document.get("stack_key") or document.get("group_key"),
"run_key": run_context.get("run_id") or _clean_dict(document.get("identity")).get("run_key"),
"plan_id": run_context.get("plan_id") or _clean_dict(document.get("identity")).get("plan_id"),
}
document["engine"] = {
**_clean_dict(document.get("engine")),
backend/app/services/psinsar_catalog_service.py
+14
View File
@@ -246,6 +246,12 @@ class PsinsarCatalogService:
"product_family": "timeseries",
"stack_key": stack_key,
"group_key": group_key,
"run_id": str(manifest.get("run_id") or "").strip() or None,
"batch_id": str(manifest.get("batch_id") or "").strip() or None,
"plan_id": str(manifest.get("plan_id") or "").strip() or None,
"plan_strategy": str(manifest.get("plan_strategy") or "").strip() or None,
"task_id": str(manifest.get("task_id") or "").strip() or None,
"workflow_run_id": str(manifest.get("workflow_run_id") or "").strip() or None,
"reference_date": reference_date,
"reference_point": manifest.get("reference_point"),
"stack_dates": stack_dates,
@@ -256,6 +262,7 @@ class PsinsarCatalogService:
"summaries": manifest.get("summaries"),
"canonical": canonical_payload,
"runtime": runtime_payload,
"source_summary": manifest.get("source_summary"),
}
published_at = _parse_datetime(temporal.get("published_at") or manifest.get("published_at"))
if published_at is None:
@@ -679,6 +686,12 @@ class PsinsarCatalogService:
"product_type": product.product_type,
"display_name": product.display_name,
"run_key": product.run_key,
"run_id": summary.get("run_id") or product.run_key,
"batch_id": summary.get("batch_id"),
"plan_id": summary.get("plan_id"),
"plan_strategy": summary.get("plan_strategy"),
"task_id": summary.get("task_id"),
"workflow_run_id": summary.get("workflow_run_id"),
"profile_code": product.profile_code,
"engine_code": product.engine_code,
"package_schema": product.package_schema,
@@ -699,6 +712,7 @@ class PsinsarCatalogService:
"stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []),
"quality": summary.get("quality"),
"summaries": summary.get("summaries"),
"source_summary": summary.get("source_summary"),
"coverage_polygon": product.coverage_polygon,
"min_lon": product.min_lon,
"min_lat": product.min_lat,
backend/app/services/spatial_service.py
+442 -36
View File
@@ -5,9 +5,11 @@
"""
import hashlib
import json
import logging
import math
import uuid
from collections import defaultdict
from itertools import combinations
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession
@@ -33,6 +35,8 @@ from ..models import (
RadarDataORM,
RadarPair,
ResultProductORM,
TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM,
)
from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_aliases
from .pairing_state_service import pairing_state_service
@@ -40,6 +44,7 @@ from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.04.phase3.v1"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
logger = logging.getLogger(__name__)
class SpatialService:
@@ -360,6 +365,142 @@ class SpatialService:
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()
def _build_timeseries_stack_identity(
self,
direction: Optional[str],
scenes: List[RadarDataORM],
) -> Dict[str, Any]:
sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or ""))
first = sorted_scenes[0]
satellite = self._normalize_timeseries_satellite_family(first)
imaging_mode = str(first.imaging_mode or "").strip() or "UNKNOWN"
polarization = str(first.polarization or "").strip() or "UNKNOWN"
orbit_direction = (
str(direction or first.orbit_direction or "").strip().upper() or "UNKNOWN"
)
group_key = "_".join(
part
for part in (satellite, imaging_mode, polarization, orbit_direction)
if str(part).strip()
)
stack_dates = [
str(item.imaging_date or "").strip()
for item in sorted_scenes
if str(item.imaging_date or "").strip()
]
digest = self._stable_sha1(
{
"direction": orbit_direction,
"scene_ids": [int(item.id) for item in sorted_scenes],
"stack_dates": stack_dates,
}
)[:10]
date_start = stack_dates[0] if stack_dates else "NA"
date_end = stack_dates[-1] if stack_dates else "NA"
return {
"direction": orbit_direction,
"group_key": group_key,
"stack_key": f"{group_key}_{date_start}_{date_end}_{digest}",
"stack_dates": stack_dates,
}
async def _persist_timeseries_stack_plan(
self,
db: AsyncSession,
*,
direction: Optional[str],
params: PsRequest,
aoi_wkt: Optional[str],
scenes: List[RadarDataORM],
common_aoi_coverage_ratio: Optional[float] = None,
coverage_consistency_ratio: Optional[float] = None,
threshold_satisfied: Optional[bool] = None,
selection_mode: Optional[str] = None,
) -> Dict[str, Any]:
request_payload = params.model_dump(exclude_none=True)
aoi_hash = self._stable_sha1(aoi_wkt) if aoi_wkt else None
identity = self._build_timeseries_stack_identity(direction, scenes)
plan = TimeseriesStackPlanORM(
plan_id=f"tsp_{uuid.uuid4().hex[:24]}",
strategy="sbas_stack",
request_hash=self._stable_sha1(
{
"params": request_payload,
"aoi_hash": aoi_hash,
"direction": identity.get("direction"),
"scene_ids": [int(item.id) for item in scenes],
}
),
request_params_json=request_payload,
aoi_source="wkt" if aoi_wkt else None,
aoi_hash=aoi_hash,
aoi_summary_json=self._build_aoi_summary(aoi_wkt),
direction=identity.get("direction"),
scene_count=len(scenes),
stack_key=identity.get("stack_key"),
group_key=identity.get("group_key"),
status="READY",
created_by="system:find_ps_timeseries",
)
db.add(plan)
await db.flush()
sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or ""))
scene_payloads: List[RadarData] = []
for rank, item in enumerate(sorted_scenes, start=1):
plan_item = TimeseriesStackPlanItemORM(
plan_ref_id=plan.id,
radar_data_ref_id=int(item.id) if item.id is not None else None,
scene_rank=rank,
file_path=item.file_path,
satellite=item.satellite,
imaging_date=item.imaging_date,
imaging_mode=item.imaging_mode,
polarization=item.polarization,
has_orbit_data=bool(item.has_orbit_data),
selection_meta_json={
"source": "find_ps_timeseries",
"direction": identity.get("direction"),
"group_key": identity.get("group_key"),
"stack_key": identity.get("stack_key"),
"initial_overlap_threshold": params.initial_overlap_threshold,
"final_overlap_threshold": params.final_overlap_threshold,
"common_aoi_coverage_ratio": common_aoi_coverage_ratio,
"coverage_consistency_ratio": coverage_consistency_ratio,
"threshold_satisfied": threshold_satisfied,
"selection_mode": selection_mode,
"orbit_direction": item.orbit_direction,
"satellite_family": self._normalize_timeseries_satellite_family(item),
"bbox": [item.min_lon, item.min_lat, item.max_lon, item.max_lat],
"scene_unique_id": item.unique_id,
},
)
db.add(plan_item)
await db.flush()
scene_payloads.append(
RadarData.model_validate(item).model_copy(
update={
"orbit_direction": identity.get("direction") or item.orbit_direction,
"stack_plan_id": plan.plan_id,
"stack_plan_item_id": int(plan_item.id),
"stack_scene_rank": rank,
"stack_group_key": identity.get("group_key"),
"stack_key": identity.get("stack_key"),
"stack_common_aoi_coverage_ratio": common_aoi_coverage_ratio,
"stack_coverage_consistency_ratio": coverage_consistency_ratio,
"stack_threshold_satisfied": threshold_satisfied,
"stack_selection_mode": selection_mode,
}
)
)
return {
"plan_id": plan.plan_id,
"group_key": identity.get("group_key"),
"stack_key": identity.get("stack_key"),
"scenes": scene_payloads,
}
def _apply_strategy(
self,
candidate_pool: List[dict],
@@ -833,6 +974,220 @@ class SpatialService:
def _generate_task_names(self, pairs: List[RadarPair]) -> List[RadarPair]:
return ensure_unique_task_aliases(pairs)
def _normalize_timeseries_direction(self, image: RadarDataORM) -> str:
raw_direction = str(image.orbit_direction or "").strip().upper()
if raw_direction in {"ASC", "ASCENDING"}:
return "ASC"
if raw_direction in {"DSC", "DESC", "DESCENDING"}:
return "DSC"
if "ASC" in raw_direction:
return "ASC"
if "DSC" in raw_direction or "DESC" in raw_direction:
return "DSC"
return raw_direction or "UNKNOWN"
def _normalize_timeseries_satellite_family(self, image: RadarDataORM) -> str:
raw_satellite = str(image.satellite or "").strip().upper()
compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
return "LT1"
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
return "S1"
return raw_satellite or "UNKNOWN"
def _timeseries_compatibility_key(self, image: RadarDataORM) -> Tuple[str, str, str, str]:
return (
self._normalize_timeseries_direction(image),
self._normalize_timeseries_satellite_family(image),
str(image.imaging_mode or "UNKNOWN").strip().upper() or "UNKNOWN",
str(image.polarization or "UNKNOWN").strip().upper() or "UNKNOWN",
)
def _format_timeseries_group_label(self, group_key: Tuple[str, str, str, str]) -> str:
return "_".join(part for part in group_key if part and part != "UNKNOWN") or "STACK"
async def _calculate_wkt_area(self, db: AsyncSession, geom_wkt: str) -> float:
geom = func.ST_GeomFromText(geom_wkt, 4326)
result = await db.execute(select(ST_Area(cast(geom, Geography))))
return float(result.scalar() or 0.0)
async def _select_stable_timeseries_stack(
self,
db: AsyncSession,
images: List[RadarDataORM],
params: PsRequest,
*,
aoi_wkt: str,
aoi_area: float,
) -> Tuple[List[RadarDataORM], float, float, bool, str]:
original_images = sorted(images, key=lambda item: (str(item.imaging_date or ""), int(item.id or 0)))
remaining = list(original_images)
best_stack: List[RadarDataORM] = []
best_consistency_ratio = 0.0
best_common_aoi_ratio = 0.0
min_stack_size = 3
target_ratio = float(params.final_overlap_threshold)
scene_aoi_areas: Dict[int, float] = {}
for img in remaining:
if img.id is None:
continue
scene_aoi_areas[int(img.id)] = await self._calculate_overlap_area(db, int(img.id), aoi_wkt)
def _score_stack(stack: List[RadarDataORM], common_area: float) -> Tuple[float, float]:
scene_areas = [
float(scene_aoi_areas.get(int(img.id or 0)) or 0.0)
for img in stack
if img.id is not None
]
min_scene_area = min(scene_areas) if scene_areas else 0.0
consistency_ratio = common_area / min_scene_area if min_scene_area > 0 else 0.0
common_aoi_ratio = common_area / aoi_area if aoi_area > 0 else 0.0
return (
max(0.0, min(consistency_ratio, 1.0)),
max(0.0, min(common_aoi_ratio, 1.0)),
)
while len(remaining) >= min_stack_size:
common_overlap = await self._find_common_overlap(
db,
[int(img.id) for img in remaining if img.id is not None],
clip_wkt=aoi_wkt,
)
common_area = float((common_overlap or {}).get("area") or 0.0)
consistency_ratio, common_aoi_ratio = _score_stack(remaining, common_area)
if (
consistency_ratio > best_consistency_ratio + 1e-9
or (
abs(consistency_ratio - best_consistency_ratio) <= 1e-9
and common_aoi_ratio > best_common_aoi_ratio + 1e-9
)
or (
abs(consistency_ratio - best_consistency_ratio) <= 1e-9
and abs(common_aoi_ratio - best_common_aoi_ratio) <= 1e-9
and len(remaining) > len(best_stack)
)
):
best_stack = list(remaining)
best_consistency_ratio = consistency_ratio
best_common_aoi_ratio = common_aoi_ratio
if consistency_ratio >= target_ratio:
return remaining, consistency_ratio, common_aoi_ratio, True, "common_overlap"
if len(remaining) == min_stack_size:
break
trial_options: List[Tuple[float, float, int, List[RadarDataORM]]] = []
for remove_index, _ in enumerate(remaining):
trial = remaining[:remove_index] + remaining[remove_index + 1:]
trial_overlap = await self._find_common_overlap(
db,
[int(img.id) for img in trial if img.id is not None],
clip_wkt=aoi_wkt,
)
trial_area = float((trial_overlap or {}).get("area") or 0.0)
trial_consistency_ratio, trial_common_aoi_ratio = _score_stack(trial, trial_area)
removed_id = int(remaining[remove_index].id or 0)
trial_options.append((trial_consistency_ratio, trial_common_aoi_ratio, -removed_id, trial))
if not trial_options:
break
_, _, _, remaining = max(trial_options, key=lambda item: (item[0], item[1], item[2]))
if best_consistency_ratio >= target_ratio and len(best_stack) >= min_stack_size:
return best_stack, best_consistency_ratio, best_common_aoi_ratio, True, "common_overlap"
network_stack, network_ratio = await self._select_pairwise_sbas_network_stack(
db,
original_images,
scene_aoi_areas,
target_ratio,
aoi_wkt=aoi_wkt,
)
if len(network_stack) >= min_stack_size:
return network_stack, network_ratio, best_common_aoi_ratio, True, "pairwise_sbas_network"
return [], best_consistency_ratio, best_common_aoi_ratio, False, "none"
async def _select_pairwise_sbas_network_stack(
self,
db: AsyncSession,
images: List[RadarDataORM],
scene_aoi_areas: Dict[int, float],
target_ratio: float,
*,
aoi_wkt: str,
) -> Tuple[List[RadarDataORM], float]:
if len(images) < 3:
return [], 0.0
image_by_id = {int(img.id): img for img in images if img.id is not None}
adjacency: Dict[int, List[Tuple[int, float]]] = {scene_id: [] for scene_id in image_by_id}
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
for left, right in combinations(images, 2):
if left.id is None or right.id is None:
continue
left_id = int(left.id)
right_id = int(right.id)
left_area = float(scene_aoi_areas.get(left_id) or 0.0)
right_area = float(scene_aoi_areas.get(right_id) or 0.0)
min_scene_area = min(left_area, right_area)
if min_scene_area <= 0:
continue
left_geom = select(RadarDataORM.geom).where(RadarDataORM.id == left_id).scalar_subquery()
right_geom = select(RadarDataORM.geom).where(RadarDataORM.id == right_id).scalar_subquery()
pair_geom = ST_Intersection(ST_Intersection(left_geom, right_geom), aoi_geom)
result = await db.execute(select(ST_Area(cast(pair_geom, Geography))))
pair_area = float(result.scalar() or 0.0)
pair_ratio = max(0.0, min(pair_area / min_scene_area, 1.0))
if pair_ratio >= target_ratio:
adjacency[left_id].append((right_id, pair_ratio))
adjacency[right_id].append((left_id, pair_ratio))
visited: set[int] = set()
best_component: List[int] = []
best_component_ratio = 0.0
for scene_id in sorted(adjacency):
if scene_id in visited:
continue
stack = [scene_id]
visited.add(scene_id)
component: List[int] = []
component_edge_ratios: List[float] = []
while stack:
current = stack.pop()
component.append(current)
for neighbor, ratio in adjacency.get(current, []):
component_edge_ratios.append(float(ratio))
if neighbor not in visited:
visited.add(neighbor)
stack.append(neighbor)
if len(component) < 3:
continue
component_ratio = min(component_edge_ratios) if component_edge_ratios else 0.0
if (
len(component) > len(best_component)
or (
len(component) == len(best_component)
and component_ratio > best_component_ratio
)
):
best_component = component
best_component_ratio = component_ratio
if len(best_component) < 3:
return [], 0.0
selected = [image_by_id[scene_id] for scene_id in best_component if scene_id in image_by_id]
selected.sort(key=lambda item: (str(item.imaging_date or ""), int(item.id or 0)))
return selected, best_component_ratio
async def find_ps_timeseries_data(
self,
db: AsyncSession,
@@ -850,9 +1205,14 @@ class SpatialService:
Returns:
按轨道方向分组的影像字典
"""
# 1. 初始筛选：找到与 AOI 相交的影像
# 1. 初始筛选：找到与 AOI 相交且单景覆盖率达标的影像
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
aoi_geog = cast(aoi_geom, Geography)
aoi_area = await self._calculate_wkt_area(db, aoi_wkt)
if aoi_area <= 0:
logger.warning("timeseries stack planning skipped: AOI area is empty")
return {}
intersection_geog = cast(ST_Intersection(RadarDataORM.geom, aoi_geom), Geography)
stmt = select(RadarDataORM).where(
and_(
@@ -863,49 +1223,93 @@ class SpatialService:
result = await db.execute(stmt)
candidates = result.scalars().all()
logger.info(
"timeseries stack planning: candidates_after_aoi_gate=%s initial_threshold=%.3f final_consistency_threshold=%.3f",
len(candidates),
float(params.initial_overlap_threshold),
float(params.final_overlap_threshold),
)
if not candidates:
return {}
# 2. 按轨道分组
images_by_orbit: Dict[str, List[RadarDataORM]] = {}
# 2. 按轨道方向、卫星、成像模式、极化分组，避免混入不兼容场景。
images_by_group: Dict[Tuple[str, str, str, str], List[RadarDataORM]] = {}
for img in candidates:
direction = img.orbit_direction or ("ASC" if "ASC" in img.satellite else "DSC")
images_by_orbit.setdefault(direction, []).append(img)
images_by_group.setdefault(self._timeseries_compatibility_key(img), []).append(img)
logger.info(
"timeseries stack planning: compatible_groups=%s group_sizes=%s",
len(images_by_group),
{
self._format_timeseries_group_label(group_key): len(items)
for group_key, items in images_by_group.items()
},
)
# 3. 计算每个轨道的公共重叠区
# 3. 每个兼容组内寻找满足公共 AOI 覆盖阈值的最大稳定候选栈。
final_results: Dict[str, List[RadarData]] = {}
plans_created = False
for direction, images in images_by_orbit.items():
if len(images) < 2:
for group_key, images in images_by_group.items():
if len(images) < 3:
logger.info(
"timeseries stack planning: group=%s skipped because scene_count=%s < 3",
self._format_timeseries_group_label(group_key),
len(images),
)
continue
try:
# 查找公共重叠区
common_overlap = await self._find_common_overlap(db, [img.id for img in images])
if not common_overlap or common_overlap["area"] < 1e-6:
continue
common_geom = common_overlap["geom"]
common_area = common_overlap["area"]
# 4. 最终筛选：覆盖公共区域一定比例的影像
final_stack = []
for img in images:
img_overlap = await self._calculate_overlap_area(db, img.id, common_geom.wkt)
if img_overlap / common_area >= params.final_overlap_threshold:
final_stack.append(RadarData.model_validate(img))
if len(final_stack) >= 2:
final_stack.sort(key=lambda x: x.imaging_date)
final_results[direction] = final_stack
(
final_stack,
consistency_ratio,
common_aoi_ratio,
threshold_satisfied,
selection_mode,
) = await self._select_stable_timeseries_stack(
db,
images,
params,
aoi_wkt=aoi_wkt,
aoi_area=aoi_area,
)
logger.info(
"timeseries stack planning: group=%s input_scenes=%s selected_scenes=%s consistency=%.4f common_aoi=%.4f threshold_satisfied=%s mode=%s",
self._format_timeseries_group_label(group_key),
len(images),
len(final_stack),
consistency_ratio,
common_aoi_ratio,
threshold_satisfied,
selection_mode,
)
if len(final_stack) >= 3:
final_stack.sort(key=lambda x: str(x.imaging_date or ""))
direction = group_key[0]
persisted_plan = await self._persist_timeseries_stack_plan(
db,
direction=direction,
params=params,
aoi_wkt=aoi_wkt,
scenes=final_stack,
common_aoi_coverage_ratio=common_aoi_ratio,
coverage_consistency_ratio=consistency_ratio,
threshold_satisfied=threshold_satisfied,
selection_mode=selection_mode,
)
result_key = persisted_plan.get("group_key") or self._format_timeseries_group_label(group_key)
if result_key in final_results:
result_key = f"{result_key}_{len(final_results) + 1}"
final_results[result_key] = persisted_plan["scenes"]
plans_created = True
except Exception as e:
print(f"处理轨道 {direction} 时出错: {e}")
print(f"处理时序候选组 {self._format_timeseries_group_label(group_key)} 时出错: {e}")
continue
if plans_created:
await db.commit()
return final_results
async def find_hazard_points_in_area(
@@ -1044,7 +1448,8 @@ class SpatialService:
async def _find_common_overlap(
self,
db: AsyncSession,
image_ids: List[int]
image_ids: List[int],
clip_wkt: Optional[str] = None,
) -> Optional[dict]:
"""
Compute common overlap geometry and area using DB aggregation.
@@ -1052,7 +1457,12 @@ class SpatialService:
if not image_ids:
return None
intersection_expr = func.st_intersection_agg(RadarDataORM.geom)
geom_expr = RadarDataORM.geom
if clip_wkt:
clip_geom = func.ST_GeomFromText(clip_wkt, 4326)
geom_expr = ST_Intersection(RadarDataORM.geom, clip_geom)
intersection_expr = func.st_intersection_agg(geom_expr)
stmt = select(
ST_Area(cast(intersection_expr, Geography)).label("common_area"),
intersection_expr.label("common_geom")
@@ -1062,11 +1472,7 @@ class SpatialService:
if not row or not row.common_geom:
return None
try:
return {"geom": to_shape(row.common_geom), "area": float(row.common_area or 0)}
except Exception as exc:
print(f"[WARN] _compute_footprint: {exc}")
return None
return {"geom": row.common_geom, "area": float(row.common_area or 0)}
def _optimize_coverage_diversity(
self,
backend/app/services/task_service.py
+3
View File
@@ -273,6 +273,9 @@ class TaskService:
else:
# 显式更新心跳时间，防止 SQLAlchemy 因属性未变而跳过 UPDATE
task.updated_at = datetime.now()
task.ended_at = None
if status == "RUNNING" and task.started_at is None:
task.started_at = datetime.now()
await db.commit()
else:
backend/app/services/timeseries_service.py
+869 -2
View File
File diff suppressed because it is too large Load Diff
backend/app/services/workflow_service.py
+91
View File
@@ -23,6 +23,28 @@ class WorkflowService:
Lightweight workflow orchestration service backed by DB.
"""
@staticmethod
def _collect_downstream_step_ids(
target_step_id: str,
steps: List[WorkflowStepORM],
) -> set[str]:
reverse_graph: Dict[str, List[str]] = {}
for step in steps:
for dependency in step.depends_on or []:
reverse_graph.setdefault(str(dependency), []).append(step.step_id)
pending = [target_step_id]
visited: set[str] = set()
while pending:
current = pending.pop()
if current in visited:
continue
visited.add(current)
for child_step_id in reverse_graph.get(current, []):
if child_step_id not in visited:
pending.append(child_step_id)
return visited
async def create_run(
self,
workflow_name: str,
@@ -201,6 +223,75 @@ class WorkflowService:
if gen_db:
await db.close()
async def retry_step(
self,
run_id: str,
step_id: str,
db: Optional[AsyncSession] = None,
) -> Dict[str, Any]:
gen_db = db is None
if gen_db:
db = _new_session()
try:
run_result = await db.execute(
select(WorkflowRunORM).where(WorkflowRunORM.run_id == run_id)
)
run = run_result.scalar_one_or_none()
if run is None:
raise ValueError(f"Workflow run not found: {run_id}")
steps_result = await db.execute(
select(WorkflowStepORM)
.where(WorkflowStepORM.run_id == run_id)
.order_by(WorkflowStepORM.id.asc())
)
steps = steps_result.scalars().all()
if not steps:
raise ValueError(f"Workflow run has no steps: {run_id}")
step_map = {step.step_id: step for step in steps}
target = step_map.get(step_id)
if target is None:
raise ValueError(f"Workflow step not found: {step_id}")
if any(step.status == "RUNNING" for step in steps):
raise ValueError("Workflow still has running steps and cannot be retried.")
retryable_statuses = {"FAILED", "COMPLETED", "CANCELLED", "SKIPPED"}
if target.status not in retryable_statuses:
raise ValueError(
f"Workflow step '{step_id}' is not retryable from status '{target.status}'."
)
reset_step_ids = self._collect_downstream_step_ids(step_id, steps)
for step in steps:
if step.step_id not in reset_step_ids:
continue
step.status = "READY" if step.step_id == step_id else "PENDING"
step.error = None
step.outputs = None
step.started_at = None
step.ended_at = None
run.status = "RUNNING"
run.ended_at = None
if gen_db:
await db.commit()
else:
await db.flush()
finally:
if gen_db:
await db.close()
await self.enqueue_ready_steps(run_id, db=None if gen_db else db)
return {
"run_id": run_id,
"step_id": step_id,
"reset_steps": sorted(reset_step_ids),
}
async def _advance_ready_steps(self, run_id: str, db: AsyncSession) -> None:
result = await db.execute(
select(WorkflowStepORM).where(WorkflowStepORM.run_id == run_id)
backend/app/services/wsl_service.py
+24
View File
@@ -193,6 +193,9 @@ def check_wsl_environment(
"bash -lc 可执行",
"Python 可执行",
"ISCE2 可 import",
"astropy.convolution import",
"cv2 import",
"scipy import",
"stripmapApp 存在",
"生产脚本存在",
"DEM 路径可读",
@@ -279,6 +282,27 @@ def check_wsl_environment(
isce_ok = rc == 0
add("ISCE2 可 import", isce_ok, out or err)
rc, out, err = run_wsl_command(
f'{python_cmd} -c "from astropy.convolution import convolve; print(\'astropy_ok\')"',
distro=distro, timeout=30,
)
astropy_ok = rc == 0 and "astropy_ok" in out
add("astropy.convolution import", astropy_ok, out or err)
rc, out, err = run_wsl_command(
f'{python_cmd} -c "import cv2; print(\'cv2_ok\')"',
distro=distro, timeout=30,
)
cv2_ok = rc == 0 and "cv2_ok" in out
add("cv2 import", cv2_ok, out or err)
rc, out, err = run_wsl_command(
f'{python_cmd} -c "import scipy; print(\'scipy_ok\')"',
distro=distro, timeout=30,
)
scipy_ok = rc == 0 and "scipy_ok" in out
add("scipy import", scipy_ok, out or err)
# 8. stripmapApp.py 存在（全路径检查）
if stripmap_app_path:
rc, out, err = run_wsl_command(
backend/migrations/007_timeseries_stack_plan_trace.sql
+17
View File
@@ -0,0 +1,17 @@
-- Additive indexes for the phase-2 timeseries stack plan trace chain.
-- Tables/columns are created by SQLAlchemy metadata and db_maintenance missing-column repair.
CREATE INDEX IF NOT EXISTS idx_ps_task_batches_plan_id
ON ps_task_batches (plan_id);
CREATE INDEX IF NOT EXISTS idx_ps_task_items_plan_item_ref_id
ON ps_task_items (plan_item_ref_id);
CREATE INDEX IF NOT EXISTS idx_ps_timeseries_runs_plan_id
ON ps_timeseries_runs (plan_id);
CREATE INDEX IF NOT EXISTS idx_timeseries_stack_plans_request_hash
ON timeseries_stack_plans (request_hash);
CREATE INDEX IF NOT EXISTS idx_timeseries_stack_plan_items_radar_ref
ON timeseries_stack_plan_items (radar_data_ref_id);