Apply current workspace changes
This commit is contained in:
@@ -219,12 +219,40 @@ class Settings(BaseSettings):
|
||||
ISCE2_DEM_PATH: str = "D:\\SRTM30m\\SRTMDEM_RSP_SARscape.wgs84"
|
||||
ISCE2_WORK_ROOT: str = ""
|
||||
ISCE2_OUTPUT_ROOT: str = ""
|
||||
ISCE2_PER_TASK_TIMEOUT_SECONDS: int = 43200
|
||||
ISCE2_SMOKE_TEST_ENABLED: bool = False
|
||||
ISCE2_STRIPMAP_APP: str = (
|
||||
"/home/administrator/miniconda3/envs/isce2/lib/python3.11/"
|
||||
"site-packages/isce/applications/stripmapApp.py"
|
||||
)
|
||||
ISCE2_PIPELINE_SCRIPT: str = ""
|
||||
PYINT_ENABLED: bool = False
|
||||
PYINT_WSL_DISTRO: str = ""
|
||||
PYINT_WSL_PYTHON: str = ""
|
||||
PYINT_HOME: str = ""
|
||||
PYINT_APP_SCRIPT: str = ""
|
||||
PYINT_TEMPLATE_ROOT: str = ""
|
||||
PYINT_WORK_ROOT: str = ""
|
||||
PYINT_OUTPUT_ROOT: str = ""
|
||||
PYINT_DEM_ROOT: str = ""
|
||||
PYINT_DEM_MODE: str = "local_fabdem"
|
||||
PYINT_FABDEM_ROOT: str = ""
|
||||
PYINT_PREPARED_DEM_PATH: str = ""
|
||||
PYINT_OPENTOPO_DEM_TYPE: str = "SRTMGL1"
|
||||
PYINT_OPENTOPO_API_KEY: str = ""
|
||||
PYINT_DEM_STRICT: bool = True
|
||||
PYINT_ORBIT_POLICY: str = "require_txt"
|
||||
PYINT_ORBIT_POOL_TXT: str = ""
|
||||
PYINT_RECORD_INPUT_ASSETS: bool = True
|
||||
PYINT_LT1_PRECISE_ORBIT_ENABLED: bool = True
|
||||
PYINT_LT1_PRECISE_ORBIT_MODE: str = "replace"
|
||||
PYINT_LT1_PRECISE_ORBIT_STRICT: bool = True
|
||||
PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT: bool = False
|
||||
PYINT_LT1_PRECISE_ORBIT_BACKUP: bool = True
|
||||
PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE: int = 5
|
||||
PYINT_GAMMA_ENV_SCRIPT: str = ""
|
||||
PYINT_DEFAULT_TIMEOUT_SECONDS: int = 43200
|
||||
PYINT_SMOKE_TEST_ENABLED: bool = False
|
||||
JOB_WORKER_HEALTH_TIMEOUT: int = 60
|
||||
JOB_WORKER_JOB_HEARTBEAT_INTERVAL: float = 5.0
|
||||
JOB_WORKER_STALE_RECOVER_INTERVAL: float = 15.0
|
||||
@@ -330,6 +358,67 @@ class Settings(BaseSettings):
|
||||
"ISCE2_PIPELINE_SCRIPT",
|
||||
_windows_path_to_wsl_mount(local_pipeline),
|
||||
)
|
||||
if not self.PYINT_WSL_DISTRO:
|
||||
object.__setattr__(self, "PYINT_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
|
||||
if not self.PYINT_WSL_PYTHON:
|
||||
object.__setattr__(self, "PYINT_WSL_PYTHON", self.ISCE2_PYTHON)
|
||||
if not self.PYINT_HOME:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_HOME",
|
||||
os.path.join(project_root, "third_party", "PyINT"),
|
||||
)
|
||||
if not self.PYINT_APP_SCRIPT and self.PYINT_HOME:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_APP_SCRIPT",
|
||||
os.path.join(self.PYINT_HOME, "pyint", "pyintApp.py"),
|
||||
)
|
||||
if not self.PYINT_TEMPLATE_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_TEMPLATE_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "pyint_templates"),
|
||||
)
|
||||
if not self.PYINT_WORK_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_WORK_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "pyint_work"),
|
||||
)
|
||||
if not self.PYINT_OUTPUT_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_OUTPUT_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "pyint_output"),
|
||||
)
|
||||
if not self.PYINT_DEM_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_DEM_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "pyint_dem"),
|
||||
)
|
||||
pyint_dem_mode = str(self.PYINT_DEM_MODE or "local_fabdem").strip().lower() or "local_fabdem"
|
||||
if pyint_dem_mode not in {"local_fabdem", "opentopo", "prepared_file"}:
|
||||
pyint_dem_mode = "local_fabdem"
|
||||
object.__setattr__(self, "PYINT_DEM_MODE", pyint_dem_mode)
|
||||
if not self.PYINT_OPENTOPO_DEM_TYPE:
|
||||
object.__setattr__(self, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1")
|
||||
pyint_orbit_policy = str(self.PYINT_ORBIT_POLICY or "require_txt").strip().lower() or "require_txt"
|
||||
if pyint_orbit_policy not in {"validate_only", "require_txt", "stage_txt"}:
|
||||
pyint_orbit_policy = "require_txt"
|
||||
object.__setattr__(self, "PYINT_ORBIT_POLICY", pyint_orbit_policy)
|
||||
pyint_precise_orbit_mode = str(self.PYINT_LT1_PRECISE_ORBIT_MODE or "replace").strip().lower() or "replace"
|
||||
if pyint_precise_orbit_mode not in {"replace", "replace_and_validate"}:
|
||||
pyint_precise_orbit_mode = "replace"
|
||||
object.__setattr__(self, "PYINT_LT1_PRECISE_ORBIT_MODE", pyint_precise_orbit_mode)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE",
|
||||
max(1, int(self.PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE or 5)),
|
||||
)
|
||||
if not self.PYINT_ORBIT_POOL_TXT:
|
||||
object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI)
|
||||
if not self.TIMESERIES_WSL_DISTRO:
|
||||
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
|
||||
if not self.TIMESERIES_ENV_NAME:
|
||||
@@ -431,6 +520,10 @@ class Settings(BaseSettings):
|
||||
os.makedirs(settings.DINSAR_PRODUCT_DIR, exist_ok=True)
|
||||
os.makedirs(settings.PSINSAR_PRODUCT_DIR, exist_ok=True)
|
||||
os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_DEM_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True)
|
||||
|
||||
|
||||
@@ -601,6 +694,92 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
if not settings.ORBIT_POOL_ISCE2:
|
||||
warnings.append("ISCE2_ENABLED=true 但 ORBIT_POOL_ISCE2 未配置。")
|
||||
|
||||
if settings.PYINT_ENABLED:
|
||||
if not settings.PYINT_WSL_DISTRO:
|
||||
errors.append("PYINT_ENABLED=true but PYINT_WSL_DISTRO is not configured.")
|
||||
if not settings.PYINT_WSL_PYTHON:
|
||||
errors.append("PYINT_ENABLED=true but PYINT_WSL_PYTHON is not configured.")
|
||||
if not settings.PYINT_HOME:
|
||||
errors.append("PYINT_ENABLED=true but PYINT_HOME is not configured.")
|
||||
if not settings.PYINT_APP_SCRIPT:
|
||||
errors.append("PYINT_ENABLED=true but PYINT_APP_SCRIPT is not configured.")
|
||||
_check_path(
|
||||
label="PYINT_HOME",
|
||||
value=settings.PYINT_HOME,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_APP_SCRIPT",
|
||||
value=settings.PYINT_APP_SCRIPT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_TEMPLATE_ROOT",
|
||||
value=settings.PYINT_TEMPLATE_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_WORK_ROOT",
|
||||
value=settings.PYINT_WORK_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_OUTPUT_ROOT",
|
||||
value=settings.PYINT_OUTPUT_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_DEM_ROOT",
|
||||
value=settings.PYINT_DEM_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
if settings.PYINT_DEM_MODE == "local_fabdem":
|
||||
_check_path(
|
||||
label="PYINT_FABDEM_ROOT",
|
||||
value=settings.PYINT_FABDEM_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
elif settings.PYINT_DEM_MODE == "prepared_file":
|
||||
_check_path(
|
||||
label="PYINT_PREPARED_DEM_PATH",
|
||||
value=(
|
||||
settings.PYINT_PREPARED_DEM_PATH
|
||||
or settings.ISCE2_DEM_PATH
|
||||
or settings.IDL_DINSAR_DEM_BASE_FILE
|
||||
),
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_ORBIT_POOL_TXT",
|
||||
value=settings.PYINT_ORBIT_POOL_TXT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(
|
||||
label="PYINT_GAMMA_ENV_SCRIPT",
|
||||
value=settings.PYINT_GAMMA_ENV_SCRIPT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
)
|
||||
|
||||
if settings.TIMESERIES_ENABLED:
|
||||
_check_path(
|
||||
label="TIMESERIES_PYTHON",
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -38,6 +38,7 @@ class RunRequest:
|
||||
num_to_process: int = 0
|
||||
timeout_seconds: Optional[int] = None
|
||||
extra: Dict[str, Any] = field(default_factory=dict)
|
||||
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -80,12 +81,19 @@ class DinsarEngine(ABC):
|
||||
def run(self, request: RunRequest) -> RunResult:
|
||||
"""Executes a production run synchronously."""
|
||||
|
||||
@property
|
||||
def default_timeout_seconds(self) -> Optional[int]:
|
||||
"""Returns the engine's default timeout when the caller leaves it empty."""
|
||||
|
||||
return None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Serializes the engine definition for API responses."""
|
||||
|
||||
return {
|
||||
"engine_code": self.engine_code,
|
||||
"engine_label": self.engine_label,
|
||||
"default_timeout_seconds": self.default_timeout_seconds,
|
||||
"profiles": [
|
||||
{
|
||||
"code": profile.code,
|
||||
|
||||
@@ -6,7 +6,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ..config import get_env_text, read_bool_env
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
|
||||
from ..services.dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
@@ -56,6 +56,10 @@ class Isce2Engine(DinsarEngine):
|
||||
def engine_label(self) -> str:
|
||||
return "ISCE2（WSL）"
|
||||
|
||||
@property
|
||||
def default_timeout_seconds(self) -> int:
|
||||
return max(60, int(settings.ISCE2_PER_TASK_TIMEOUT_SECONDS or 43200))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Config helpers
|
||||
# ------------------------------------------------------------------
|
||||
@@ -410,11 +414,21 @@ class Isce2Engine(DinsarEngine):
|
||||
extra = self.normalize_extra(request.extra)
|
||||
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
|
||||
task_dirs: List[str] = validation["task_dirs"]
|
||||
total_tasks = len(task_dirs)
|
||||
run_started_at = datetime.utcnow()
|
||||
run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z"
|
||||
run_key = build_run_key(self.engine_code, request.profile, started_at=run_started_at)
|
||||
progress_callback = request.progress_callback
|
||||
|
||||
timeout = request.timeout_seconds or 21600
|
||||
def emit_progress(event_type: str, **payload: Any) -> None:
|
||||
if not callable(progress_callback):
|
||||
return
|
||||
try:
|
||||
progress_callback({"event": event_type, **payload})
|
||||
except Exception:
|
||||
return
|
||||
|
||||
timeout = max(60, int(request.timeout_seconds or self.default_timeout_seconds))
|
||||
force = bool(extra.get("force"))
|
||||
target_grid_size_m = int(extra.get("target_grid_size_m", DEFAULT_TARGET_GRID_SIZE_M))
|
||||
bbox = extra.get("bbox", "")
|
||||
@@ -436,7 +450,7 @@ class Isce2Engine(DinsarEngine):
|
||||
pairs_processed = 0
|
||||
pairs_failed = 0
|
||||
|
||||
for task_dir in task_dirs:
|
||||
for pair_index, task_dir in enumerate(task_dirs, start=1):
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
@@ -450,8 +464,31 @@ class Isce2Engine(DinsarEngine):
|
||||
wsl_work_dir = windows_path_to_wsl(work_dir, distro=self._distro)
|
||||
wsl_output_dir = windows_path_to_wsl(output_dir, distro=self._distro)
|
||||
wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=self._distro)
|
||||
emit_progress(
|
||||
"pair_started",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
task_dir=task_dir,
|
||||
work_dir=work_dir,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
if not wsl_task_dir:
|
||||
pairs_failed += 1
|
||||
error_text = f"Unable to convert task dir to WSL path: {task_dir}"
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-2,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
@@ -463,7 +500,7 @@ class Isce2Engine(DinsarEngine):
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -2,
|
||||
"error": f"Unable to convert task dir to WSL path: {task_dir}",
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
@@ -475,6 +512,18 @@ class Isce2Engine(DinsarEngine):
|
||||
continue
|
||||
if not wsl_work_dir or not wsl_output_dir or not wsl_orbit_output_dir:
|
||||
pairs_failed += 1
|
||||
error_text = "Unable to convert ISCE2 work/output paths to WSL paths."
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-2,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
@@ -486,7 +535,7 @@ class Isce2Engine(DinsarEngine):
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -2,
|
||||
"error": "Unable to convert ISCE2 work/output paths to WSL paths.",
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
@@ -585,6 +634,17 @@ class Isce2Engine(DinsarEngine):
|
||||
else:
|
||||
pairs_failed += 1
|
||||
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=success,
|
||||
returncode=rc,
|
||||
error=stderr.strip() if stderr else "",
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
|
||||
@@ -0,0 +1,814 @@
|
||||
"""PyINT D-InSAR engine backed by a WSL wrapper pipeline."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from ..services.dinsar_naming import write_run_metadata
|
||||
from ..services.pyint_input_assets_service import (
|
||||
get_pyint_dem_summary,
|
||||
get_pyint_orbit_context,
|
||||
materialize_pyint_input_assets,
|
||||
resolve_pyint_task_input_assets,
|
||||
)
|
||||
from ..services.pyint_service import (
|
||||
DEFAULT_AZIMUTH_LOOKS,
|
||||
DEFAULT_PARALLEL_WORKERS,
|
||||
DEFAULT_RANGE_LOOKS,
|
||||
MAX_LOOKS,
|
||||
MAX_PARALLEL_WORKERS,
|
||||
build_project_name,
|
||||
check_pyint_environment,
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
quote_shell,
|
||||
resolve_time_baseline_days,
|
||||
to_wsl_path,
|
||||
validate_pyint_root_dir,
|
||||
)
|
||||
from ..services.wsl_service import run_wsl_command
|
||||
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
return get_env_text(name, default) or default
|
||||
|
||||
|
||||
def _read_bool_env(name: str, default: bool = False) -> bool:
|
||||
return read_bool_env(name, default)
|
||||
|
||||
|
||||
def _windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
drive, tail = os.path.splitdrive(os.path.normpath(text))
|
||||
if not drive:
|
||||
return text.replace("\\", "/")
|
||||
drive_letter = drive.rstrip(":").lower()
|
||||
normalized_tail = tail.replace("\\", "/")
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
class PyintEngine(DinsarEngine):
|
||||
@property
|
||||
def engine_code(self) -> str:
|
||||
return "pyint"
|
||||
|
||||
@property
|
||||
def engine_label(self) -> str:
|
||||
return "PyINT / Gamma"
|
||||
|
||||
@property
|
||||
def default_timeout_seconds(self) -> int:
|
||||
return max(60, int(settings.PYINT_DEFAULT_TIMEOUT_SECONDS or 43200))
|
||||
|
||||
@property
|
||||
def _enabled(self) -> bool:
|
||||
return _read_bool_env("PYINT_ENABLED", False)
|
||||
|
||||
@property
|
||||
def _distro(self) -> str:
|
||||
return _read_env("PYINT_WSL_DISTRO", settings.ISCE2_WSL_DISTRO)
|
||||
|
||||
@property
|
||||
def _python(self) -> str:
|
||||
return _read_env("PYINT_WSL_PYTHON", settings.ISCE2_PYTHON)
|
||||
|
||||
@property
|
||||
def _pyint_home(self) -> str:
|
||||
return _read_env("PYINT_HOME", "")
|
||||
|
||||
@property
|
||||
def _pyint_app_script(self) -> str:
|
||||
explicit = _read_env("PYINT_APP_SCRIPT", "")
|
||||
if explicit:
|
||||
return explicit
|
||||
home = self._pyint_home
|
||||
if not home:
|
||||
return ""
|
||||
return os.path.join(home, "pyint", "pyintApp.py")
|
||||
|
||||
@property
|
||||
def _template_root(self) -> str:
|
||||
return _read_env("PYINT_TEMPLATE_ROOT", "")
|
||||
|
||||
@property
|
||||
def _work_root(self) -> str:
|
||||
return _read_env("PYINT_WORK_ROOT", "")
|
||||
|
||||
@property
|
||||
def _output_root(self) -> str:
|
||||
return _read_env("PYINT_OUTPUT_ROOT", "")
|
||||
|
||||
@property
|
||||
def _dem_root(self) -> str:
|
||||
return _read_env("PYINT_DEM_ROOT", "")
|
||||
|
||||
@property
|
||||
def _dem_mode(self) -> str:
|
||||
return str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
|
||||
|
||||
@property
|
||||
def _fabdem_root(self) -> str:
|
||||
return _read_env("PYINT_FABDEM_ROOT", "")
|
||||
|
||||
@property
|
||||
def _opentopo_dem_type(self) -> str:
|
||||
return _read_env("PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1")
|
||||
|
||||
@property
|
||||
def _opentopo_api_key(self) -> str:
|
||||
return _read_env("PYINT_OPENTOPO_API_KEY", "")
|
||||
|
||||
@property
|
||||
def _orbit_policy(self) -> str:
|
||||
return str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower()
|
||||
|
||||
@property
|
||||
def _orbit_pool_txt(self) -> str:
|
||||
return _read_env("PYINT_ORBIT_POOL_TXT", settings.ORBIT_POOL_ENVI)
|
||||
|
||||
@property
|
||||
def _record_input_assets(self) -> bool:
|
||||
return _read_bool_env("PYINT_RECORD_INPUT_ASSETS", True)
|
||||
|
||||
@property
|
||||
def _gamma_env_script(self) -> str:
|
||||
return _read_env("PYINT_GAMMA_ENV_SCRIPT", "")
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_enabled(self) -> bool:
|
||||
return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True)
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_mode(self) -> str:
|
||||
return str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower()
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_strict(self) -> bool:
|
||||
return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_STRICT", True)
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_validate_with_orb_filt(self) -> bool:
|
||||
return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False)
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_backup(self) -> bool:
|
||||
return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_BACKUP", True)
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_orb_filt_degree(self) -> int:
|
||||
return max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5))
|
||||
|
||||
@property
|
||||
def _smoke_test(self) -> bool:
|
||||
return _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False)
|
||||
|
||||
@property
|
||||
def _pipeline_script(self) -> str:
|
||||
local_script = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "pyint_pipeline"
|
||||
/ "run_lt1_pyint_pipeline.py"
|
||||
)
|
||||
return _windows_path_to_wsl_mount(str(local_script))
|
||||
|
||||
def get_profiles(self) -> List[EngineProfile]:
|
||||
return [
|
||||
EngineProfile(
|
||||
code="lt1_gamma_dinsar",
|
||||
label="LT-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
|
||||
params_schema={
|
||||
"force": {
|
||||
"label": "强制重跑",
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "删除当前 run_key 对应的工作区后重跑。",
|
||||
},
|
||||
"range_looks": {
|
||||
"label": "距离向多视",
|
||||
"type": "number",
|
||||
"default": DEFAULT_RANGE_LOOKS,
|
||||
"step": 1,
|
||||
"min": 1,
|
||||
"max": MAX_LOOKS,
|
||||
"description": "PyINT 模板中的 range_looks。",
|
||||
},
|
||||
"azimuth_looks": {
|
||||
"label": "方位向多视",
|
||||
"type": "number",
|
||||
"default": DEFAULT_AZIMUTH_LOOKS,
|
||||
"step": 1,
|
||||
"min": 1,
|
||||
"max": MAX_LOOKS,
|
||||
"description": "PyINT 模板中的 azimuth_looks。",
|
||||
},
|
||||
"parallel_workers": {
|
||||
"label": "并行数",
|
||||
"type": "number",
|
||||
"default": DEFAULT_PARALLEL_WORKERS,
|
||||
"step": 1,
|
||||
"min": 1,
|
||||
"max": MAX_PARALLEL_WORKERS,
|
||||
"description": "同步控制 raw2slc/coreg/diff/unwrap/geocode 的并行数。",
|
||||
},
|
||||
"unwrap": {
|
||||
"label": "执行解缠",
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "关闭后仅做到差分干涉图，不做解缠。",
|
||||
},
|
||||
"geocode": {
|
||||
"label": "执行地理编码",
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "关闭后不导出地理编码结果。",
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
def normalize_extra(self, extra: Dict[str, Any] | None) -> Dict[str, Any]:
|
||||
normalized: Dict[str, Any] = dict(extra or {})
|
||||
|
||||
def _coerce_bool(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
text = str(value or "").strip().lower()
|
||||
if text in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "off", ""}:
|
||||
return False
|
||||
return bool(value)
|
||||
|
||||
for key in ("force", "unwrap", "geocode"):
|
||||
if key in normalized:
|
||||
normalized[key] = _coerce_bool(normalized[key])
|
||||
|
||||
for key, maximum, label in (
|
||||
("range_looks", MAX_LOOKS, "距离向多视"),
|
||||
("azimuth_looks", MAX_LOOKS, "方位向多视"),
|
||||
("parallel_workers", MAX_PARALLEL_WORKERS, "并行数"),
|
||||
):
|
||||
if key not in normalized or normalized[key] is None:
|
||||
continue
|
||||
try:
|
||||
parsed = int(normalized[key])
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{label}必须为整数。") from exc
|
||||
if parsed < 1 or parsed > maximum:
|
||||
raise ValueError(f"{label}必须在 1 到 {maximum} 之间。")
|
||||
normalized[key] = parsed
|
||||
|
||||
return normalized
|
||||
|
||||
def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
return validate_pyint_root_dir(root_dir, num_to_process)
|
||||
|
||||
def check_available(self) -> EngineAvailability:
|
||||
report = check_pyint_environment(
|
||||
enabled=self._enabled,
|
||||
distro=self._distro,
|
||||
python_cmd=self._python,
|
||||
pyint_home=self._pyint_home,
|
||||
pyint_app_script=self._pyint_app_script,
|
||||
template_root=self._template_root,
|
||||
work_root=self._work_root,
|
||||
output_root=self._output_root,
|
||||
dem_root=self._dem_root,
|
||||
gamma_env_script=self._gamma_env_script,
|
||||
smoke_test=self._smoke_test,
|
||||
)
|
||||
checks_list = [
|
||||
{
|
||||
"name": check.name,
|
||||
"ok": check.ok,
|
||||
"detail": check.detail,
|
||||
"skipped": check.skipped,
|
||||
}
|
||||
for check in report.checks
|
||||
]
|
||||
if report.overall_ok:
|
||||
status = "ok"
|
||||
available = True
|
||||
else:
|
||||
critical_failed = [check for check in report.checks if not check.ok and not check.skipped]
|
||||
status = "degraded" if critical_failed else "unavailable"
|
||||
available = False
|
||||
return EngineAvailability(
|
||||
engine_code=self.engine_code,
|
||||
status=status,
|
||||
available=available,
|
||||
checks=checks_list,
|
||||
message=report.message,
|
||||
)
|
||||
|
||||
def run(self, request: RunRequest) -> RunResult:
|
||||
if not self._enabled:
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
profile=request.profile,
|
||||
job_id=request.job_id,
|
||||
error="PyINT is disabled.",
|
||||
)
|
||||
|
||||
if request.profile != "lt1_gamma_dinsar":
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
profile=request.profile,
|
||||
job_id=request.job_id,
|
||||
error=f"Unknown profile: {request.profile}",
|
||||
)
|
||||
|
||||
return self._run_lt1_gamma_dinsar(request)
|
||||
|
||||
def _run_lt1_gamma_dinsar(self, request: RunRequest) -> RunResult:
|
||||
extra = self.normalize_extra(request.extra)
|
||||
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
|
||||
task_dirs: List[str] = validation["task_dirs"]
|
||||
total_tasks = len(task_dirs)
|
||||
run_started_at = datetime.utcnow()
|
||||
run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z"
|
||||
run_key = f"run_{run_started_at.strftime('%Y%m%dT%H%M%SZ')}_{self.engine_code}_{request.profile}"
|
||||
progress_callback = request.progress_callback
|
||||
|
||||
def emit_progress(event_type: str, **payload: Any) -> None:
|
||||
if not callable(progress_callback):
|
||||
return
|
||||
try:
|
||||
progress_callback({"event": event_type, **payload})
|
||||
except Exception:
|
||||
return
|
||||
|
||||
timeout = max(60, int(request.timeout_seconds or self.default_timeout_seconds))
|
||||
force = bool(extra.get("force"))
|
||||
range_looks = int(extra.get("range_looks", DEFAULT_RANGE_LOOKS))
|
||||
azimuth_looks = int(extra.get("azimuth_looks", DEFAULT_AZIMUTH_LOOKS))
|
||||
parallel_workers = int(extra.get("parallel_workers", DEFAULT_PARALLEL_WORKERS))
|
||||
unwrap = bool(extra.get("unwrap", True))
|
||||
geocode = bool(extra.get("geocode", True))
|
||||
|
||||
wsl_pyint_home = to_wsl_path(self._pyint_home)
|
||||
wsl_pyint_app = to_wsl_path(self._pyint_app_script)
|
||||
wsl_dem_root = to_wsl_path(self._dem_root)
|
||||
wsl_fabdem_root = to_wsl_path(self._fabdem_root) if self._fabdem_root else ""
|
||||
wsl_orbit_pool = to_wsl_path(self._orbit_pool_txt) if self._orbit_pool_txt else ""
|
||||
shared_dem_summary = get_pyint_dem_summary()
|
||||
prepared_dem_path = str(shared_dem_summary.get("prepared_dem_path") or "").strip()
|
||||
prepared_dem_kind = str(shared_dem_summary.get("prepared_dem_kind") or "").strip()
|
||||
wsl_prepared_dem_path = to_wsl_path(prepared_dem_path) if prepared_dem_path else ""
|
||||
shared_orbit_context = get_pyint_orbit_context()
|
||||
|
||||
task_results: List[Dict[str, Any]] = []
|
||||
output_dirs: List[str] = []
|
||||
pairs_processed = 0
|
||||
pairs_failed = 0
|
||||
|
||||
for pair_index, task_dir in enumerate(task_dirs, start=1):
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
task_name = task_identity["task_name"]
|
||||
task_alias = task_identity["task_alias"]
|
||||
pair_key = task_identity["pair_key"]
|
||||
pair_meta = task_identity["pair_meta"]
|
||||
master_date = task_identity["master_date"]
|
||||
slave_date = task_identity["slave_date"]
|
||||
|
||||
work_run_root = os.path.normpath(os.path.join(self._work_root, pair_key, run_key))
|
||||
output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, run_key, "native"))
|
||||
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
|
||||
project_name = build_project_name(pair_key, run_key)
|
||||
project_dir = os.path.join(work_run_root, project_name)
|
||||
input_assets_dir = os.path.join(work_run_root, "input_assets")
|
||||
|
||||
wsl_task_dir = to_wsl_path(task_dir)
|
||||
wsl_project_dir = to_wsl_path(project_dir)
|
||||
wsl_output_dir = to_wsl_path(output_dir)
|
||||
wsl_template_root = to_wsl_path(template_root)
|
||||
|
||||
emit_progress(
|
||||
"pair_started",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
task_dir=task_dir,
|
||||
work_dir=work_run_root,
|
||||
output_dir=output_dir,
|
||||
)
|
||||
|
||||
if not all((wsl_task_dir, wsl_project_dir, wsl_output_dir, wsl_template_root, wsl_pyint_home, wsl_pyint_app, wsl_dem_root)):
|
||||
pairs_failed += 1
|
||||
error_text = "Unable to convert PyINT paths to WSL paths."
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-2,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"work_dir": work_run_root,
|
||||
"project_dir": project_dir,
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -2,
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_project_dir": wsl_project_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
archives = self._discover_archives(task_dir)
|
||||
master_archives = archives.get("master", [])
|
||||
slave_archives = archives.get("slave", [])
|
||||
if not master_date:
|
||||
master_date = infer_scene_date_from_archives(master_archives)
|
||||
if not slave_date:
|
||||
slave_date = infer_scene_date_from_archives(slave_archives)
|
||||
time_baseline_days = resolve_time_baseline_days(master_date, slave_date, pair_meta)
|
||||
try:
|
||||
task_input_assets = resolve_pyint_task_input_assets(
|
||||
task_dir,
|
||||
dem_summary=shared_dem_summary,
|
||||
orbit_context=shared_orbit_context,
|
||||
)
|
||||
except Exception as exc:
|
||||
pairs_failed += 1
|
||||
error_text = f"Failed to resolve PyINT input assets: {exc}"
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-3,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"work_dir": work_run_root,
|
||||
"project_dir": project_dir,
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -3,
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_project_dir": wsl_project_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if not task_input_assets.get("allow_submit"):
|
||||
pairs_failed += 1
|
||||
error_text = "; ".join(task_input_assets.get("blockers") or []) or "PyINT input assets are incomplete."
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-4,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"work_dir": work_run_root,
|
||||
"project_dir": project_dir,
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -4,
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
"input_assets": task_input_assets.get("input_assets"),
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_project_dir": wsl_project_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
materialized_input_assets = materialize_pyint_input_assets(
|
||||
task_summary=task_input_assets,
|
||||
input_assets_dir=input_assets_dir,
|
||||
project_name=project_name,
|
||||
)
|
||||
except Exception as exc:
|
||||
pairs_failed += 1
|
||||
error_text = f"Failed to materialize PyINT input assets: {exc}"
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=False,
|
||||
returncode=-5,
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"work_dir": work_run_root,
|
||||
"project_dir": project_dir,
|
||||
"output_dir": output_dir,
|
||||
"success": False,
|
||||
"returncode": -5,
|
||||
"error": error_text,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"command": "",
|
||||
"input_assets": task_input_assets.get("input_assets"),
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_project_dir": wsl_project_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
input_assets_summary = materialized_input_assets.get("input_assets") or task_input_assets.get("input_assets") or {}
|
||||
wsl_input_assets_dir = (
|
||||
to_wsl_path(materialized_input_assets.get("input_assets_dir", ""))
|
||||
if materialized_input_assets.get("input_assets_dir")
|
||||
else ""
|
||||
)
|
||||
wsl_input_assets_json = (
|
||||
to_wsl_path(materialized_input_assets.get("task_manifest_path", ""))
|
||||
if materialized_input_assets.get("task_manifest_path")
|
||||
else ""
|
||||
)
|
||||
|
||||
cmd_parts = [
|
||||
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}",
|
||||
f"--project-dir {quote_shell(wsl_project_dir)}",
|
||||
f"--template-root {quote_shell(wsl_template_root)}",
|
||||
f"--output-dir {quote_shell(wsl_output_dir)}",
|
||||
f"--pyint-home {quote_shell(wsl_pyint_home)}",
|
||||
f"--pyint-app-script {quote_shell(wsl_pyint_app)}",
|
||||
f"--python {quote_shell(self._python)}",
|
||||
f"--dem-root {quote_shell(wsl_dem_root)}",
|
||||
f"--dem-mode {quote_shell(self._dem_mode)}",
|
||||
f"--project-name {quote_shell(project_name)}",
|
||||
f"--pair-key {quote_shell(pair_key)}",
|
||||
f"--task-alias {quote_shell(task_alias)}",
|
||||
f"--orbit-policy {quote_shell(self._orbit_policy)}",
|
||||
f"--range-looks {range_looks}",
|
||||
f"--azimuth-looks {azimuth_looks}",
|
||||
f"--parallel-workers {parallel_workers}",
|
||||
f"--master-date {quote_shell(master_date)}" if master_date else "",
|
||||
f"--slave-date {quote_shell(slave_date)}" if slave_date else "",
|
||||
f"--time-baseline-days {time_baseline_days}",
|
||||
f"--input-assets-dir {quote_shell(wsl_input_assets_dir)}" if wsl_input_assets_dir else "",
|
||||
f"--input-assets-json {quote_shell(wsl_input_assets_json)}" if wsl_input_assets_json else "",
|
||||
f"--lt1-precise-orbit-enabled {'true' if self._lt1_precise_orbit_enabled else 'false'}",
|
||||
f"--lt1-precise-orbit-mode {quote_shell(self._lt1_precise_orbit_mode)}",
|
||||
f"--lt1-precise-orbit-strict {'true' if self._lt1_precise_orbit_strict else 'false'}",
|
||||
(
|
||||
f"--lt1-precise-orbit-validate-with-orb-filt "
|
||||
f"{'true' if self._lt1_precise_orbit_validate_with_orb_filt else 'false'}"
|
||||
),
|
||||
f"--lt1-precise-orbit-backup {'true' if self._lt1_precise_orbit_backup else 'false'}",
|
||||
f"--lt1-precise-orbit-orb-filt-degree {self._lt1_precise_orbit_orb_filt_degree}",
|
||||
"--unwrap" if unwrap else "--no-unwrap",
|
||||
"--geocode" if geocode else "--no-geocode",
|
||||
]
|
||||
if self._dem_mode == "local_fabdem" and wsl_fabdem_root:
|
||||
cmd_parts.append(f"--fabdem-root {quote_shell(wsl_fabdem_root)}")
|
||||
if self._dem_mode == "prepared_file" and wsl_prepared_dem_path:
|
||||
cmd_parts.append(f"--prepared-dem-path {quote_shell(wsl_prepared_dem_path)}")
|
||||
if self._dem_mode == "opentopo":
|
||||
if self._opentopo_dem_type:
|
||||
cmd_parts.append(f"--opentopo-dem-type {quote_shell(self._opentopo_dem_type)}")
|
||||
if self._opentopo_api_key:
|
||||
cmd_parts.append(f"--opentopo-api-key {quote_shell(self._opentopo_api_key)}")
|
||||
if self._gamma_env_script:
|
||||
cmd_parts.append(f"--gamma-env-script {quote_shell(to_wsl_path(self._gamma_env_script))}")
|
||||
if force:
|
||||
cmd_parts.append("--force")
|
||||
|
||||
cmd = " ".join(part for part in cmd_parts if part)
|
||||
rc, stdout, stderr = run_wsl_command(
|
||||
cmd,
|
||||
distro=self._distro,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
success = rc == 0
|
||||
if success:
|
||||
pairs_processed += 1
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
write_run_metadata(
|
||||
output_dir,
|
||||
{
|
||||
"run_key": run_key,
|
||||
"pair_key": pair_key,
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"engine_code": self.engine_code,
|
||||
"profile_code": request.profile,
|
||||
"source_root": os.path.normpath(request.root_dir),
|
||||
"task_dir": os.path.normpath(task_dir),
|
||||
"work_dir": work_run_root,
|
||||
"output_dir": output_dir,
|
||||
"project_dir": project_dir,
|
||||
"started_at": run_started_at_text,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"params": {
|
||||
"force": force,
|
||||
"range_looks": range_looks,
|
||||
"azimuth_looks": azimuth_looks,
|
||||
"parallel_workers": parallel_workers,
|
||||
"unwrap": unwrap,
|
||||
"geocode": geocode,
|
||||
},
|
||||
"master_path": pair_meta.get("master_path"),
|
||||
"slave_path": pair_meta.get("slave_path"),
|
||||
"master_satellite": task_input_assets.get("master_satellite") or pair_meta.get("master_satellite"),
|
||||
"slave_satellite": task_input_assets.get("slave_satellite") or pair_meta.get("slave_satellite"),
|
||||
"master_imaging_date": pair_meta.get("master_imaging_date") or master_date,
|
||||
"slave_imaging_date": pair_meta.get("slave_imaging_date") or slave_date,
|
||||
"master_imaging_mode": pair_meta.get("master_imaging_mode"),
|
||||
"slave_imaging_mode": pair_meta.get("slave_imaging_mode"),
|
||||
"master_polarization": pair_meta.get("master_polarization"),
|
||||
"slave_polarization": pair_meta.get("slave_polarization"),
|
||||
"time_baseline_days": pair_meta.get("time_baseline_days") or time_baseline_days,
|
||||
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
|
||||
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
|
||||
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
|
||||
"network_run_id": pair_meta.get("network_run_id"),
|
||||
"network_edge_id": pair_meta.get("network_edge_id"),
|
||||
"policy_version": pair_meta.get("policy_version"),
|
||||
"selection_strategy": pair_meta.get("selection_strategy"),
|
||||
"input_assets": input_assets_summary,
|
||||
},
|
||||
)
|
||||
output_dirs.append(output_dir)
|
||||
else:
|
||||
pairs_failed += 1
|
||||
|
||||
emit_progress(
|
||||
"pair_finished",
|
||||
pair_index=pair_index,
|
||||
pair_total=total_tasks,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
success=success,
|
||||
returncode=rc,
|
||||
error=stderr.strip() if stderr else "",
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"work_dir": work_run_root,
|
||||
"project_dir": project_dir,
|
||||
"output_dir": output_dir,
|
||||
"command": cmd,
|
||||
"success": success,
|
||||
"returncode": rc,
|
||||
"stdout_tail": stdout[-3000:] if stdout else "",
|
||||
"stderr_tail": stderr[-3000:] if stderr else "",
|
||||
"error": stderr.strip() if stderr else "",
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_project_dir": wsl_project_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
"wsl_template_root": wsl_template_root,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"archive_counts": {
|
||||
"master": len(master_archives),
|
||||
"slave": len(slave_archives),
|
||||
},
|
||||
"input_assets": input_assets_summary,
|
||||
"wsl_input_assets_dir": wsl_input_assets_dir,
|
||||
}
|
||||
)
|
||||
|
||||
invalid_candidates = validation.get("invalid_candidates", [])
|
||||
pairs_failed += len(invalid_candidates)
|
||||
overall_success = pairs_processed > 0 or (pairs_processed == 0 and pairs_failed == 0)
|
||||
failed_task_names = [
|
||||
item["task_name"]
|
||||
for item in task_results
|
||||
if not item.get("success")
|
||||
] + [item["name"] for item in invalid_candidates]
|
||||
|
||||
error = None
|
||||
if not overall_success:
|
||||
if failed_task_names:
|
||||
error = f"All PyINT tasks failed: {', '.join(failed_task_names[:10])}"
|
||||
else:
|
||||
error = "PyINT run failed."
|
||||
|
||||
last_task_result = task_results[-1] if task_results else {}
|
||||
return RunResult(
|
||||
success=overall_success,
|
||||
engine_code=self.engine_code,
|
||||
profile=request.profile,
|
||||
job_id=request.job_id,
|
||||
pairs_processed=pairs_processed,
|
||||
pairs_failed=pairs_failed,
|
||||
output_dirs=output_dirs,
|
||||
error=error,
|
||||
detail={
|
||||
"mode": validation["mode"],
|
||||
"task_count": len(task_dirs),
|
||||
"selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results],
|
||||
"invalid_candidates": invalid_candidates,
|
||||
"task_results": task_results,
|
||||
"run_key": run_key,
|
||||
"started_at": run_started_at_text,
|
||||
"force": force,
|
||||
"timeout_seconds": timeout,
|
||||
"range_looks": range_looks,
|
||||
"azimuth_looks": azimuth_looks,
|
||||
"parallel_workers": parallel_workers,
|
||||
"unwrap": unwrap,
|
||||
"geocode": geocode,
|
||||
"command": last_task_result.get("command", ""),
|
||||
"stdout_tail": last_task_result.get("stdout_tail", ""),
|
||||
"stderr_tail": last_task_result.get("stderr_tail", ""),
|
||||
"wsl_task_dir": last_task_result.get("wsl_task_dir", ""),
|
||||
"wsl_project_dir": last_task_result.get("wsl_project_dir", ""),
|
||||
"wsl_output_dir": last_task_result.get("wsl_output_dir", ""),
|
||||
"wsl_template_root": last_task_result.get("wsl_template_root", ""),
|
||||
"wsl_dem_root": wsl_dem_root,
|
||||
"wsl_dem": wsl_dem_root,
|
||||
"wsl_pyint_home": wsl_pyint_home,
|
||||
"wsl_orbit_pool": wsl_orbit_pool,
|
||||
"wsl_work_root": to_wsl_path(self._work_root) if self._work_root else "",
|
||||
"wsl_output_root": to_wsl_path(self._output_root) if self._output_root else "",
|
||||
"dem_mode": self._dem_mode,
|
||||
"prepared_dem_path": prepared_dem_path,
|
||||
"prepared_dem_kind": prepared_dem_kind,
|
||||
"wsl_prepared_dem_path": wsl_prepared_dem_path,
|
||||
"orbit_policy": self._orbit_policy,
|
||||
"lt1_precise_orbit_enabled": self._lt1_precise_orbit_enabled,
|
||||
"lt1_precise_orbit_mode": self._lt1_precise_orbit_mode,
|
||||
"lt1_precise_orbit_strict": self._lt1_precise_orbit_strict,
|
||||
"lt1_precise_orbit_validate_with_orb_filt": self._lt1_precise_orbit_validate_with_orb_filt,
|
||||
"lt1_precise_orbit_backup": self._lt1_precise_orbit_backup,
|
||||
"lt1_precise_orbit_orb_filt_degree": self._lt1_precise_orbit_orb_filt_degree,
|
||||
"record_input_assets": self._record_input_assets,
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _discover_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
from ..services.pyint_service import discover_lt1_archives
|
||||
|
||||
return discover_lt1_archives(task_dir)
|
||||
@@ -31,9 +31,10 @@ def _bootstrap() -> None:
|
||||
|
||||
from .isce2_engine import Isce2Engine
|
||||
from .landsar_engine import LandsarEngine
|
||||
from .pyint_engine import PyintEngine
|
||||
from .sarscape_engine import SarscapeEngine
|
||||
|
||||
for engine in (SarscapeEngine(), Isce2Engine(), LandsarEngine()):
|
||||
for engine in (SarscapeEngine(), Isce2Engine(), PyintEngine(), LandsarEngine()):
|
||||
register(engine)
|
||||
|
||||
|
||||
|
||||
@@ -71,10 +71,12 @@ class DinsarResult(BaseModel):
|
||||
task_alias: Optional[str] = None
|
||||
pair_key: Optional[str] = None
|
||||
pair_uid: Optional[str] = None
|
||||
run_key: Optional[str] = None
|
||||
network_run_id: Optional[str] = None
|
||||
network_edge_id: Optional[int] = None
|
||||
policy_version: Optional[str] = None
|
||||
selection_strategy: Optional[str] = None
|
||||
engine_code: Optional[str] = None
|
||||
file_path: str
|
||||
min_lon: float
|
||||
min_lat: float
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""PyINT pipeline helpers."""
|
||||
@@ -0,0 +1,575 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
ISCE2_PIPELINE_DIR = SCRIPT_DIR.parent / "isce2_pipeline"
|
||||
if str(ISCE2_PIPELINE_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ISCE2_PIPELINE_DIR))
|
||||
|
||||
from convert_lt1_orbit_to_isce_xml import StateVector, parse_orbit_file # type: ignore
|
||||
|
||||
|
||||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||||
VECTOR_POS_RE = re.compile(r"^state_vector_position_(\d+):")
|
||||
VECTOR_VEL_RE = re.compile(r"^state_vector_velocity_(\d+):")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedSlcPar:
|
||||
path: Path
|
||||
lines: List[str]
|
||||
trailing_newline: bool
|
||||
acquisition_date: date
|
||||
number_of_state_vectors: int
|
||||
time_of_first_state_vector: float
|
||||
state_vector_interval: float
|
||||
position_line_indexes: Dict[int, int]
|
||||
velocity_line_indexes: Dict[int, int]
|
||||
|
||||
|
||||
def utc_now_text() -> str:
|
||||
return datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
|
||||
def read_bool(value: Any, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
return str(value).strip().lower() in TRUE_VALUES
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
normalized = text.replace("\\", "/")
|
||||
if normalized.startswith("/"):
|
||||
return normalized
|
||||
if normalized.startswith("//"):
|
||||
return ""
|
||||
match = re.match(r"^([A-Za-z]):/(.*)$", normalized)
|
||||
if not match:
|
||||
return normalized
|
||||
drive_letter = match.group(1).lower()
|
||||
tail = match.group(2).lstrip("/")
|
||||
return f"/mnt/{drive_letter}/{tail}"
|
||||
|
||||
|
||||
def resolve_existing_path(path: str) -> Optional[Path]:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
direct = Path(text)
|
||||
if direct.exists():
|
||||
return direct.resolve()
|
||||
converted = windows_path_to_wsl_mount(text)
|
||||
if converted:
|
||||
candidate = Path(converted)
|
||||
if candidate.exists():
|
||||
return candidate.resolve()
|
||||
return None
|
||||
|
||||
|
||||
def load_json_file(path: Path | None) -> Dict[str, Any]:
|
||||
if path is None or not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Apply LT-1 precise orbit TXT to Gamma .slc.par state vectors.")
|
||||
parser.add_argument("--date", required=True, help="Scene date in YYYYMMDD format.")
|
||||
parser.add_argument("--manifest-json", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST", ""), help="task_manifest.json path.")
|
||||
parser.add_argument("--summary-json", default="", help="Summary JSON path. Defaults to <slc_dir>/orbit_bridge_summary.json.")
|
||||
parser.add_argument("--role", choices=("auto", "master", "slave"), default="auto")
|
||||
parser.add_argument("--operation-tag", default="raw2slc")
|
||||
parser.add_argument("--mode", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MODE", "replace"))
|
||||
parser.add_argument("--slc-par", dest="slc_par_files", action="append", default=[], help="Target .slc.par or .slc.update.par file.")
|
||||
parser.add_argument("--backup", dest="backup", action="store_true")
|
||||
parser.add_argument("--no-backup", dest="backup", action="store_false")
|
||||
parser.add_argument("--strict", dest="strict", action="store_true")
|
||||
parser.add_argument("--no-strict", dest="strict", action="store_false")
|
||||
parser.add_argument("--validate-with-orb-filt", dest="validate_with_orb_filt", action="store_true")
|
||||
parser.add_argument("--no-validate-with-orb-filt", dest="validate_with_orb_filt", action="store_false")
|
||||
parser.add_argument(
|
||||
"--orb-filt-degree",
|
||||
type=int,
|
||||
default=int(str(os.getenv("PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", "5")).strip() or "5"),
|
||||
)
|
||||
parser.set_defaults(
|
||||
backup=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_BACKUP"), True),
|
||||
strict=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_STRICT"), True),
|
||||
validate_with_orb_filt=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT"), False),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def get_orbits_payload(manifest: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if isinstance(manifest.get("orbits"), dict):
|
||||
return manifest["orbits"]
|
||||
input_assets = manifest.get("input_assets")
|
||||
if isinstance(input_assets, dict) and isinstance(input_assets.get("orbits"), dict):
|
||||
return input_assets["orbits"]
|
||||
return {}
|
||||
|
||||
|
||||
def resolve_orbit_entry(manifest: Dict[str, Any], date_text: str, role: str) -> Dict[str, Any]:
|
||||
orbits = get_orbits_payload(manifest)
|
||||
candidates: List[tuple[str, Dict[str, Any]]] = []
|
||||
for role_name in ("master", "slave"):
|
||||
item = orbits.get(role_name)
|
||||
if isinstance(item, dict):
|
||||
candidates.append((role_name, item))
|
||||
|
||||
if role in {"master", "slave"}:
|
||||
item = dict(orbits.get(role) or {})
|
||||
if not item:
|
||||
raise RuntimeError(f"Missing orbit entry for role={role}")
|
||||
item["role"] = role
|
||||
return item
|
||||
|
||||
matched: List[Dict[str, Any]] = []
|
||||
for role_name, item in candidates:
|
||||
item_date = str(item.get("date") or "").strip()
|
||||
expected_name = str(item.get("expected_name") or "").strip()
|
||||
if item_date == date_text or date_text in expected_name:
|
||||
candidate = dict(item)
|
||||
candidate["role"] = role_name
|
||||
matched.append(candidate)
|
||||
|
||||
if len(matched) == 1:
|
||||
return matched[0]
|
||||
if not matched:
|
||||
raise RuntimeError(f"Unable to match precise orbit entry for date={date_text}")
|
||||
raise RuntimeError(f"Ambiguous precise orbit entries for date={date_text}")
|
||||
|
||||
|
||||
def resolve_orbit_txt_path(entry: Dict[str, Any]) -> Path:
|
||||
for key in ("staged_path", "path"):
|
||||
candidate = resolve_existing_path(str(entry.get(key) or ""))
|
||||
if candidate is not None and candidate.is_file():
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Precise orbit TXT does not exist: expected {entry.get('expected_name') or '<unknown>'}"
|
||||
)
|
||||
|
||||
|
||||
def parse_float_field(lines: Iterable[str], prefix: str) -> float:
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith(prefix):
|
||||
continue
|
||||
_, _, value = stripped.partition(":")
|
||||
first_token = value.strip().split()[0]
|
||||
return float(first_token)
|
||||
raise ValueError(f"Missing field: {prefix}")
|
||||
|
||||
|
||||
def parse_int_field(lines: Iterable[str], prefix: str) -> int:
|
||||
return int(round(parse_float_field(lines, prefix)))
|
||||
|
||||
|
||||
def parse_slc_date(lines: Iterable[str]) -> date:
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("date:"):
|
||||
continue
|
||||
match = re.match(r"^date:\s+(\d+)\s+(\d+)\s+(\d+)", stripped)
|
||||
if not match:
|
||||
raise ValueError(f"Unable to parse date line: {stripped}")
|
||||
return date(int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
raise ValueError("Missing date: field in .slc.par")
|
||||
|
||||
|
||||
def parse_slc_par(path: Path) -> ParsedSlcPar:
|
||||
raw_text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
trailing_newline = raw_text.endswith("\n")
|
||||
lines = raw_text.splitlines()
|
||||
acquisition_date = parse_slc_date(lines)
|
||||
number_of_state_vectors = parse_int_field(lines, "number_of_state_vectors")
|
||||
time_of_first_state_vector = parse_float_field(lines, "time_of_first_state_vector")
|
||||
state_vector_interval = parse_float_field(lines, "state_vector_interval")
|
||||
|
||||
position_line_indexes: Dict[int, int] = {}
|
||||
velocity_line_indexes: Dict[int, int] = {}
|
||||
for idx, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
pos_match = VECTOR_POS_RE.match(stripped)
|
||||
if pos_match:
|
||||
position_line_indexes[int(pos_match.group(1))] = idx
|
||||
continue
|
||||
vel_match = VECTOR_VEL_RE.match(stripped)
|
||||
if vel_match:
|
||||
velocity_line_indexes[int(vel_match.group(1))] = idx
|
||||
|
||||
missing_positions = [index for index in range(1, number_of_state_vectors + 1) if index not in position_line_indexes]
|
||||
missing_velocities = [index for index in range(1, number_of_state_vectors + 1) if index not in velocity_line_indexes]
|
||||
if missing_positions or missing_velocities:
|
||||
raise ValueError(
|
||||
"Incomplete state vector block in .slc.par: "
|
||||
f"missing positions={missing_positions[:5]}, missing velocities={missing_velocities[:5]}"
|
||||
)
|
||||
|
||||
return ParsedSlcPar(
|
||||
path=path,
|
||||
lines=lines,
|
||||
trailing_newline=trailing_newline,
|
||||
acquisition_date=acquisition_date,
|
||||
number_of_state_vectors=number_of_state_vectors,
|
||||
time_of_first_state_vector=time_of_first_state_vector,
|
||||
state_vector_interval=state_vector_interval,
|
||||
position_line_indexes=position_line_indexes,
|
||||
velocity_line_indexes=velocity_line_indexes,
|
||||
)
|
||||
|
||||
|
||||
def build_target_times(parsed: ParsedSlcPar) -> List[datetime]:
|
||||
start_time = datetime(parsed.acquisition_date.year, parsed.acquisition_date.month, parsed.acquisition_date.day)
|
||||
return [
|
||||
start_time + timedelta(seconds=parsed.time_of_first_state_vector + parsed.state_vector_interval * index)
|
||||
for index in range(parsed.number_of_state_vectors)
|
||||
]
|
||||
|
||||
|
||||
def norm3(values: Iterable[float]) -> float:
|
||||
items = [float(item) for item in values]
|
||||
return math.sqrt(sum(item * item for item in items))
|
||||
|
||||
|
||||
def interpolate_state_vector(target_time: datetime, vectors: List[StateVector]) -> StateVector:
|
||||
if not vectors:
|
||||
raise ValueError("No precise orbit vectors available for interpolation")
|
||||
|
||||
times = [vector.time for vector in vectors]
|
||||
if target_time < times[0] or target_time > times[-1]:
|
||||
raise ValueError(
|
||||
f"Target time {target_time.isoformat()} is outside orbit range {times[0].isoformat()} - {times[-1].isoformat()}"
|
||||
)
|
||||
|
||||
right_index = bisect.bisect_left(times, target_time)
|
||||
if right_index < len(vectors) and times[right_index] == target_time:
|
||||
return vectors[right_index]
|
||||
if right_index == 0:
|
||||
return vectors[0]
|
||||
if right_index >= len(vectors):
|
||||
return vectors[-1]
|
||||
|
||||
left = vectors[right_index - 1]
|
||||
right = vectors[right_index]
|
||||
interval_seconds = (right.time - left.time).total_seconds()
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("Orbit vectors are not strictly increasing in time")
|
||||
|
||||
offset_seconds = (target_time - left.time).total_seconds()
|
||||
u = offset_seconds / interval_seconds
|
||||
|
||||
h00 = 2 * u * u * u - 3 * u * u + 1
|
||||
h10 = u * u * u - 2 * u * u + u
|
||||
h01 = -2 * u * u * u + 3 * u * u
|
||||
h11 = u * u * u - u * u
|
||||
|
||||
dh00 = 6 * u * u - 6 * u
|
||||
dh10 = 3 * u * u - 4 * u + 1
|
||||
dh01 = -6 * u * u + 6 * u
|
||||
dh11 = 3 * u * u - 2 * u
|
||||
|
||||
p0 = (left.x, left.y, left.z)
|
||||
p1 = (right.x, right.y, right.z)
|
||||
v0 = (left.vx, left.vy, left.vz)
|
||||
v1 = (right.vx, right.vy, right.vz)
|
||||
|
||||
position = []
|
||||
velocity = []
|
||||
for axis in range(3):
|
||||
pos = (
|
||||
h00 * p0[axis]
|
||||
+ h10 * interval_seconds * v0[axis]
|
||||
+ h01 * p1[axis]
|
||||
+ h11 * interval_seconds * v1[axis]
|
||||
)
|
||||
vel = (
|
||||
dh00 * p0[axis]
|
||||
+ dh10 * interval_seconds * v0[axis]
|
||||
+ dh01 * p1[axis]
|
||||
+ dh11 * interval_seconds * v1[axis]
|
||||
) / interval_seconds
|
||||
position.append(pos)
|
||||
velocity.append(vel)
|
||||
|
||||
return StateVector(
|
||||
time=target_time,
|
||||
x=position[0],
|
||||
y=position[1],
|
||||
z=position[2],
|
||||
vx=velocity[0],
|
||||
vy=velocity[1],
|
||||
vz=velocity[2],
|
||||
)
|
||||
|
||||
|
||||
def format_position_line(index: int, vector: StateVector) -> str:
|
||||
return (
|
||||
f"state_vector_position_{index}:"
|
||||
f" {vector.x:14.4f} {vector.y:14.4f} {vector.z:14.4f} m m m"
|
||||
)
|
||||
|
||||
|
||||
def format_velocity_line(index: int, vector: StateVector) -> str:
|
||||
return (
|
||||
f"state_vector_velocity_{index}:"
|
||||
f" {vector.vx:13.5f} {vector.vy:13.5f} {vector.vz:13.5f} m/s m/s m/s"
|
||||
)
|
||||
|
||||
|
||||
def backup_slc_par(path: Path) -> str:
|
||||
backup_path = path.with_name(path.name + ".orbit_bridge.bak")
|
||||
if not backup_path.exists():
|
||||
shutil.copy2(path, backup_path)
|
||||
return str(backup_path)
|
||||
|
||||
|
||||
def write_bridged_slc_par(
|
||||
parsed: ParsedSlcPar,
|
||||
vectors: List[StateVector],
|
||||
*,
|
||||
backup_enabled: bool,
|
||||
) -> Dict[str, Any]:
|
||||
if len(vectors) != parsed.number_of_state_vectors:
|
||||
raise ValueError("Interpolated vector count does not match .slc.par state vector count")
|
||||
|
||||
backup_path = ""
|
||||
if backup_enabled:
|
||||
backup_path = backup_slc_par(parsed.path)
|
||||
|
||||
updated_lines = list(parsed.lines)
|
||||
for index, vector in enumerate(vectors, start=1):
|
||||
updated_lines[parsed.position_line_indexes[index]] = format_position_line(index, vector)
|
||||
updated_lines[parsed.velocity_line_indexes[index]] = format_velocity_line(index, vector)
|
||||
|
||||
text = "\n".join(updated_lines)
|
||||
if parsed.trailing_newline:
|
||||
text += "\n"
|
||||
parsed.path.write_text(text, encoding="utf-8")
|
||||
|
||||
return {
|
||||
"backup_path": backup_path,
|
||||
}
|
||||
|
||||
|
||||
def run_orb_filt_validation(path: Path, degree: int) -> Dict[str, Any]:
|
||||
command = shutil.which("ORB_filt_spline.py")
|
||||
if not command:
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": False,
|
||||
"status": "missing_command",
|
||||
"command": "ORB_filt_spline.py",
|
||||
}
|
||||
|
||||
validate_path = path.with_name(path.name + ".orb_filt_validate.par")
|
||||
result = subprocess.run(
|
||||
[command, str(path), str(validate_path), "--degree", str(int(degree))],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0 or not validate_path.exists():
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": False,
|
||||
"status": "command_failed",
|
||||
"command": " ".join(result.args),
|
||||
"returncode": int(result.returncode),
|
||||
"stdout": (result.stdout or "")[-2000:],
|
||||
"stderr": (result.stderr or "")[-2000:],
|
||||
"output_par": str(validate_path),
|
||||
}
|
||||
|
||||
current_parsed = parse_slc_par(path)
|
||||
validated_parsed = parse_slc_par(validate_path)
|
||||
position_corrections: List[float] = []
|
||||
velocity_corrections: List[float] = []
|
||||
for index in range(1, current_parsed.number_of_state_vectors + 1):
|
||||
cur_position = parse_vector_values(current_parsed.lines[current_parsed.position_line_indexes[index]])
|
||||
val_position = parse_vector_values(validated_parsed.lines[validated_parsed.position_line_indexes[index]])
|
||||
cur_velocity = parse_vector_values(current_parsed.lines[current_parsed.velocity_line_indexes[index]])
|
||||
val_velocity = parse_vector_values(validated_parsed.lines[validated_parsed.velocity_line_indexes[index]])
|
||||
position_corrections.append(norm3([val_position[i] - cur_position[i] for i in range(3)]))
|
||||
velocity_corrections.append(norm3([val_velocity[i] - cur_velocity[i] for i in range(3)]))
|
||||
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": True,
|
||||
"status": "ok",
|
||||
"command": " ".join(result.args),
|
||||
"returncode": int(result.returncode),
|
||||
"output_par": str(validate_path),
|
||||
"max_position_correction_m": max(position_corrections) if position_corrections else 0.0,
|
||||
"max_velocity_correction_mps": max(velocity_corrections) if velocity_corrections else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def parse_vector_values(line: str) -> List[float]:
|
||||
_, _, payload = line.partition(":")
|
||||
values: List[float] = []
|
||||
for token in payload.split():
|
||||
try:
|
||||
values.append(float(token))
|
||||
except ValueError:
|
||||
break
|
||||
if len(values) == 3:
|
||||
break
|
||||
if len(values) != 3:
|
||||
raise ValueError(f"Unable to parse state vector values from line: {line}")
|
||||
return values
|
||||
|
||||
|
||||
def build_operation_record(args: argparse.Namespace, summary_path: Path, manifest_path: Path | None) -> Dict[str, Any]:
|
||||
return {
|
||||
"generated_at": utc_now_text(),
|
||||
"date": str(args.date or "").strip(),
|
||||
"role": args.role,
|
||||
"operation_tag": str(args.operation_tag or "").strip(),
|
||||
"mode": str(args.mode or "").strip(),
|
||||
"strict": bool(args.strict),
|
||||
"backup": bool(args.backup),
|
||||
"validate_with_orb_filt": bool(args.validate_with_orb_filt),
|
||||
"orb_filt_degree": int(args.orb_filt_degree),
|
||||
"manifest_json": str(manifest_path) if manifest_path else "",
|
||||
"summary_json": str(summary_path),
|
||||
"slc_par_files": [str(path) for path in args.slc_par_files],
|
||||
"ok": False,
|
||||
"error": "",
|
||||
"orbit_source": {},
|
||||
"results": [],
|
||||
}
|
||||
|
||||
|
||||
def append_operation_summary(summary_path: Path, operation: Dict[str, Any]) -> None:
|
||||
existing = load_json_file(summary_path)
|
||||
operations = existing.get("operations")
|
||||
if not isinstance(operations, list):
|
||||
operations = []
|
||||
operations.append(operation)
|
||||
payload = {
|
||||
"generated_at": existing.get("generated_at") or utc_now_text(),
|
||||
"last_updated_at": utc_now_text(),
|
||||
"ok": all(bool(item.get("ok")) for item in operations),
|
||||
"operation_count": len(operations),
|
||||
"operations": operations,
|
||||
}
|
||||
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
summary_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def default_summary_path(slc_par_files: List[str]) -> Path:
|
||||
first_path = Path(slc_par_files[0]).resolve()
|
||||
return first_path.parent / "orbit_bridge_summary.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if not args.slc_par_files:
|
||||
raise SystemExit("--slc-par must be specified at least once")
|
||||
|
||||
manifest_path = resolve_existing_path(args.manifest_json) if args.manifest_json else None
|
||||
summary_path = Path(args.summary_json).resolve() if args.summary_json else default_summary_path(args.slc_par_files)
|
||||
operation = build_operation_record(args, summary_path, manifest_path)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
if manifest_path is None:
|
||||
raise FileNotFoundError("Precise orbit manifest JSON is not available")
|
||||
manifest = load_json_file(manifest_path)
|
||||
orbit_entry = resolve_orbit_entry(manifest, str(args.date or "").strip(), args.role)
|
||||
orbit_txt_path = resolve_orbit_txt_path(orbit_entry)
|
||||
orbit_vectors = sorted(parse_orbit_file(orbit_txt_path), key=lambda item: item.time)
|
||||
operation["orbit_source"] = {
|
||||
"role": orbit_entry.get("role"),
|
||||
"satellite": orbit_entry.get("satellite"),
|
||||
"date": orbit_entry.get("date"),
|
||||
"expected_name": orbit_entry.get("expected_name"),
|
||||
"source_txt": str(orbit_txt_path),
|
||||
"vector_count": len(orbit_vectors),
|
||||
"time_start": orbit_vectors[0].time.isoformat() if orbit_vectors else "",
|
||||
"time_stop": orbit_vectors[-1].time.isoformat() if orbit_vectors else "",
|
||||
}
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
for slc_par_text in args.slc_par_files:
|
||||
slc_par_path = resolve_existing_path(slc_par_text)
|
||||
if slc_par_path is None or not slc_par_path.is_file():
|
||||
raise FileNotFoundError(f"Target .slc.par does not exist: {slc_par_text}")
|
||||
|
||||
parsed = parse_slc_par(slc_par_path)
|
||||
target_times = build_target_times(parsed)
|
||||
bridged_vectors = [interpolate_state_vector(target_time, orbit_vectors) for target_time in target_times]
|
||||
write_info = write_bridged_slc_par(parsed, bridged_vectors, backup_enabled=bool(args.backup))
|
||||
validation = (
|
||||
run_orb_filt_validation(slc_par_path, args.orb_filt_degree)
|
||||
if args.validate_with_orb_filt
|
||||
else {"requested": False, "ok": True, "status": "skipped"}
|
||||
)
|
||||
result_item = {
|
||||
"path": str(slc_par_path),
|
||||
"status": "applied",
|
||||
"ok": bool(validation.get("ok", False)),
|
||||
"backup_path": write_info.get("backup_path", ""),
|
||||
"vector_count": parsed.number_of_state_vectors,
|
||||
"time_of_first_state_vector": parsed.time_of_first_state_vector,
|
||||
"state_vector_interval": parsed.state_vector_interval,
|
||||
"validation": validation,
|
||||
"first_target_time": target_times[0].isoformat() if target_times else "",
|
||||
"last_target_time": target_times[-1].isoformat() if target_times else "",
|
||||
"max_position_norm_m": max(norm3((vector.x, vector.y, vector.z)) for vector in bridged_vectors) if bridged_vectors else 0.0,
|
||||
"max_velocity_norm_mps": max(norm3((vector.vx, vector.vy, vector.vz)) for vector in bridged_vectors) if bridged_vectors else 0.0,
|
||||
}
|
||||
results.append(result_item)
|
||||
|
||||
operation["results"] = results
|
||||
operation["ok"] = all(bool(item.get("ok")) for item in results)
|
||||
if not operation["ok"]:
|
||||
operation["error"] = "One or more target .slc.par files failed validation"
|
||||
if args.strict:
|
||||
exit_code = 1
|
||||
except Exception as exc:
|
||||
operation["error"] = str(exc)
|
||||
operation["ok"] = False
|
||||
exit_code = 1 if args.strict else 0
|
||||
|
||||
append_operation_summary(summary_path, operation)
|
||||
if operation.get("error"):
|
||||
print(operation["error"], file=sys.stderr)
|
||||
else:
|
||||
applied_count = len(operation.get("results") or [])
|
||||
print(
|
||||
f"Applied LT-1 precise orbit bridge to {applied_count} file(s) for {operation.get('date')} "
|
||||
f"[{operation.get('operation_tag')}]"
|
||||
)
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
_pyint_gamma_die() {
|
||||
echo "$1" >&2
|
||||
return 1 2>/dev/null || exit 1
|
||||
}
|
||||
|
||||
_pyint_gamma_home=""
|
||||
if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${PYINT_GAMMA_HOME}"
|
||||
elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${GAMMA_HOME}"
|
||||
else
|
||||
for _candidate in \
|
||||
/usr/local/GAMMA_SOFTWARE-20240627 \
|
||||
/usr/local/GAMMA_SOFTWARE-* \
|
||||
/opt/GAMMA_SOFTWARE-*; do
|
||||
[ -d "${_candidate}" ] || continue
|
||||
_pyint_gamma_home="${_candidate}"
|
||||
break
|
||||
done
|
||||
fi
|
||||
|
||||
[ -n "${_pyint_gamma_home}" ] || _pyint_gamma_die "Gamma home not found."
|
||||
|
||||
export GAMMA_HOME="${_pyint_gamma_home}"
|
||||
export MSP_HOME="${GAMMA_HOME}/MSP"
|
||||
export ISP_HOME="${GAMMA_HOME}/ISP"
|
||||
export DIFF_HOME="${GAMMA_HOME}/DIFF"
|
||||
export DISP_HOME="${GAMMA_HOME}/DISP"
|
||||
export LAT_HOME="${GAMMA_HOME}/LAT"
|
||||
export IPTA_HOME="${GAMMA_HOME}/IPTA"
|
||||
export GEO_HOME="${GAMMA_HOME}/GEO"
|
||||
|
||||
_pyint_gamma_prepend_path() {
|
||||
local _dir="$1"
|
||||
[ -d "${_dir}" ] || return 0
|
||||
case ":${PATH}:" in
|
||||
*":${_dir}:"*) ;;
|
||||
*) PATH="${_dir}:${PATH}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for _gamma_dir in \
|
||||
"${MSP_HOME}/bin" \
|
||||
"${ISP_HOME}/bin" \
|
||||
"${DIFF_HOME}/bin" \
|
||||
"${DISP_HOME}/bin" \
|
||||
"${LAT_HOME}/bin" \
|
||||
"${IPTA_HOME}/bin" \
|
||||
"${GEO_HOME}/bin" \
|
||||
"${MSP_HOME}/scripts" \
|
||||
"${ISP_HOME}/scripts" \
|
||||
"${DIFF_HOME}/scripts" \
|
||||
"${DISP_HOME}/scripts" \
|
||||
"${LAT_HOME}/scripts" \
|
||||
"${IPTA_HOME}/scripts" \
|
||||
"${GEO_HOME}/scripts"; do
|
||||
_pyint_gamma_prepend_path "${_gamma_dir}"
|
||||
done
|
||||
|
||||
export PATH
|
||||
export OS="linux64"
|
||||
export HDF5_DISABLE_VERSION_CHECK="1"
|
||||
export GNUTERM="${GNUTERM:-qt}"
|
||||
export GAMMA_RASTER="${GAMMA_RASTER:-BMP}"
|
||||
export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
_pyint_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
_pyint_script_dir="${_pyint_repo_root}/third_party/PyINT/pyint"
|
||||
_pyint_gamma_prepend_path "${_pyint_script_dir}"
|
||||
|
||||
unset _pyint_gamma_home
|
||||
unset _gamma_dir
|
||||
unset _pyint_repo_root
|
||||
unset _pyint_script_dir
|
||||
unset -f _pyint_gamma_prepend_path
|
||||
unset -f _pyint_gamma_die
|
||||
@@ -0,0 +1,847 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
PAIR_META_FILENAME = ".dinsar_pair.json"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Materialize a PyINT LT-1 workspace from an existing Task_xxx pair directory."
|
||||
)
|
||||
parser.add_argument("task_dir", help="Task directory containing master/ and slave/ subdirectories.")
|
||||
parser.add_argument("--project-dir", required=True, help="Workspace directory for the generated PyINT project.")
|
||||
parser.add_argument("--template-root", required=True, help="Directory where the generated template will be written.")
|
||||
parser.add_argument("--output-dir", required=True, help="Directory where normalized native outputs will be copied.")
|
||||
parser.add_argument("--pyint-home", required=True, help="PyINT repository root inside WSL.")
|
||||
parser.add_argument("--pyint-app-script", required=True, help="pyintApp.py path inside WSL.")
|
||||
parser.add_argument("--python", required=True, help="Python interpreter used to run PyINT inside WSL.")
|
||||
parser.add_argument("--dem-root", required=True, help="DEMDIR root used by PyINT.")
|
||||
parser.add_argument("--dem-mode", default="local_fabdem", help="DEM strategy used for this run.")
|
||||
parser.add_argument("--fabdem-root", default="", help="Optional FABDEM tile root inside WSL.")
|
||||
parser.add_argument("--prepared-dem-path", default="", help="Optional existing DEM path inside WSL.")
|
||||
parser.add_argument("--opentopo-dem-type", default="SRTMGL1", help="DEM type when using OpenTopography.")
|
||||
parser.add_argument("--opentopo-api-key", default="", help="Optional OpenTopography API key.")
|
||||
parser.add_argument("--project-name", required=True, help="Unique PyINT project name for this run.")
|
||||
parser.add_argument("--gamma-env-script", default="", help="Optional shell script used to expose GAMMA commands.")
|
||||
parser.add_argument("--pair-key", default="", help="Pair key recorded into the run summary.")
|
||||
parser.add_argument("--task-alias", default="", help="Task alias recorded into the run summary.")
|
||||
parser.add_argument("--orbit-policy", default="require_txt", help="Orbit governance policy recorded into the run summary.")
|
||||
parser.add_argument("--input-assets-dir", default="", help="Optional input_assets directory for this run.")
|
||||
parser.add_argument("--input-assets-json", default="", help="Optional task_manifest.json path for this run.")
|
||||
parser.add_argument("--master-date", default="", help="Master date in YYYYMMDD format.")
|
||||
parser.add_argument("--slave-date", default="", help="Slave date in YYYYMMDD format.")
|
||||
parser.add_argument("--time-baseline-days", type=int, default=0, help="Time baseline to record in ifgram_list.txt.")
|
||||
parser.add_argument("--range-looks", type=int, default=2)
|
||||
parser.add_argument("--azimuth-looks", type=int, default=2)
|
||||
parser.add_argument("--parallel-workers", type=int, default=1)
|
||||
parser.add_argument("--lt1-precise-orbit-enabled", default="true", help="Enable LT-1 precise orbit bridge.")
|
||||
parser.add_argument("--lt1-precise-orbit-mode", default="replace", help="LT-1 precise orbit bridge mode.")
|
||||
parser.add_argument("--lt1-precise-orbit-strict", default="true", help="Fail the run if precise orbit bridge fails.")
|
||||
parser.add_argument(
|
||||
"--lt1-precise-orbit-validate-with-orb-filt",
|
||||
default="false",
|
||||
help="Run ORB_filt_spline.py on a validation copy after rewriting state vectors.",
|
||||
)
|
||||
parser.add_argument("--lt1-precise-orbit-backup", default="true", help="Backup original .slc.par before rewrite.")
|
||||
parser.add_argument("--lt1-precise-orbit-orb-filt-degree", type=int, default=5)
|
||||
parser.add_argument("--unwrap", dest="unwrap", action="store_true")
|
||||
parser.add_argument("--no-unwrap", dest="unwrap", action="store_false")
|
||||
parser.add_argument("--geocode", dest="geocode", action="store_true")
|
||||
parser.add_argument("--no-geocode", dest="geocode", action="store_false")
|
||||
parser.add_argument("--force", action="store_true", help="Delete an existing run root before rebuilding it.")
|
||||
parser.set_defaults(unwrap=True, geocode=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalize_date_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
digits = "".join(ch for ch in text if ch.isdigit())
|
||||
if len(digits) >= 8 and digits.startswith("20"):
|
||||
return digits[:8]
|
||||
return ""
|
||||
|
||||
|
||||
def normalize_bool_text(value: Any, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def ensure_directory(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def safe_rmtree(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
resolved = path.resolve()
|
||||
if len(resolved.parts) < 4:
|
||||
raise RuntimeError(f"Refusing to remove an unsafe path: {resolved}")
|
||||
shutil.rmtree(resolved)
|
||||
|
||||
|
||||
def load_pair_meta(task_dir: Path) -> Dict[str, Any]:
|
||||
path = task_dir / PAIR_META_FILENAME
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def load_json_file(path: Path | None) -> Dict[str, Any]:
|
||||
if path is None or not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def discover_lt1_archives(scene_dir: Path) -> List[Path]:
|
||||
if not scene_dir.is_dir():
|
||||
return []
|
||||
items: List[Path] = []
|
||||
for pattern in LT1_INPUT_GLOBS:
|
||||
items.extend(path.resolve() for path in scene_dir.rglob(pattern) if path.is_file())
|
||||
return sorted(set(items))
|
||||
|
||||
|
||||
def infer_scene_date(paths: Iterable[Path]) -> str:
|
||||
dates = {
|
||||
normalize_date_text(path.name)
|
||||
for path in paths
|
||||
if normalize_date_text(path.name)
|
||||
}
|
||||
if len(dates) == 1:
|
||||
return next(iter(dates))
|
||||
return ""
|
||||
|
||||
|
||||
def hardlink_or_copy(src: Path, dst: Path) -> str:
|
||||
ensure_directory(dst.parent)
|
||||
if dst.exists():
|
||||
return "skipped"
|
||||
try:
|
||||
os.link(src, dst)
|
||||
return "hardlinked"
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
dst.symlink_to(src)
|
||||
return "symlinked"
|
||||
except OSError:
|
||||
pass
|
||||
shutil.copy2(src, dst)
|
||||
return "copied"
|
||||
|
||||
|
||||
def collect_related_lt1_input_files(path: Path) -> List[Path]:
|
||||
resolved = path.resolve()
|
||||
if resolved.suffix.lower() != ".tiff":
|
||||
return [resolved]
|
||||
|
||||
stem = resolved.stem
|
||||
files = [
|
||||
candidate.resolve()
|
||||
for candidate in resolved.parent.iterdir()
|
||||
if candidate.is_file() and (candidate.name == resolved.name or candidate.name.startswith(stem))
|
||||
]
|
||||
return sorted(set(files))
|
||||
|
||||
|
||||
def write_text(path: Path, content: str) -> Path:
|
||||
ensure_directory(path.parent)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def inspect_prepared_dem_path(path_text: str) -> Dict[str, str]:
|
||||
text = str(path_text or "").strip()
|
||||
if not text:
|
||||
return {
|
||||
"path": "",
|
||||
"kind": "",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
path = Path(text)
|
||||
try:
|
||||
resolved_path = path.resolve()
|
||||
except Exception:
|
||||
resolved_path = path
|
||||
|
||||
gamma_par_path = Path(str(resolved_path) + ".par")
|
||||
vrt_path = Path(str(resolved_path) + ".vrt")
|
||||
xml_path = Path(str(resolved_path) + ".xml")
|
||||
hdr_path = Path(str(resolved_path) + ".hdr")
|
||||
|
||||
if resolved_path.is_file() and gamma_par_path.is_file():
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "gamma_ready",
|
||||
"direct_dem_path": str(resolved_path),
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
if resolved_path.is_file() and (vrt_path.is_file() or xml_path.is_file() or hdr_path.is_file()):
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "source_dem",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": str(resolved_path),
|
||||
"source_dem_open_path": str(vrt_path if vrt_path.is_file() else resolved_path),
|
||||
}
|
||||
|
||||
if resolved_path.suffix.lower() == ".vrt" and resolved_path.is_file():
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "source_dem",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": str(resolved_path),
|
||||
"source_dem_open_path": str(resolved_path),
|
||||
}
|
||||
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
|
||||
def build_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
dem_mode: str,
|
||||
fabdem_root: str,
|
||||
prepared_dem_path: str,
|
||||
opentopo_dem_type: str,
|
||||
opentopo_api_key: str,
|
||||
) -> str:
|
||||
prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
"satelite=LT",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"extract_burst_all=0",
|
||||
f"extract_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
"pot_all=0",
|
||||
f"pot_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
"atmcor_all=0",
|
||||
f"atmcor_all_parallel={int(parallel_workers)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
"gacos_correction=0",
|
||||
"load_data=0",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
]
|
||||
if dem_mode == "local_fabdem" and fabdem_root:
|
||||
lines.append(f"fabdem_dir={fabdem_root}")
|
||||
else:
|
||||
lines.append("fabdem_dir=-")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready":
|
||||
lines.append(f"DEM={prepared_dem['direct_dem_path']}")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem":
|
||||
lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}")
|
||||
else:
|
||||
lines.append("prepared_dem_source=-")
|
||||
if dem_mode == "opentopo":
|
||||
lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}")
|
||||
lines.append(f"opentopo_api_key={opentopo_api_key or '-'}")
|
||||
else:
|
||||
lines.append("opentopo_dem_type=-")
|
||||
lines.append("opentopo_api_key=-")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_ifgram_list(path: Path, master_date: str, slave_date: str, time_baseline_days: int) -> Path:
|
||||
content = f"{master_date}-{slave_date} {int(time_baseline_days)} 0.0\n"
|
||||
return write_text(path, content)
|
||||
|
||||
|
||||
def write_wrapper_scripts(
|
||||
*,
|
||||
wrappers_dir: Path,
|
||||
pyint_home: Path,
|
||||
python_cmd: str,
|
||||
gamma_env_script: str,
|
||||
) -> List[Path]:
|
||||
scripts_dir = pyint_home / "pyint"
|
||||
if not scripts_dir.is_dir():
|
||||
raise FileNotFoundError(f"PyINT scripts directory not found: {scripts_dir}")
|
||||
|
||||
ensure_directory(wrappers_dir)
|
||||
created: List[Path] = []
|
||||
pyint_scripts = sorted(path for path in scripts_dir.glob("*.py") if path.is_file())
|
||||
for script_path in pyint_scripts:
|
||||
wrapper_path = wrappers_dir / script_path.name
|
||||
lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"set -e",
|
||||
]
|
||||
if gamma_env_script:
|
||||
lines.append(f". '{gamma_env_script}' >/dev/null 2>&1")
|
||||
lines.extend(
|
||||
[
|
||||
f"export PATH='{wrappers_dir}':'{scripts_dir}':\"$PATH\"",
|
||||
f"export PYTHONPATH='{pyint_home}':\"${{PYTHONPATH:-}}\"",
|
||||
f"exec '{python_cmd}' '{script_path}' \"$@\"",
|
||||
"",
|
||||
]
|
||||
)
|
||||
write_text(wrapper_path, "\n".join(lines))
|
||||
wrapper_path.chmod(wrapper_path.stat().st_mode | stat.S_IEXEC)
|
||||
created.append(wrapper_path)
|
||||
return created
|
||||
|
||||
|
||||
def run_logged(command: List[str], *, env: Dict[str, str], cwd: Path, stdout_path: Path, stderr_path: Path) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
write_text(stdout_path, result.stdout or "")
|
||||
write_text(stderr_path, result.stderr or "")
|
||||
return result
|
||||
|
||||
|
||||
def require_task_layout(task_dir: Path) -> None:
|
||||
missing = [name for name in ("master", "slave") if not (task_dir / name).is_dir()]
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Task directory is missing required subdirectories: {', '.join(missing)}")
|
||||
|
||||
|
||||
def collect_expected_outputs(project_dir: Path, pair_name: str, range_looks: int) -> Dict[str, str]:
|
||||
pair_dir = project_dir / "ifgrams" / pair_name
|
||||
look_text = f"{int(range_looks)}rlks"
|
||||
return {
|
||||
"pair_dir": str(pair_dir),
|
||||
"diff_filt": str(pair_dir / f"{pair_name}_{look_text}.diff_filt"),
|
||||
"coh": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.cor"),
|
||||
"unw": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.unw"),
|
||||
"geo_unw": str(pair_dir / f"geo_{pair_name}_{look_text}.diff_filt.unw"),
|
||||
"geo_los": str(pair_dir / f"geo_{pair_name}_{look_text}.los_disp"),
|
||||
}
|
||||
|
||||
|
||||
def assert_required_outputs(outputs: Dict[str, str], *, unwrap: bool, geocode: bool) -> None:
|
||||
required = ["pair_dir", "diff_filt", "coh"]
|
||||
if unwrap:
|
||||
required.append("unw")
|
||||
if geocode:
|
||||
required.append("geo_unw")
|
||||
missing = [name for name in required if not Path(outputs[name]).exists()]
|
||||
if missing:
|
||||
raise RuntimeError(f"PyINT run finished but required outputs are missing: {', '.join(missing)}")
|
||||
|
||||
|
||||
def is_binary_all_zero(path: Path, *, chunk_size: int = 1024 * 1024) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
return True
|
||||
if any(chunk):
|
||||
return False
|
||||
|
||||
|
||||
def collect_output_sanity_checks(
|
||||
outputs: Dict[str, str],
|
||||
*,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
) -> List[Dict[str, Any]]:
|
||||
targets = [
|
||||
("diff_filt", "wrapped differential interferogram"),
|
||||
("coh", "coherence"),
|
||||
]
|
||||
if unwrap:
|
||||
targets.append(("unw", "unwrapped interferogram"))
|
||||
if geocode:
|
||||
targets.extend(
|
||||
[
|
||||
("geo_unw", "geocoded unwrapped interferogram"),
|
||||
("geo_los", "geocoded LOS displacement"),
|
||||
]
|
||||
)
|
||||
|
||||
checks: List[Dict[str, Any]] = []
|
||||
for name, label in targets:
|
||||
path = Path(outputs[name])
|
||||
exists = path.exists()
|
||||
size_bytes = path.stat().st_size if exists else 0
|
||||
all_zero = exists and is_binary_all_zero(path)
|
||||
checks.append(
|
||||
{
|
||||
"name": name,
|
||||
"label": label,
|
||||
"path": str(path),
|
||||
"exists": exists,
|
||||
"size_bytes": int(size_bytes),
|
||||
"all_zero": bool(all_zero),
|
||||
"ok": bool(exists and size_bytes > 0 and not all_zero),
|
||||
}
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def assert_output_sanity(checks: List[Dict[str, Any]]) -> None:
|
||||
failed = [item for item in checks if not item.get("ok")]
|
||||
if not failed:
|
||||
return
|
||||
detail = ", ".join(f"{item['name']}={item['path']}" for item in failed)
|
||||
raise RuntimeError(f"PyINT run produced invalid all-zero binary outputs: {detail}")
|
||||
|
||||
|
||||
def collect_stage_error_logs(project_dir: Path) -> Dict[str, str]:
|
||||
logs: Dict[str, str] = {}
|
||||
for filename in (
|
||||
"coreg_gamma_all.err",
|
||||
"diff_gamma_all.err",
|
||||
"unwrap_gamma_all.err",
|
||||
"geocode_gamma_all.err",
|
||||
):
|
||||
path = project_dir / filename
|
||||
if path.is_file():
|
||||
logs[filename] = str(path)
|
||||
return logs
|
||||
|
||||
|
||||
def copy_native_outputs(
|
||||
*,
|
||||
project_dir: Path,
|
||||
output_dir: Path,
|
||||
pair_name: str,
|
||||
template_path: Path,
|
||||
ifgram_list_path: Path,
|
||||
stdout_path: Path,
|
||||
stderr_path: Path,
|
||||
) -> Dict[str, str]:
|
||||
ensure_directory(output_dir)
|
||||
native_pair_dir = project_dir / "ifgrams" / pair_name
|
||||
target_pair_dir = output_dir / "ifgrams" / pair_name
|
||||
if native_pair_dir.is_dir():
|
||||
shutil.copytree(native_pair_dir, target_pair_dir, dirs_exist_ok=True)
|
||||
|
||||
target_template = output_dir / template_path.name
|
||||
shutil.copy2(template_path, target_template)
|
||||
target_ifgram_list = output_dir / ifgram_list_path.name
|
||||
shutil.copy2(ifgram_list_path, target_ifgram_list)
|
||||
target_stdout = output_dir / stdout_path.name
|
||||
target_stderr = output_dir / stderr_path.name
|
||||
shutil.copy2(stdout_path, target_stdout)
|
||||
shutil.copy2(stderr_path, target_stderr)
|
||||
|
||||
return {
|
||||
"pair_dir": str(target_pair_dir),
|
||||
"template_path": str(target_template),
|
||||
"ifgram_list_path": str(target_ifgram_list),
|
||||
"stdout_path": str(target_stdout),
|
||||
"stderr_path": str(target_stderr),
|
||||
}
|
||||
|
||||
|
||||
def collect_orbit_bridge_summaries(project_dir: Path) -> List[Dict[str, Any]]:
|
||||
summaries: List[Dict[str, Any]] = []
|
||||
slc_root = project_dir / "SLC"
|
||||
if not slc_root.is_dir():
|
||||
return summaries
|
||||
|
||||
for summary_path in sorted(slc_root.glob("*/orbit_bridge_summary.json")):
|
||||
payload = load_json_file(summary_path)
|
||||
operations = payload.get("operations") if isinstance(payload.get("operations"), list) else []
|
||||
failed_operations = [item for item in operations if not item.get("ok")]
|
||||
summaries.append(
|
||||
{
|
||||
"path": str(summary_path),
|
||||
"date_dir": summary_path.parent.name,
|
||||
"ok": bool(payload.get("ok", not failed_operations)),
|
||||
"operation_count": len(operations),
|
||||
"failed_operation_count": len(failed_operations),
|
||||
"operations": operations,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
return summaries
|
||||
|
||||
|
||||
def copy_orbit_bridge_summaries(summaries: List[Dict[str, Any]], output_dir: Path) -> Dict[str, str]:
|
||||
if not summaries:
|
||||
return {}
|
||||
|
||||
target_dir = ensure_directory(output_dir / "orbit_bridge")
|
||||
copied: Dict[str, str] = {}
|
||||
for item in summaries:
|
||||
source_path = Path(item["path"])
|
||||
target_path = target_dir / f"{item['date_dir']}_orbit_bridge_summary.json"
|
||||
shutil.copy2(source_path, target_path)
|
||||
copied[item["date_dir"]] = str(target_path)
|
||||
return copied
|
||||
|
||||
|
||||
def assert_orbit_bridge_ok(
|
||||
*,
|
||||
enabled: bool,
|
||||
strict: bool,
|
||||
summaries: List[Dict[str, Any]],
|
||||
expected_dates: Iterable[str],
|
||||
) -> None:
|
||||
if not enabled:
|
||||
return
|
||||
if not strict:
|
||||
return
|
||||
|
||||
expected = {str(item).strip() for item in expected_dates if str(item).strip()}
|
||||
found = {str(item.get("date_dir") or "").strip() for item in summaries if str(item.get("date_dir") or "").strip()}
|
||||
missing = sorted(expected - found)
|
||||
if missing:
|
||||
raise RuntimeError(f"LT-1 precise orbit bridge summary is missing for: {', '.join(missing)}")
|
||||
|
||||
failed = [item for item in summaries if not item.get("ok")]
|
||||
if failed:
|
||||
failed_dates = ", ".join(sorted(str(item.get("date_dir") or "") for item in failed))
|
||||
raise RuntimeError(f"LT-1 precise orbit bridge reported failures for: {failed_dates}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
task_dir = Path(args.task_dir).resolve()
|
||||
project_dir = Path(args.project_dir).resolve()
|
||||
run_root = project_dir.parent
|
||||
template_root = Path(args.template_root).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
pyint_home = Path(args.pyint_home).resolve()
|
||||
pyint_app_script = Path(args.pyint_app_script).resolve()
|
||||
dem_root = Path(args.dem_root).resolve()
|
||||
input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None
|
||||
input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None
|
||||
input_assets_payload = load_json_file(input_assets_json)
|
||||
precise_orbit_enabled = normalize_bool_text(args.lt1_precise_orbit_enabled, True)
|
||||
precise_orbit_strict = normalize_bool_text(args.lt1_precise_orbit_strict, True)
|
||||
precise_orbit_validate_with_orb_filt = normalize_bool_text(
|
||||
args.lt1_precise_orbit_validate_with_orb_filt,
|
||||
False,
|
||||
)
|
||||
precise_orbit_backup = normalize_bool_text(args.lt1_precise_orbit_backup, True)
|
||||
precise_orbit_mode = str(args.lt1_precise_orbit_mode or "replace").strip().lower() or "replace"
|
||||
precise_orbit_helper = (Path(__file__).resolve().parent / "apply_lt1_precise_orbit.py").resolve()
|
||||
dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem"
|
||||
prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
|
||||
require_task_layout(task_dir)
|
||||
if not pyint_app_script.is_file():
|
||||
raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}")
|
||||
if precise_orbit_enabled and not precise_orbit_helper.is_file():
|
||||
raise FileNotFoundError(f"Precise orbit bridge helper not found: {precise_orbit_helper}")
|
||||
if precise_orbit_enabled and input_assets_json is None:
|
||||
raise RuntimeError("LT-1 precise orbit bridge requires --input-assets-json.")
|
||||
if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"):
|
||||
raise RuntimeError(
|
||||
"Prepared DEM mode requires either a Gamma DEM with .par, "
|
||||
"or a source DEM with .xml/.hdr/.vrt sidecars."
|
||||
)
|
||||
|
||||
if args.force:
|
||||
safe_rmtree(run_root)
|
||||
safe_rmtree(template_root)
|
||||
safe_rmtree(output_dir)
|
||||
|
||||
if run_root.exists():
|
||||
raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}")
|
||||
|
||||
pair_meta = load_pair_meta(task_dir)
|
||||
master_archives = discover_lt1_archives(task_dir / "master")
|
||||
slave_archives = discover_lt1_archives(task_dir / "slave")
|
||||
if not master_archives:
|
||||
raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'master'}")
|
||||
if not slave_archives:
|
||||
raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'slave'}")
|
||||
|
||||
master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date")) or infer_scene_date(master_archives)
|
||||
slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date")) or infer_scene_date(slave_archives)
|
||||
if not master_date or not slave_date:
|
||||
raise RuntimeError("Unable to determine master/slave dates from pair metadata or archive names.")
|
||||
|
||||
pair_name = f"{master_date}-{slave_date}"
|
||||
task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name
|
||||
pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip()
|
||||
time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0)
|
||||
|
||||
ensure_directory(run_root)
|
||||
ensure_directory(template_root)
|
||||
ensure_directory(output_dir)
|
||||
ensure_directory(dem_root)
|
||||
|
||||
pyint_scripts_dir = pyint_home / "pyint"
|
||||
wrappers_dir = ensure_directory(run_root / "wrappers")
|
||||
write_wrapper_scripts(
|
||||
wrappers_dir=wrappers_dir,
|
||||
pyint_home=pyint_home,
|
||||
python_cmd=args.python,
|
||||
gamma_env_script=args.gamma_env_script,
|
||||
)
|
||||
|
||||
template_path = write_text(
|
||||
template_root / f"{args.project_name}.template",
|
||||
build_template_text(
|
||||
project_name=args.project_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
parallel_workers=args.parallel_workers,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
dem_mode=dem_mode,
|
||||
fabdem_root=str(args.fabdem_root or "").strip(),
|
||||
prepared_dem_path=str(args.prepared_dem_path or "").strip(),
|
||||
opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
opentopo_api_key=str(args.opentopo_api_key or "").strip(),
|
||||
),
|
||||
)
|
||||
|
||||
scratch_root = ensure_directory(project_dir.parent)
|
||||
archive_materialization: List[Dict[str, str]] = []
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"SCRATCHDIR": str(scratch_root),
|
||||
"TEMPLATEDIR": str(template_root),
|
||||
"DEMDIR": str(dem_root),
|
||||
"PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}",
|
||||
"PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ENABLED": "true" if precise_orbit_enabled else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_MODE": precise_orbit_mode,
|
||||
"PYINT_LT1_PRECISE_ORBIT_STRICT": "true" if precise_orbit_strict else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT": "true" if precise_orbit_validate_with_orb_filt else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_BACKUP": "true" if precise_orbit_backup else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE": str(int(args.lt1_precise_orbit_orb_filt_degree)),
|
||||
"PYINT_LT1_PRECISE_ORBIT_HELPER": str(precise_orbit_helper),
|
||||
"PYINT_LT1_PRECISE_ORBIT_MANIFEST": str(input_assets_json) if input_assets_json else "",
|
||||
}
|
||||
)
|
||||
|
||||
generate_stdout = run_root / "pyint_generate.stdout.log"
|
||||
generate_stderr = run_root / "pyint_generate.stderr.log"
|
||||
generate_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), "-g", args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=generate_stdout,
|
||||
stderr_path=generate_stderr,
|
||||
)
|
||||
if generate_result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py -g failed with rc={generate_result.returncode}: "
|
||||
f"{(generate_result.stderr or generate_result.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
pyint_project_dir = project_dir
|
||||
download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD")
|
||||
ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days)
|
||||
for role, archives in (("master", master_archives), ("slave", slave_archives)):
|
||||
for src_path in archives:
|
||||
related_files = collect_related_lt1_input_files(src_path)
|
||||
for related_path in related_files:
|
||||
target_path = download_dir / related_path.name
|
||||
op = hardlink_or_copy(related_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(related_path),
|
||||
"group_source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
|
||||
run_stdout = run_root / "pyint.stdout.log"
|
||||
run_stderr = run_root / "pyint.stderr.log"
|
||||
run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
run_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
if run_result.returncode != 0:
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
detail_text = (run_result.stderr or run_result.stdout or "").strip()
|
||||
if stage_error_logs:
|
||||
log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items())
|
||||
detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}"
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py failed with rc={run_result.returncode}: "
|
||||
f"{detail_text}"
|
||||
)
|
||||
|
||||
orbit_bridge_summaries = collect_orbit_bridge_summaries(pyint_project_dir)
|
||||
assert_orbit_bridge_ok(
|
||||
enabled=precise_orbit_enabled,
|
||||
strict=precise_orbit_strict,
|
||||
summaries=orbit_bridge_summaries,
|
||||
expected_dates=(master_date, slave_date),
|
||||
)
|
||||
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(
|
||||
outputs,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
)
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
|
||||
copied_paths = copy_native_outputs(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
template_path=template_path,
|
||||
ifgram_list_path=ifgram_list_path,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
copied_orbit_bridge_paths = copy_orbit_bridge_summaries(orbit_bridge_summaries, output_dir)
|
||||
|
||||
summary = {
|
||||
"ok": True,
|
||||
"task_dir": str(task_dir),
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"project_name": args.project_name,
|
||||
"project_dir": str(pyint_project_dir),
|
||||
"run_root": str(run_root),
|
||||
"template_root": str(template_root),
|
||||
"output_dir": str(output_dir),
|
||||
"pyint_home": str(pyint_home),
|
||||
"pyint_app_script": str(pyint_app_script),
|
||||
"gamma_env_script": args.gamma_env_script,
|
||||
"dem": {
|
||||
"mode": dem_mode,
|
||||
"dem_root": str(dem_root),
|
||||
"fabdem_root": str(args.fabdem_root or "").strip(),
|
||||
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"prepared_dem_direct_path": str(prepared_dem_info.get("direct_dem_path") or ""),
|
||||
"prepared_dem_source_path": str(prepared_dem_info.get("source_dem_path") or ""),
|
||||
"prepared_dem_open_path": str(prepared_dem_info.get("source_dem_open_path") or ""),
|
||||
"opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
"opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()),
|
||||
},
|
||||
"orbit_policy": str(args.orbit_policy or "require_txt").strip().lower(),
|
||||
"precise_orbit_bridge": {
|
||||
"enabled": precise_orbit_enabled,
|
||||
"mode": precise_orbit_mode,
|
||||
"strict": precise_orbit_strict,
|
||||
"validate_with_orb_filt": precise_orbit_validate_with_orb_filt,
|
||||
"backup": precise_orbit_backup,
|
||||
"orb_filt_degree": int(args.lt1_precise_orbit_orb_filt_degree),
|
||||
"helper_path": str(precise_orbit_helper),
|
||||
"manifest_json": str(input_assets_json) if input_assets_json else "",
|
||||
"summaries": [
|
||||
{
|
||||
"path": item["path"],
|
||||
"date_dir": item["date_dir"],
|
||||
"ok": item["ok"],
|
||||
"operation_count": item["operation_count"],
|
||||
"failed_operation_count": item["failed_operation_count"],
|
||||
"copied_summary_path": copied_orbit_bridge_paths.get(item["date_dir"], ""),
|
||||
}
|
||||
for item in orbit_bridge_summaries
|
||||
],
|
||||
},
|
||||
"input_assets_dir": str(input_assets_dir) if input_assets_dir else "",
|
||||
"input_assets_json": str(input_assets_json) if input_assets_json else "",
|
||||
"input_assets": input_assets_payload,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"pair_name": pair_name,
|
||||
"time_baseline_days": time_baseline_days,
|
||||
"range_looks": int(args.range_looks),
|
||||
"azimuth_looks": int(args.azimuth_looks),
|
||||
"parallel_workers": int(args.parallel_workers),
|
||||
"unwrap": bool(args.unwrap),
|
||||
"geocode": bool(args.geocode),
|
||||
"archives": {
|
||||
"master": [str(path) for path in master_archives],
|
||||
"slave": [str(path) for path in slave_archives],
|
||||
},
|
||||
"archive_materialization": archive_materialization,
|
||||
"workspace_outputs": outputs,
|
||||
"output_sanity_checks": output_sanity_checks,
|
||||
"copied_outputs": copied_paths,
|
||||
"copied_orbit_bridge_paths": copied_orbit_bridge_paths,
|
||||
"logs": {
|
||||
"generate_stdout": str(generate_stdout),
|
||||
"generate_stderr": str(generate_stderr),
|
||||
"run_stdout": str(run_stdout),
|
||||
"run_stderr": str(run_stderr),
|
||||
"stage_error_logs": stage_error_logs,
|
||||
},
|
||||
"started_at": run_started_at,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
}
|
||||
|
||||
summary_path = output_dir / "pyint_run_summary.json"
|
||||
write_text(summary_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
raise
|
||||
@@ -166,10 +166,12 @@ def _build_dinsar_result_payload(record: DinsarCatalogReadRecord) -> DinsarResul
|
||||
task_alias=product.task_alias,
|
||||
pair_key=product.pair_key,
|
||||
pair_uid=product.pair_uid,
|
||||
run_key=product.run_key,
|
||||
network_run_id=product.network_run_id,
|
||||
network_edge_id=product.network_edge_id,
|
||||
policy_version=product.policy_version,
|
||||
selection_strategy=product.selection_strategy,
|
||||
engine_code=product.engine_code,
|
||||
file_path=file_path,
|
||||
min_lon=float(min_lon or 0.0),
|
||||
min_lat=float(min_lat or 0.0),
|
||||
|
||||
@@ -13,6 +13,7 @@ from ..config import read_int_env, settings
|
||||
from ..models import AuthUserORM
|
||||
from ..services.dinsar_production_service import dinsar_production_service
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.pyint_input_assets_service import build_pyint_input_preview, summarize_preview_blockers
|
||||
from ..services.task_service import task_service
|
||||
|
||||
router = APIRouter(prefix="/dinsar-production", tags=["dinsar-production"])
|
||||
@@ -29,11 +30,17 @@ ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env(
|
||||
minimum=1,
|
||||
maximum=10,
|
||||
)
|
||||
PYINT_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env(
|
||||
"PYINT_PRODUCTION_JOB_MAX_ATTEMPTS",
|
||||
1,
|
||||
minimum=1,
|
||||
maximum=10,
|
||||
)
|
||||
|
||||
|
||||
class RunJobRequest(BaseModel):
|
||||
engine_code: str = Field(..., description="Engine code: sarscape / isce2 / landsar")
|
||||
profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap")
|
||||
engine_code: str = Field(..., description="Engine code: sarscape / isce2 / pyint / landsar")
|
||||
profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap / lt1_gamma_dinsar")
|
||||
root_dir: str = Field(..., description="Windows root directory")
|
||||
num_to_process: int = Field(default=0, ge=0, description="How many tasks to process; 0 means all")
|
||||
timeout_seconds: Optional[int] = Field(default=None, ge=60)
|
||||
@@ -45,6 +52,11 @@ class WslCheckRequest(BaseModel):
|
||||
smoke_test: bool = Field(default=False)
|
||||
|
||||
|
||||
class PreviewInputAssetsRequest(BaseModel):
|
||||
root_dir: str = Field(..., description="Windows root directory or a single Task_* directory")
|
||||
num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all")
|
||||
|
||||
|
||||
def _get_registry():
|
||||
from ..dinsar_engines import registry
|
||||
|
||||
@@ -116,6 +128,22 @@ async def run_wsl_check(
|
||||
return report.to_dict()
|
||||
|
||||
|
||||
@router.post("/engines/pyint/preview-input-assets")
|
||||
async def preview_pyint_input_assets(
|
||||
req: PreviewInputAssetsRequest,
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
):
|
||||
try:
|
||||
preview = await asyncio.to_thread(
|
||||
build_pyint_input_preview,
|
||||
req.root_dir,
|
||||
req.num_to_process,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return preview
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
async def submit_run(
|
||||
req: RunJobRequest,
|
||||
@@ -144,7 +172,7 @@ async def submit_run(
|
||||
)
|
||||
|
||||
validation_summary = None
|
||||
if req.engine_code == "isce2" and hasattr(engine, "validate_root_dir"):
|
||||
if hasattr(engine, "validate_root_dir"):
|
||||
try:
|
||||
validation_summary = await asyncio.to_thread(
|
||||
engine.validate_root_dir,
|
||||
@@ -154,33 +182,69 @@ async def submit_run(
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
effective_timeout_seconds = req.timeout_seconds
|
||||
if effective_timeout_seconds is None:
|
||||
engine_default_timeout = getattr(engine, "default_timeout_seconds", None)
|
||||
if engine_default_timeout:
|
||||
effective_timeout_seconds = int(engine_default_timeout)
|
||||
|
||||
pyint_preview = None
|
||||
if req.engine_code == "pyint":
|
||||
try:
|
||||
pyint_preview = await asyncio.to_thread(
|
||||
build_pyint_input_preview,
|
||||
req.root_dir,
|
||||
req.num_to_process,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if not pyint_preview.get("allow_submit"):
|
||||
detail = summarize_preview_blockers(pyint_preview)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"PyINT 输入资产预检未通过: {detail or '请先修复阻塞项。'}",
|
||||
)
|
||||
if not pyint_preview.get("allow_submit"):
|
||||
detail = summarize_preview_blockers(pyint_preview)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"PyINT 输入资产预检未通过: {detail or '请先修复阻塞项。'}",
|
||||
)
|
||||
|
||||
payload = {
|
||||
"engine_code": req.engine_code,
|
||||
"profile": req.profile,
|
||||
"root_dir": req.root_dir,
|
||||
"num_to_process": req.num_to_process,
|
||||
"timeout_seconds": req.timeout_seconds,
|
||||
"timeout_seconds": effective_timeout_seconds,
|
||||
"extra": dict(req.extra or {}),
|
||||
}
|
||||
|
||||
from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN
|
||||
from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN, JOB_TYPE_PYINT_RUN
|
||||
|
||||
if req.engine_code == "sarscape":
|
||||
payload["mode"] = "custom" if req.profile == "custom6" else "metatask"
|
||||
job_type = JOB_TYPE_IDL_RUN_DINSAR
|
||||
max_attempts = DINSAR_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
elif req.engine_code == "isce2":
|
||||
elif req.engine_code in {"isce2", "pyint"}:
|
||||
if hasattr(engine, "normalize_extra"):
|
||||
try:
|
||||
payload["extra"] = engine.normalize_extra(payload["extra"])
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
job_type = JOB_TYPE_ISCE2_RUN
|
||||
max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
if req.engine_code == "isce2":
|
||||
job_type = JOB_TYPE_ISCE2_RUN
|
||||
max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
else:
|
||||
job_type = JOB_TYPE_PYINT_RUN
|
||||
max_attempts = PYINT_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
if validation_summary is not None:
|
||||
validated_task_count = validation_summary.get("task_count", 0)
|
||||
if pyint_preview is not None:
|
||||
validated_task_count = int(pyint_preview.get("selected_task_count", validated_task_count) or 0)
|
||||
payload["extra"].update(
|
||||
{
|
||||
"__validated_task_count": validation_summary.get("task_count", 0),
|
||||
"__validated_task_count": validated_task_count,
|
||||
"__validated_mode": validation_summary.get("mode", ""),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,32 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import AuthUserORM
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.task_service import task_service
|
||||
from ..services.unpack_service import get_unpack_config
|
||||
from ..services.unpack_service import build_unpack_run_config, get_unpack_config
|
||||
from .dependencies import _require_admin
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class UnpackRunRequest(BaseModel):
|
||||
max_files_per_run: Optional[int] = Field(default=None, ge=0)
|
||||
max_runtime_minutes: Optional[int] = Field(default=None, ge=0)
|
||||
|
||||
|
||||
@router.get("/unpack/config")
|
||||
async def get_unpack_config_endpoint():
|
||||
"""
|
||||
获取解包配置 (来源 .env)。
|
||||
"""
|
||||
return get_unpack_config()
|
||||
|
||||
|
||||
@router.post("/unpack/run", status_code=202)
|
||||
async def run_unpack_endpoint(background_tasks: BackgroundTasks, admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
"""
|
||||
触发一次解包任务。
|
||||
"""
|
||||
config = get_unpack_config()
|
||||
async def run_unpack_endpoint(
|
||||
request: Optional[UnpackRunRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
del admin_user
|
||||
|
||||
overrides = request.model_dump(exclude_none=True) if request else None
|
||||
config = build_unpack_run_config(overrides)
|
||||
if not config.get("source_dirs"):
|
||||
raise HTTPException(status_code=400, detail="UNPACK_SOURCE_DIRS is not configured.")
|
||||
|
||||
@@ -36,7 +42,11 @@ async def run_unpack_endpoint(background_tasks: BackgroundTasks, admin_user: Aut
|
||||
"Archive unpack",
|
||||
params=config,
|
||||
)
|
||||
await job_queue_service.create_job("UNPACK_ARCHIVES", payload=config, task_id=task_id)
|
||||
await job_queue_service.create_job(
|
||||
"UNPACK_ARCHIVES",
|
||||
payload=config,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Unpack task queued", "task_id": task_id}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
@@ -74,6 +74,7 @@ JOB_TYPE_WATER_DETECT = "WATER_DETECT"
|
||||
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
@@ -1878,8 +1879,13 @@ async def _handle_idl_run_dinsar(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
"""ISCE2 生产任务 handler — 通过 WSL 执行 run_lt1_dinsar_pipeline.py。"""
|
||||
async def _handle_queued_engine_run(
|
||||
job: SystemJobORM,
|
||||
*,
|
||||
engine_title: str,
|
||||
fallback_timeout_seconds: int,
|
||||
) -> None:
|
||||
"""Run a queued D-InSAR engine task through the shared WSL execution path."""
|
||||
payload = job.payload or {}
|
||||
engine_code = payload.get("engine_code", "isce2")
|
||||
profile = payload.get("profile", "lt1_stripmap")
|
||||
@@ -1887,17 +1893,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
num_to_process = payload.get("num_to_process", 0)
|
||||
timeout_seconds = payload.get("timeout_seconds")
|
||||
extra = payload.get("extra", {})
|
||||
selected_task_count = max(1, int(extra.get("__validated_task_count") or 0 or 1))
|
||||
pair_timeout_seconds = int(timeout_seconds or fallback_timeout_seconds)
|
||||
|
||||
await task_service.start_task(
|
||||
job.task_id,
|
||||
message=f"[{engine_code}/{profile}] 启动 ISCE2 处理...",
|
||||
message=f"[{engine_code}/{profile}] 启动 {engine_title} 处理...",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"ISCE2 job accepted. root_dir={root_dir}, profile={profile}, timeout={timeout_seconds or 21600}s, extra={extra}",
|
||||
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, timeout={pair_timeout_seconds}s, extra={extra}",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"{engine_title} batch contains {selected_task_count} pair task(s). "
|
||||
f"Pairs run sequentially and each pair uses timeout={pair_timeout_seconds}s."
|
||||
),
|
||||
)
|
||||
|
||||
from ..dinsar_engines.base import RunRequest
|
||||
from ..dinsar_engines import registry
|
||||
|
||||
@@ -1905,6 +1920,25 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
if not engine:
|
||||
raise RuntimeError(f"引擎 '{engine_code}' 未注册")
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
progress_queue: asyncio.Queue[Optional[Dict[str, Any]]] = asyncio.Queue()
|
||||
progress_state: Dict[str, Any] = {
|
||||
"progress": 5,
|
||||
"message": f"[{engine_code}/{profile}] Running in WSL...",
|
||||
"pair_index": 0,
|
||||
"pair_total": selected_task_count,
|
||||
"pair_label": "",
|
||||
"pair_started_monotonic": None,
|
||||
}
|
||||
|
||||
def _emit_progress(event: Dict[str, Any]) -> None:
|
||||
if not event:
|
||||
return
|
||||
try:
|
||||
loop.call_soon_threadsafe(progress_queue.put_nowait, dict(event))
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
request = RunRequest(
|
||||
engine_code=engine_code,
|
||||
profile=profile,
|
||||
@@ -1913,6 +1947,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
num_to_process=num_to_process,
|
||||
timeout_seconds=timeout_seconds,
|
||||
extra=extra,
|
||||
progress_callback=_emit_progress,
|
||||
)
|
||||
|
||||
await task_service.update_task(
|
||||
@@ -1921,18 +1956,119 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
message=f"[{engine_code}/{profile}] 正在执行，请等待...",
|
||||
)
|
||||
|
||||
async def _consume_progress() -> None:
|
||||
while True:
|
||||
event = await progress_queue.get()
|
||||
if event is None:
|
||||
return
|
||||
|
||||
event_type = str(event.get("event") or "").strip().lower()
|
||||
pair_total = max(1, int(event.get("pair_total") or progress_state["pair_total"] or 1))
|
||||
pair_index = max(0, int(event.get("pair_index") or 0))
|
||||
task_label = str(event.get("task_alias") or event.get("task_name") or "").strip()
|
||||
|
||||
if event_type == "pair_started":
|
||||
progress = min(
|
||||
90,
|
||||
max(
|
||||
int(progress_state["progress"] or 5),
|
||||
5 + int((max(pair_index - 1, 0) / pair_total) * 80),
|
||||
),
|
||||
)
|
||||
progress_state.update(
|
||||
{
|
||||
"progress": progress,
|
||||
"pair_index": pair_index,
|
||||
"pair_total": pair_total,
|
||||
"pair_label": task_label,
|
||||
"pair_started_monotonic": time.monotonic(),
|
||||
"message": f"[{engine_code}/{profile}] Running {pair_index}/{pair_total}: {task_label}",
|
||||
}
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"{engine_title} pair {pair_index}/{pair_total} started: {task_label} "
|
||||
f"(work_dir={event.get('work_dir')})"
|
||||
),
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress_state["progress"],
|
||||
message=progress_state["message"],
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type == "pair_finished":
|
||||
success = bool(event.get("success"))
|
||||
returncode = int(event.get("returncode") or 0)
|
||||
progress = min(
|
||||
90,
|
||||
max(
|
||||
int(progress_state["progress"] or 5),
|
||||
5 + int((max(pair_index, 0) / pair_total) * 80) if success else int(progress_state["progress"] or 5),
|
||||
),
|
||||
)
|
||||
progress_state.update(
|
||||
{
|
||||
"progress": progress,
|
||||
"pair_index": pair_index,
|
||||
"pair_total": pair_total,
|
||||
"pair_label": task_label,
|
||||
"pair_started_monotonic": None,
|
||||
}
|
||||
)
|
||||
if success:
|
||||
progress_state["message"] = f"[{engine_code}/{profile}] Finished {pair_index}/{pair_total}: {task_label}"
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"{engine_title} pair {pair_index}/{pair_total} completed: {task_label}",
|
||||
)
|
||||
else:
|
||||
error_text = str(event.get("error") or "").strip()
|
||||
timeout_note = " (timeout)" if returncode == -1 else ""
|
||||
progress_state["message"] = f"[{engine_code}/{profile}] Failed {pair_index}/{pair_total}: {task_label}"
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
(
|
||||
f"{engine_title} pair {pair_index}/{pair_total} failed{timeout_note}: "
|
||||
f"{task_label} (rc={returncode})"
|
||||
f"{f', error={error_text}' if error_text else ''}"
|
||||
),
|
||||
)
|
||||
if pair_index < pair_total:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"{engine_title} will continue with the next pair ({pair_index + 1}/{pair_total}).",
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress_state["progress"],
|
||||
message=progress_state["message"],
|
||||
)
|
||||
|
||||
async def _task_keepalive():
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
message = str(progress_state.get("message") or f"[{engine_code}/{profile}] Running in WSL...")
|
||||
started_monotonic = progress_state.get("pair_started_monotonic")
|
||||
if isinstance(started_monotonic, (int, float)):
|
||||
elapsed_seconds = max(0, int(time.monotonic() - float(started_monotonic)))
|
||||
message = f"{message} (elapsed={elapsed_seconds}s)"
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=5,
|
||||
message=f"[{engine_code}/{profile}] Running in WSL...",
|
||||
progress=int(progress_state.get("progress") or 5),
|
||||
message=message,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[keepalive] WARNING: failed to update task {job.task_id}: {exc}")
|
||||
|
||||
progress_task = asyncio.create_task(_consume_progress())
|
||||
keepalive_task = asyncio.create_task(_task_keepalive())
|
||||
try:
|
||||
result = await asyncio.to_thread(engine.run, request)
|
||||
@@ -1942,6 +2078,8 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await progress_queue.put(None)
|
||||
await progress_task
|
||||
|
||||
detail = result.detail or {}
|
||||
|
||||
@@ -1949,7 +2087,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"ISCE2 run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}",
|
||||
f"{engine_title} run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}",
|
||||
)
|
||||
|
||||
for invalid in detail.get("invalid_candidates", []) or []:
|
||||
@@ -2062,16 +2200,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"Auto-published ISCE2 results from {len(output_dirs)} directory(s). "
|
||||
f"Auto-published {engine_title} results from {len(output_dirs)} directory(s). "
|
||||
f"processed={publish_result.get('processed', 0)} "
|
||||
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
|
||||
),
|
||||
)
|
||||
if int(publish_result.get("processed", 0) or 0) <= 0:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
(
|
||||
f"No publishable {engine_title} result bundle was detected under "
|
||||
f"{len(output_dirs)} output director"
|
||||
f"{'y' if len(output_dirs) == 1 else 'ies'}."
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"Auto-publish ISCE2 results failed: {exc}",
|
||||
f"Auto-publish {engine_title} results failed: {exc}",
|
||||
)
|
||||
|
||||
if result.success:
|
||||
@@ -2080,7 +2228,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
f"[{engine_code}/{profile}] 完成 — "
|
||||
f"[{engine_code}/{profile}] 完成，"
|
||||
f"成功 {result.pairs_processed} 对，失败 {result.pairs_failed} 对"
|
||||
),
|
||||
)
|
||||
@@ -2095,6 +2243,22 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await _handle_queued_engine_run(
|
||||
job,
|
||||
engine_title="ISCE2",
|
||||
fallback_timeout_seconds=settings.ISCE2_PER_TASK_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_pyint_run(job: SystemJobORM) -> None:
|
||||
await _handle_queued_engine_run(
|
||||
job,
|
||||
engine_title="PyINT",
|
||||
fallback_timeout_seconds=settings.PYINT_DEFAULT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_water_geocode(job: SystemJobORM) -> None:
|
||||
"""单景 SAR 地理编码 job handler（多视 + 地理编码 + 辐射定标）。"""
|
||||
from .water_service import run_geocoding_workflow, WATER_RESULTS_DIR
|
||||
@@ -2860,6 +3024,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_IDL_RUN_IMPORT: _handle_idl_run_import,
|
||||
JOB_TYPE_IDL_RUN_DINSAR: _handle_idl_run_dinsar,
|
||||
JOB_TYPE_ISCE2_RUN: _handle_isce2_run,
|
||||
JOB_TYPE_PYINT_RUN: _handle_pyint_run,
|
||||
JOB_TYPE_WATER_GEOCODE: _handle_water_geocode,
|
||||
JOB_TYPE_WATER_FLOOD: _handle_water_flood,
|
||||
JOB_TYPE_WATER_DETECT: _handle_water_detect,
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
"""PyINT input-asset resolution and materialization helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ..config import settings
|
||||
from .orbit_converter import get_source_orbit_inventory
|
||||
from .pyint_service import (
|
||||
discover_lt1_archives,
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
validate_pyint_root_dir,
|
||||
)
|
||||
|
||||
|
||||
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
|
||||
VALID_ORBIT_POLICIES = {"validate_only", "require_txt", "stage_txt"}
|
||||
VALID_PRECISE_ORBIT_MODES = {"replace", "replace_and_validate"}
|
||||
|
||||
|
||||
def _utc_now_text() -> str:
|
||||
return datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
|
||||
def _normalize_path(path: Any) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _copy_json_safe(value: Any) -> Any:
|
||||
return json.loads(json.dumps(value, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def _normalize_lt1_satellite(value: Any) -> str:
|
||||
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
|
||||
if "LT1A" in text:
|
||||
return "LT1A"
|
||||
if "LT1B" in text:
|
||||
return "LT1B"
|
||||
if text in {"A", "LTA"}:
|
||||
return "LT1A"
|
||||
if text in {"B", "LTB"}:
|
||||
return "LT1B"
|
||||
return ""
|
||||
|
||||
|
||||
def _infer_satellite_from_archives(paths: List[str]) -> str:
|
||||
satellites = {
|
||||
satellite
|
||||
for path in paths
|
||||
for satellite in [_normalize_lt1_satellite(os.path.basename(path))]
|
||||
if satellite
|
||||
}
|
||||
if len(satellites) == 1:
|
||||
return next(iter(satellites))
|
||||
return ""
|
||||
|
||||
|
||||
def _get_dem_mode() -> str:
|
||||
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
|
||||
if raw_mode not in VALID_DEM_MODES:
|
||||
return "local_fabdem"
|
||||
return raw_mode
|
||||
|
||||
|
||||
def _get_orbit_policy() -> str:
|
||||
raw_policy = str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower()
|
||||
if raw_policy not in VALID_ORBIT_POLICIES:
|
||||
return "require_txt"
|
||||
return raw_policy
|
||||
|
||||
|
||||
def _get_orbit_pool_root() -> str:
|
||||
explicit = _normalize_path(getattr(settings, "PYINT_ORBIT_POOL_TXT", ""))
|
||||
if explicit:
|
||||
return explicit
|
||||
return _normalize_path(settings.ORBIT_POOL_ENVI)
|
||||
|
||||
|
||||
def get_pyint_precise_orbit_bridge_summary() -> Dict[str, Any]:
|
||||
mode = str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower()
|
||||
if mode not in VALID_PRECISE_ORBIT_MODES:
|
||||
mode = "replace"
|
||||
return {
|
||||
"enabled": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ENABLED", True)),
|
||||
"mode": mode,
|
||||
"strict": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_STRICT", True)),
|
||||
"validate_with_orb_filt": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False)),
|
||||
"backup": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_BACKUP", True)),
|
||||
"orb_filt_degree": max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5)),
|
||||
}
|
||||
|
||||
|
||||
def _prepared_dem_variants(value: Any) -> List[str]:
|
||||
text = str(value or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return []
|
||||
|
||||
normalized = _normalize_path(text)
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
candidates = [normalized]
|
||||
root, ext = os.path.splitext(normalized)
|
||||
if ext.lower() == ".wgs84":
|
||||
candidates.append(root)
|
||||
elif not ext:
|
||||
candidates.append(normalized + ".wgs84")
|
||||
|
||||
unique: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
item = _normalize_path(candidate)
|
||||
if not item or item in seen:
|
||||
continue
|
||||
seen.add(item)
|
||||
unique.append(item)
|
||||
return unique
|
||||
|
||||
|
||||
def _resolve_prepared_dem_path() -> Dict[str, str]:
|
||||
explicit_value = getattr(settings, "PYINT_PREPARED_DEM_PATH", "")
|
||||
explicit_candidates = _prepared_dem_variants(explicit_value)
|
||||
if str(explicit_value or "").strip():
|
||||
for candidate in explicit_candidates:
|
||||
if os.path.isfile(candidate):
|
||||
return {
|
||||
"path": candidate,
|
||||
"resolved_from": "explicit",
|
||||
}
|
||||
return {
|
||||
"path": "",
|
||||
"resolved_from": "explicit",
|
||||
}
|
||||
|
||||
sources = [
|
||||
("isce2_dem_path", getattr(settings, "ISCE2_DEM_PATH", "")),
|
||||
("idl_dinsar_dem_base_file", getattr(settings, "IDL_DINSAR_DEM_BASE_FILE", "")),
|
||||
]
|
||||
for source_name, raw_value in sources:
|
||||
for candidate in _prepared_dem_variants(raw_value):
|
||||
if os.path.isfile(candidate):
|
||||
return {
|
||||
"path": candidate,
|
||||
"resolved_from": source_name,
|
||||
}
|
||||
return {
|
||||
"path": "",
|
||||
"resolved_from": "",
|
||||
}
|
||||
|
||||
|
||||
def _inspect_prepared_dem_path(path: Any) -> Dict[str, Any]:
|
||||
normalized = _normalize_path(path)
|
||||
if not normalized:
|
||||
return {
|
||||
"path": "",
|
||||
"exists": False,
|
||||
"kind": "",
|
||||
"gamma_par_path": "",
|
||||
"gamma_par_exists": False,
|
||||
"xml_path": "",
|
||||
"xml_exists": False,
|
||||
"hdr_path": "",
|
||||
"hdr_exists": False,
|
||||
"vrt_path": "",
|
||||
"vrt_exists": False,
|
||||
"open_path": "",
|
||||
}
|
||||
|
||||
gamma_par_path = normalized + ".par"
|
||||
xml_path = normalized + ".xml"
|
||||
hdr_path = normalized + ".hdr"
|
||||
vrt_path = normalized + ".vrt"
|
||||
|
||||
path_exists = os.path.isfile(normalized)
|
||||
gamma_par_exists = os.path.isfile(gamma_par_path)
|
||||
xml_exists = os.path.isfile(xml_path)
|
||||
hdr_exists = os.path.isfile(hdr_path)
|
||||
vrt_exists = os.path.isfile(vrt_path)
|
||||
|
||||
kind = ""
|
||||
open_path = ""
|
||||
if path_exists and gamma_par_exists:
|
||||
kind = "gamma_ready"
|
||||
open_path = normalized
|
||||
elif path_exists and (xml_exists or hdr_exists or vrt_exists):
|
||||
kind = "source_dem"
|
||||
open_path = vrt_path if vrt_exists else normalized
|
||||
|
||||
return {
|
||||
"path": normalized,
|
||||
"exists": path_exists,
|
||||
"kind": kind,
|
||||
"gamma_par_path": gamma_par_path,
|
||||
"gamma_par_exists": gamma_par_exists,
|
||||
"xml_path": xml_path,
|
||||
"xml_exists": xml_exists,
|
||||
"hdr_path": hdr_path,
|
||||
"hdr_exists": hdr_exists,
|
||||
"vrt_path": vrt_path,
|
||||
"vrt_exists": vrt_exists,
|
||||
"open_path": open_path,
|
||||
}
|
||||
|
||||
|
||||
def get_pyint_dem_summary() -> Dict[str, Any]:
|
||||
mode = _get_dem_mode()
|
||||
strict = bool(getattr(settings, "PYINT_DEM_STRICT", True))
|
||||
cache_root = _normalize_path(settings.PYINT_DEM_ROOT)
|
||||
fabdem_root = _normalize_path(getattr(settings, "PYINT_FABDEM_ROOT", ""))
|
||||
prepared_dem_resolution = _resolve_prepared_dem_path()
|
||||
prepared_dem_info = _inspect_prepared_dem_path(prepared_dem_resolution.get("path"))
|
||||
opentopo_dem_type = str(getattr(settings, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") or "SRTMGL1").strip() or "SRTMGL1"
|
||||
opentopo_api_key = str(getattr(settings, "PYINT_OPENTOPO_API_KEY", "") or "").strip()
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
source_root = ""
|
||||
source_exists = False
|
||||
if mode == "local_fabdem":
|
||||
source_root = fabdem_root
|
||||
source_exists = bool(source_root and os.path.isdir(source_root))
|
||||
elif mode == "prepared_file":
|
||||
source_root = str(prepared_dem_info.get("path") or "")
|
||||
source_exists = bool(prepared_dem_info.get("exists"))
|
||||
cache_root_exists = bool(cache_root and os.path.isdir(cache_root))
|
||||
|
||||
if mode == "local_fabdem":
|
||||
if not fabdem_root:
|
||||
message = "未配置 PYINT_FABDEM_ROOT。"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif not os.path.isdir(fabdem_root):
|
||||
message = f"本地 FABDEM 根目录不存在: {fabdem_root}"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif mode == "opentopo":
|
||||
if not opentopo_api_key:
|
||||
message = "DEM 策略为 OpenTopography，但未配置 PYINT_OPENTOPO_API_KEY。"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif mode == "prepared_file":
|
||||
if not prepared_dem_info.get("path"):
|
||||
if prepared_dem_resolution.get("resolved_from") == "explicit":
|
||||
message = "PYINT_PREPARED_DEM_PATH 已配置，但目标文件不存在。"
|
||||
else:
|
||||
message = (
|
||||
"未配置 PYINT_PREPARED_DEM_PATH，且未能从 ISCE2_DEM_PATH / "
|
||||
"IDL_DINSAR_DEM_BASE_FILE 解析现有 DEM。"
|
||||
)
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif prepared_dem_info.get("kind") not in {"gamma_ready", "source_dem"}:
|
||||
message = (
|
||||
"现有 DEM 缺少可识别 sidecar，至少需要同名 .par，或 .xml/.hdr/.vrt 中的一个: "
|
||||
+ str(prepared_dem_info.get("path") or "")
|
||||
)
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
|
||||
if not cache_root:
|
||||
blockers.append("未配置 PYINT_DEM_ROOT。")
|
||||
elif not cache_root_exists:
|
||||
warnings.append(f"DEM 缓存目录当前不存在，运行时将尝试创建: {cache_root}")
|
||||
|
||||
status = "ok"
|
||||
if blockers:
|
||||
status = "blocked"
|
||||
elif warnings:
|
||||
status = "warning"
|
||||
|
||||
if mode == "local_fabdem":
|
||||
detail = "使用本地 FABDEM 瓦片目录，由 PyINT 在 DEMDIR 中生成运行期 DEM。"
|
||||
elif mode == "prepared_file":
|
||||
if prepared_dem_info.get("kind") == "gamma_ready":
|
||||
detail = "使用现有 Gamma DEM，运行时将直接注入到 PyINT 模板。"
|
||||
else:
|
||||
detail = "使用现有系统 DEM，运行时将按任务覆盖区裁剪并转换为本次任务的 Gamma DEM。"
|
||||
else:
|
||||
detail = f"使用 OpenTopography 在线 DEM 源，DEM 类型为 {opentopo_dem_type}。"
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"strict": strict,
|
||||
"source_root": source_root,
|
||||
"source_exists": source_exists,
|
||||
"cache_root": cache_root,
|
||||
"cache_root_exists": cache_root_exists,
|
||||
"fabdem_root": fabdem_root,
|
||||
"prepared_dem_path": str(prepared_dem_info.get("path") or ""),
|
||||
"prepared_dem_resolved_from": str(prepared_dem_resolution.get("resolved_from") or ""),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"prepared_dem_open_path": str(prepared_dem_info.get("open_path") or ""),
|
||||
"prepared_dem_support": {
|
||||
"gamma_par_exists": bool(prepared_dem_info.get("gamma_par_exists")),
|
||||
"xml_exists": bool(prepared_dem_info.get("xml_exists")),
|
||||
"hdr_exists": bool(prepared_dem_info.get("hdr_exists")),
|
||||
"vrt_exists": bool(prepared_dem_info.get("vrt_exists")),
|
||||
},
|
||||
"opentopo_dem_type": opentopo_dem_type,
|
||||
"opentopo_api_key_configured": bool(opentopo_api_key),
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": not blockers,
|
||||
}
|
||||
|
||||
|
||||
def _load_orbit_inventory() -> Dict[str, Any]:
|
||||
pool_root = _get_orbit_pool_root()
|
||||
if not pool_root:
|
||||
return {
|
||||
"pool_root": "",
|
||||
"pool_exists": False,
|
||||
"files": {},
|
||||
"warnings": ["未配置 PYINT_ORBIT_POOL_TXT，且 ORBIT_POOL_ENVI 为空。"],
|
||||
}
|
||||
if not os.path.isdir(pool_root):
|
||||
return {
|
||||
"pool_root": pool_root,
|
||||
"pool_exists": False,
|
||||
"files": {},
|
||||
"warnings": [f"轨道池目录不存在: {pool_root}"],
|
||||
}
|
||||
|
||||
inventory = get_source_orbit_inventory(pool_root, recursive=True)
|
||||
return {
|
||||
"pool_root": pool_root,
|
||||
"pool_exists": True,
|
||||
"files": inventory.get("files", {}),
|
||||
"warnings": list(inventory.get("errors", []) or []),
|
||||
"duplicate_count": int(inventory.get("duplicate_count", 0) or 0),
|
||||
}
|
||||
|
||||
|
||||
def get_pyint_orbit_context() -> Dict[str, Any]:
|
||||
return _load_orbit_inventory()
|
||||
|
||||
|
||||
def _resolve_orbit_file(
|
||||
*,
|
||||
role: str,
|
||||
satellite: str,
|
||||
date_text: str,
|
||||
pool_root: str,
|
||||
orbit_files: Dict[str, Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
satellite_text = _normalize_lt1_satellite(satellite)
|
||||
normalized_date = str(date_text or "").strip()
|
||||
expected_name = (
|
||||
f"{satellite_text}_GpsData_GAS_C_{normalized_date}.txt"
|
||||
if satellite_text and normalized_date
|
||||
else ""
|
||||
)
|
||||
result: Dict[str, Any] = {
|
||||
"role": role,
|
||||
"satellite": satellite_text,
|
||||
"date": normalized_date,
|
||||
"expected_name": expected_name,
|
||||
"pool_root": pool_root,
|
||||
"resolved": False,
|
||||
"path": "",
|
||||
"resolution_method": "",
|
||||
"staged_path": "",
|
||||
}
|
||||
if not satellite_text:
|
||||
result["error"] = f"{role} 场景未能识别 LT-1 卫星型号。"
|
||||
return result
|
||||
if not normalized_date:
|
||||
result["error"] = f"{role} 场景未能识别成像日期。"
|
||||
return result
|
||||
|
||||
stem = os.path.splitext(expected_name)[0]
|
||||
item = orbit_files.get(stem)
|
||||
if item and os.path.isfile(item.get("path", "")):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(item["path"]),
|
||||
"resolution_method": "indexed_pool_scan",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
direct_candidate = os.path.join(pool_root, satellite_text, expected_name)
|
||||
if os.path.isfile(direct_candidate):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(direct_candidate),
|
||||
"resolution_method": "direct_satellite_subdir",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
flat_candidate = os.path.join(pool_root, expected_name)
|
||||
if os.path.isfile(flat_candidate):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(flat_candidate),
|
||||
"resolution_method": "direct_pool_root",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
result["error"] = f"轨道池中缺少 {expected_name}"
|
||||
return result
|
||||
|
||||
|
||||
def resolve_pyint_task_input_assets(
|
||||
task_dir: str,
|
||||
*,
|
||||
dem_summary: Dict[str, Any] | None = None,
|
||||
orbit_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
task_dir = _normalize_path(task_dir)
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
pair_meta = task_identity["pair_meta"]
|
||||
archives = discover_lt1_archives(task_dir)
|
||||
master_archives = list(archives.get("master", []) or [])
|
||||
slave_archives = list(archives.get("slave", []) or [])
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
|
||||
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
|
||||
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
orbit_policy = _get_orbit_policy()
|
||||
orbit_context = orbit_context or get_pyint_orbit_context()
|
||||
orbit_pool_root = orbit_context.get("pool_root", "")
|
||||
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
|
||||
orbit_files = orbit_context.get("files", {}) or {}
|
||||
|
||||
orbit_warnings: List[str] = []
|
||||
if orbit_context.get("warnings"):
|
||||
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
|
||||
|
||||
master_orbit = _resolve_orbit_file(
|
||||
role="master",
|
||||
satellite=master_satellite,
|
||||
date_text=master_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
slave_orbit = _resolve_orbit_file(
|
||||
role="slave",
|
||||
satellite=slave_satellite,
|
||||
date_text=slave_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
|
||||
for orbit_item in (master_orbit, slave_orbit):
|
||||
if orbit_item.get("resolved"):
|
||||
continue
|
||||
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} 轨道缺失").strip()
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(message)
|
||||
else:
|
||||
blockers.append(message)
|
||||
|
||||
if not orbit_pool_root:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append("轨道池未配置，当前仅记录警告。")
|
||||
else:
|
||||
blockers.append("轨道池未配置。")
|
||||
elif not orbit_pool_exists:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
else:
|
||||
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
|
||||
warnings.extend(orbit_warnings)
|
||||
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
}
|
||||
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
orbits_summary = {
|
||||
"policy": orbit_policy,
|
||||
"pool_root": orbit_pool_root,
|
||||
"pool_exists": orbit_pool_exists,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": orbit_warnings,
|
||||
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
}
|
||||
|
||||
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
|
||||
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
|
||||
|
||||
return {
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"task_dir": task_dir,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archive_counts": {
|
||||
"master": len(master_archives),
|
||||
"slave": len(slave_archives),
|
||||
},
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": allow_submit,
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbit_resolution": {
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
},
|
||||
"input_assets": {
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbits": orbits_summary,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
orbit_context = get_pyint_orbit_context()
|
||||
|
||||
warnings: List[str] = list(dem_summary.get("warnings") or [])
|
||||
blockers: List[str] = list(dem_summary.get("blockers") or [])
|
||||
task_summaries: List[Dict[str, Any]] = []
|
||||
resolved_task_count = 0
|
||||
missing_task_count = 0
|
||||
|
||||
for task_dir in validation.get("task_dirs", []) or []:
|
||||
task_summary = resolve_pyint_task_input_assets(
|
||||
task_dir,
|
||||
dem_summary=dem_summary,
|
||||
orbit_context=orbit_context,
|
||||
)
|
||||
task_summaries.append(task_summary)
|
||||
if task_summary.get("warnings"):
|
||||
warnings.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["warnings"]
|
||||
)
|
||||
if task_summary.get("blockers"):
|
||||
blockers.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["blockers"]
|
||||
)
|
||||
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
|
||||
resolved_task_count += 1
|
||||
else:
|
||||
missing_task_count += 1
|
||||
|
||||
allow_submit = not blockers
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
return {
|
||||
"root_dir": validation["root_dir"],
|
||||
"mode": validation["mode"],
|
||||
"task_count": len(task_summaries),
|
||||
"selected_task_count": len(task_summaries),
|
||||
"allow_submit": allow_submit,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"invalid_candidates": validation.get("invalid_candidates", []),
|
||||
"dem": dem_summary,
|
||||
"orbits": {
|
||||
"policy": _get_orbit_policy(),
|
||||
"pool_root": orbit_context.get("pool_root", ""),
|
||||
"pool_exists": bool(orbit_context.get("pool_exists")),
|
||||
"resolved_task_count": resolved_task_count,
|
||||
"missing_task_count": missing_task_count,
|
||||
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
|
||||
"warnings": list(orbit_context.get("warnings") or []),
|
||||
},
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Helpers for integrating the external PyINT workflow."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
|
||||
from .wsl_service import run_wsl_command
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
DEFAULT_RANGE_LOOKS = 2
|
||||
DEFAULT_AZIMUTH_LOOKS = 2
|
||||
DEFAULT_PARALLEL_WORKERS = 1
|
||||
MAX_LOOKS = 32
|
||||
MAX_PARALLEL_WORKERS = 16
|
||||
|
||||
_DATE_TOKEN_RE = re.compile(r"(20\d{6})")
|
||||
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PyintCheck:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str = ""
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class PyintEnvironmentReport:
|
||||
overall_ok: bool
|
||||
checks: List[PyintCheck] = field(default_factory=list)
|
||||
message: str = ""
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
return get_env_text(name, default) or default
|
||||
|
||||
|
||||
def _read_bool_env(name: str, default: bool = False) -> bool:
|
||||
return read_bool_env(name, default)
|
||||
|
||||
|
||||
def normalize_date_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
match = _DATE_TOKEN_RE.search(re.sub(r"\D", "", text))
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = _DATE_TOKEN_RE.search(text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def slugify_text(value: Any, *, default: str = "item", max_len: int = 96) -> str:
|
||||
text = _SAFE_TEXT_RE.sub("_", str(value or "").strip()).strip("._")
|
||||
if not text:
|
||||
text = default
|
||||
return text[:max_len]
|
||||
|
||||
|
||||
def build_project_name(pair_key: str, run_key: str) -> str:
|
||||
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
normalized = os.path.normpath(text)
|
||||
if normalized.startswith("/"):
|
||||
return normalized.replace("\\", "/")
|
||||
if normalized.startswith("\\\\"):
|
||||
return ""
|
||||
drive, tail = os.path.splitdrive(normalized)
|
||||
if not drive:
|
||||
return normalized.replace("\\", "/")
|
||||
drive_letter = drive.rstrip(":").lower()
|
||||
normalized_tail = tail.replace("\\", "/")
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
def to_wsl_path(path: str) -> str:
|
||||
return windows_path_to_wsl_mount(path)
|
||||
|
||||
|
||||
def quote_shell(value: str) -> str:
|
||||
return shlex.quote(str(value or ""))
|
||||
|
||||
|
||||
def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
|
||||
result: Dict[str, List[str]] = {"master": [], "slave": []}
|
||||
for role in ("master", "slave"):
|
||||
role_dir = task_path / role
|
||||
if not role_dir.is_dir():
|
||||
continue
|
||||
inputs = []
|
||||
for pattern in LT1_INPUT_GLOBS:
|
||||
inputs.extend(
|
||||
str(path.resolve())
|
||||
for path in role_dir.rglob(pattern)
|
||||
if path.is_file()
|
||||
)
|
||||
result[role] = sorted(set(inputs))
|
||||
return result
|
||||
|
||||
|
||||
def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
|
||||
dates = {
|
||||
date_text
|
||||
for path in paths
|
||||
for date_text in [normalize_date_text(os.path.basename(path))]
|
||||
if date_text
|
||||
}
|
||||
if len(dates) == 1:
|
||||
return next(iter(dates))
|
||||
return ""
|
||||
|
||||
|
||||
def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
return {
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"pair_meta": pair_meta,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
}
|
||||
|
||||
|
||||
def build_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
"satelite=LT",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def validate_pyint_root_dir(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
normalized_root = os.path.normpath(os.path.abspath(str(root_dir or "").strip()))
|
||||
if not root_dir or not os.path.isdir(normalized_root):
|
||||
raise ValueError(f"PyINT root_dir does not exist or is not a directory: {root_dir}")
|
||||
|
||||
def _missing_task_subdirs(task_dir: str) -> List[str]:
|
||||
missing: List[str] = []
|
||||
for subdir in ("master", "slave"):
|
||||
if not os.path.isdir(os.path.join(task_dir, subdir)):
|
||||
missing.append(subdir)
|
||||
return missing
|
||||
|
||||
def _iter_child_dirs(directory: str):
|
||||
with os.scandir(directory) as entries:
|
||||
child_dirs = [entry for entry in entries if entry.is_dir()]
|
||||
child_dirs.sort(key=lambda entry: entry.name.lower())
|
||||
return child_dirs
|
||||
|
||||
if not _missing_task_subdirs(normalized_root):
|
||||
task_dirs = [normalized_root]
|
||||
invalid_candidates: List[Dict[str, Any]] = []
|
||||
mode = "single_task_dir"
|
||||
else:
|
||||
task_dirs = []
|
||||
invalid_candidates = []
|
||||
for entry in _iter_child_dirs(normalized_root):
|
||||
if not entry.name.lower().startswith("task_"):
|
||||
continue
|
||||
missing = _missing_task_subdirs(entry.path)
|
||||
if missing:
|
||||
invalid_candidates.append(
|
||||
{"name": entry.name, "path": entry.path, "missing_subdirs": missing}
|
||||
)
|
||||
continue
|
||||
task_dirs.append(os.path.normpath(entry.path))
|
||||
mode = "task_root_dir"
|
||||
|
||||
if not task_dirs:
|
||||
detail = ""
|
||||
if invalid_candidates:
|
||||
formatted = ", ".join(
|
||||
f"{item['name']} missing {','.join(item['missing_subdirs'])}"
|
||||
for item in invalid_candidates[:5]
|
||||
)
|
||||
detail = f" Invalid candidates: {formatted}."
|
||||
raise ValueError(
|
||||
"PyINT root_dir must be either a single task directory containing "
|
||||
"'master' and 'slave', or a parent directory containing valid Task_* subdirectories."
|
||||
f"{detail}"
|
||||
)
|
||||
|
||||
selected_count = int(num_to_process or 0)
|
||||
if selected_count > 0:
|
||||
task_dirs = task_dirs[:selected_count]
|
||||
|
||||
return {
|
||||
"root_dir": normalized_root,
|
||||
"mode": mode,
|
||||
"task_dirs": task_dirs,
|
||||
"task_count": len(task_dirs),
|
||||
"invalid_candidates": invalid_candidates,
|
||||
}
|
||||
|
||||
|
||||
def resolve_time_baseline_days(master_date: str, slave_date: str, pair_meta: Dict[str, Any]) -> int:
|
||||
raw_days = pair_meta.get("time_baseline_days")
|
||||
try:
|
||||
if raw_days not in (None, ""):
|
||||
return int(raw_days)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if not master_date or not slave_date:
|
||||
return 0
|
||||
try:
|
||||
master_dt = datetime.strptime(master_date, "%Y%m%d")
|
||||
slave_dt = datetime.strptime(slave_date, "%Y%m%d")
|
||||
except ValueError:
|
||||
return 0
|
||||
return (slave_dt - master_dt).days
|
||||
|
||||
|
||||
def _gamma_prefix(gamma_env_script_wsl: str) -> str:
|
||||
script = str(gamma_env_script_wsl or "").strip()
|
||||
if not script:
|
||||
return ""
|
||||
return f". {quote_shell(script)} >/dev/null 2>&1 && "
|
||||
|
||||
|
||||
def check_pyint_environment(
|
||||
*,
|
||||
enabled: Optional[bool] = None,
|
||||
distro: Optional[str] = None,
|
||||
python_cmd: Optional[str] = None,
|
||||
pyint_home: Optional[str] = None,
|
||||
pyint_app_script: Optional[str] = None,
|
||||
template_root: Optional[str] = None,
|
||||
work_root: Optional[str] = None,
|
||||
output_root: Optional[str] = None,
|
||||
dem_root: Optional[str] = None,
|
||||
gamma_env_script: Optional[str] = None,
|
||||
smoke_test: Optional[bool] = None,
|
||||
) -> PyintEnvironmentReport:
|
||||
enabled_value = _read_bool_env("PYINT_ENABLED", False) if enabled is None else bool(enabled)
|
||||
if not enabled_value:
|
||||
return PyintEnvironmentReport(
|
||||
overall_ok=False,
|
||||
checks=[PyintCheck(name="PYINT_ENABLED", ok=False, detail="PYINT_ENABLED=false")],
|
||||
message="PyINT is disabled. Set PYINT_ENABLED=true to enable it.",
|
||||
)
|
||||
|
||||
distro_value = str(distro or _read_env("PYINT_WSL_DISTRO", settings.ISCE2_WSL_DISTRO)).strip()
|
||||
python_value = str(python_cmd or _read_env("PYINT_WSL_PYTHON", settings.ISCE2_PYTHON)).strip()
|
||||
pyint_home_wsl = to_wsl_path(str(pyint_home or _read_env("PYINT_HOME", "")))
|
||||
pyint_app_wsl = to_wsl_path(str(pyint_app_script or _read_env("PYINT_APP_SCRIPT", "")))
|
||||
template_root_wsl = to_wsl_path(str(template_root or _read_env("PYINT_TEMPLATE_ROOT", "")))
|
||||
work_root_wsl = to_wsl_path(str(work_root or _read_env("PYINT_WORK_ROOT", "")))
|
||||
output_root_wsl = to_wsl_path(str(output_root or _read_env("PYINT_OUTPUT_ROOT", "")))
|
||||
dem_root_wsl = to_wsl_path(str(dem_root or _read_env("PYINT_DEM_ROOT", "")))
|
||||
gamma_env_wsl = to_wsl_path(str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")))
|
||||
smoke_enabled = _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) if smoke_test is None else bool(smoke_test)
|
||||
precise_orbit_enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True)
|
||||
|
||||
checks: List[PyintCheck] = []
|
||||
|
||||
def add(name: str, ok: bool, detail: str = "", skipped: bool = False) -> None:
|
||||
checks.append(PyintCheck(name=name, ok=ok, detail=detail, skipped=skipped))
|
||||
|
||||
rc, out, err = run_wsl_command("echo pyint_alive", distro=distro_value, timeout=15)
|
||||
wsl_ok = rc == 0 and "pyint_alive" in out
|
||||
add("WSL distro", wsl_ok, out or err or distro_value)
|
||||
|
||||
if not wsl_ok:
|
||||
return PyintEnvironmentReport(
|
||||
overall_ok=False,
|
||||
checks=checks,
|
||||
message=f"WSL distro is unavailable: {distro_value}",
|
||||
)
|
||||
|
||||
rc, out, err = run_wsl_command(
|
||||
f"{quote_shell(python_value)} --version",
|
||||
distro=distro_value,
|
||||
timeout=15,
|
||||
)
|
||||
add("WSL Python", rc == 0, out or err or python_value)
|
||||
|
||||
if pyint_home_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -d {quote_shell(pyint_home_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("PYINT_HOME", rc == 0 and "ok" in out, pyint_home_wsl or err)
|
||||
else:
|
||||
add("PYINT_HOME", False, "PYINT_HOME is empty")
|
||||
|
||||
if pyint_app_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -f {quote_shell(pyint_app_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("pyintApp.py", rc == 0 and "ok" in out, pyint_app_wsl or err)
|
||||
else:
|
||||
add("pyintApp.py", False, "PYINT_APP_SCRIPT is empty")
|
||||
|
||||
for name, path_text in (
|
||||
("PYINT_TEMPLATE_ROOT", template_root_wsl),
|
||||
("PYINT_WORK_ROOT", work_root_wsl),
|
||||
("PYINT_OUTPUT_ROOT", output_root_wsl),
|
||||
("PYINT_DEM_ROOT", dem_root_wsl),
|
||||
):
|
||||
if not path_text:
|
||||
add(name, False, f"{name} is empty")
|
||||
continue
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -d {quote_shell(path_text)} && test -w {quote_shell(path_text)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add(name, rc == 0 and "ok" in out, path_text or err)
|
||||
|
||||
if gamma_env_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -f {quote_shell(gamma_env_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("GAMMA env script", rc == 0 and "ok" in out, gamma_env_wsl or err)
|
||||
else:
|
||||
add("GAMMA env script", True, "Not configured; using current PATH", skipped=True)
|
||||
|
||||
gamma_prefix = _gamma_prefix(gamma_env_wsl)
|
||||
for name, command_name in (
|
||||
("GAMMA LT1 import", "LT1_import_SLC_from_zipfiles1"),
|
||||
("GAMMA geocode_back", "geocode_back"),
|
||||
):
|
||||
rc, out, err = run_wsl_command(
|
||||
gamma_prefix + f"command -v {quote_shell(command_name)}",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add(name, rc == 0 and bool(out.strip()), out or err or command_name)
|
||||
|
||||
helper_path = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "pyint_pipeline"
|
||||
/ "apply_lt1_precise_orbit.py"
|
||||
)
|
||||
if precise_orbit_enabled:
|
||||
add("LT1 precise orbit bridge helper", helper_path.is_file(), str(helper_path))
|
||||
else:
|
||||
add("LT1 precise orbit bridge helper", True, "Skipped", skipped=True)
|
||||
|
||||
if smoke_enabled:
|
||||
smoke_cmd = (
|
||||
f"export PYTHONPATH={quote_shell(pyint_home_wsl)}:$PYTHONPATH && "
|
||||
+ gamma_prefix
|
||||
+ f"{quote_shell(python_value)} {quote_shell(pyint_app_wsl)} -h >/dev/null"
|
||||
)
|
||||
rc, out, err = run_wsl_command(smoke_cmd, distro=distro_value, timeout=60)
|
||||
add("PyINT smoke test", rc == 0, out or err or "pyintApp.py -h")
|
||||
else:
|
||||
add("PyINT smoke test", True, "Skipped", skipped=True)
|
||||
|
||||
required_checks = [check for check in checks if not check.skipped]
|
||||
overall_ok = all(check.ok for check in required_checks)
|
||||
failed_names = [check.name for check in required_checks if not check.ok]
|
||||
message = "All PyINT checks passed." if overall_ok else f"Failed checks: {', '.join(failed_names)}"
|
||||
return PyintEnvironmentReport(overall_ok=overall_ok, checks=checks, message=message)
|
||||
@@ -54,12 +54,55 @@ def get_unpack_config() -> Dict[str, Any]:
|
||||
minimum=1,
|
||||
maximum=32,
|
||||
),
|
||||
"max_files_per_run": module.parse_int(
|
||||
env.get("UNPACK_MAX_FILES_PER_RUN"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
),
|
||||
"max_runtime_minutes": module.parse_int(
|
||||
env.get("UNPACK_MAX_RUNTIME_MINUTES"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def run_unpack_task(task_id: str):
|
||||
def _normalize_unpack_run_limits(raw_config: Optional[Dict[str, Any]]) -> Dict[str, int]:
|
||||
if not isinstance(raw_config, dict):
|
||||
return {}
|
||||
|
||||
module = _load_unpack_module()
|
||||
normalized: Dict[str, int] = {}
|
||||
|
||||
if raw_config.get("max_files_per_run") is not None:
|
||||
normalized["max_files_per_run"] = module.parse_int(
|
||||
raw_config.get("max_files_per_run"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
)
|
||||
if raw_config.get("max_runtime_minutes") is not None:
|
||||
normalized["max_runtime_minutes"] = module.parse_int(
|
||||
raw_config.get("max_runtime_minutes"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def build_unpack_run_config(overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
config = get_unpack_config()
|
||||
config.update(_normalize_unpack_run_limits(overrides))
|
||||
return config
|
||||
|
||||
|
||||
async def run_unpack_task(task_id: str, task_config: Optional[Dict[str, Any]] = None):
|
||||
module = _load_unpack_module()
|
||||
loop = asyncio.get_running_loop()
|
||||
config_overrides = _normalize_unpack_run_limits(task_config)
|
||||
if not config_overrides:
|
||||
task_record = await task_service.get_task(task_id)
|
||||
config_overrides = _normalize_unpack_run_limits(getattr(task_record, "params", None))
|
||||
|
||||
def _submit(coro):
|
||||
try:
|
||||
@@ -88,14 +131,19 @@ async def run_unpack_task(task_id: str):
|
||||
module.run_unpack_job,
|
||||
log_callback=log_cb,
|
||||
progress_callback=progress_cb,
|
||||
config_overrides=config_overrides or None,
|
||||
)
|
||||
|
||||
if not result:
|
||||
result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0}
|
||||
result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0, "remaining": 0, "message": "completed"}
|
||||
|
||||
summary = (
|
||||
"Unpack complete: processed {processed}, failed {failed}, skipped {skipped}"
|
||||
).format(**result)
|
||||
summary = "Unpack complete: processed {processed}, failed {failed}, skipped {skipped}".format(**result)
|
||||
remaining = int(result.get("remaining") or 0)
|
||||
if remaining > 0:
|
||||
summary = f"{summary}, remaining {remaining}"
|
||||
message_text = str(result.get("message") or "").strip()
|
||||
if message_text and message_text != "completed":
|
||||
summary = f"{summary} ({message_text})"
|
||||
await task_service.update_task(task_id, status="COMPLETED", progress=100, message=summary)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(task_id, status="FAILED", message=f"Unpack failed: {exc}")
|
||||
|
||||
Reference in New Issue
Block a user