Add SARscape SBAS prepared stack workflow

This commit is contained in:
2026-04-30 09:42:01 +08:00
parent 4c0d1f2c2b
commit 6696fe90fa
27 changed files with 4915 additions and 95 deletions
+20 -2
View File
@@ -25,9 +25,12 @@ def _parse_args() -> argparse.Namespace:
)
parser.add_argument(
"--workflow",
required=True,
required=False,
choices=["import", "dinsar", "dinsar_custom"],
)
parser.add_argument("--inspect-sarscape-sbas", action="store_true")
parser.add_argument("--include-parameters", action="store_true")
parser.add_argument("--task-name", action="append", default=[])
parser.add_argument("--root-dir", required=False)
parser.add_argument("--task-dir", required=False)
parser.add_argument("--output-dir", required=False)
@@ -44,7 +47,22 @@ def main() -> int:
ensure_project_env_loaded()
args = _parse_args()
try:
from .envi_service import run_single_task_workflow, run_workflow
from .envi_service import (
inspect_sarscape_sbas_tasks,
run_single_task_workflow,
run_workflow,
)
if args.inspect_sarscape_sbas:
record = inspect_sarscape_sbas_tasks(
args.task_name or None,
include_parameters=bool(args.include_parameters),
)
print(json.dumps(record, ensure_ascii=False))
return 0 if record.get("ok") else 2
if not args.workflow:
raise ValueError("--workflow is required unless --inspect-sarscape-sbas is used.")
if args.task_dir:
if not args.output_dir:
+558 -20
View File
@@ -13,6 +13,7 @@ import subprocess
import sys
import time
import defusedxml.ElementTree as ET
from contextlib import contextmanager
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from datetime import datetime
from glob import glob
@@ -67,13 +68,43 @@ def get_envi_runner_python() -> str:
return os.path.normpath(sys.executable)
def get_envi_taskengine_cwd() -> str:
"""Dedicated cwd for envipyengine/taskengine temp files.
SARscape can create zero-byte env_*.xyz and IDL*.tmp files in the current
working directory. Keep those files under runtime instead of the repo root.
"""
base_dir = _to_local_path(
getattr(settings, "IDL_WORKER_RUNTIME_DIR", "")
or os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
)
cwd = os.path.join(base_dir, "envi_cwd")
os.makedirs(cwd, exist_ok=True)
return os.path.normpath(os.path.abspath(cwd))
def get_envi_custom_code_dir() -> str:
envi_root = _envi_install_root()
if not envi_root:
return ""
candidates = [
os.path.join(envi_root, "user_custom_code"),
os.path.join(envi_root, "custom_code"),
]
for path in candidates:
if os.path.isdir(path):
return os.path.normpath(os.path.abspath(path))
return ""
def get_envi_runner_cwd() -> str:
return os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
return get_envi_taskengine_cwd()
def get_envi_runner_env() -> Dict[str, str]:
env = os.environ.copy()
project_root = get_envi_runner_cwd()
project_root = os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
taskengine_cwd = get_envi_taskengine_cwd()
existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()]
ordered = [project_root, *existing]
deduped: List[str] = []
@@ -88,9 +119,36 @@ def get_envi_runner_env() -> Dict[str, str]:
seen.add(key)
deduped.append(str(raw_path))
env["PYTHONPATH"] = os.pathsep.join(deduped)
env["TEMP"] = taskengine_cwd
env["TMP"] = taskengine_cwd
env["IDL_TMPDIR"] = taskengine_cwd
custom_code_dir = get_envi_custom_code_dir()
if custom_code_dir:
env["ENVI_CUSTOM_CODE"] = custom_code_dir
return env
@contextmanager
def _envi_taskengine_runtime_context():
"""Run in-process ENVI calls from the dedicated runtime cwd."""
target_cwd = get_envi_taskengine_cwd()
old_cwd = os.getcwd()
old_env = {name: os.environ.get(name) for name in ("TEMP", "TMP", "IDL_TMPDIR")}
os.environ["TEMP"] = target_cwd
os.environ["TMP"] = target_cwd
os.environ["IDL_TMPDIR"] = target_cwd
try:
os.chdir(target_cwd)
yield target_cwd
finally:
os.chdir(old_cwd)
for name, value in old_env.items():
if value is None:
os.environ.pop(name, None)
else:
os.environ[name] = value
def build_envi_runner_command(*args: Any) -> List[str]:
command = [
get_envi_runner_python(),
@@ -101,6 +159,79 @@ def build_envi_runner_command(*args: Any) -> List[str]:
return command
def _list_taskengine_pids() -> set[int]:
"""Return taskengine.exe PIDs on Windows without importing optional deps."""
if os.name != "nt":
return set()
try:
completed = subprocess.run(
[
"powershell.exe",
"-NoProfile",
"-Command",
"Get-Process taskengine -ErrorAction SilentlyContinue | ForEach-Object { $_.Id }",
],
capture_output=True,
text=True,
timeout=5,
check=False,
)
except Exception:
return set()
pids: set[int] = set()
for line in str(completed.stdout or "").splitlines():
raw = line.strip()
if raw.isdigit():
pids.add(int(raw))
return pids
def _stop_taskengine_pids(pids: set[int]) -> List[int]:
"""Stop specific taskengine.exe PIDs; avoids killing pre-existing sessions."""
stopped: List[int] = []
if os.name != "nt":
return stopped
for pid in sorted(pids):
try:
subprocess.run(
[
"powershell.exe",
"-NoProfile",
"-Command",
f"Stop-Process -Id {int(pid)} -Force -ErrorAction SilentlyContinue",
],
capture_output=True,
text=True,
timeout=10,
check=False,
)
stopped.append(int(pid))
except Exception:
continue
return stopped
def _cleanup_new_taskengine_processes(existing_pids: set[int]) -> Dict[str, Any]:
"""Best-effort cleanup for taskengine.exe children spawned by a timed-out runner."""
existing = set(existing_pids or set())
first_targets = _list_taskengine_pids() - existing
stopped = _stop_taskengine_pids(first_targets)
# taskengine can take a moment to detach from the runner process. Re-check once.
time.sleep(1)
second_targets = _list_taskengine_pids() - existing
stopped.extend(pid for pid in _stop_taskengine_pids(second_targets) if pid not in stopped)
time.sleep(1)
remaining = sorted(_list_taskengine_pids() - existing)
return {
"taskengine_cleanup_attempted": True,
"taskengine_stopped_pids": sorted(set(stopped)),
"taskengine_remaining_new_pids": remaining,
}
def probe_envi_runner() -> Dict[str, Any]:
python_path = get_envi_runner_python()
project_root = get_envi_runner_cwd()
@@ -238,6 +369,39 @@ import threading
_ENVI_GLOBAL_LOCK = threading.Lock()
SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES = [
"wf_sbas",
"wf_esbas",
]
SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES = [
"SARscape_setting_output_folders",
"SARsLoadPreferences",
"SARsImportSarSelector",
"SARscapeSuggestLooks",
"SARscapeEnviuriToShape",
]
SARSCAPE_SBAS_STACK_TASK_CANDIDATES = [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
SARSCAPE_SBAS_TASK_CANDIDATES = [
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
*SARSCAPE_SBAS_STACK_TASK_CANDIDATES,
]
# ---------------------------------------------------------------------------
# Progress file for subprocess ↔ job handler communication
# ---------------------------------------------------------------------------
@@ -417,26 +581,32 @@ def execute_envi_task(task_name: str, parameters: Dict[str, Any]) -> Dict[str, A
) from exc
with _ENVI_GLOBAL_LOCK:
engine = Engine("ENVI")
task = engine.task(task_name)
with _envi_taskengine_runtime_context():
engine = Engine("ENVI")
task = engine.task(task_name)
existing_taskengine_pids = _list_taskengine_pids()
# Run with timeout to handle envipyengine hangs
with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(task.execute, parameters)
try:
result = future.result(timeout=_ENVI_TASK_TIMEOUT)
except FuturesTimeoutError:
# Run with timeout to handle envipyengine hangs
with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(task.execute, parameters)
try:
import subprocess as _sp
_sp.run(["taskkill", "/F", "/IM", "taskengine.exe"],
capture_output=True, timeout=10)
print("[WARN] execute_envi_task: killed taskengine after timeout")
except Exception as _exc:
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}")
raise RuntimeError(
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s "
f"(envipyengine hung). Output files may still exist."
)
result = future.result(timeout=_ENVI_TASK_TIMEOUT)
except FuturesTimeoutError:
try:
cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
stopped = cleanup.get("taskengine_stopped_pids") or []
remaining = cleanup.get("taskengine_remaining_new_pids") or []
print(
"[WARN] execute_envi_task: task timed out; "
f"stopped_new_taskengine_pids={stopped}; "
f"remaining_new_taskengine_pids={remaining}"
)
except Exception as _exc:
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}")
raise RuntimeError(
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s "
f"(envipyengine hung). Output files may still exist."
)
# taskengine returns {"outputParameters": {...}, ...}
return result.get("outputParameters", result)
@@ -453,6 +623,374 @@ def _unwrap_sarscapedata(value: Any) -> Any:
return value
def _configured_sarscape_sbas_task_candidates() -> List[str]:
configured = str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
if not configured:
return list(SARSCAPE_SBAS_TASK_CANDIDATES)
names: List[str] = []
for raw in configured.replace(";", ",").split(","):
name = raw.strip()
if name and name not in names:
names.append(name)
return names or list(SARSCAPE_SBAS_TASK_CANDIDATES)
def _envi_install_root() -> str:
executable = _to_local_path(IDL_EXECUTABLE)
if not executable:
return ""
return os.path.abspath(os.path.join(os.path.dirname(executable), "..", "..", ".."))
def _static_envi_task_template_path(task_name: str) -> str:
name = str(task_name or "").strip()
if not name:
return ""
envi_root = _envi_install_root()
if not envi_root:
return ""
candidates = [
os.path.join(envi_root, "user_custom_code", f"{name}.task"),
os.path.join(envi_root, "resource", "templates", "tasks", "SARscape", f"{name}.task"),
os.path.join(envi_root, "resource", "templates", "tasks", f"{name}.task"),
]
for path in candidates:
if os.path.isfile(path):
return path
return ""
def _json_safe_parameter(value: Any) -> Any:
if isinstance(value, dict):
return {str(key): _json_safe_parameter(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe_parameter(item) for item in value]
if value is None or isinstance(value, (str, int, float, bool)):
return value
return str(value)
def _summarize_task_parameters(raw_parameters: Any) -> Dict[str, Any]:
safe_parameters = _json_safe_parameter(raw_parameters)
input_names: List[str] = []
output_names: List[str] = []
required_input_names: List[str] = []
if isinstance(safe_parameters, dict):
iterable = safe_parameters.values()
elif isinstance(safe_parameters, list):
iterable = safe_parameters
else:
iterable = []
for item in iterable:
if not isinstance(item, dict):
continue
name = str(item.get("name") or item.get("NAME") or "").strip()
direction = str(item.get("direction") or item.get("DIRECTION") or "").strip().lower()
required = bool(item.get("required") or item.get("REQUIRED"))
if not name:
continue
if direction == "input":
input_names.append(name)
if required:
required_input_names.append(name)
elif direction == "output":
output_names.append(name)
return {
"parameter_count": (
len(safe_parameters)
if isinstance(safe_parameters, (dict, list))
else 0
),
"input_names": input_names,
"required_input_names": required_input_names,
"output_names": output_names,
"parameters": safe_parameters,
}
def list_envi_tasks() -> Dict[str, Any]:
"""List ENVI task names without instantiating individual task parameters."""
result: Dict[str, Any] = {
"ok": False,
"engine": "envipyengine",
"task_count": 0,
"tasks": [],
"error": None,
}
try:
_ensure_envipyengine_config()
from envipyengine import Engine
except ImportError:
result["error"] = (
"envipyengine is not installed. Install it with: pip install envipyengine"
)
return result
with _ENVI_GLOBAL_LOCK:
with _envi_taskengine_runtime_context():
try:
names = Engine("ENVI").tasks()
except Exception as exc:
result["error"] = str(exc)
return result
result["tasks"] = [str(name) for name in names]
result["task_count"] = len(result["tasks"])
result["ok"] = True
return result
def discover_sarscape_sbas_tasks() -> Dict[str, Any]:
"""Discover installed SARscape SBAS/E-SBAS task names by filtering Engine.tasks()."""
report = list_envi_tasks()
task_names = list(report.get("tasks") or [])
keywords = (
"StackSBAS",
"StackESBAS",
"ConnectionGraphESBAS",
)
explicit_names = set(SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES) | set(SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES)
matches = [
name
for name in task_names
if (
str(name) in explicit_names
or (
str(name).startswith("SARsInSAR")
and any(keyword.lower() in str(name).lower() for keyword in keywords)
)
)
]
static_task_files: Dict[str, str] = {}
for name in SARSCAPE_SBAS_TASK_CANDIDATES:
path = _static_envi_task_template_path(name)
if path:
static_task_files[name] = path
if name not in matches:
matches.append(name)
preferred_order = [
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
ordered: List[str] = []
for name in preferred_order:
if name in matches and name not in ordered:
ordered.append(name)
for name in sorted(matches):
if name not in ordered:
ordered.append(name)
return {
"ok": bool(report.get("ok")) and bool(ordered),
"engine": report.get("engine"),
"task_count": int(report.get("task_count") or 0),
"sarscape_sbas_task_count": len(ordered),
"sarscape_sbas_tasks": ordered,
"static_task_files": {
name: static_task_files[name]
for name in ordered
if name in static_task_files
},
"error": report.get("error"),
}
def inspect_envi_tasks(task_names: List[str]) -> Dict[str, Any]:
"""Inspect ENVI/SARscape tasks without executing them."""
started_at = _utc_now_text()
deduped_names: List[str] = []
for raw_name in task_names:
name = str(raw_name or "").strip()
if name and name not in deduped_names:
deduped_names.append(name)
result: Dict[str, Any] = {
"ok": False,
"engine": "envipyengine",
"started_at": started_at,
"finished_at": None,
"task_count": len(deduped_names),
"available_count": 0,
"missing_count": 0,
"tasks": [],
"error": None,
}
if not deduped_names:
result["error"] = "No task names provided."
result["finished_at"] = _utc_now_text()
return result
try:
_ensure_envipyengine_config()
from envipyengine import Engine
except ImportError as exc:
result["error"] = (
"envipyengine is not installed. Install it with: pip install envipyengine"
)
result["finished_at"] = _utc_now_text()
return result
with _ENVI_GLOBAL_LOCK:
with _envi_taskengine_runtime_context():
try:
engine = Engine("ENVI")
except Exception as exc:
result["error"] = f"Failed to initialize ENVI engine: {exc}"
result["finished_at"] = _utc_now_text()
return result
for task_name in deduped_names:
item: Dict[str, Any] = {
"name": task_name,
"available": False,
"error": None,
"parameter_count": 0,
"input_names": [],
"required_input_names": [],
"output_names": [],
"parameters": [],
}
try:
task = engine.task(task_name)
summary = _summarize_task_parameters(getattr(task, "parameters", []))
item.update(summary)
item["available"] = True
except Exception as exc:
item["error"] = str(exc)
result["tasks"].append(item)
result["available_count"] = sum(1 for item in result["tasks"] if item.get("available"))
result["missing_count"] = sum(1 for item in result["tasks"] if not item.get("available"))
result["ok"] = result["available_count"] > 0
result["finished_at"] = _utc_now_text()
return result
def inspect_sarscape_sbas_tasks(
task_names: Optional[List[str]] = None,
*,
include_parameters: bool = False,
) -> Dict[str, Any]:
"""Inspect likely SARscape SBAS/E-SBAS task names for the installed version."""
status = get_status()
discovery = discover_sarscape_sbas_tasks()
names = task_names or list(discovery.get("sarscape_sbas_tasks") or _configured_sarscape_sbas_task_candidates())
if include_parameters:
task_report = inspect_envi_tasks(names)
else:
discovered_set = set(discovery.get("sarscape_sbas_tasks") or [])
task_report = {
"ok": bool(discovery.get("ok")),
"engine": "envipyengine",
"task_count": len(names),
"available_count": sum(1 for name in names if name in discovered_set),
"missing_count": sum(1 for name in names if name not in discovered_set),
"tasks": [
{
"name": name,
"available": name in discovered_set,
"error": None if name in discovered_set else "Task name not listed by Engine.tasks().",
"parameter_count": None,
"input_names": [],
"required_input_names": [],
"output_names": [],
"parameters": [],
}
for name in names
],
"error": discovery.get("error"),
}
task_report["status"] = {
"idl_installed": status.get("idl_installed"),
"idl_executable": status.get("idl_executable"),
"runner_ready": status.get("runner_ready"),
"runner_python": status.get("runner_python"),
"runner_message": status.get("runner_message"),
"dem_base_file": status.get("dem_base_file"),
"dem_exists": status.get("dem_exists"),
}
task_report["candidate_source"] = (
"SARSCAPE_SBAS_TASK_NAMES"
if str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
else "engine_task_list"
)
task_report["include_parameters"] = bool(include_parameters)
task_report["discovery"] = discovery
task_report["ready_for_pipeline_design"] = bool(task_report.get("ok"))
return task_report
def inspect_sarscape_sbas_tasks_subprocess(
task_names: Optional[List[str]] = None,
*,
timeout_seconds: int = 120,
include_parameters: bool = False,
) -> Dict[str, Any]:
"""Run SARscape SBAS task inspection through the isolated ENVI runner."""
command = build_envi_runner_command("--inspect-sarscape-sbas")
if include_parameters:
command.append("--include-parameters")
for name in task_names or []:
if str(name or "").strip():
command.extend(["--task-name", str(name).strip()])
existing_taskengine_pids = _list_taskengine_pids()
try:
completed = subprocess.run(
command,
cwd=get_envi_runner_cwd(),
env=get_envi_runner_env(),
capture_output=True,
text=True,
timeout=max(10, int(timeout_seconds or 120)),
check=False,
)
except subprocess.TimeoutExpired as exc:
cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
stdout_text = str(exc.stdout or "").strip()
stderr_text = str(exc.stderr or "").strip()
return {
"ok": False,
"returncode": None,
"timeout": True,
"timeout_seconds": max(10, int(timeout_seconds or 120)),
"stdout": stdout_text[:2000],
"stderr": stderr_text[:2000],
"error": (
"SARscape SBAS task inspection timed out. "
"Use lightweight discovery without include_parameters, or provide a manually verified task template."
),
"runner_command": command,
**cleanup,
}
stdout_text = str(completed.stdout or "").strip()
stderr_text = str(completed.stderr or "").strip()
try:
payload = json.loads(stdout_text) if stdout_text else {}
except Exception:
payload = {}
payload.setdefault("returncode", int(completed.returncode))
payload.setdefault("stdout", stdout_text[:2000])
payload.setdefault("stderr", stderr_text[:2000])
payload["runner_command"] = command
if completed.returncode != 0:
payload["ok"] = False
payload.setdefault("error", stderr_text or stdout_text or f"returncode={completed.returncode}")
return payload
# ---------------------------------------------------------------------------
# Import workflow
# ---------------------------------------------------------------------------
+71
View File
@@ -43,6 +43,8 @@ from .timeseries_service import (
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT,
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK,
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS,
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS,
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT,
JOB_TYPE_TIMESERIES_STACK_PREP,
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH,
timeseries_service,
@@ -3550,6 +3552,73 @@ async def _handle_timeseries_run_mintpy_sbas(job: SystemJobORM) -> None:
raise
async def _handle_timeseries_sarscape_preflight(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires task_id for progress tracking.")
payload = job.payload or {}
run_id = str(payload.get("run_id") or "").strip()
if not run_id:
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires run_id payload.")
async with AsyncSessionLocal() as db:
try:
await task_service.update_task(
job.task_id,
progress=45,
message="Building SARscape SBAS processor manifest...",
db=db,
)
result = await timeseries_service.build_sarscape_processor_preflight(run_id, db=db)
ready_text = "ready" if result.get("ready_for_execution") else "planning_only"
is_preflight_only = str(result.get("execution_mode") or "").strip() == "preflight_only"
await task_service.update_task(
job.task_id,
status="COMPLETED" if is_preflight_only else None,
progress=100 if is_preflight_only else 55,
message=(
f"SARscape SBAS preflight complete: state={ready_text} "
f"manifest={result.get('processor_manifest_path')}"
),
db=db,
)
except Exception as exc:
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
raise
async def _handle_timeseries_run_sarscape_sbas(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires task_id for progress tracking.")
payload = job.payload or {}
run_id = str(payload.get("run_id") or "").strip()
if not run_id:
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires run_id payload.")
async with AsyncSessionLocal() as db:
try:
await task_service.update_task(
job.task_id,
progress=90,
message="Running SARscape SBAS pipeline...",
db=db,
)
async with engine_lock_service.acquire("sarscape_sbas_timeseries"):
result = await timeseries_service.run_sarscape_sbas(run_id, db=db)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
f"SARscape SBAS complete: tasks={result.get('task_count', 0)} "
f"report={result.get('report_path')}"
),
db=db,
)
except Exception as exc:
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
raise
async def _handle_timeseries_export_publish(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("TIMESERIES_EXPORT_PUBLISH requires task_id for progress tracking.")
@@ -3648,6 +3717,8 @@ _HANDLERS = {
JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize,
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK: _handle_timeseries_run_isce2_stack,
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS: _handle_timeseries_run_mintpy_sbas,
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT: _handle_timeseries_sarscape_preflight,
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS: _handle_timeseries_run_sarscape_sbas,
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH: _handle_timeseries_export_publish,
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT: _handle_timeseries_register_product,
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
+7 -1
View File
@@ -17,7 +17,13 @@ from .. import database
from ..config import settings
from ..models import SystemWorkerHeartbeatORM
IDL_JOB_TYPES = {"IDL_RUN_IMPORT", "IDL_RUN_DINSAR", "WATER_GEOCODE", "WATER_FLOOD"}
IDL_JOB_TYPES = {
"IDL_RUN_IMPORT",
"IDL_RUN_DINSAR",
"WATER_GEOCODE",
"WATER_FLOOD",
"TIMESERIES_RUN_SARSCAPE_SBAS",
}
def _default_worker_id() -> str:
@@ -0,0 +1,681 @@
from __future__ import annotations
import hashlib
import json
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from ..config import settings
from . import envi_service
PROCESSOR_CODE = "sarscape_sbas"
ENGINE_CODE = "sarscape"
PREPARED_STACK_SCHEMA = "insar.prepared-sbas-stack/v1"
NATIVE_WORKFLOW_TASK = "wf_sbas"
NATIVE_ESBAS_WORKFLOW_TASK = "wf_esbas"
TEMPLATE_STRATEGY_NATIVE = "native_workflow_metatask"
TEMPLATE_STRATEGY_EXPLICIT = "explicit_stack_tasks"
SUPPORTED_TEMPLATE_STRATEGIES = {
TEMPLATE_STRATEGY_NATIVE,
TEMPLATE_STRATEGY_EXPLICIT,
}
REQUIRED_STACK_TASKS = [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
]
REQUIRED_TASKS = REQUIRED_STACK_TASKS
OPTIONAL_TASKS = [
NATIVE_WORKFLOW_TASK,
NATIVE_ESBAS_WORKFLOW_TASK,
"SARscape_setting_output_folders",
"SARsLoadPreferences",
"SARsImportSarSelector",
"SARscapeSuggestLooks",
"SARscapeEnviuriToShape",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
PIPELINE_PHASES = [
{
"phase_id": "connection_graph",
"task_name": "SARsInSARStackSBASGenerateConnectionGraph",
"purpose": "Build or ingest the SBAS connection graph.",
},
{
"phase_id": "interferogram_generation",
"task_name": "SARsInSARStackSBASInterferogramGeneration",
"purpose": "Generate interferograms for the selected SBAS graph.",
},
{
"phase_id": "inversion_step1",
"task_name": "SARsInSARStackSBASInversionStep1",
"purpose": "Run SARscape SBAS inversion step 1.",
},
{
"phase_id": "inversion_step2",
"task_name": "SARsInSARStackSBASInversionStep2",
"purpose": "Run SARscape SBAS inversion step 2.",
},
{
"phase_id": "geocode_export",
"task_name": "SARsInSARStackSBASGeocode",
"purpose": "Geocode velocity, displacement, and quality outputs.",
},
{
"phase_id": "variogram_optional",
"task_name": "SARsInSARStackSBASVariogram",
"purpose": "Optional SARscape variogram/quality analysis.",
"optional": True,
},
]
REQUIRED_RESULT_ROLES = [
"stack_manifest",
"processor_manifest",
"selected_network_edges",
"velocity_product",
"timeseries_product",
"temporal_coherence",
"geocoded_raster",
"preview_png",
"logs",
]
def default_parameter_template_path() -> str:
configured = str(getattr(settings, "SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH", "") or "").strip()
if configured:
return configured
return str(
Path(__file__).resolve().parents[2]
/ "templates"
/ "sarscape_sbas_parameter_template.example.json"
)
def _utcnow_iso() -> str:
return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
def _canonical_json(payload: Dict[str, Any]) -> str:
return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _sha256_payload(payload: Dict[str, Any]) -> str:
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
def _available_task_names(discovery_report: Optional[Dict[str, Any]]) -> set[str]:
if not isinstance(discovery_report, dict):
return set()
names: set[str] = set()
for item in discovery_report.get("tasks") or []:
if isinstance(item, dict) and bool(item.get("available")):
name = str(item.get("name") or "").strip()
if name:
names.add(name)
discovered = discovery_report.get("discovery") or {}
for name in discovered.get("sarscape_sbas_tasks") or []:
text = str(name or "").strip()
if text:
names.add(text)
return names
def _numeric_values(items: List[Dict[str, Any]], key: str) -> List[float]:
values: List[float] = []
for item in items:
try:
if item.get(key) is not None:
values.append(float(item.get(key)))
except Exception:
continue
return values
def load_parameter_template(parameter_template_path: Optional[str] = None) -> Dict[str, Any]:
path = str(parameter_template_path or default_parameter_template_path() or "").strip()
result: Dict[str, Any] = {
"path": path or None,
"exists": False,
"readable": False,
"schema": None,
"validated": False,
"execution_strategy": TEMPLATE_STRATEGY_NATIVE,
"native_workflow_task": None,
"task_count": 0,
"missing_required_tasks": list(REQUIRED_STACK_TASKS),
"tasks_without_parameters": [],
"errors": [],
"template": None,
}
if not path:
result["errors"].append("SARscape SBAS parameter template path is empty.")
return result
template_file = Path(path)
result["exists"] = template_file.is_file()
if not template_file.is_file():
result["errors"].append(f"SARscape SBAS parameter template not found: {path}")
return result
try:
payload = json.loads(template_file.read_text(encoding="utf-8"))
except Exception as exc:
result["errors"].append(f"Failed to read SARscape SBAS parameter template: {exc}")
return result
if not isinstance(payload, dict):
result["errors"].append("SARscape SBAS parameter template must be a JSON object.")
return result
raw_strategy = str(payload.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
execution_strategy = (
raw_strategy if raw_strategy in SUPPORTED_TEMPLATE_STRATEGIES else TEMPLATE_STRATEGY_NATIVE
)
native_workflow = payload.get("native_workflow") if isinstance(payload.get("native_workflow"), dict) else {}
native_workflow_task = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
native_workflow_parameters = native_workflow.get("parameters")
tasks = payload.get("tasks") if isinstance(payload.get("tasks"), list) else []
task_names = {
str(item.get("task_name") or "").strip()
for item in tasks
if isinstance(item, dict) and str(item.get("task_name") or "").strip()
}
missing_required = [name for name in REQUIRED_STACK_TASKS if name not in task_names]
tasks_without_parameters = [
str(item.get("task_name") or item.get("phase_id") or "<unnamed>")
for item in tasks
if isinstance(item, dict)
and bool(item.get("enabled", True))
and not isinstance(item.get("parameters"), dict)
]
result.update(
{
"readable": True,
"schema": payload.get("schema"),
"validated": bool(payload.get("validated")),
"execution_strategy": execution_strategy,
"native_workflow_task": native_workflow_task,
"task_count": len(tasks),
"missing_required_tasks": missing_required,
"tasks_without_parameters": tasks_without_parameters,
"template": payload,
}
)
if str(payload.get("schema") or "") != "insar.sarscape-sbas-template/v1":
result["errors"].append("Unsupported SARscape SBAS parameter template schema.")
if raw_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
result["errors"].append(
"Unsupported SARscape SBAS execution_strategy: " + (raw_strategy or "<empty>")
)
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
if not native_workflow_task:
result["errors"].append("Native SARscape workflow task name is empty.")
if not isinstance(native_workflow_parameters, dict):
result["errors"].append("Native SARscape workflow parameters must be a JSON object.")
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and missing_required:
result["errors"].append("Template is missing required tasks: " + ", ".join(missing_required))
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and tasks_without_parameters:
result["errors"].append("Template tasks without parameters object: " + ", ".join(tasks_without_parameters))
if not payload.get("validated"):
result["errors"].append("Template is not marked validated=true.")
return result
def summarize_network_edges(network_edges: List[Dict[str, Any]]) -> Dict[str, Any]:
enabled_edges = [item for item in network_edges if bool(item.get("enabled", True))]
temporal = _numeric_values(enabled_edges, "temporal_baseline_days")
spatial = _numeric_values(enabled_edges, "spatial_baseline_meters")
overlap = _numeric_values(enabled_edges, "pair_aoi_overlap_ratio")
return {
"edge_count": len(network_edges),
"enabled_edge_count": len(enabled_edges),
"temporal_baseline_days": {
"min": min(temporal) if temporal else None,
"max": max(temporal) if temporal else None,
},
"spatial_baseline_meters": {
"min": min(spatial) if spatial else None,
"max": max(spatial) if spatial else None,
},
"pair_aoi_overlap_ratio": {
"min": min(overlap) if overlap else None,
"max": max(overlap) if overlap else None,
},
}
def build_processor_manifest(
stack_manifest: Dict[str, Any],
*,
discovery_report: Optional[Dict[str, Any]] = None,
parameter_template_path: Optional[str] = None,
) -> Dict[str, Any]:
"""Build the SARscape SBAS processor contract without executing ENVI tasks."""
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
network_edges = (
stack_manifest.get("network_edges")
if isinstance(stack_manifest.get("network_edges"), list)
else []
)
template_status = load_parameter_template(parameter_template_path)
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
template_strategy = str(
template_status.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE
).strip()
native_workflow_task = str(
template_status.get("native_workflow_task") or NATIVE_WORKFLOW_TASK
).strip()
available_tasks = _available_task_names(discovery_report)
missing_stack_tasks = [name for name in REQUIRED_STACK_TASKS if name not in available_tasks]
missing_native_tasks = [native_workflow_task] if native_workflow_task not in available_tasks else []
missing_required_tasks = (
missing_native_tasks
if template_strategy == TEMPLATE_STRATEGY_NATIVE
else missing_stack_tasks
)
template_path = str(template_status.get("path") or "").strip()
template_exists = bool(template_status.get("exists"))
template_validated = bool(template_status.get("validated")) and not template_status.get("errors")
execution_enabled = bool(getattr(settings, "SARSCAPE_SBAS_ALLOW_EXECUTION", False))
native_workflow_available = not missing_native_tasks
explicit_stack_available = not missing_stack_tasks
blockers: List[str] = []
if len(scenes) < 3:
blockers.append("SARscape SBAS requires at least 3 stack scenes.")
if not network_edges:
blockers.append("No SBAS network_edges are present in the stack manifest.")
if missing_required_tasks:
blockers.append(
"Missing required SARscape SBAS tasks for "
f"{template_strategy}: " + ", ".join(missing_required_tasks)
)
if not template_exists:
blockers.append(
"SARscape SBAS parameter template is not configured. "
"Live task.parameters is intentionally not used because it can hang taskengine."
)
elif not template_validated:
blockers.extend(str(item) for item in (template_status.get("errors") or []))
if not execution_enabled:
blockers.append("SARSCAPE_SBAS_ALLOW_EXECUTION is false; SARscape SBAS production execution is disabled.")
parameter_template_state = (
"validated"
if template_validated
else ("configured_unvalidated" if template_exists else "required")
)
task_sequence = [
{
"phase_id": "native_wf_sbas",
"task_name": native_workflow_task,
"purpose": "Run SARscape's installed end-to-end SBAS metatask.",
"available": native_workflow_available,
"required": template_strategy == TEMPLATE_STRATEGY_NATIVE,
"parameter_template_status": parameter_template_state,
"has_template_parameters": isinstance(
(template.get("native_workflow") or {}).get("parameters")
if isinstance(template.get("native_workflow"), dict)
else None,
dict,
),
"supports_system_selected_edges": False,
"ready": (
native_workflow_available
and template_strategy == TEMPLATE_STRATEGY_NATIVE
and template_validated
and execution_enabled
),
}
]
template_tasks = {
str(item.get("task_name") or "").strip(): item
for item in (template.get("tasks") or [])
if isinstance(item, dict)
}
for phase in PIPELINE_PHASES:
task_name = str(phase["task_name"])
optional = bool(phase.get("optional", False))
template_task = template_tasks.get(task_name) or {}
has_template_parameters = isinstance(template_task.get("parameters"), dict)
task_sequence.append(
{
**phase,
"available": task_name in available_tasks,
"required": not optional,
"template_phase_id": template_task.get("phase_id"),
"parameter_template_status": parameter_template_state,
"has_template_parameters": has_template_parameters,
"ready": (
(task_name in available_tasks or optional)
and template_strategy == TEMPLATE_STRATEGY_EXPLICIT
and template_validated
and execution_enabled
),
}
)
return {
"schema": "insar.sarscape-sbas-processor/v1",
"created_at_utc": _utcnow_iso(),
"engine_code": ENGINE_CODE,
"processor_code": PROCESSOR_CODE,
"execution_enabled": execution_enabled,
"ready_for_pipeline_design": bool(discovery_report and discovery_report.get("ok")),
"ready_for_execution": not blockers,
"blockers": blockers,
"stack_manifest_checksum": _sha256_payload(stack_manifest),
"stack_manifest_summary": {
"schema": stack_manifest.get("schema"),
"prepared_stack_schema": stack_manifest.get("prepared_stack_schema"),
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
"manifest_role": stack_manifest.get("manifest_role"),
"batch_id": stack_manifest.get("batch_id"),
"plan_id": stack_manifest.get("plan_id"),
"plan_strategy": stack_manifest.get("plan_strategy"),
"reference_date": stack_manifest.get("reference_date"),
"scene_count": len(scenes),
"stack_key": stack_manifest.get("stack_key"),
"group_key": stack_manifest.get("group_key"),
},
"network_summary": summarize_network_edges(network_edges),
"execution_strategy": template_strategy,
"execution_strategies": {
TEMPLATE_STRATEGY_NATIVE: {
"preferred": template_strategy == TEMPLATE_STRATEGY_NATIVE,
"task_name": native_workflow_task,
"available": native_workflow_available,
"required_tasks": [native_workflow_task],
"missing_tasks": missing_native_tasks,
"supports_system_selected_edges": False,
"graph_policy": "SARscape wf_sbas builds the connection graph internally; system network_edges are retained for audit and comparison.",
},
TEMPLATE_STRATEGY_EXPLICIT: {
"preferred": template_strategy == TEMPLATE_STRATEGY_EXPLICIT,
"available": explicit_stack_available,
"required_tasks": list(REQUIRED_STACK_TASKS),
"missing_tasks": missing_stack_tasks,
"supports_system_selected_edges": "not_verified",
"graph_policy": "Explicit task chaining can expose the connection graph step, but direct injection of the system-selected edge list still needs SARscape parameter validation.",
},
},
"required_tasks": [native_workflow_task] if template_strategy == TEMPLATE_STRATEGY_NATIVE else list(REQUIRED_STACK_TASKS),
"required_stack_tasks": list(REQUIRED_STACK_TASKS),
"optional_tasks": list(OPTIONAL_TASKS),
"available_tasks": sorted(available_tasks),
"missing_required_tasks": missing_required_tasks,
"parameter_template": {
"path": template_path or None,
"exists": template_exists,
"readable": bool(template_status.get("readable")),
"validated": bool(template_status.get("validated")),
"execution_strategy": template_strategy,
"native_workflow_task": native_workflow_task,
"task_count": int(template_status.get("task_count") or 0),
"errors": template_status.get("errors") or [],
"source": "manual_sarscape_template",
},
"task_sequence": task_sequence,
"input_contract": {
"required_manifest_role_for_execution": "prepared_sbas_stack",
"prepared_stack_schema": PREPARED_STACK_SCHEMA,
"production_input_policy": "prepared_stack_manifest_only",
"scene_path_fields": ["folder_path", "tiff_path", "meta_path"],
"network_edge_source": "stack_manifest.network_edges",
"dem_source": "IDL_DINSAR_DEM_BASE_FILE",
"orbit_source": "ORBIT_POOL_ENVI",
},
"result_contract": {
"catalog_name": "psinsar",
"required_roles": list(REQUIRED_RESULT_ROLES),
"publish_manifest_schema": "psinsar.publish.v2",
},
"notes": [
"This manifest is a planning contract only; it does not execute SARscape tasks.",
"Execution must use checked-in SARscape parameter templates, not live task.parameters.",
"The native wf_sbas strategy is the preferred first integration path on this workstation.",
],
}
def build_preflight_report(
stack_manifest: Dict[str, Any],
*,
include_task_discovery: bool = True,
discovery_timeout_seconds: int = 120,
parameter_template_path: Optional[str] = None,
) -> Dict[str, Any]:
status = envi_service.get_status()
discovery_report: Optional[Dict[str, Any]] = None
if include_task_discovery:
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
timeout_seconds=discovery_timeout_seconds,
include_parameters=False,
)
processor_manifest = build_processor_manifest(
stack_manifest,
discovery_report=discovery_report,
parameter_template_path=parameter_template_path,
)
env_blockers: List[str] = []
if not status.get("idl_installed"):
env_blockers.append("IDL/ENVI executable is not installed or not configured.")
if not status.get("runner_ready"):
env_blockers.append("ENVI runner is not ready: " + str(status.get("runner_message") or "unknown"))
if not status.get("dem_exists"):
env_blockers.append("SARscape DEM base file is missing: " + str(status.get("dem_base_file") or ""))
if discovery_report is not None and not discovery_report.get("ok"):
env_blockers.append("SARscape SBAS task discovery failed: " + str(discovery_report.get("error") or "unknown"))
all_blockers = [*env_blockers, *(processor_manifest.get("blockers") or [])]
return {
"schema": "insar.sarscape-sbas-preflight/v1",
"created_at_utc": _utcnow_iso(),
"engine_code": ENGINE_CODE,
"processor_code": PROCESSOR_CODE,
"ready_for_pipeline_design": bool(
status.get("idl_installed")
and status.get("runner_ready")
and (discovery_report is None or discovery_report.get("ok"))
),
"ready_for_execution": not all_blockers,
"blockers": all_blockers,
"environment": status,
"task_discovery": discovery_report,
"processor_manifest": processor_manifest,
}
def _resolve_template_value(value: Any, context: Dict[str, Any]) -> Any:
if isinstance(value, str):
text = value.strip()
if text in context:
return context[text]
resolved = value
for key, replacement in context.items():
if key in resolved and isinstance(replacement, (str, int, float, bool)):
resolved = resolved.replace(key, str(replacement))
return resolved
if isinstance(value, list):
return [_resolve_template_value(item, context) for item in value]
if isinstance(value, dict):
return {
str(key): _resolve_template_value(item, context)
for key, item in value.items()
}
return value
def _scene_input_uris(scenes: List[Dict[str, Any]]) -> List[str]:
uris: List[str] = []
for item in scenes:
for key in ("meta_path", "tiff_path", "folder_path"):
text = str(item.get(key) or "").strip()
if text:
uris.append(text)
break
return uris
def execute_template_workflow(
stack_manifest: Dict[str, Any],
*,
work_root: str,
selected_manifest_path: str,
timeout_seconds: Optional[int] = None,
) -> Dict[str, Any]:
"""Execute a validated SARscape SBAS template.
This path is intentionally gated by SARSCAPE_SBAS_ALLOW_EXECUTION and
template validated=true. The default checked-in template is not executable.
"""
if stack_manifest.get("prepared_stack_schema") != PREPARED_STACK_SCHEMA:
raise ValueError(
f"SARscape SBAS execution requires a prepared stack manifest ({PREPARED_STACK_SCHEMA})."
)
if not str(stack_manifest.get("prepared_stack_id") or "").strip():
raise ValueError("SARscape SBAS execution requires prepared_stack_id.")
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
timeout_seconds=int(getattr(settings, "SARSCAPE_SBAS_DISCOVERY_TIMEOUT_SECONDS", 120) or 120),
include_parameters=False,
)
processor_manifest = build_processor_manifest(
stack_manifest,
discovery_report=discovery_report,
)
if not processor_manifest.get("ready_for_execution"):
blockers = "; ".join(str(item) for item in (processor_manifest.get("blockers") or []))
raise ValueError("SARscape SBAS execution is not ready: " + (blockers or "unknown blocker"))
template_status = load_parameter_template()
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
tasks = [item for item in (template.get("tasks") or []) if isinstance(item, dict)]
output_root = Path(work_root) / "sarscape_sbas"
output_root.mkdir(parents=True, exist_ok=True)
artifacts = stack_manifest.get("artifacts") if isinstance(stack_manifest.get("artifacts"), dict) else {}
prepared_edges_path = str(artifacts.get("selected_network_edges_path_windows") or "").strip()
if prepared_edges_path:
network_edges_path = Path(prepared_edges_path)
if not network_edges_path.is_file():
raise FileNotFoundError(f"Prepared selected_network_edges.json not found: {network_edges_path}")
else:
network_edges_path = output_root / "selected_network_edges.json"
network_edges_path.write_text(
json.dumps(stack_manifest.get("network_edges") or [], ensure_ascii=False, indent=2),
encoding="utf-8",
)
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
context: Dict[str, Any] = {
"${work_root}": str(work_root),
"${output_root}": str(output_root),
"${selected_stack_manifest}": str(selected_manifest_path),
"${selected_network_edges}": str(network_edges_path),
"${scene_meta_paths}": [
str(item.get("meta_path"))
for item in scenes
if str(item.get("meta_path") or "").strip()
],
"${scene_input_uris}": _scene_input_uris(scenes),
"${scene_folder_paths}": [
str(item.get("folder_path"))
for item in scenes
if str(item.get("folder_path") or "").strip()
],
"${selection_params}": stack_manifest.get("selection_params") or {},
"${dem_sarscapedata}": envi_service._build_sarscapedata(envi_service.DEM_BASE_FILE), # noqa: SLF001
}
executed: List[Dict[str, Any]] = []
previous_outputs: Dict[str, Any] = {}
execution_strategy = str(template.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
if execution_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
execution_strategy = TEMPLATE_STRATEGY_NATIVE
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
native_workflow = template.get("native_workflow") if isinstance(template.get("native_workflow"), dict) else {}
task_name = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
if not task_name:
raise ValueError("Native SARscape SBAS workflow task_name is empty.")
phase_id = str(native_workflow.get("phase_id") or "native_wf_sbas").strip()
phase_output_dir = output_root / phase_id
phase_output_dir.mkdir(parents=True, exist_ok=True)
phase_context = {
**context,
"${phase_id}": phase_id,
"${phase_output_dir}": str(phase_output_dir),
"${previous_outputs}": previous_outputs,
}
parameters = _resolve_template_value(native_workflow.get("parameters") or {}, phase_context)
result = envi_service.execute_envi_task(task_name, parameters)
previous_outputs[phase_id] = result
executed.append(
{
"phase_id": phase_id,
"task_name": task_name,
"output_dir": str(phase_output_dir),
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
}
)
tasks = []
for item in tasks:
if not bool(item.get("enabled", True)):
continue
task_name = str(item.get("task_name") or "").strip()
phase_id = str(item.get("phase_id") or task_name).strip()
if not task_name:
raise ValueError(f"SARscape template task is missing task_name: {phase_id}")
phase_output_dir = output_root / phase_id
phase_output_dir.mkdir(parents=True, exist_ok=True)
phase_context = {
**context,
"${phase_id}": phase_id,
"${phase_output_dir}": str(phase_output_dir),
"${previous_outputs}": previous_outputs,
}
parameters = _resolve_template_value(item.get("parameters") or {}, phase_context)
result = envi_service.execute_envi_task(task_name, parameters)
previous_outputs[phase_id] = result
executed.append(
{
"phase_id": phase_id,
"task_name": task_name,
"output_dir": str(phase_output_dir),
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
}
)
return {
"schema": "insar.sarscape-sbas-execution/v1",
"created_at_utc": _utcnow_iso(),
"processor_code": PROCESSOR_CODE,
"execution_strategy": execution_strategy,
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
"work_root": str(work_root),
"output_root": str(output_root),
"selected_network_edges_path": str(network_edges_path),
"task_count": len(executed),
"executed_tasks": executed,
"processor_manifest": processor_manifest,
}
def write_processor_manifest(path: str | Path, manifest: Dict[str, Any]) -> str:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
return str(target)
+204
View File
@@ -35,6 +35,7 @@ from ..models import (
RadarDataORM,
RadarPair,
ResultProductORM,
TimeseriesStackPlanEdgeORM,
TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM,
)
@@ -404,6 +405,138 @@ class SpatialService:
"stack_dates": stack_dates,
}
async def _build_timeseries_network_edges(
self,
db: AsyncSession,
scenes: List[RadarDataORM],
params: PsRequest,
*,
aoi_wkt: Optional[str],
selection_mode: Optional[str],
) -> Tuple[List[Dict[str, Any]], List[str]]:
scene_ids = [int(item.id) for item in scenes if item.id is not None]
if len(scene_ids) < 2:
return [], []
master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM)
stmt = (
select(PairingMetricCacheORM, master_alias, slave_alias)
.join(master_alias, master_alias.id == PairingMetricCacheORM.master_scene_ref_id)
.join(slave_alias, slave_alias.id == PairingMetricCacheORM.slave_scene_ref_id)
.where(
PairingMetricCacheORM.metric_version == pairing_state_service.metric_version,
PairingMetricCacheORM.status == "READY",
PairingMetricCacheORM.master_scene_ref_id.in_(scene_ids),
PairingMetricCacheORM.slave_scene_ref_id.in_(scene_ids),
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters,
PairingMetricCacheORM.scene_overlap_ratio >= params.network_overlap_threshold,
)
.order_by(
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
PairingMetricCacheORM.time_baseline_days.asc(),
PairingMetricCacheORM.spatial_baseline_meters.asc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
PairingMetricCacheORM.id.asc(),
)
)
result = await db.execute(stmt)
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
candidate_pool.append(
{
"metric_cache_ref_id": int(metric_row.id),
"pair_uid": metric_row.pair_uid,
"master_scene_uid": metric_row.master_scene_uid,
"slave_scene_uid": metric_row.slave_scene_uid,
"master": RadarData.model_validate(master_row),
"slave": RadarData.model_validate(slave_row),
"days": int(metric_row.time_baseline_days or 0),
"dist": float(metric_row.spatial_baseline_meters or 0.0),
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0.0),
}
)
warnings: List[str] = []
if not candidate_pool:
warnings.append(
"No pairing_metric_cache edges matched the time-series SBAS network thresholds."
)
return [], warnings
candidate_scene_ids = {
int(candidate[role].id)
for candidate in candidate_pool
for role in ("master", "slave")
if candidate.get(role) is not None
}
missing_scene_count = len(set(scene_ids) - candidate_scene_ids)
if missing_scene_count > 0:
warnings.append(
f"{missing_scene_count} selected scenes have no metric-cache edge under the current SBAS thresholds."
)
pairing_params = PairingRequest(
time_baseline_min=params.time_baseline_min,
time_baseline_max=params.time_baseline_max,
overlap_threshold=params.network_overlap_threshold,
spatial_baseline_max_meters=params.spatial_baseline_max_meters,
coverage_diversity_penalty=0.3,
require_same_imaging_mode=True,
require_same_polarization=True,
strategy="sbas",
num_connections=params.num_connections,
)
selected_candidates, strategy_warnings = self._apply_sbas_strategy(
candidate_pool,
pairing_params,
aoi_wkt=aoi_wkt,
)
warnings.extend(strategy_warnings)
edges: List[Dict[str, Any]] = []
for edge_rank, candidate in enumerate(self._sorted_candidates(selected_candidates), start=1):
master = candidate["master"]
slave = candidate["slave"]
edges.append(
{
"edge_rank": edge_rank,
"metric_cache_ref_id": candidate.get("metric_cache_ref_id"),
"master_scene_ref_id": int(master.id),
"slave_scene_ref_id": int(slave.id),
"master_imaging_date": master.imaging_date,
"slave_imaging_date": slave.imaging_date,
"temporal_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
"selection_reason": candidate.get("selection_reason"),
"selection_score": (
float(candidate["selection_score"])
if candidate.get("selection_score") is not None
else None
),
"selection_meta_json": {
"source": "pairing_metric_cache",
"selection_mode": selection_mode,
"pair_uid": candidate.get("pair_uid"),
"metric_version": pairing_state_service.metric_version,
"time_baseline_min": params.time_baseline_min,
"time_baseline_max": params.time_baseline_max,
"spatial_baseline_max_meters": params.spatial_baseline_max_meters,
"network_overlap_threshold": params.network_overlap_threshold,
"num_connections": params.num_connections,
},
"enabled": True,
}
)
if not edges:
warnings.append("SBAS strategy did not select any network edges for this stack.")
return edges, warnings
async def _persist_timeseries_stack_plan(
self,
db: AsyncSession,
@@ -416,6 +549,8 @@ class SpatialService:
coverage_consistency_ratio: Optional[float] = None,
threshold_satisfied: Optional[bool] = None,
selection_mode: Optional[str] = None,
network_edges: Optional[List[Dict[str, Any]]] = None,
network_warnings: Optional[List[str]] = None,
) -> Dict[str, Any]:
request_payload = params.model_dump(exclude_none=True)
aoi_hash = self._stable_sha1(aoi_wkt) if aoi_wkt else None
@@ -447,6 +582,9 @@ class SpatialService:
sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or ""))
scene_payloads: List[RadarData] = []
plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
safe_network_edges = list(network_edges or [])
safe_network_warnings = [str(item) for item in (network_warnings or []) if str(item).strip()]
for rank, item in enumerate(sorted_scenes, start=1):
plan_item = TimeseriesStackPlanItemORM(
plan_ref_id=plan.id,
@@ -469,6 +607,8 @@ class SpatialService:
"coverage_consistency_ratio": coverage_consistency_ratio,
"threshold_satisfied": threshold_satisfied,
"selection_mode": selection_mode,
"network_edge_count": len(safe_network_edges),
"network_warnings": safe_network_warnings,
"orbit_direction": item.orbit_direction,
"satellite_family": self._normalize_timeseries_satellite_family(item),
"bbox": [item.min_lon, item.min_lat, item.max_lon, item.max_lat],
@@ -477,6 +617,8 @@ class SpatialService:
)
db.add(plan_item)
await db.flush()
if item.id is not None:
plan_item_by_scene_id[int(item.id)] = plan_item
scene_payloads.append(
RadarData.model_validate(item).model_copy(
update={
@@ -490,14 +632,61 @@ class SpatialService:
"stack_coverage_consistency_ratio": coverage_consistency_ratio,
"stack_threshold_satisfied": threshold_satisfied,
"stack_selection_mode": selection_mode,
"stack_network_edge_count": len(safe_network_edges),
"stack_network_warnings": safe_network_warnings,
}
)
)
for edge_payload in safe_network_edges:
master_scene_id = edge_payload.get("master_scene_ref_id")
slave_scene_id = edge_payload.get("slave_scene_ref_id")
master_plan_item = (
plan_item_by_scene_id.get(int(master_scene_id))
if master_scene_id is not None
else None
)
slave_plan_item = (
plan_item_by_scene_id.get(int(slave_scene_id))
if slave_scene_id is not None
else None
)
edge = TimeseriesStackPlanEdgeORM(
plan_ref_id=plan.id,
master_plan_item_ref_id=(
int(master_plan_item.id)
if master_plan_item is not None and master_plan_item.id is not None
else None
),
slave_plan_item_ref_id=(
int(slave_plan_item.id)
if slave_plan_item is not None and slave_plan_item.id is not None
else None
),
metric_cache_ref_id=edge_payload.get("metric_cache_ref_id"),
master_scene_ref_id=master_scene_id,
slave_scene_ref_id=slave_scene_id,
edge_rank=int(edge_payload.get("edge_rank") or 0),
master_imaging_date=edge_payload.get("master_imaging_date"),
slave_imaging_date=edge_payload.get("slave_imaging_date"),
temporal_baseline_days=edge_payload.get("temporal_baseline_days"),
spatial_baseline_meters=edge_payload.get("spatial_baseline_meters"),
perpendicular_baseline_meters=edge_payload.get("perpendicular_baseline_meters"),
scene_overlap_ratio=edge_payload.get("scene_overlap_ratio"),
pair_aoi_overlap_ratio=edge_payload.get("pair_aoi_overlap_ratio"),
selection_reason=edge_payload.get("selection_reason"),
selection_score=edge_payload.get("selection_score"),
selection_meta_json=edge_payload.get("selection_meta_json"),
enabled=bool(edge_payload.get("enabled", True)),
)
db.add(edge)
return {
"plan_id": plan.plan_id,
"group_key": identity.get("group_key"),
"stack_key": identity.get("stack_key"),
"edge_count": len(safe_network_edges),
"network_warnings": safe_network_warnings,
"scenes": scene_payloads,
}
@@ -1286,6 +1475,19 @@ class SpatialService:
if len(final_stack) >= 3:
final_stack.sort(key=lambda x: str(x.imaging_date or ""))
direction = group_key[0]
network_edges, network_warnings = await self._build_timeseries_network_edges(
db,
final_stack,
params,
aoi_wkt=aoi_wkt,
selection_mode=selection_mode,
)
logger.info(
"timeseries stack planning: group=%s network_edges=%s network_warnings=%s",
self._format_timeseries_group_label(group_key),
len(network_edges),
len(network_warnings),
)
persisted_plan = await self._persist_timeseries_stack_plan(
db,
direction=direction,
@@ -1296,6 +1498,8 @@ class SpatialService:
coverage_consistency_ratio=consistency_ratio,
threshold_satisfied=threshold_satisfied,
selection_mode=selection_mode,
network_edges=network_edges,
network_warnings=network_warnings,
)
result_key = persisted_plan.get("group_key") or self._format_timeseries_group_label(group_key)
if result_key in final_results:
File diff suppressed because it is too large Load Diff
+1
View File
@@ -173,6 +173,7 @@ class WorkflowService:
step.outputs = outputs
await self._advance_ready_steps(run_id, db)
await db.flush()
await self.enqueue_ready_steps(run_id, db=db)
if gen_db: