Add SARscape SBAS prepared stack workflow
This commit is contained in:
@@ -25,9 +25,12 @@ def _parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workflow",
|
||||
required=True,
|
||||
required=False,
|
||||
choices=["import", "dinsar", "dinsar_custom"],
|
||||
)
|
||||
parser.add_argument("--inspect-sarscape-sbas", action="store_true")
|
||||
parser.add_argument("--include-parameters", action="store_true")
|
||||
parser.add_argument("--task-name", action="append", default=[])
|
||||
parser.add_argument("--root-dir", required=False)
|
||||
parser.add_argument("--task-dir", required=False)
|
||||
parser.add_argument("--output-dir", required=False)
|
||||
@@ -44,7 +47,22 @@ def main() -> int:
|
||||
ensure_project_env_loaded()
|
||||
args = _parse_args()
|
||||
try:
|
||||
from .envi_service import run_single_task_workflow, run_workflow
|
||||
from .envi_service import (
|
||||
inspect_sarscape_sbas_tasks,
|
||||
run_single_task_workflow,
|
||||
run_workflow,
|
||||
)
|
||||
|
||||
if args.inspect_sarscape_sbas:
|
||||
record = inspect_sarscape_sbas_tasks(
|
||||
args.task_name or None,
|
||||
include_parameters=bool(args.include_parameters),
|
||||
)
|
||||
print(json.dumps(record, ensure_ascii=False))
|
||||
return 0 if record.get("ok") else 2
|
||||
|
||||
if not args.workflow:
|
||||
raise ValueError("--workflow is required unless --inspect-sarscape-sbas is used.")
|
||||
|
||||
if args.task_dir:
|
||||
if not args.output_dir:
|
||||
|
||||
@@ -13,6 +13,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
import defusedxml.ElementTree as ET
|
||||
from contextlib import contextmanager
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
@@ -67,13 +68,43 @@ def get_envi_runner_python() -> str:
|
||||
return os.path.normpath(sys.executable)
|
||||
|
||||
|
||||
def get_envi_taskengine_cwd() -> str:
|
||||
"""Dedicated cwd for envipyengine/taskengine temp files.
|
||||
|
||||
SARscape can create zero-byte env_*.xyz and IDL*.tmp files in the current
|
||||
working directory. Keep those files under runtime instead of the repo root.
|
||||
"""
|
||||
base_dir = _to_local_path(
|
||||
getattr(settings, "IDL_WORKER_RUNTIME_DIR", "")
|
||||
or os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
|
||||
)
|
||||
cwd = os.path.join(base_dir, "envi_cwd")
|
||||
os.makedirs(cwd, exist_ok=True)
|
||||
return os.path.normpath(os.path.abspath(cwd))
|
||||
|
||||
|
||||
def get_envi_custom_code_dir() -> str:
|
||||
envi_root = _envi_install_root()
|
||||
if not envi_root:
|
||||
return ""
|
||||
candidates = [
|
||||
os.path.join(envi_root, "user_custom_code"),
|
||||
os.path.join(envi_root, "custom_code"),
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isdir(path):
|
||||
return os.path.normpath(os.path.abspath(path))
|
||||
return ""
|
||||
|
||||
|
||||
def get_envi_runner_cwd() -> str:
|
||||
return os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
|
||||
return get_envi_taskengine_cwd()
|
||||
|
||||
|
||||
def get_envi_runner_env() -> Dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
project_root = get_envi_runner_cwd()
|
||||
project_root = os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
|
||||
taskengine_cwd = get_envi_taskengine_cwd()
|
||||
existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()]
|
||||
ordered = [project_root, *existing]
|
||||
deduped: List[str] = []
|
||||
@@ -88,9 +119,36 @@ def get_envi_runner_env() -> Dict[str, str]:
|
||||
seen.add(key)
|
||||
deduped.append(str(raw_path))
|
||||
env["PYTHONPATH"] = os.pathsep.join(deduped)
|
||||
env["TEMP"] = taskengine_cwd
|
||||
env["TMP"] = taskengine_cwd
|
||||
env["IDL_TMPDIR"] = taskengine_cwd
|
||||
custom_code_dir = get_envi_custom_code_dir()
|
||||
if custom_code_dir:
|
||||
env["ENVI_CUSTOM_CODE"] = custom_code_dir
|
||||
return env
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _envi_taskengine_runtime_context():
|
||||
"""Run in-process ENVI calls from the dedicated runtime cwd."""
|
||||
target_cwd = get_envi_taskengine_cwd()
|
||||
old_cwd = os.getcwd()
|
||||
old_env = {name: os.environ.get(name) for name in ("TEMP", "TMP", "IDL_TMPDIR")}
|
||||
os.environ["TEMP"] = target_cwd
|
||||
os.environ["TMP"] = target_cwd
|
||||
os.environ["IDL_TMPDIR"] = target_cwd
|
||||
try:
|
||||
os.chdir(target_cwd)
|
||||
yield target_cwd
|
||||
finally:
|
||||
os.chdir(old_cwd)
|
||||
for name, value in old_env.items():
|
||||
if value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
|
||||
|
||||
def build_envi_runner_command(*args: Any) -> List[str]:
|
||||
command = [
|
||||
get_envi_runner_python(),
|
||||
@@ -101,6 +159,79 @@ def build_envi_runner_command(*args: Any) -> List[str]:
|
||||
return command
|
||||
|
||||
|
||||
def _list_taskengine_pids() -> set[int]:
|
||||
"""Return taskengine.exe PIDs on Windows without importing optional deps."""
|
||||
if os.name != "nt":
|
||||
return set()
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
"Get-Process taskengine -ErrorAction SilentlyContinue | ForEach-Object { $_.Id }",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
pids: set[int] = set()
|
||||
for line in str(completed.stdout or "").splitlines():
|
||||
raw = line.strip()
|
||||
if raw.isdigit():
|
||||
pids.add(int(raw))
|
||||
return pids
|
||||
|
||||
|
||||
def _stop_taskengine_pids(pids: set[int]) -> List[int]:
|
||||
"""Stop specific taskengine.exe PIDs; avoids killing pre-existing sessions."""
|
||||
stopped: List[int] = []
|
||||
if os.name != "nt":
|
||||
return stopped
|
||||
for pid in sorted(pids):
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"powershell.exe",
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
f"Stop-Process -Id {int(pid)} -Force -ErrorAction SilentlyContinue",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
stopped.append(int(pid))
|
||||
except Exception:
|
||||
continue
|
||||
return stopped
|
||||
|
||||
|
||||
def _cleanup_new_taskengine_processes(existing_pids: set[int]) -> Dict[str, Any]:
|
||||
"""Best-effort cleanup for taskengine.exe children spawned by a timed-out runner."""
|
||||
existing = set(existing_pids or set())
|
||||
first_targets = _list_taskengine_pids() - existing
|
||||
stopped = _stop_taskengine_pids(first_targets)
|
||||
|
||||
# taskengine can take a moment to detach from the runner process. Re-check once.
|
||||
time.sleep(1)
|
||||
second_targets = _list_taskengine_pids() - existing
|
||||
stopped.extend(pid for pid in _stop_taskengine_pids(second_targets) if pid not in stopped)
|
||||
|
||||
time.sleep(1)
|
||||
remaining = sorted(_list_taskengine_pids() - existing)
|
||||
return {
|
||||
"taskengine_cleanup_attempted": True,
|
||||
"taskengine_stopped_pids": sorted(set(stopped)),
|
||||
"taskengine_remaining_new_pids": remaining,
|
||||
}
|
||||
|
||||
|
||||
def probe_envi_runner() -> Dict[str, Any]:
|
||||
python_path = get_envi_runner_python()
|
||||
project_root = get_envi_runner_cwd()
|
||||
@@ -238,6 +369,39 @@ import threading
|
||||
_ENVI_GLOBAL_LOCK = threading.Lock()
|
||||
|
||||
|
||||
SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES = [
|
||||
"wf_sbas",
|
||||
"wf_esbas",
|
||||
]
|
||||
|
||||
SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES = [
|
||||
"SARscape_setting_output_folders",
|
||||
"SARsLoadPreferences",
|
||||
"SARsImportSarSelector",
|
||||
"SARscapeSuggestLooks",
|
||||
"SARscapeEnviuriToShape",
|
||||
]
|
||||
|
||||
SARSCAPE_SBAS_STACK_TASK_CANDIDATES = [
|
||||
"SARsInSARStackSBASGenerateConnectionGraph",
|
||||
"SARsInSARStackSBASInterferogramGeneration",
|
||||
"SARsInSARStackSBASInversionStep1",
|
||||
"SARsInSARStackSBASInversionStep2",
|
||||
"SARsInSARStackSBASGeocode",
|
||||
"SARsInSARStackSBASVariogram",
|
||||
"SARsInSARStackESBASInterferogramGeneration",
|
||||
"SARsInSARStackESBASInversion",
|
||||
"SARsInSARStackESBASGeocode",
|
||||
"SARsInSARConnectionGraphESBAS",
|
||||
]
|
||||
|
||||
SARSCAPE_SBAS_TASK_CANDIDATES = [
|
||||
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
|
||||
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
|
||||
*SARSCAPE_SBAS_STACK_TASK_CANDIDATES,
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Progress file for subprocess ↔ job handler communication
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -417,26 +581,32 @@ def execute_envi_task(task_name: str, parameters: Dict[str, Any]) -> Dict[str, A
|
||||
) from exc
|
||||
|
||||
with _ENVI_GLOBAL_LOCK:
|
||||
engine = Engine("ENVI")
|
||||
task = engine.task(task_name)
|
||||
with _envi_taskengine_runtime_context():
|
||||
engine = Engine("ENVI")
|
||||
task = engine.task(task_name)
|
||||
existing_taskengine_pids = _list_taskengine_pids()
|
||||
|
||||
# Run with timeout to handle envipyengine hangs
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(task.execute, parameters)
|
||||
try:
|
||||
result = future.result(timeout=_ENVI_TASK_TIMEOUT)
|
||||
except FuturesTimeoutError:
|
||||
# Run with timeout to handle envipyengine hangs
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(task.execute, parameters)
|
||||
try:
|
||||
import subprocess as _sp
|
||||
_sp.run(["taskkill", "/F", "/IM", "taskengine.exe"],
|
||||
capture_output=True, timeout=10)
|
||||
print("[WARN] execute_envi_task: killed taskengine after timeout")
|
||||
except Exception as _exc:
|
||||
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}")
|
||||
raise RuntimeError(
|
||||
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s "
|
||||
f"(envipyengine hung). Output files may still exist."
|
||||
)
|
||||
result = future.result(timeout=_ENVI_TASK_TIMEOUT)
|
||||
except FuturesTimeoutError:
|
||||
try:
|
||||
cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
|
||||
stopped = cleanup.get("taskengine_stopped_pids") or []
|
||||
remaining = cleanup.get("taskengine_remaining_new_pids") or []
|
||||
print(
|
||||
"[WARN] execute_envi_task: task timed out; "
|
||||
f"stopped_new_taskengine_pids={stopped}; "
|
||||
f"remaining_new_taskengine_pids={remaining}"
|
||||
)
|
||||
except Exception as _exc:
|
||||
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}")
|
||||
raise RuntimeError(
|
||||
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s "
|
||||
f"(envipyengine hung). Output files may still exist."
|
||||
)
|
||||
|
||||
# taskengine returns {"outputParameters": {...}, ...}
|
||||
return result.get("outputParameters", result)
|
||||
@@ -453,6 +623,374 @@ def _unwrap_sarscapedata(value: Any) -> Any:
|
||||
return value
|
||||
|
||||
|
||||
def _configured_sarscape_sbas_task_candidates() -> List[str]:
|
||||
configured = str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
|
||||
if not configured:
|
||||
return list(SARSCAPE_SBAS_TASK_CANDIDATES)
|
||||
names: List[str] = []
|
||||
for raw in configured.replace(";", ",").split(","):
|
||||
name = raw.strip()
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
return names or list(SARSCAPE_SBAS_TASK_CANDIDATES)
|
||||
|
||||
|
||||
def _envi_install_root() -> str:
|
||||
executable = _to_local_path(IDL_EXECUTABLE)
|
||||
if not executable:
|
||||
return ""
|
||||
return os.path.abspath(os.path.join(os.path.dirname(executable), "..", "..", ".."))
|
||||
|
||||
|
||||
def _static_envi_task_template_path(task_name: str) -> str:
|
||||
name = str(task_name or "").strip()
|
||||
if not name:
|
||||
return ""
|
||||
envi_root = _envi_install_root()
|
||||
if not envi_root:
|
||||
return ""
|
||||
candidates = [
|
||||
os.path.join(envi_root, "user_custom_code", f"{name}.task"),
|
||||
os.path.join(envi_root, "resource", "templates", "tasks", "SARscape", f"{name}.task"),
|
||||
os.path.join(envi_root, "resource", "templates", "tasks", f"{name}.task"),
|
||||
]
|
||||
for path in candidates:
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
return ""
|
||||
|
||||
|
||||
def _json_safe_parameter(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {str(key): _json_safe_parameter(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_safe_parameter(item) for item in value]
|
||||
if value is None or isinstance(value, (str, int, float, bool)):
|
||||
return value
|
||||
return str(value)
|
||||
|
||||
|
||||
def _summarize_task_parameters(raw_parameters: Any) -> Dict[str, Any]:
|
||||
safe_parameters = _json_safe_parameter(raw_parameters)
|
||||
input_names: List[str] = []
|
||||
output_names: List[str] = []
|
||||
required_input_names: List[str] = []
|
||||
|
||||
if isinstance(safe_parameters, dict):
|
||||
iterable = safe_parameters.values()
|
||||
elif isinstance(safe_parameters, list):
|
||||
iterable = safe_parameters
|
||||
else:
|
||||
iterable = []
|
||||
|
||||
for item in iterable:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = str(item.get("name") or item.get("NAME") or "").strip()
|
||||
direction = str(item.get("direction") or item.get("DIRECTION") or "").strip().lower()
|
||||
required = bool(item.get("required") or item.get("REQUIRED"))
|
||||
if not name:
|
||||
continue
|
||||
if direction == "input":
|
||||
input_names.append(name)
|
||||
if required:
|
||||
required_input_names.append(name)
|
||||
elif direction == "output":
|
||||
output_names.append(name)
|
||||
|
||||
return {
|
||||
"parameter_count": (
|
||||
len(safe_parameters)
|
||||
if isinstance(safe_parameters, (dict, list))
|
||||
else 0
|
||||
),
|
||||
"input_names": input_names,
|
||||
"required_input_names": required_input_names,
|
||||
"output_names": output_names,
|
||||
"parameters": safe_parameters,
|
||||
}
|
||||
|
||||
|
||||
def list_envi_tasks() -> Dict[str, Any]:
|
||||
"""List ENVI task names without instantiating individual task parameters."""
|
||||
result: Dict[str, Any] = {
|
||||
"ok": False,
|
||||
"engine": "envipyengine",
|
||||
"task_count": 0,
|
||||
"tasks": [],
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
_ensure_envipyengine_config()
|
||||
from envipyengine import Engine
|
||||
except ImportError:
|
||||
result["error"] = (
|
||||
"envipyengine is not installed. Install it with: pip install envipyengine"
|
||||
)
|
||||
return result
|
||||
|
||||
with _ENVI_GLOBAL_LOCK:
|
||||
with _envi_taskengine_runtime_context():
|
||||
try:
|
||||
names = Engine("ENVI").tasks()
|
||||
except Exception as exc:
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
|
||||
result["tasks"] = [str(name) for name in names]
|
||||
result["task_count"] = len(result["tasks"])
|
||||
result["ok"] = True
|
||||
return result
|
||||
|
||||
|
||||
def discover_sarscape_sbas_tasks() -> Dict[str, Any]:
|
||||
"""Discover installed SARscape SBAS/E-SBAS task names by filtering Engine.tasks()."""
|
||||
report = list_envi_tasks()
|
||||
task_names = list(report.get("tasks") or [])
|
||||
keywords = (
|
||||
"StackSBAS",
|
||||
"StackESBAS",
|
||||
"ConnectionGraphESBAS",
|
||||
)
|
||||
explicit_names = set(SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES) | set(SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES)
|
||||
matches = [
|
||||
name
|
||||
for name in task_names
|
||||
if (
|
||||
str(name) in explicit_names
|
||||
or (
|
||||
str(name).startswith("SARsInSAR")
|
||||
and any(keyword.lower() in str(name).lower() for keyword in keywords)
|
||||
)
|
||||
)
|
||||
]
|
||||
static_task_files: Dict[str, str] = {}
|
||||
for name in SARSCAPE_SBAS_TASK_CANDIDATES:
|
||||
path = _static_envi_task_template_path(name)
|
||||
if path:
|
||||
static_task_files[name] = path
|
||||
if name not in matches:
|
||||
matches.append(name)
|
||||
preferred_order = [
|
||||
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
|
||||
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
|
||||
"SARsInSARStackSBASGenerateConnectionGraph",
|
||||
"SARsInSARStackSBASInterferogramGeneration",
|
||||
"SARsInSARStackSBASInversionStep1",
|
||||
"SARsInSARStackSBASInversionStep2",
|
||||
"SARsInSARStackSBASGeocode",
|
||||
"SARsInSARStackSBASVariogram",
|
||||
"SARsInSARStackESBASInterferogramGeneration",
|
||||
"SARsInSARStackESBASInversion",
|
||||
"SARsInSARStackESBASGeocode",
|
||||
"SARsInSARConnectionGraphESBAS",
|
||||
]
|
||||
ordered: List[str] = []
|
||||
for name in preferred_order:
|
||||
if name in matches and name not in ordered:
|
||||
ordered.append(name)
|
||||
for name in sorted(matches):
|
||||
if name not in ordered:
|
||||
ordered.append(name)
|
||||
|
||||
return {
|
||||
"ok": bool(report.get("ok")) and bool(ordered),
|
||||
"engine": report.get("engine"),
|
||||
"task_count": int(report.get("task_count") or 0),
|
||||
"sarscape_sbas_task_count": len(ordered),
|
||||
"sarscape_sbas_tasks": ordered,
|
||||
"static_task_files": {
|
||||
name: static_task_files[name]
|
||||
for name in ordered
|
||||
if name in static_task_files
|
||||
},
|
||||
"error": report.get("error"),
|
||||
}
|
||||
|
||||
|
||||
def inspect_envi_tasks(task_names: List[str]) -> Dict[str, Any]:
|
||||
"""Inspect ENVI/SARscape tasks without executing them."""
|
||||
started_at = _utc_now_text()
|
||||
deduped_names: List[str] = []
|
||||
for raw_name in task_names:
|
||||
name = str(raw_name or "").strip()
|
||||
if name and name not in deduped_names:
|
||||
deduped_names.append(name)
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"ok": False,
|
||||
"engine": "envipyengine",
|
||||
"started_at": started_at,
|
||||
"finished_at": None,
|
||||
"task_count": len(deduped_names),
|
||||
"available_count": 0,
|
||||
"missing_count": 0,
|
||||
"tasks": [],
|
||||
"error": None,
|
||||
}
|
||||
if not deduped_names:
|
||||
result["error"] = "No task names provided."
|
||||
result["finished_at"] = _utc_now_text()
|
||||
return result
|
||||
|
||||
try:
|
||||
_ensure_envipyengine_config()
|
||||
from envipyengine import Engine
|
||||
except ImportError as exc:
|
||||
result["error"] = (
|
||||
"envipyengine is not installed. Install it with: pip install envipyengine"
|
||||
)
|
||||
result["finished_at"] = _utc_now_text()
|
||||
return result
|
||||
|
||||
with _ENVI_GLOBAL_LOCK:
|
||||
with _envi_taskengine_runtime_context():
|
||||
try:
|
||||
engine = Engine("ENVI")
|
||||
except Exception as exc:
|
||||
result["error"] = f"Failed to initialize ENVI engine: {exc}"
|
||||
result["finished_at"] = _utc_now_text()
|
||||
return result
|
||||
|
||||
for task_name in deduped_names:
|
||||
item: Dict[str, Any] = {
|
||||
"name": task_name,
|
||||
"available": False,
|
||||
"error": None,
|
||||
"parameter_count": 0,
|
||||
"input_names": [],
|
||||
"required_input_names": [],
|
||||
"output_names": [],
|
||||
"parameters": [],
|
||||
}
|
||||
try:
|
||||
task = engine.task(task_name)
|
||||
summary = _summarize_task_parameters(getattr(task, "parameters", []))
|
||||
item.update(summary)
|
||||
item["available"] = True
|
||||
except Exception as exc:
|
||||
item["error"] = str(exc)
|
||||
result["tasks"].append(item)
|
||||
|
||||
result["available_count"] = sum(1 for item in result["tasks"] if item.get("available"))
|
||||
result["missing_count"] = sum(1 for item in result["tasks"] if not item.get("available"))
|
||||
result["ok"] = result["available_count"] > 0
|
||||
result["finished_at"] = _utc_now_text()
|
||||
return result
|
||||
|
||||
|
||||
def inspect_sarscape_sbas_tasks(
|
||||
task_names: Optional[List[str]] = None,
|
||||
*,
|
||||
include_parameters: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Inspect likely SARscape SBAS/E-SBAS task names for the installed version."""
|
||||
status = get_status()
|
||||
discovery = discover_sarscape_sbas_tasks()
|
||||
names = task_names or list(discovery.get("sarscape_sbas_tasks") or _configured_sarscape_sbas_task_candidates())
|
||||
if include_parameters:
|
||||
task_report = inspect_envi_tasks(names)
|
||||
else:
|
||||
discovered_set = set(discovery.get("sarscape_sbas_tasks") or [])
|
||||
task_report = {
|
||||
"ok": bool(discovery.get("ok")),
|
||||
"engine": "envipyengine",
|
||||
"task_count": len(names),
|
||||
"available_count": sum(1 for name in names if name in discovered_set),
|
||||
"missing_count": sum(1 for name in names if name not in discovered_set),
|
||||
"tasks": [
|
||||
{
|
||||
"name": name,
|
||||
"available": name in discovered_set,
|
||||
"error": None if name in discovered_set else "Task name not listed by Engine.tasks().",
|
||||
"parameter_count": None,
|
||||
"input_names": [],
|
||||
"required_input_names": [],
|
||||
"output_names": [],
|
||||
"parameters": [],
|
||||
}
|
||||
for name in names
|
||||
],
|
||||
"error": discovery.get("error"),
|
||||
}
|
||||
task_report["status"] = {
|
||||
"idl_installed": status.get("idl_installed"),
|
||||
"idl_executable": status.get("idl_executable"),
|
||||
"runner_ready": status.get("runner_ready"),
|
||||
"runner_python": status.get("runner_python"),
|
||||
"runner_message": status.get("runner_message"),
|
||||
"dem_base_file": status.get("dem_base_file"),
|
||||
"dem_exists": status.get("dem_exists"),
|
||||
}
|
||||
task_report["candidate_source"] = (
|
||||
"SARSCAPE_SBAS_TASK_NAMES"
|
||||
if str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
|
||||
else "engine_task_list"
|
||||
)
|
||||
task_report["include_parameters"] = bool(include_parameters)
|
||||
task_report["discovery"] = discovery
|
||||
task_report["ready_for_pipeline_design"] = bool(task_report.get("ok"))
|
||||
return task_report
|
||||
|
||||
|
||||
def inspect_sarscape_sbas_tasks_subprocess(
|
||||
task_names: Optional[List[str]] = None,
|
||||
*,
|
||||
timeout_seconds: int = 120,
|
||||
include_parameters: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run SARscape SBAS task inspection through the isolated ENVI runner."""
|
||||
command = build_envi_runner_command("--inspect-sarscape-sbas")
|
||||
if include_parameters:
|
||||
command.append("--include-parameters")
|
||||
for name in task_names or []:
|
||||
if str(name or "").strip():
|
||||
command.extend(["--task-name", str(name).strip()])
|
||||
existing_taskengine_pids = _list_taskengine_pids()
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=get_envi_runner_cwd(),
|
||||
env=get_envi_runner_env(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=max(10, int(timeout_seconds or 120)),
|
||||
check=False,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
|
||||
stdout_text = str(exc.stdout or "").strip()
|
||||
stderr_text = str(exc.stderr or "").strip()
|
||||
return {
|
||||
"ok": False,
|
||||
"returncode": None,
|
||||
"timeout": True,
|
||||
"timeout_seconds": max(10, int(timeout_seconds or 120)),
|
||||
"stdout": stdout_text[:2000],
|
||||
"stderr": stderr_text[:2000],
|
||||
"error": (
|
||||
"SARscape SBAS task inspection timed out. "
|
||||
"Use lightweight discovery without include_parameters, or provide a manually verified task template."
|
||||
),
|
||||
"runner_command": command,
|
||||
**cleanup,
|
||||
}
|
||||
stdout_text = str(completed.stdout or "").strip()
|
||||
stderr_text = str(completed.stderr or "").strip()
|
||||
try:
|
||||
payload = json.loads(stdout_text) if stdout_text else {}
|
||||
except Exception:
|
||||
payload = {}
|
||||
payload.setdefault("returncode", int(completed.returncode))
|
||||
payload.setdefault("stdout", stdout_text[:2000])
|
||||
payload.setdefault("stderr", stderr_text[:2000])
|
||||
payload["runner_command"] = command
|
||||
if completed.returncode != 0:
|
||||
payload["ok"] = False
|
||||
payload.setdefault("error", stderr_text or stdout_text or f"returncode={completed.returncode}")
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import workflow
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -43,6 +43,8 @@ from .timeseries_service import (
|
||||
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT,
|
||||
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK,
|
||||
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS,
|
||||
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS,
|
||||
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT,
|
||||
JOB_TYPE_TIMESERIES_STACK_PREP,
|
||||
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH,
|
||||
timeseries_service,
|
||||
@@ -3550,6 +3552,73 @@ async def _handle_timeseries_run_mintpy_sbas(job: SystemJobORM) -> None:
|
||||
raise
|
||||
|
||||
|
||||
async def _handle_timeseries_sarscape_preflight(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
run_id = str(payload.get("run_id") or "").strip()
|
||||
if not run_id:
|
||||
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires run_id payload.")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=45,
|
||||
message="Building SARscape SBAS processor manifest...",
|
||||
db=db,
|
||||
)
|
||||
result = await timeseries_service.build_sarscape_processor_preflight(run_id, db=db)
|
||||
ready_text = "ready" if result.get("ready_for_execution") else "planning_only"
|
||||
is_preflight_only = str(result.get("execution_mode") or "").strip() == "preflight_only"
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED" if is_preflight_only else None,
|
||||
progress=100 if is_preflight_only else 55,
|
||||
message=(
|
||||
f"SARscape SBAS preflight complete: state={ready_text} "
|
||||
f"manifest={result.get('processor_manifest_path')}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
except Exception as exc:
|
||||
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
|
||||
raise
|
||||
|
||||
|
||||
async def _handle_timeseries_run_sarscape_sbas(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
run_id = str(payload.get("run_id") or "").strip()
|
||||
if not run_id:
|
||||
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires run_id payload.")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=90,
|
||||
message="Running SARscape SBAS pipeline...",
|
||||
db=db,
|
||||
)
|
||||
async with engine_lock_service.acquire("sarscape_sbas_timeseries"):
|
||||
result = await timeseries_service.run_sarscape_sbas(run_id, db=db)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
f"SARscape SBAS complete: tasks={result.get('task_count', 0)} "
|
||||
f"report={result.get('report_path')}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
except Exception as exc:
|
||||
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
|
||||
raise
|
||||
|
||||
|
||||
async def _handle_timeseries_export_publish(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("TIMESERIES_EXPORT_PUBLISH requires task_id for progress tracking.")
|
||||
@@ -3648,6 +3717,8 @@ _HANDLERS = {
|
||||
JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize,
|
||||
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK: _handle_timeseries_run_isce2_stack,
|
||||
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS: _handle_timeseries_run_mintpy_sbas,
|
||||
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT: _handle_timeseries_sarscape_preflight,
|
||||
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS: _handle_timeseries_run_sarscape_sbas,
|
||||
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH: _handle_timeseries_export_publish,
|
||||
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT: _handle_timeseries_register_product,
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
|
||||
|
||||
@@ -17,7 +17,13 @@ from .. import database
|
||||
from ..config import settings
|
||||
from ..models import SystemWorkerHeartbeatORM
|
||||
|
||||
IDL_JOB_TYPES = {"IDL_RUN_IMPORT", "IDL_RUN_DINSAR", "WATER_GEOCODE", "WATER_FLOOD"}
|
||||
IDL_JOB_TYPES = {
|
||||
"IDL_RUN_IMPORT",
|
||||
"IDL_RUN_DINSAR",
|
||||
"WATER_GEOCODE",
|
||||
"WATER_FLOOD",
|
||||
"TIMESERIES_RUN_SARSCAPE_SBAS",
|
||||
}
|
||||
|
||||
|
||||
def _default_worker_id() -> str:
|
||||
|
||||
@@ -0,0 +1,681 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..config import settings
|
||||
from . import envi_service
|
||||
|
||||
|
||||
PROCESSOR_CODE = "sarscape_sbas"
|
||||
ENGINE_CODE = "sarscape"
|
||||
PREPARED_STACK_SCHEMA = "insar.prepared-sbas-stack/v1"
|
||||
|
||||
NATIVE_WORKFLOW_TASK = "wf_sbas"
|
||||
NATIVE_ESBAS_WORKFLOW_TASK = "wf_esbas"
|
||||
TEMPLATE_STRATEGY_NATIVE = "native_workflow_metatask"
|
||||
TEMPLATE_STRATEGY_EXPLICIT = "explicit_stack_tasks"
|
||||
SUPPORTED_TEMPLATE_STRATEGIES = {
|
||||
TEMPLATE_STRATEGY_NATIVE,
|
||||
TEMPLATE_STRATEGY_EXPLICIT,
|
||||
}
|
||||
|
||||
REQUIRED_STACK_TASKS = [
|
||||
"SARsInSARStackSBASGenerateConnectionGraph",
|
||||
"SARsInSARStackSBASInterferogramGeneration",
|
||||
"SARsInSARStackSBASInversionStep1",
|
||||
"SARsInSARStackSBASInversionStep2",
|
||||
"SARsInSARStackSBASGeocode",
|
||||
]
|
||||
|
||||
REQUIRED_TASKS = REQUIRED_STACK_TASKS
|
||||
|
||||
OPTIONAL_TASKS = [
|
||||
NATIVE_WORKFLOW_TASK,
|
||||
NATIVE_ESBAS_WORKFLOW_TASK,
|
||||
"SARscape_setting_output_folders",
|
||||
"SARsLoadPreferences",
|
||||
"SARsImportSarSelector",
|
||||
"SARscapeSuggestLooks",
|
||||
"SARscapeEnviuriToShape",
|
||||
"SARsInSARStackSBASVariogram",
|
||||
"SARsInSARStackESBASInterferogramGeneration",
|
||||
"SARsInSARStackESBASInversion",
|
||||
"SARsInSARStackESBASGeocode",
|
||||
"SARsInSARConnectionGraphESBAS",
|
||||
]
|
||||
|
||||
PIPELINE_PHASES = [
|
||||
{
|
||||
"phase_id": "connection_graph",
|
||||
"task_name": "SARsInSARStackSBASGenerateConnectionGraph",
|
||||
"purpose": "Build or ingest the SBAS connection graph.",
|
||||
},
|
||||
{
|
||||
"phase_id": "interferogram_generation",
|
||||
"task_name": "SARsInSARStackSBASInterferogramGeneration",
|
||||
"purpose": "Generate interferograms for the selected SBAS graph.",
|
||||
},
|
||||
{
|
||||
"phase_id": "inversion_step1",
|
||||
"task_name": "SARsInSARStackSBASInversionStep1",
|
||||
"purpose": "Run SARscape SBAS inversion step 1.",
|
||||
},
|
||||
{
|
||||
"phase_id": "inversion_step2",
|
||||
"task_name": "SARsInSARStackSBASInversionStep2",
|
||||
"purpose": "Run SARscape SBAS inversion step 2.",
|
||||
},
|
||||
{
|
||||
"phase_id": "geocode_export",
|
||||
"task_name": "SARsInSARStackSBASGeocode",
|
||||
"purpose": "Geocode velocity, displacement, and quality outputs.",
|
||||
},
|
||||
{
|
||||
"phase_id": "variogram_optional",
|
||||
"task_name": "SARsInSARStackSBASVariogram",
|
||||
"purpose": "Optional SARscape variogram/quality analysis.",
|
||||
"optional": True,
|
||||
},
|
||||
]
|
||||
|
||||
REQUIRED_RESULT_ROLES = [
|
||||
"stack_manifest",
|
||||
"processor_manifest",
|
||||
"selected_network_edges",
|
||||
"velocity_product",
|
||||
"timeseries_product",
|
||||
"temporal_coherence",
|
||||
"geocoded_raster",
|
||||
"preview_png",
|
||||
"logs",
|
||||
]
|
||||
|
||||
|
||||
def default_parameter_template_path() -> str:
|
||||
configured = str(getattr(settings, "SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH", "") or "").strip()
|
||||
if configured:
|
||||
return configured
|
||||
return str(
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "templates"
|
||||
/ "sarscape_sbas_parameter_template.example.json"
|
||||
)
|
||||
|
||||
|
||||
def _utcnow_iso() -> str:
|
||||
return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
|
||||
|
||||
|
||||
def _canonical_json(payload: Dict[str, Any]) -> str:
|
||||
return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _sha256_payload(payload: Dict[str, Any]) -> str:
|
||||
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _available_task_names(discovery_report: Optional[Dict[str, Any]]) -> set[str]:
|
||||
if not isinstance(discovery_report, dict):
|
||||
return set()
|
||||
names: set[str] = set()
|
||||
for item in discovery_report.get("tasks") or []:
|
||||
if isinstance(item, dict) and bool(item.get("available")):
|
||||
name = str(item.get("name") or "").strip()
|
||||
if name:
|
||||
names.add(name)
|
||||
discovered = discovery_report.get("discovery") or {}
|
||||
for name in discovered.get("sarscape_sbas_tasks") or []:
|
||||
text = str(name or "").strip()
|
||||
if text:
|
||||
names.add(text)
|
||||
return names
|
||||
|
||||
|
||||
def _numeric_values(items: List[Dict[str, Any]], key: str) -> List[float]:
|
||||
values: List[float] = []
|
||||
for item in items:
|
||||
try:
|
||||
if item.get(key) is not None:
|
||||
values.append(float(item.get(key)))
|
||||
except Exception:
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def load_parameter_template(parameter_template_path: Optional[str] = None) -> Dict[str, Any]:
|
||||
path = str(parameter_template_path or default_parameter_template_path() or "").strip()
|
||||
result: Dict[str, Any] = {
|
||||
"path": path or None,
|
||||
"exists": False,
|
||||
"readable": False,
|
||||
"schema": None,
|
||||
"validated": False,
|
||||
"execution_strategy": TEMPLATE_STRATEGY_NATIVE,
|
||||
"native_workflow_task": None,
|
||||
"task_count": 0,
|
||||
"missing_required_tasks": list(REQUIRED_STACK_TASKS),
|
||||
"tasks_without_parameters": [],
|
||||
"errors": [],
|
||||
"template": None,
|
||||
}
|
||||
if not path:
|
||||
result["errors"].append("SARscape SBAS parameter template path is empty.")
|
||||
return result
|
||||
template_file = Path(path)
|
||||
result["exists"] = template_file.is_file()
|
||||
if not template_file.is_file():
|
||||
result["errors"].append(f"SARscape SBAS parameter template not found: {path}")
|
||||
return result
|
||||
try:
|
||||
payload = json.loads(template_file.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
result["errors"].append(f"Failed to read SARscape SBAS parameter template: {exc}")
|
||||
return result
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
result["errors"].append("SARscape SBAS parameter template must be a JSON object.")
|
||||
return result
|
||||
|
||||
raw_strategy = str(payload.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
|
||||
execution_strategy = (
|
||||
raw_strategy if raw_strategy in SUPPORTED_TEMPLATE_STRATEGIES else TEMPLATE_STRATEGY_NATIVE
|
||||
)
|
||||
native_workflow = payload.get("native_workflow") if isinstance(payload.get("native_workflow"), dict) else {}
|
||||
native_workflow_task = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
|
||||
native_workflow_parameters = native_workflow.get("parameters")
|
||||
tasks = payload.get("tasks") if isinstance(payload.get("tasks"), list) else []
|
||||
task_names = {
|
||||
str(item.get("task_name") or "").strip()
|
||||
for item in tasks
|
||||
if isinstance(item, dict) and str(item.get("task_name") or "").strip()
|
||||
}
|
||||
missing_required = [name for name in REQUIRED_STACK_TASKS if name not in task_names]
|
||||
tasks_without_parameters = [
|
||||
str(item.get("task_name") or item.get("phase_id") or "<unnamed>")
|
||||
for item in tasks
|
||||
if isinstance(item, dict)
|
||||
and bool(item.get("enabled", True))
|
||||
and not isinstance(item.get("parameters"), dict)
|
||||
]
|
||||
result.update(
|
||||
{
|
||||
"readable": True,
|
||||
"schema": payload.get("schema"),
|
||||
"validated": bool(payload.get("validated")),
|
||||
"execution_strategy": execution_strategy,
|
||||
"native_workflow_task": native_workflow_task,
|
||||
"task_count": len(tasks),
|
||||
"missing_required_tasks": missing_required,
|
||||
"tasks_without_parameters": tasks_without_parameters,
|
||||
"template": payload,
|
||||
}
|
||||
)
|
||||
if str(payload.get("schema") or "") != "insar.sarscape-sbas-template/v1":
|
||||
result["errors"].append("Unsupported SARscape SBAS parameter template schema.")
|
||||
if raw_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
|
||||
result["errors"].append(
|
||||
"Unsupported SARscape SBAS execution_strategy: " + (raw_strategy or "<empty>")
|
||||
)
|
||||
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
|
||||
if not native_workflow_task:
|
||||
result["errors"].append("Native SARscape workflow task name is empty.")
|
||||
if not isinstance(native_workflow_parameters, dict):
|
||||
result["errors"].append("Native SARscape workflow parameters must be a JSON object.")
|
||||
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and missing_required:
|
||||
result["errors"].append("Template is missing required tasks: " + ", ".join(missing_required))
|
||||
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and tasks_without_parameters:
|
||||
result["errors"].append("Template tasks without parameters object: " + ", ".join(tasks_without_parameters))
|
||||
if not payload.get("validated"):
|
||||
result["errors"].append("Template is not marked validated=true.")
|
||||
return result
|
||||
|
||||
|
||||
def summarize_network_edges(network_edges: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
enabled_edges = [item for item in network_edges if bool(item.get("enabled", True))]
|
||||
temporal = _numeric_values(enabled_edges, "temporal_baseline_days")
|
||||
spatial = _numeric_values(enabled_edges, "spatial_baseline_meters")
|
||||
overlap = _numeric_values(enabled_edges, "pair_aoi_overlap_ratio")
|
||||
return {
|
||||
"edge_count": len(network_edges),
|
||||
"enabled_edge_count": len(enabled_edges),
|
||||
"temporal_baseline_days": {
|
||||
"min": min(temporal) if temporal else None,
|
||||
"max": max(temporal) if temporal else None,
|
||||
},
|
||||
"spatial_baseline_meters": {
|
||||
"min": min(spatial) if spatial else None,
|
||||
"max": max(spatial) if spatial else None,
|
||||
},
|
||||
"pair_aoi_overlap_ratio": {
|
||||
"min": min(overlap) if overlap else None,
|
||||
"max": max(overlap) if overlap else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_processor_manifest(
|
||||
stack_manifest: Dict[str, Any],
|
||||
*,
|
||||
discovery_report: Optional[Dict[str, Any]] = None,
|
||||
parameter_template_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Build the SARscape SBAS processor contract without executing ENVI tasks."""
|
||||
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
|
||||
network_edges = (
|
||||
stack_manifest.get("network_edges")
|
||||
if isinstance(stack_manifest.get("network_edges"), list)
|
||||
else []
|
||||
)
|
||||
template_status = load_parameter_template(parameter_template_path)
|
||||
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
|
||||
template_strategy = str(
|
||||
template_status.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE
|
||||
).strip()
|
||||
native_workflow_task = str(
|
||||
template_status.get("native_workflow_task") or NATIVE_WORKFLOW_TASK
|
||||
).strip()
|
||||
available_tasks = _available_task_names(discovery_report)
|
||||
missing_stack_tasks = [name for name in REQUIRED_STACK_TASKS if name not in available_tasks]
|
||||
missing_native_tasks = [native_workflow_task] if native_workflow_task not in available_tasks else []
|
||||
missing_required_tasks = (
|
||||
missing_native_tasks
|
||||
if template_strategy == TEMPLATE_STRATEGY_NATIVE
|
||||
else missing_stack_tasks
|
||||
)
|
||||
template_path = str(template_status.get("path") or "").strip()
|
||||
template_exists = bool(template_status.get("exists"))
|
||||
template_validated = bool(template_status.get("validated")) and not template_status.get("errors")
|
||||
execution_enabled = bool(getattr(settings, "SARSCAPE_SBAS_ALLOW_EXECUTION", False))
|
||||
native_workflow_available = not missing_native_tasks
|
||||
explicit_stack_available = not missing_stack_tasks
|
||||
|
||||
blockers: List[str] = []
|
||||
if len(scenes) < 3:
|
||||
blockers.append("SARscape SBAS requires at least 3 stack scenes.")
|
||||
if not network_edges:
|
||||
blockers.append("No SBAS network_edges are present in the stack manifest.")
|
||||
if missing_required_tasks:
|
||||
blockers.append(
|
||||
"Missing required SARscape SBAS tasks for "
|
||||
f"{template_strategy}: " + ", ".join(missing_required_tasks)
|
||||
)
|
||||
if not template_exists:
|
||||
blockers.append(
|
||||
"SARscape SBAS parameter template is not configured. "
|
||||
"Live task.parameters is intentionally not used because it can hang taskengine."
|
||||
)
|
||||
elif not template_validated:
|
||||
blockers.extend(str(item) for item in (template_status.get("errors") or []))
|
||||
if not execution_enabled:
|
||||
blockers.append("SARSCAPE_SBAS_ALLOW_EXECUTION is false; SARscape SBAS production execution is disabled.")
|
||||
|
||||
parameter_template_state = (
|
||||
"validated"
|
||||
if template_validated
|
||||
else ("configured_unvalidated" if template_exists else "required")
|
||||
)
|
||||
task_sequence = [
|
||||
{
|
||||
"phase_id": "native_wf_sbas",
|
||||
"task_name": native_workflow_task,
|
||||
"purpose": "Run SARscape's installed end-to-end SBAS metatask.",
|
||||
"available": native_workflow_available,
|
||||
"required": template_strategy == TEMPLATE_STRATEGY_NATIVE,
|
||||
"parameter_template_status": parameter_template_state,
|
||||
"has_template_parameters": isinstance(
|
||||
(template.get("native_workflow") or {}).get("parameters")
|
||||
if isinstance(template.get("native_workflow"), dict)
|
||||
else None,
|
||||
dict,
|
||||
),
|
||||
"supports_system_selected_edges": False,
|
||||
"ready": (
|
||||
native_workflow_available
|
||||
and template_strategy == TEMPLATE_STRATEGY_NATIVE
|
||||
and template_validated
|
||||
and execution_enabled
|
||||
),
|
||||
}
|
||||
]
|
||||
template_tasks = {
|
||||
str(item.get("task_name") or "").strip(): item
|
||||
for item in (template.get("tasks") or [])
|
||||
if isinstance(item, dict)
|
||||
}
|
||||
for phase in PIPELINE_PHASES:
|
||||
task_name = str(phase["task_name"])
|
||||
optional = bool(phase.get("optional", False))
|
||||
template_task = template_tasks.get(task_name) or {}
|
||||
has_template_parameters = isinstance(template_task.get("parameters"), dict)
|
||||
task_sequence.append(
|
||||
{
|
||||
**phase,
|
||||
"available": task_name in available_tasks,
|
||||
"required": not optional,
|
||||
"template_phase_id": template_task.get("phase_id"),
|
||||
"parameter_template_status": parameter_template_state,
|
||||
"has_template_parameters": has_template_parameters,
|
||||
"ready": (
|
||||
(task_name in available_tasks or optional)
|
||||
and template_strategy == TEMPLATE_STRATEGY_EXPLICIT
|
||||
and template_validated
|
||||
and execution_enabled
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema": "insar.sarscape-sbas-processor/v1",
|
||||
"created_at_utc": _utcnow_iso(),
|
||||
"engine_code": ENGINE_CODE,
|
||||
"processor_code": PROCESSOR_CODE,
|
||||
"execution_enabled": execution_enabled,
|
||||
"ready_for_pipeline_design": bool(discovery_report and discovery_report.get("ok")),
|
||||
"ready_for_execution": not blockers,
|
||||
"blockers": blockers,
|
||||
"stack_manifest_checksum": _sha256_payload(stack_manifest),
|
||||
"stack_manifest_summary": {
|
||||
"schema": stack_manifest.get("schema"),
|
||||
"prepared_stack_schema": stack_manifest.get("prepared_stack_schema"),
|
||||
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
|
||||
"manifest_role": stack_manifest.get("manifest_role"),
|
||||
"batch_id": stack_manifest.get("batch_id"),
|
||||
"plan_id": stack_manifest.get("plan_id"),
|
||||
"plan_strategy": stack_manifest.get("plan_strategy"),
|
||||
"reference_date": stack_manifest.get("reference_date"),
|
||||
"scene_count": len(scenes),
|
||||
"stack_key": stack_manifest.get("stack_key"),
|
||||
"group_key": stack_manifest.get("group_key"),
|
||||
},
|
||||
"network_summary": summarize_network_edges(network_edges),
|
||||
"execution_strategy": template_strategy,
|
||||
"execution_strategies": {
|
||||
TEMPLATE_STRATEGY_NATIVE: {
|
||||
"preferred": template_strategy == TEMPLATE_STRATEGY_NATIVE,
|
||||
"task_name": native_workflow_task,
|
||||
"available": native_workflow_available,
|
||||
"required_tasks": [native_workflow_task],
|
||||
"missing_tasks": missing_native_tasks,
|
||||
"supports_system_selected_edges": False,
|
||||
"graph_policy": "SARscape wf_sbas builds the connection graph internally; system network_edges are retained for audit and comparison.",
|
||||
},
|
||||
TEMPLATE_STRATEGY_EXPLICIT: {
|
||||
"preferred": template_strategy == TEMPLATE_STRATEGY_EXPLICIT,
|
||||
"available": explicit_stack_available,
|
||||
"required_tasks": list(REQUIRED_STACK_TASKS),
|
||||
"missing_tasks": missing_stack_tasks,
|
||||
"supports_system_selected_edges": "not_verified",
|
||||
"graph_policy": "Explicit task chaining can expose the connection graph step, but direct injection of the system-selected edge list still needs SARscape parameter validation.",
|
||||
},
|
||||
},
|
||||
"required_tasks": [native_workflow_task] if template_strategy == TEMPLATE_STRATEGY_NATIVE else list(REQUIRED_STACK_TASKS),
|
||||
"required_stack_tasks": list(REQUIRED_STACK_TASKS),
|
||||
"optional_tasks": list(OPTIONAL_TASKS),
|
||||
"available_tasks": sorted(available_tasks),
|
||||
"missing_required_tasks": missing_required_tasks,
|
||||
"parameter_template": {
|
||||
"path": template_path or None,
|
||||
"exists": template_exists,
|
||||
"readable": bool(template_status.get("readable")),
|
||||
"validated": bool(template_status.get("validated")),
|
||||
"execution_strategy": template_strategy,
|
||||
"native_workflow_task": native_workflow_task,
|
||||
"task_count": int(template_status.get("task_count") or 0),
|
||||
"errors": template_status.get("errors") or [],
|
||||
"source": "manual_sarscape_template",
|
||||
},
|
||||
"task_sequence": task_sequence,
|
||||
"input_contract": {
|
||||
"required_manifest_role_for_execution": "prepared_sbas_stack",
|
||||
"prepared_stack_schema": PREPARED_STACK_SCHEMA,
|
||||
"production_input_policy": "prepared_stack_manifest_only",
|
||||
"scene_path_fields": ["folder_path", "tiff_path", "meta_path"],
|
||||
"network_edge_source": "stack_manifest.network_edges",
|
||||
"dem_source": "IDL_DINSAR_DEM_BASE_FILE",
|
||||
"orbit_source": "ORBIT_POOL_ENVI",
|
||||
},
|
||||
"result_contract": {
|
||||
"catalog_name": "psinsar",
|
||||
"required_roles": list(REQUIRED_RESULT_ROLES),
|
||||
"publish_manifest_schema": "psinsar.publish.v2",
|
||||
},
|
||||
"notes": [
|
||||
"This manifest is a planning contract only; it does not execute SARscape tasks.",
|
||||
"Execution must use checked-in SARscape parameter templates, not live task.parameters.",
|
||||
"The native wf_sbas strategy is the preferred first integration path on this workstation.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_preflight_report(
|
||||
stack_manifest: Dict[str, Any],
|
||||
*,
|
||||
include_task_discovery: bool = True,
|
||||
discovery_timeout_seconds: int = 120,
|
||||
parameter_template_path: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
status = envi_service.get_status()
|
||||
discovery_report: Optional[Dict[str, Any]] = None
|
||||
if include_task_discovery:
|
||||
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
|
||||
timeout_seconds=discovery_timeout_seconds,
|
||||
include_parameters=False,
|
||||
)
|
||||
processor_manifest = build_processor_manifest(
|
||||
stack_manifest,
|
||||
discovery_report=discovery_report,
|
||||
parameter_template_path=parameter_template_path,
|
||||
)
|
||||
env_blockers: List[str] = []
|
||||
if not status.get("idl_installed"):
|
||||
env_blockers.append("IDL/ENVI executable is not installed or not configured.")
|
||||
if not status.get("runner_ready"):
|
||||
env_blockers.append("ENVI runner is not ready: " + str(status.get("runner_message") or "unknown"))
|
||||
if not status.get("dem_exists"):
|
||||
env_blockers.append("SARscape DEM base file is missing: " + str(status.get("dem_base_file") or ""))
|
||||
if discovery_report is not None and not discovery_report.get("ok"):
|
||||
env_blockers.append("SARscape SBAS task discovery failed: " + str(discovery_report.get("error") or "unknown"))
|
||||
|
||||
all_blockers = [*env_blockers, *(processor_manifest.get("blockers") or [])]
|
||||
return {
|
||||
"schema": "insar.sarscape-sbas-preflight/v1",
|
||||
"created_at_utc": _utcnow_iso(),
|
||||
"engine_code": ENGINE_CODE,
|
||||
"processor_code": PROCESSOR_CODE,
|
||||
"ready_for_pipeline_design": bool(
|
||||
status.get("idl_installed")
|
||||
and status.get("runner_ready")
|
||||
and (discovery_report is None or discovery_report.get("ok"))
|
||||
),
|
||||
"ready_for_execution": not all_blockers,
|
||||
"blockers": all_blockers,
|
||||
"environment": status,
|
||||
"task_discovery": discovery_report,
|
||||
"processor_manifest": processor_manifest,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_template_value(value: Any, context: Dict[str, Any]) -> Any:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text in context:
|
||||
return context[text]
|
||||
resolved = value
|
||||
for key, replacement in context.items():
|
||||
if key in resolved and isinstance(replacement, (str, int, float, bool)):
|
||||
resolved = resolved.replace(key, str(replacement))
|
||||
return resolved
|
||||
if isinstance(value, list):
|
||||
return [_resolve_template_value(item, context) for item in value]
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(key): _resolve_template_value(item, context)
|
||||
for key, item in value.items()
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def _scene_input_uris(scenes: List[Dict[str, Any]]) -> List[str]:
|
||||
uris: List[str] = []
|
||||
for item in scenes:
|
||||
for key in ("meta_path", "tiff_path", "folder_path"):
|
||||
text = str(item.get(key) or "").strip()
|
||||
if text:
|
||||
uris.append(text)
|
||||
break
|
||||
return uris
|
||||
|
||||
|
||||
def execute_template_workflow(
|
||||
stack_manifest: Dict[str, Any],
|
||||
*,
|
||||
work_root: str,
|
||||
selected_manifest_path: str,
|
||||
timeout_seconds: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Execute a validated SARscape SBAS template.
|
||||
|
||||
This path is intentionally gated by SARSCAPE_SBAS_ALLOW_EXECUTION and
|
||||
template validated=true. The default checked-in template is not executable.
|
||||
"""
|
||||
if stack_manifest.get("prepared_stack_schema") != PREPARED_STACK_SCHEMA:
|
||||
raise ValueError(
|
||||
f"SARscape SBAS execution requires a prepared stack manifest ({PREPARED_STACK_SCHEMA})."
|
||||
)
|
||||
if not str(stack_manifest.get("prepared_stack_id") or "").strip():
|
||||
raise ValueError("SARscape SBAS execution requires prepared_stack_id.")
|
||||
|
||||
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
|
||||
timeout_seconds=int(getattr(settings, "SARSCAPE_SBAS_DISCOVERY_TIMEOUT_SECONDS", 120) or 120),
|
||||
include_parameters=False,
|
||||
)
|
||||
processor_manifest = build_processor_manifest(
|
||||
stack_manifest,
|
||||
discovery_report=discovery_report,
|
||||
)
|
||||
if not processor_manifest.get("ready_for_execution"):
|
||||
blockers = "; ".join(str(item) for item in (processor_manifest.get("blockers") or []))
|
||||
raise ValueError("SARscape SBAS execution is not ready: " + (blockers or "unknown blocker"))
|
||||
|
||||
template_status = load_parameter_template()
|
||||
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
|
||||
tasks = [item for item in (template.get("tasks") or []) if isinstance(item, dict)]
|
||||
output_root = Path(work_root) / "sarscape_sbas"
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
artifacts = stack_manifest.get("artifacts") if isinstance(stack_manifest.get("artifacts"), dict) else {}
|
||||
prepared_edges_path = str(artifacts.get("selected_network_edges_path_windows") or "").strip()
|
||||
if prepared_edges_path:
|
||||
network_edges_path = Path(prepared_edges_path)
|
||||
if not network_edges_path.is_file():
|
||||
raise FileNotFoundError(f"Prepared selected_network_edges.json not found: {network_edges_path}")
|
||||
else:
|
||||
network_edges_path = output_root / "selected_network_edges.json"
|
||||
network_edges_path.write_text(
|
||||
json.dumps(stack_manifest.get("network_edges") or [], ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
|
||||
context: Dict[str, Any] = {
|
||||
"${work_root}": str(work_root),
|
||||
"${output_root}": str(output_root),
|
||||
"${selected_stack_manifest}": str(selected_manifest_path),
|
||||
"${selected_network_edges}": str(network_edges_path),
|
||||
"${scene_meta_paths}": [
|
||||
str(item.get("meta_path"))
|
||||
for item in scenes
|
||||
if str(item.get("meta_path") or "").strip()
|
||||
],
|
||||
"${scene_input_uris}": _scene_input_uris(scenes),
|
||||
"${scene_folder_paths}": [
|
||||
str(item.get("folder_path"))
|
||||
for item in scenes
|
||||
if str(item.get("folder_path") or "").strip()
|
||||
],
|
||||
"${selection_params}": stack_manifest.get("selection_params") or {},
|
||||
"${dem_sarscapedata}": envi_service._build_sarscapedata(envi_service.DEM_BASE_FILE), # noqa: SLF001
|
||||
}
|
||||
|
||||
executed: List[Dict[str, Any]] = []
|
||||
previous_outputs: Dict[str, Any] = {}
|
||||
execution_strategy = str(template.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
|
||||
if execution_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
|
||||
execution_strategy = TEMPLATE_STRATEGY_NATIVE
|
||||
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
|
||||
native_workflow = template.get("native_workflow") if isinstance(template.get("native_workflow"), dict) else {}
|
||||
task_name = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
|
||||
if not task_name:
|
||||
raise ValueError("Native SARscape SBAS workflow task_name is empty.")
|
||||
phase_id = str(native_workflow.get("phase_id") or "native_wf_sbas").strip()
|
||||
phase_output_dir = output_root / phase_id
|
||||
phase_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
phase_context = {
|
||||
**context,
|
||||
"${phase_id}": phase_id,
|
||||
"${phase_output_dir}": str(phase_output_dir),
|
||||
"${previous_outputs}": previous_outputs,
|
||||
}
|
||||
parameters = _resolve_template_value(native_workflow.get("parameters") or {}, phase_context)
|
||||
result = envi_service.execute_envi_task(task_name, parameters)
|
||||
previous_outputs[phase_id] = result
|
||||
executed.append(
|
||||
{
|
||||
"phase_id": phase_id,
|
||||
"task_name": task_name,
|
||||
"output_dir": str(phase_output_dir),
|
||||
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
|
||||
}
|
||||
)
|
||||
tasks = []
|
||||
|
||||
for item in tasks:
|
||||
if not bool(item.get("enabled", True)):
|
||||
continue
|
||||
task_name = str(item.get("task_name") or "").strip()
|
||||
phase_id = str(item.get("phase_id") or task_name).strip()
|
||||
if not task_name:
|
||||
raise ValueError(f"SARscape template task is missing task_name: {phase_id}")
|
||||
phase_output_dir = output_root / phase_id
|
||||
phase_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
phase_context = {
|
||||
**context,
|
||||
"${phase_id}": phase_id,
|
||||
"${phase_output_dir}": str(phase_output_dir),
|
||||
"${previous_outputs}": previous_outputs,
|
||||
}
|
||||
parameters = _resolve_template_value(item.get("parameters") or {}, phase_context)
|
||||
result = envi_service.execute_envi_task(task_name, parameters)
|
||||
previous_outputs[phase_id] = result
|
||||
executed.append(
|
||||
{
|
||||
"phase_id": phase_id,
|
||||
"task_name": task_name,
|
||||
"output_dir": str(phase_output_dir),
|
||||
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"schema": "insar.sarscape-sbas-execution/v1",
|
||||
"created_at_utc": _utcnow_iso(),
|
||||
"processor_code": PROCESSOR_CODE,
|
||||
"execution_strategy": execution_strategy,
|
||||
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
|
||||
"work_root": str(work_root),
|
||||
"output_root": str(output_root),
|
||||
"selected_network_edges_path": str(network_edges_path),
|
||||
"task_count": len(executed),
|
||||
"executed_tasks": executed,
|
||||
"processor_manifest": processor_manifest,
|
||||
}
|
||||
|
||||
|
||||
def write_processor_manifest(path: str | Path, manifest: Dict[str, Any]) -> str:
|
||||
target = Path(path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return str(target)
|
||||
@@ -35,6 +35,7 @@ from ..models import (
|
||||
RadarDataORM,
|
||||
RadarPair,
|
||||
ResultProductORM,
|
||||
TimeseriesStackPlanEdgeORM,
|
||||
TimeseriesStackPlanItemORM,
|
||||
TimeseriesStackPlanORM,
|
||||
)
|
||||
@@ -404,6 +405,138 @@ class SpatialService:
|
||||
"stack_dates": stack_dates,
|
||||
}
|
||||
|
||||
async def _build_timeseries_network_edges(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
scenes: List[RadarDataORM],
|
||||
params: PsRequest,
|
||||
*,
|
||||
aoi_wkt: Optional[str],
|
||||
selection_mode: Optional[str],
|
||||
) -> Tuple[List[Dict[str, Any]], List[str]]:
|
||||
scene_ids = [int(item.id) for item in scenes if item.id is not None]
|
||||
if len(scene_ids) < 2:
|
||||
return [], []
|
||||
|
||||
master_alias = aliased(RadarDataORM)
|
||||
slave_alias = aliased(RadarDataORM)
|
||||
stmt = (
|
||||
select(PairingMetricCacheORM, master_alias, slave_alias)
|
||||
.join(master_alias, master_alias.id == PairingMetricCacheORM.master_scene_ref_id)
|
||||
.join(slave_alias, slave_alias.id == PairingMetricCacheORM.slave_scene_ref_id)
|
||||
.where(
|
||||
PairingMetricCacheORM.metric_version == pairing_state_service.metric_version,
|
||||
PairingMetricCacheORM.status == "READY",
|
||||
PairingMetricCacheORM.master_scene_ref_id.in_(scene_ids),
|
||||
PairingMetricCacheORM.slave_scene_ref_id.in_(scene_ids),
|
||||
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
|
||||
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
|
||||
PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters,
|
||||
PairingMetricCacheORM.scene_overlap_ratio >= params.network_overlap_threshold,
|
||||
)
|
||||
.order_by(
|
||||
PairingMetricCacheORM.master_imaging_date.asc(),
|
||||
PairingMetricCacheORM.slave_imaging_date.asc(),
|
||||
PairingMetricCacheORM.time_baseline_days.asc(),
|
||||
PairingMetricCacheORM.spatial_baseline_meters.asc(),
|
||||
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
|
||||
PairingMetricCacheORM.id.asc(),
|
||||
)
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
|
||||
candidate_pool: List[dict] = []
|
||||
for metric_row, master_row, slave_row in result.all():
|
||||
candidate_pool.append(
|
||||
{
|
||||
"metric_cache_ref_id": int(metric_row.id),
|
||||
"pair_uid": metric_row.pair_uid,
|
||||
"master_scene_uid": metric_row.master_scene_uid,
|
||||
"slave_scene_uid": metric_row.slave_scene_uid,
|
||||
"master": RadarData.model_validate(master_row),
|
||||
"slave": RadarData.model_validate(slave_row),
|
||||
"days": int(metric_row.time_baseline_days or 0),
|
||||
"dist": float(metric_row.spatial_baseline_meters or 0.0),
|
||||
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0.0),
|
||||
}
|
||||
)
|
||||
|
||||
warnings: List[str] = []
|
||||
if not candidate_pool:
|
||||
warnings.append(
|
||||
"No pairing_metric_cache edges matched the time-series SBAS network thresholds."
|
||||
)
|
||||
return [], warnings
|
||||
candidate_scene_ids = {
|
||||
int(candidate[role].id)
|
||||
for candidate in candidate_pool
|
||||
for role in ("master", "slave")
|
||||
if candidate.get(role) is not None
|
||||
}
|
||||
missing_scene_count = len(set(scene_ids) - candidate_scene_ids)
|
||||
if missing_scene_count > 0:
|
||||
warnings.append(
|
||||
f"{missing_scene_count} selected scenes have no metric-cache edge under the current SBAS thresholds."
|
||||
)
|
||||
|
||||
pairing_params = PairingRequest(
|
||||
time_baseline_min=params.time_baseline_min,
|
||||
time_baseline_max=params.time_baseline_max,
|
||||
overlap_threshold=params.network_overlap_threshold,
|
||||
spatial_baseline_max_meters=params.spatial_baseline_max_meters,
|
||||
coverage_diversity_penalty=0.3,
|
||||
require_same_imaging_mode=True,
|
||||
require_same_polarization=True,
|
||||
strategy="sbas",
|
||||
num_connections=params.num_connections,
|
||||
)
|
||||
selected_candidates, strategy_warnings = self._apply_sbas_strategy(
|
||||
candidate_pool,
|
||||
pairing_params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
)
|
||||
warnings.extend(strategy_warnings)
|
||||
|
||||
edges: List[Dict[str, Any]] = []
|
||||
for edge_rank, candidate in enumerate(self._sorted_candidates(selected_candidates), start=1):
|
||||
master = candidate["master"]
|
||||
slave = candidate["slave"]
|
||||
edges.append(
|
||||
{
|
||||
"edge_rank": edge_rank,
|
||||
"metric_cache_ref_id": candidate.get("metric_cache_ref_id"),
|
||||
"master_scene_ref_id": int(master.id),
|
||||
"slave_scene_ref_id": int(slave.id),
|
||||
"master_imaging_date": master.imaging_date,
|
||||
"slave_imaging_date": slave.imaging_date,
|
||||
"temporal_baseline_days": int(candidate.get("days") or 0),
|
||||
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
|
||||
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
|
||||
"selection_reason": candidate.get("selection_reason"),
|
||||
"selection_score": (
|
||||
float(candidate["selection_score"])
|
||||
if candidate.get("selection_score") is not None
|
||||
else None
|
||||
),
|
||||
"selection_meta_json": {
|
||||
"source": "pairing_metric_cache",
|
||||
"selection_mode": selection_mode,
|
||||
"pair_uid": candidate.get("pair_uid"),
|
||||
"metric_version": pairing_state_service.metric_version,
|
||||
"time_baseline_min": params.time_baseline_min,
|
||||
"time_baseline_max": params.time_baseline_max,
|
||||
"spatial_baseline_max_meters": params.spatial_baseline_max_meters,
|
||||
"network_overlap_threshold": params.network_overlap_threshold,
|
||||
"num_connections": params.num_connections,
|
||||
},
|
||||
"enabled": True,
|
||||
}
|
||||
)
|
||||
|
||||
if not edges:
|
||||
warnings.append("SBAS strategy did not select any network edges for this stack.")
|
||||
return edges, warnings
|
||||
|
||||
async def _persist_timeseries_stack_plan(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
@@ -416,6 +549,8 @@ class SpatialService:
|
||||
coverage_consistency_ratio: Optional[float] = None,
|
||||
threshold_satisfied: Optional[bool] = None,
|
||||
selection_mode: Optional[str] = None,
|
||||
network_edges: Optional[List[Dict[str, Any]]] = None,
|
||||
network_warnings: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
request_payload = params.model_dump(exclude_none=True)
|
||||
aoi_hash = self._stable_sha1(aoi_wkt) if aoi_wkt else None
|
||||
@@ -447,6 +582,9 @@ class SpatialService:
|
||||
|
||||
sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or ""))
|
||||
scene_payloads: List[RadarData] = []
|
||||
plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
|
||||
safe_network_edges = list(network_edges or [])
|
||||
safe_network_warnings = [str(item) for item in (network_warnings or []) if str(item).strip()]
|
||||
for rank, item in enumerate(sorted_scenes, start=1):
|
||||
plan_item = TimeseriesStackPlanItemORM(
|
||||
plan_ref_id=plan.id,
|
||||
@@ -469,6 +607,8 @@ class SpatialService:
|
||||
"coverage_consistency_ratio": coverage_consistency_ratio,
|
||||
"threshold_satisfied": threshold_satisfied,
|
||||
"selection_mode": selection_mode,
|
||||
"network_edge_count": len(safe_network_edges),
|
||||
"network_warnings": safe_network_warnings,
|
||||
"orbit_direction": item.orbit_direction,
|
||||
"satellite_family": self._normalize_timeseries_satellite_family(item),
|
||||
"bbox": [item.min_lon, item.min_lat, item.max_lon, item.max_lat],
|
||||
@@ -477,6 +617,8 @@ class SpatialService:
|
||||
)
|
||||
db.add(plan_item)
|
||||
await db.flush()
|
||||
if item.id is not None:
|
||||
plan_item_by_scene_id[int(item.id)] = plan_item
|
||||
scene_payloads.append(
|
||||
RadarData.model_validate(item).model_copy(
|
||||
update={
|
||||
@@ -490,14 +632,61 @@ class SpatialService:
|
||||
"stack_coverage_consistency_ratio": coverage_consistency_ratio,
|
||||
"stack_threshold_satisfied": threshold_satisfied,
|
||||
"stack_selection_mode": selection_mode,
|
||||
"stack_network_edge_count": len(safe_network_edges),
|
||||
"stack_network_warnings": safe_network_warnings,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
for edge_payload in safe_network_edges:
|
||||
master_scene_id = edge_payload.get("master_scene_ref_id")
|
||||
slave_scene_id = edge_payload.get("slave_scene_ref_id")
|
||||
master_plan_item = (
|
||||
plan_item_by_scene_id.get(int(master_scene_id))
|
||||
if master_scene_id is not None
|
||||
else None
|
||||
)
|
||||
slave_plan_item = (
|
||||
plan_item_by_scene_id.get(int(slave_scene_id))
|
||||
if slave_scene_id is not None
|
||||
else None
|
||||
)
|
||||
edge = TimeseriesStackPlanEdgeORM(
|
||||
plan_ref_id=plan.id,
|
||||
master_plan_item_ref_id=(
|
||||
int(master_plan_item.id)
|
||||
if master_plan_item is not None and master_plan_item.id is not None
|
||||
else None
|
||||
),
|
||||
slave_plan_item_ref_id=(
|
||||
int(slave_plan_item.id)
|
||||
if slave_plan_item is not None and slave_plan_item.id is not None
|
||||
else None
|
||||
),
|
||||
metric_cache_ref_id=edge_payload.get("metric_cache_ref_id"),
|
||||
master_scene_ref_id=master_scene_id,
|
||||
slave_scene_ref_id=slave_scene_id,
|
||||
edge_rank=int(edge_payload.get("edge_rank") or 0),
|
||||
master_imaging_date=edge_payload.get("master_imaging_date"),
|
||||
slave_imaging_date=edge_payload.get("slave_imaging_date"),
|
||||
temporal_baseline_days=edge_payload.get("temporal_baseline_days"),
|
||||
spatial_baseline_meters=edge_payload.get("spatial_baseline_meters"),
|
||||
perpendicular_baseline_meters=edge_payload.get("perpendicular_baseline_meters"),
|
||||
scene_overlap_ratio=edge_payload.get("scene_overlap_ratio"),
|
||||
pair_aoi_overlap_ratio=edge_payload.get("pair_aoi_overlap_ratio"),
|
||||
selection_reason=edge_payload.get("selection_reason"),
|
||||
selection_score=edge_payload.get("selection_score"),
|
||||
selection_meta_json=edge_payload.get("selection_meta_json"),
|
||||
enabled=bool(edge_payload.get("enabled", True)),
|
||||
)
|
||||
db.add(edge)
|
||||
|
||||
return {
|
||||
"plan_id": plan.plan_id,
|
||||
"group_key": identity.get("group_key"),
|
||||
"stack_key": identity.get("stack_key"),
|
||||
"edge_count": len(safe_network_edges),
|
||||
"network_warnings": safe_network_warnings,
|
||||
"scenes": scene_payloads,
|
||||
}
|
||||
|
||||
@@ -1286,6 +1475,19 @@ class SpatialService:
|
||||
if len(final_stack) >= 3:
|
||||
final_stack.sort(key=lambda x: str(x.imaging_date or ""))
|
||||
direction = group_key[0]
|
||||
network_edges, network_warnings = await self._build_timeseries_network_edges(
|
||||
db,
|
||||
final_stack,
|
||||
params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
selection_mode=selection_mode,
|
||||
)
|
||||
logger.info(
|
||||
"timeseries stack planning: group=%s network_edges=%s network_warnings=%s",
|
||||
self._format_timeseries_group_label(group_key),
|
||||
len(network_edges),
|
||||
len(network_warnings),
|
||||
)
|
||||
persisted_plan = await self._persist_timeseries_stack_plan(
|
||||
db,
|
||||
direction=direction,
|
||||
@@ -1296,6 +1498,8 @@ class SpatialService:
|
||||
coverage_consistency_ratio=consistency_ratio,
|
||||
threshold_satisfied=threshold_satisfied,
|
||||
selection_mode=selection_mode,
|
||||
network_edges=network_edges,
|
||||
network_warnings=network_warnings,
|
||||
)
|
||||
result_key = persisted_plan.get("group_key") or self._format_timeseries_group_label(group_key)
|
||||
if result_key in final_results:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -173,6 +173,7 @@ class WorkflowService:
|
||||
step.outputs = outputs
|
||||
|
||||
await self._advance_ready_steps(run_id, db)
|
||||
await db.flush()
|
||||
await self.enqueue_ready_steps(run_id, db=db)
|
||||
|
||||
if gen_db:
|
||||
|
||||
Reference in New Issue
Block a user