Apply current workspace changes
This commit is contained in:
@@ -74,6 +74,7 @@ JOB_TYPE_WATER_DETECT = "WATER_DETECT"
|
||||
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
@@ -1878,8 +1879,13 @@ async def _handle_idl_run_dinsar(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
"""ISCE2 生产任务 handler — 通过 WSL 执行 run_lt1_dinsar_pipeline.py。"""
|
||||
async def _handle_queued_engine_run(
|
||||
job: SystemJobORM,
|
||||
*,
|
||||
engine_title: str,
|
||||
fallback_timeout_seconds: int,
|
||||
) -> None:
|
||||
"""Run a queued D-InSAR engine task through the shared WSL execution path."""
|
||||
payload = job.payload or {}
|
||||
engine_code = payload.get("engine_code", "isce2")
|
||||
profile = payload.get("profile", "lt1_stripmap")
|
||||
@@ -1887,17 +1893,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
num_to_process = payload.get("num_to_process", 0)
|
||||
timeout_seconds = payload.get("timeout_seconds")
|
||||
extra = payload.get("extra", {})
|
||||
selected_task_count = max(1, int(extra.get("__validated_task_count") or 0 or 1))
|
||||
pair_timeout_seconds = int(timeout_seconds or fallback_timeout_seconds)
|
||||
|
||||
await task_service.start_task(
|
||||
job.task_id,
|
||||
message=f"[{engine_code}/{profile}] 启动 ISCE2 处理...",
|
||||
message=f"[{engine_code}/{profile}] 启动 {engine_title} 处理...",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"ISCE2 job accepted. root_dir={root_dir}, profile={profile}, timeout={timeout_seconds or 21600}s, extra={extra}",
|
||||
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, timeout={pair_timeout_seconds}s, extra={extra}",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"{engine_title} batch contains {selected_task_count} pair task(s). "
|
||||
f"Pairs run sequentially and each pair uses timeout={pair_timeout_seconds}s."
|
||||
),
|
||||
)
|
||||
|
||||
from ..dinsar_engines.base import RunRequest
|
||||
from ..dinsar_engines import registry
|
||||
|
||||
@@ -1905,6 +1920,25 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
if not engine:
|
||||
raise RuntimeError(f"引擎 '{engine_code}' 未注册")
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
progress_queue: asyncio.Queue[Optional[Dict[str, Any]]] = asyncio.Queue()
|
||||
progress_state: Dict[str, Any] = {
|
||||
"progress": 5,
|
||||
"message": f"[{engine_code}/{profile}] Running in WSL...",
|
||||
"pair_index": 0,
|
||||
"pair_total": selected_task_count,
|
||||
"pair_label": "",
|
||||
"pair_started_monotonic": None,
|
||||
}
|
||||
|
||||
def _emit_progress(event: Dict[str, Any]) -> None:
|
||||
if not event:
|
||||
return
|
||||
try:
|
||||
loop.call_soon_threadsafe(progress_queue.put_nowait, dict(event))
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
request = RunRequest(
|
||||
engine_code=engine_code,
|
||||
profile=profile,
|
||||
@@ -1913,6 +1947,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
num_to_process=num_to_process,
|
||||
timeout_seconds=timeout_seconds,
|
||||
extra=extra,
|
||||
progress_callback=_emit_progress,
|
||||
)
|
||||
|
||||
await task_service.update_task(
|
||||
@@ -1921,18 +1956,119 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
message=f"[{engine_code}/{profile}] 正在执行，请等待...",
|
||||
)
|
||||
|
||||
async def _consume_progress() -> None:
|
||||
while True:
|
||||
event = await progress_queue.get()
|
||||
if event is None:
|
||||
return
|
||||
|
||||
event_type = str(event.get("event") or "").strip().lower()
|
||||
pair_total = max(1, int(event.get("pair_total") or progress_state["pair_total"] or 1))
|
||||
pair_index = max(0, int(event.get("pair_index") or 0))
|
||||
task_label = str(event.get("task_alias") or event.get("task_name") or "").strip()
|
||||
|
||||
if event_type == "pair_started":
|
||||
progress = min(
|
||||
90,
|
||||
max(
|
||||
int(progress_state["progress"] or 5),
|
||||
5 + int((max(pair_index - 1, 0) / pair_total) * 80),
|
||||
),
|
||||
)
|
||||
progress_state.update(
|
||||
{
|
||||
"progress": progress,
|
||||
"pair_index": pair_index,
|
||||
"pair_total": pair_total,
|
||||
"pair_label": task_label,
|
||||
"pair_started_monotonic": time.monotonic(),
|
||||
"message": f"[{engine_code}/{profile}] Running {pair_index}/{pair_total}: {task_label}",
|
||||
}
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"{engine_title} pair {pair_index}/{pair_total} started: {task_label} "
|
||||
f"(work_dir={event.get('work_dir')})"
|
||||
),
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress_state["progress"],
|
||||
message=progress_state["message"],
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type == "pair_finished":
|
||||
success = bool(event.get("success"))
|
||||
returncode = int(event.get("returncode") or 0)
|
||||
progress = min(
|
||||
90,
|
||||
max(
|
||||
int(progress_state["progress"] or 5),
|
||||
5 + int((max(pair_index, 0) / pair_total) * 80) if success else int(progress_state["progress"] or 5),
|
||||
),
|
||||
)
|
||||
progress_state.update(
|
||||
{
|
||||
"progress": progress,
|
||||
"pair_index": pair_index,
|
||||
"pair_total": pair_total,
|
||||
"pair_label": task_label,
|
||||
"pair_started_monotonic": None,
|
||||
}
|
||||
)
|
||||
if success:
|
||||
progress_state["message"] = f"[{engine_code}/{profile}] Finished {pair_index}/{pair_total}: {task_label}"
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"{engine_title} pair {pair_index}/{pair_total} completed: {task_label}",
|
||||
)
|
||||
else:
|
||||
error_text = str(event.get("error") or "").strip()
|
||||
timeout_note = " (timeout)" if returncode == -1 else ""
|
||||
progress_state["message"] = f"[{engine_code}/{profile}] Failed {pair_index}/{pair_total}: {task_label}"
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
(
|
||||
f"{engine_title} pair {pair_index}/{pair_total} failed{timeout_note}: "
|
||||
f"{task_label} (rc={returncode})"
|
||||
f"{f', error={error_text}' if error_text else ''}"
|
||||
),
|
||||
)
|
||||
if pair_index < pair_total:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"{engine_title} will continue with the next pair ({pair_index + 1}/{pair_total}).",
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress_state["progress"],
|
||||
message=progress_state["message"],
|
||||
)
|
||||
|
||||
async def _task_keepalive():
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
message = str(progress_state.get("message") or f"[{engine_code}/{profile}] Running in WSL...")
|
||||
started_monotonic = progress_state.get("pair_started_monotonic")
|
||||
if isinstance(started_monotonic, (int, float)):
|
||||
elapsed_seconds = max(0, int(time.monotonic() - float(started_monotonic)))
|
||||
message = f"{message} (elapsed={elapsed_seconds}s)"
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=5,
|
||||
message=f"[{engine_code}/{profile}] Running in WSL...",
|
||||
progress=int(progress_state.get("progress") or 5),
|
||||
message=message,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[keepalive] WARNING: failed to update task {job.task_id}: {exc}")
|
||||
|
||||
progress_task = asyncio.create_task(_consume_progress())
|
||||
keepalive_task = asyncio.create_task(_task_keepalive())
|
||||
try:
|
||||
result = await asyncio.to_thread(engine.run, request)
|
||||
@@ -1942,6 +2078,8 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await progress_queue.put(None)
|
||||
await progress_task
|
||||
|
||||
detail = result.detail or {}
|
||||
|
||||
@@ -1949,7 +2087,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"ISCE2 run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}",
|
||||
f"{engine_title} run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}",
|
||||
)
|
||||
|
||||
for invalid in detail.get("invalid_candidates", []) or []:
|
||||
@@ -2062,16 +2200,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"Auto-published ISCE2 results from {len(output_dirs)} directory(s). "
|
||||
f"Auto-published {engine_title} results from {len(output_dirs)} directory(s). "
|
||||
f"processed={publish_result.get('processed', 0)} "
|
||||
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
|
||||
),
|
||||
)
|
||||
if int(publish_result.get("processed", 0) or 0) <= 0:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
(
|
||||
f"No publishable {engine_title} result bundle was detected under "
|
||||
f"{len(output_dirs)} output director"
|
||||
f"{'y' if len(output_dirs) == 1 else 'ies'}."
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"Auto-publish ISCE2 results failed: {exc}",
|
||||
f"Auto-publish {engine_title} results failed: {exc}",
|
||||
)
|
||||
|
||||
if result.success:
|
||||
@@ -2080,7 +2228,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
f"[{engine_code}/{profile}] 完成 — "
|
||||
f"[{engine_code}/{profile}] 完成，"
|
||||
f"成功 {result.pairs_processed} 对，失败 {result.pairs_failed} 对"
|
||||
),
|
||||
)
|
||||
@@ -2095,6 +2243,22 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
await _handle_queued_engine_run(
|
||||
job,
|
||||
engine_title="ISCE2",
|
||||
fallback_timeout_seconds=settings.ISCE2_PER_TASK_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_pyint_run(job: SystemJobORM) -> None:
|
||||
await _handle_queued_engine_run(
|
||||
job,
|
||||
engine_title="PyINT",
|
||||
fallback_timeout_seconds=settings.PYINT_DEFAULT_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_water_geocode(job: SystemJobORM) -> None:
|
||||
"""单景 SAR 地理编码 job handler（多视 + 地理编码 + 辐射定标）。"""
|
||||
from .water_service import run_geocoding_workflow, WATER_RESULTS_DIR
|
||||
@@ -2860,6 +3024,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_IDL_RUN_IMPORT: _handle_idl_run_import,
|
||||
JOB_TYPE_IDL_RUN_DINSAR: _handle_idl_run_dinsar,
|
||||
JOB_TYPE_ISCE2_RUN: _handle_isce2_run,
|
||||
JOB_TYPE_PYINT_RUN: _handle_pyint_run,
|
||||
JOB_TYPE_WATER_GEOCODE: _handle_water_geocode,
|
||||
JOB_TYPE_WATER_FLOOD: _handle_water_flood,
|
||||
JOB_TYPE_WATER_DETECT: _handle_water_detect,
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
"""PyINT input-asset resolution and materialization helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from ..config import settings
|
||||
from .orbit_converter import get_source_orbit_inventory
|
||||
from .pyint_service import (
|
||||
discover_lt1_archives,
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
validate_pyint_root_dir,
|
||||
)
|
||||
|
||||
|
||||
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
|
||||
VALID_ORBIT_POLICIES = {"validate_only", "require_txt", "stage_txt"}
|
||||
VALID_PRECISE_ORBIT_MODES = {"replace", "replace_and_validate"}
|
||||
|
||||
|
||||
def _utc_now_text() -> str:
|
||||
return datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
|
||||
def _normalize_path(path: Any) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _copy_json_safe(value: Any) -> Any:
|
||||
return json.loads(json.dumps(value, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
def _normalize_lt1_satellite(value: Any) -> str:
|
||||
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
|
||||
if "LT1A" in text:
|
||||
return "LT1A"
|
||||
if "LT1B" in text:
|
||||
return "LT1B"
|
||||
if text in {"A", "LTA"}:
|
||||
return "LT1A"
|
||||
if text in {"B", "LTB"}:
|
||||
return "LT1B"
|
||||
return ""
|
||||
|
||||
|
||||
def _infer_satellite_from_archives(paths: List[str]) -> str:
|
||||
satellites = {
|
||||
satellite
|
||||
for path in paths
|
||||
for satellite in [_normalize_lt1_satellite(os.path.basename(path))]
|
||||
if satellite
|
||||
}
|
||||
if len(satellites) == 1:
|
||||
return next(iter(satellites))
|
||||
return ""
|
||||
|
||||
|
||||
def _get_dem_mode() -> str:
|
||||
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
|
||||
if raw_mode not in VALID_DEM_MODES:
|
||||
return "local_fabdem"
|
||||
return raw_mode
|
||||
|
||||
|
||||
def _get_orbit_policy() -> str:
|
||||
raw_policy = str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower()
|
||||
if raw_policy not in VALID_ORBIT_POLICIES:
|
||||
return "require_txt"
|
||||
return raw_policy
|
||||
|
||||
|
||||
def _get_orbit_pool_root() -> str:
|
||||
explicit = _normalize_path(getattr(settings, "PYINT_ORBIT_POOL_TXT", ""))
|
||||
if explicit:
|
||||
return explicit
|
||||
return _normalize_path(settings.ORBIT_POOL_ENVI)
|
||||
|
||||
|
||||
def get_pyint_precise_orbit_bridge_summary() -> Dict[str, Any]:
|
||||
mode = str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower()
|
||||
if mode not in VALID_PRECISE_ORBIT_MODES:
|
||||
mode = "replace"
|
||||
return {
|
||||
"enabled": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ENABLED", True)),
|
||||
"mode": mode,
|
||||
"strict": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_STRICT", True)),
|
||||
"validate_with_orb_filt": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False)),
|
||||
"backup": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_BACKUP", True)),
|
||||
"orb_filt_degree": max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5)),
|
||||
}
|
||||
|
||||
|
||||
def _prepared_dem_variants(value: Any) -> List[str]:
|
||||
text = str(value or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return []
|
||||
|
||||
normalized = _normalize_path(text)
|
||||
if not normalized:
|
||||
return []
|
||||
|
||||
candidates = [normalized]
|
||||
root, ext = os.path.splitext(normalized)
|
||||
if ext.lower() == ".wgs84":
|
||||
candidates.append(root)
|
||||
elif not ext:
|
||||
candidates.append(normalized + ".wgs84")
|
||||
|
||||
unique: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
item = _normalize_path(candidate)
|
||||
if not item or item in seen:
|
||||
continue
|
||||
seen.add(item)
|
||||
unique.append(item)
|
||||
return unique
|
||||
|
||||
|
||||
def _resolve_prepared_dem_path() -> Dict[str, str]:
|
||||
explicit_value = getattr(settings, "PYINT_PREPARED_DEM_PATH", "")
|
||||
explicit_candidates = _prepared_dem_variants(explicit_value)
|
||||
if str(explicit_value or "").strip():
|
||||
for candidate in explicit_candidates:
|
||||
if os.path.isfile(candidate):
|
||||
return {
|
||||
"path": candidate,
|
||||
"resolved_from": "explicit",
|
||||
}
|
||||
return {
|
||||
"path": "",
|
||||
"resolved_from": "explicit",
|
||||
}
|
||||
|
||||
sources = [
|
||||
("isce2_dem_path", getattr(settings, "ISCE2_DEM_PATH", "")),
|
||||
("idl_dinsar_dem_base_file", getattr(settings, "IDL_DINSAR_DEM_BASE_FILE", "")),
|
||||
]
|
||||
for source_name, raw_value in sources:
|
||||
for candidate in _prepared_dem_variants(raw_value):
|
||||
if os.path.isfile(candidate):
|
||||
return {
|
||||
"path": candidate,
|
||||
"resolved_from": source_name,
|
||||
}
|
||||
return {
|
||||
"path": "",
|
||||
"resolved_from": "",
|
||||
}
|
||||
|
||||
|
||||
def _inspect_prepared_dem_path(path: Any) -> Dict[str, Any]:
|
||||
normalized = _normalize_path(path)
|
||||
if not normalized:
|
||||
return {
|
||||
"path": "",
|
||||
"exists": False,
|
||||
"kind": "",
|
||||
"gamma_par_path": "",
|
||||
"gamma_par_exists": False,
|
||||
"xml_path": "",
|
||||
"xml_exists": False,
|
||||
"hdr_path": "",
|
||||
"hdr_exists": False,
|
||||
"vrt_path": "",
|
||||
"vrt_exists": False,
|
||||
"open_path": "",
|
||||
}
|
||||
|
||||
gamma_par_path = normalized + ".par"
|
||||
xml_path = normalized + ".xml"
|
||||
hdr_path = normalized + ".hdr"
|
||||
vrt_path = normalized + ".vrt"
|
||||
|
||||
path_exists = os.path.isfile(normalized)
|
||||
gamma_par_exists = os.path.isfile(gamma_par_path)
|
||||
xml_exists = os.path.isfile(xml_path)
|
||||
hdr_exists = os.path.isfile(hdr_path)
|
||||
vrt_exists = os.path.isfile(vrt_path)
|
||||
|
||||
kind = ""
|
||||
open_path = ""
|
||||
if path_exists and gamma_par_exists:
|
||||
kind = "gamma_ready"
|
||||
open_path = normalized
|
||||
elif path_exists and (xml_exists or hdr_exists or vrt_exists):
|
||||
kind = "source_dem"
|
||||
open_path = vrt_path if vrt_exists else normalized
|
||||
|
||||
return {
|
||||
"path": normalized,
|
||||
"exists": path_exists,
|
||||
"kind": kind,
|
||||
"gamma_par_path": gamma_par_path,
|
||||
"gamma_par_exists": gamma_par_exists,
|
||||
"xml_path": xml_path,
|
||||
"xml_exists": xml_exists,
|
||||
"hdr_path": hdr_path,
|
||||
"hdr_exists": hdr_exists,
|
||||
"vrt_path": vrt_path,
|
||||
"vrt_exists": vrt_exists,
|
||||
"open_path": open_path,
|
||||
}
|
||||
|
||||
|
||||
def get_pyint_dem_summary() -> Dict[str, Any]:
|
||||
mode = _get_dem_mode()
|
||||
strict = bool(getattr(settings, "PYINT_DEM_STRICT", True))
|
||||
cache_root = _normalize_path(settings.PYINT_DEM_ROOT)
|
||||
fabdem_root = _normalize_path(getattr(settings, "PYINT_FABDEM_ROOT", ""))
|
||||
prepared_dem_resolution = _resolve_prepared_dem_path()
|
||||
prepared_dem_info = _inspect_prepared_dem_path(prepared_dem_resolution.get("path"))
|
||||
opentopo_dem_type = str(getattr(settings, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") or "SRTMGL1").strip() or "SRTMGL1"
|
||||
opentopo_api_key = str(getattr(settings, "PYINT_OPENTOPO_API_KEY", "") or "").strip()
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
source_root = ""
|
||||
source_exists = False
|
||||
if mode == "local_fabdem":
|
||||
source_root = fabdem_root
|
||||
source_exists = bool(source_root and os.path.isdir(source_root))
|
||||
elif mode == "prepared_file":
|
||||
source_root = str(prepared_dem_info.get("path") or "")
|
||||
source_exists = bool(prepared_dem_info.get("exists"))
|
||||
cache_root_exists = bool(cache_root and os.path.isdir(cache_root))
|
||||
|
||||
if mode == "local_fabdem":
|
||||
if not fabdem_root:
|
||||
message = "未配置 PYINT_FABDEM_ROOT。"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif not os.path.isdir(fabdem_root):
|
||||
message = f"本地 FABDEM 根目录不存在: {fabdem_root}"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif mode == "opentopo":
|
||||
if not opentopo_api_key:
|
||||
message = "DEM 策略为 OpenTopography，但未配置 PYINT_OPENTOPO_API_KEY。"
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif mode == "prepared_file":
|
||||
if not prepared_dem_info.get("path"):
|
||||
if prepared_dem_resolution.get("resolved_from") == "explicit":
|
||||
message = "PYINT_PREPARED_DEM_PATH 已配置，但目标文件不存在。"
|
||||
else:
|
||||
message = (
|
||||
"未配置 PYINT_PREPARED_DEM_PATH，且未能从 ISCE2_DEM_PATH / "
|
||||
"IDL_DINSAR_DEM_BASE_FILE 解析现有 DEM。"
|
||||
)
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
elif prepared_dem_info.get("kind") not in {"gamma_ready", "source_dem"}:
|
||||
message = (
|
||||
"现有 DEM 缺少可识别 sidecar，至少需要同名 .par，或 .xml/.hdr/.vrt 中的一个: "
|
||||
+ str(prepared_dem_info.get("path") or "")
|
||||
)
|
||||
if strict:
|
||||
blockers.append(message)
|
||||
else:
|
||||
warnings.append(message)
|
||||
|
||||
if not cache_root:
|
||||
blockers.append("未配置 PYINT_DEM_ROOT。")
|
||||
elif not cache_root_exists:
|
||||
warnings.append(f"DEM 缓存目录当前不存在，运行时将尝试创建: {cache_root}")
|
||||
|
||||
status = "ok"
|
||||
if blockers:
|
||||
status = "blocked"
|
||||
elif warnings:
|
||||
status = "warning"
|
||||
|
||||
if mode == "local_fabdem":
|
||||
detail = "使用本地 FABDEM 瓦片目录，由 PyINT 在 DEMDIR 中生成运行期 DEM。"
|
||||
elif mode == "prepared_file":
|
||||
if prepared_dem_info.get("kind") == "gamma_ready":
|
||||
detail = "使用现有 Gamma DEM，运行时将直接注入到 PyINT 模板。"
|
||||
else:
|
||||
detail = "使用现有系统 DEM，运行时将按任务覆盖区裁剪并转换为本次任务的 Gamma DEM。"
|
||||
else:
|
||||
detail = f"使用 OpenTopography 在线 DEM 源，DEM 类型为 {opentopo_dem_type}。"
|
||||
|
||||
return {
|
||||
"mode": mode,
|
||||
"strict": strict,
|
||||
"source_root": source_root,
|
||||
"source_exists": source_exists,
|
||||
"cache_root": cache_root,
|
||||
"cache_root_exists": cache_root_exists,
|
||||
"fabdem_root": fabdem_root,
|
||||
"prepared_dem_path": str(prepared_dem_info.get("path") or ""),
|
||||
"prepared_dem_resolved_from": str(prepared_dem_resolution.get("resolved_from") or ""),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"prepared_dem_open_path": str(prepared_dem_info.get("open_path") or ""),
|
||||
"prepared_dem_support": {
|
||||
"gamma_par_exists": bool(prepared_dem_info.get("gamma_par_exists")),
|
||||
"xml_exists": bool(prepared_dem_info.get("xml_exists")),
|
||||
"hdr_exists": bool(prepared_dem_info.get("hdr_exists")),
|
||||
"vrt_exists": bool(prepared_dem_info.get("vrt_exists")),
|
||||
},
|
||||
"opentopo_dem_type": opentopo_dem_type,
|
||||
"opentopo_api_key_configured": bool(opentopo_api_key),
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": not blockers,
|
||||
}
|
||||
|
||||
|
||||
def _load_orbit_inventory() -> Dict[str, Any]:
|
||||
pool_root = _get_orbit_pool_root()
|
||||
if not pool_root:
|
||||
return {
|
||||
"pool_root": "",
|
||||
"pool_exists": False,
|
||||
"files": {},
|
||||
"warnings": ["未配置 PYINT_ORBIT_POOL_TXT，且 ORBIT_POOL_ENVI 为空。"],
|
||||
}
|
||||
if not os.path.isdir(pool_root):
|
||||
return {
|
||||
"pool_root": pool_root,
|
||||
"pool_exists": False,
|
||||
"files": {},
|
||||
"warnings": [f"轨道池目录不存在: {pool_root}"],
|
||||
}
|
||||
|
||||
inventory = get_source_orbit_inventory(pool_root, recursive=True)
|
||||
return {
|
||||
"pool_root": pool_root,
|
||||
"pool_exists": True,
|
||||
"files": inventory.get("files", {}),
|
||||
"warnings": list(inventory.get("errors", []) or []),
|
||||
"duplicate_count": int(inventory.get("duplicate_count", 0) or 0),
|
||||
}
|
||||
|
||||
|
||||
def get_pyint_orbit_context() -> Dict[str, Any]:
|
||||
return _load_orbit_inventory()
|
||||
|
||||
|
||||
def _resolve_orbit_file(
|
||||
*,
|
||||
role: str,
|
||||
satellite: str,
|
||||
date_text: str,
|
||||
pool_root: str,
|
||||
orbit_files: Dict[str, Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
satellite_text = _normalize_lt1_satellite(satellite)
|
||||
normalized_date = str(date_text or "").strip()
|
||||
expected_name = (
|
||||
f"{satellite_text}_GpsData_GAS_C_{normalized_date}.txt"
|
||||
if satellite_text and normalized_date
|
||||
else ""
|
||||
)
|
||||
result: Dict[str, Any] = {
|
||||
"role": role,
|
||||
"satellite": satellite_text,
|
||||
"date": normalized_date,
|
||||
"expected_name": expected_name,
|
||||
"pool_root": pool_root,
|
||||
"resolved": False,
|
||||
"path": "",
|
||||
"resolution_method": "",
|
||||
"staged_path": "",
|
||||
}
|
||||
if not satellite_text:
|
||||
result["error"] = f"{role} 场景未能识别 LT-1 卫星型号。"
|
||||
return result
|
||||
if not normalized_date:
|
||||
result["error"] = f"{role} 场景未能识别成像日期。"
|
||||
return result
|
||||
|
||||
stem = os.path.splitext(expected_name)[0]
|
||||
item = orbit_files.get(stem)
|
||||
if item and os.path.isfile(item.get("path", "")):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(item["path"]),
|
||||
"resolution_method": "indexed_pool_scan",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
direct_candidate = os.path.join(pool_root, satellite_text, expected_name)
|
||||
if os.path.isfile(direct_candidate):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(direct_candidate),
|
||||
"resolution_method": "direct_satellite_subdir",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
flat_candidate = os.path.join(pool_root, expected_name)
|
||||
if os.path.isfile(flat_candidate):
|
||||
result.update(
|
||||
{
|
||||
"resolved": True,
|
||||
"path": _normalize_path(flat_candidate),
|
||||
"resolution_method": "direct_pool_root",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
result["error"] = f"轨道池中缺少 {expected_name}"
|
||||
return result
|
||||
|
||||
|
||||
def resolve_pyint_task_input_assets(
|
||||
task_dir: str,
|
||||
*,
|
||||
dem_summary: Dict[str, Any] | None = None,
|
||||
orbit_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
task_dir = _normalize_path(task_dir)
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
pair_meta = task_identity["pair_meta"]
|
||||
archives = discover_lt1_archives(task_dir)
|
||||
master_archives = list(archives.get("master", []) or [])
|
||||
slave_archives = list(archives.get("slave", []) or [])
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
|
||||
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
|
||||
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
orbit_policy = _get_orbit_policy()
|
||||
orbit_context = orbit_context or get_pyint_orbit_context()
|
||||
orbit_pool_root = orbit_context.get("pool_root", "")
|
||||
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
|
||||
orbit_files = orbit_context.get("files", {}) or {}
|
||||
|
||||
orbit_warnings: List[str] = []
|
||||
if orbit_context.get("warnings"):
|
||||
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
|
||||
|
||||
master_orbit = _resolve_orbit_file(
|
||||
role="master",
|
||||
satellite=master_satellite,
|
||||
date_text=master_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
slave_orbit = _resolve_orbit_file(
|
||||
role="slave",
|
||||
satellite=slave_satellite,
|
||||
date_text=slave_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
|
||||
for orbit_item in (master_orbit, slave_orbit):
|
||||
if orbit_item.get("resolved"):
|
||||
continue
|
||||
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} 轨道缺失").strip()
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(message)
|
||||
else:
|
||||
blockers.append(message)
|
||||
|
||||
if not orbit_pool_root:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append("轨道池未配置，当前仅记录警告。")
|
||||
else:
|
||||
blockers.append("轨道池未配置。")
|
||||
elif not orbit_pool_exists:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
else:
|
||||
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
|
||||
warnings.extend(orbit_warnings)
|
||||
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
}
|
||||
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
orbits_summary = {
|
||||
"policy": orbit_policy,
|
||||
"pool_root": orbit_pool_root,
|
||||
"pool_exists": orbit_pool_exists,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": orbit_warnings,
|
||||
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
}
|
||||
|
||||
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
|
||||
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
|
||||
|
||||
return {
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"task_dir": task_dir,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archive_counts": {
|
||||
"master": len(master_archives),
|
||||
"slave": len(slave_archives),
|
||||
},
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": allow_submit,
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbit_resolution": {
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
},
|
||||
"input_assets": {
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbits": orbits_summary,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
orbit_context = get_pyint_orbit_context()
|
||||
|
||||
warnings: List[str] = list(dem_summary.get("warnings") or [])
|
||||
blockers: List[str] = list(dem_summary.get("blockers") or [])
|
||||
task_summaries: List[Dict[str, Any]] = []
|
||||
resolved_task_count = 0
|
||||
missing_task_count = 0
|
||||
|
||||
for task_dir in validation.get("task_dirs", []) or []:
|
||||
task_summary = resolve_pyint_task_input_assets(
|
||||
task_dir,
|
||||
dem_summary=dem_summary,
|
||||
orbit_context=orbit_context,
|
||||
)
|
||||
task_summaries.append(task_summary)
|
||||
if task_summary.get("warnings"):
|
||||
warnings.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["warnings"]
|
||||
)
|
||||
if task_summary.get("blockers"):
|
||||
blockers.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["blockers"]
|
||||
)
|
||||
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
|
||||
resolved_task_count += 1
|
||||
else:
|
||||
missing_task_count += 1
|
||||
|
||||
allow_submit = not blockers
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
return {
|
||||
"root_dir": validation["root_dir"],
|
||||
"mode": validation["mode"],
|
||||
"task_count": len(task_summaries),
|
||||
"selected_task_count": len(task_summaries),
|
||||
"allow_submit": allow_submit,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"invalid_candidates": validation.get("invalid_candidates", []),
|
||||
"dem": dem_summary,
|
||||
"orbits": {
|
||||
"policy": _get_orbit_policy(),
|
||||
"pool_root": orbit_context.get("pool_root", ""),
|
||||
"pool_exists": bool(orbit_context.get("pool_exists")),
|
||||
"resolved_task_count": resolved_task_count,
|
||||
"missing_task_count": missing_task_count,
|
||||
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
|
||||
"warnings": list(orbit_context.get("warnings") or []),
|
||||
},
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
@@ -0,0 +1,409 @@
|
||||
"""Helpers for integrating the external PyINT workflow."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
|
||||
from .wsl_service import run_wsl_command
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
DEFAULT_RANGE_LOOKS = 2
|
||||
DEFAULT_AZIMUTH_LOOKS = 2
|
||||
DEFAULT_PARALLEL_WORKERS = 1
|
||||
MAX_LOOKS = 32
|
||||
MAX_PARALLEL_WORKERS = 16
|
||||
|
||||
_DATE_TOKEN_RE = re.compile(r"(20\d{6})")
|
||||
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class PyintCheck:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str = ""
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class PyintEnvironmentReport:
|
||||
overall_ok: bool
|
||||
checks: List[PyintCheck] = field(default_factory=list)
|
||||
message: str = ""
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
return get_env_text(name, default) or default
|
||||
|
||||
|
||||
def _read_bool_env(name: str, default: bool = False) -> bool:
|
||||
return read_bool_env(name, default)
|
||||
|
||||
|
||||
def normalize_date_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
match = _DATE_TOKEN_RE.search(re.sub(r"\D", "", text))
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = _DATE_TOKEN_RE.search(text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def slugify_text(value: Any, *, default: str = "item", max_len: int = 96) -> str:
|
||||
text = _SAFE_TEXT_RE.sub("_", str(value or "").strip()).strip("._")
|
||||
if not text:
|
||||
text = default
|
||||
return text[:max_len]
|
||||
|
||||
|
||||
def build_project_name(pair_key: str, run_key: str) -> str:
|
||||
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
normalized = os.path.normpath(text)
|
||||
if normalized.startswith("/"):
|
||||
return normalized.replace("\\", "/")
|
||||
if normalized.startswith("\\\\"):
|
||||
return ""
|
||||
drive, tail = os.path.splitdrive(normalized)
|
||||
if not drive:
|
||||
return normalized.replace("\\", "/")
|
||||
drive_letter = drive.rstrip(":").lower()
|
||||
normalized_tail = tail.replace("\\", "/")
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
def to_wsl_path(path: str) -> str:
|
||||
return windows_path_to_wsl_mount(path)
|
||||
|
||||
|
||||
def quote_shell(value: str) -> str:
|
||||
return shlex.quote(str(value or ""))
|
||||
|
||||
|
||||
def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
|
||||
result: Dict[str, List[str]] = {"master": [], "slave": []}
|
||||
for role in ("master", "slave"):
|
||||
role_dir = task_path / role
|
||||
if not role_dir.is_dir():
|
||||
continue
|
||||
inputs = []
|
||||
for pattern in LT1_INPUT_GLOBS:
|
||||
inputs.extend(
|
||||
str(path.resolve())
|
||||
for path in role_dir.rglob(pattern)
|
||||
if path.is_file()
|
||||
)
|
||||
result[role] = sorted(set(inputs))
|
||||
return result
|
||||
|
||||
|
||||
def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
|
||||
dates = {
|
||||
date_text
|
||||
for path in paths
|
||||
for date_text in [normalize_date_text(os.path.basename(path))]
|
||||
if date_text
|
||||
}
|
||||
if len(dates) == 1:
|
||||
return next(iter(dates))
|
||||
return ""
|
||||
|
||||
|
||||
def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
return {
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"pair_meta": pair_meta,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
}
|
||||
|
||||
|
||||
def build_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
"satelite=LT",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def validate_pyint_root_dir(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
normalized_root = os.path.normpath(os.path.abspath(str(root_dir or "").strip()))
|
||||
if not root_dir or not os.path.isdir(normalized_root):
|
||||
raise ValueError(f"PyINT root_dir does not exist or is not a directory: {root_dir}")
|
||||
|
||||
def _missing_task_subdirs(task_dir: str) -> List[str]:
|
||||
missing: List[str] = []
|
||||
for subdir in ("master", "slave"):
|
||||
if not os.path.isdir(os.path.join(task_dir, subdir)):
|
||||
missing.append(subdir)
|
||||
return missing
|
||||
|
||||
def _iter_child_dirs(directory: str):
|
||||
with os.scandir(directory) as entries:
|
||||
child_dirs = [entry for entry in entries if entry.is_dir()]
|
||||
child_dirs.sort(key=lambda entry: entry.name.lower())
|
||||
return child_dirs
|
||||
|
||||
if not _missing_task_subdirs(normalized_root):
|
||||
task_dirs = [normalized_root]
|
||||
invalid_candidates: List[Dict[str, Any]] = []
|
||||
mode = "single_task_dir"
|
||||
else:
|
||||
task_dirs = []
|
||||
invalid_candidates = []
|
||||
for entry in _iter_child_dirs(normalized_root):
|
||||
if not entry.name.lower().startswith("task_"):
|
||||
continue
|
||||
missing = _missing_task_subdirs(entry.path)
|
||||
if missing:
|
||||
invalid_candidates.append(
|
||||
{"name": entry.name, "path": entry.path, "missing_subdirs": missing}
|
||||
)
|
||||
continue
|
||||
task_dirs.append(os.path.normpath(entry.path))
|
||||
mode = "task_root_dir"
|
||||
|
||||
if not task_dirs:
|
||||
detail = ""
|
||||
if invalid_candidates:
|
||||
formatted = ", ".join(
|
||||
f"{item['name']} missing {','.join(item['missing_subdirs'])}"
|
||||
for item in invalid_candidates[:5]
|
||||
)
|
||||
detail = f" Invalid candidates: {formatted}."
|
||||
raise ValueError(
|
||||
"PyINT root_dir must be either a single task directory containing "
|
||||
"'master' and 'slave', or a parent directory containing valid Task_* subdirectories."
|
||||
f"{detail}"
|
||||
)
|
||||
|
||||
selected_count = int(num_to_process or 0)
|
||||
if selected_count > 0:
|
||||
task_dirs = task_dirs[:selected_count]
|
||||
|
||||
return {
|
||||
"root_dir": normalized_root,
|
||||
"mode": mode,
|
||||
"task_dirs": task_dirs,
|
||||
"task_count": len(task_dirs),
|
||||
"invalid_candidates": invalid_candidates,
|
||||
}
|
||||
|
||||
|
||||
def resolve_time_baseline_days(master_date: str, slave_date: str, pair_meta: Dict[str, Any]) -> int:
|
||||
raw_days = pair_meta.get("time_baseline_days")
|
||||
try:
|
||||
if raw_days not in (None, ""):
|
||||
return int(raw_days)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if not master_date or not slave_date:
|
||||
return 0
|
||||
try:
|
||||
master_dt = datetime.strptime(master_date, "%Y%m%d")
|
||||
slave_dt = datetime.strptime(slave_date, "%Y%m%d")
|
||||
except ValueError:
|
||||
return 0
|
||||
return (slave_dt - master_dt).days
|
||||
|
||||
|
||||
def _gamma_prefix(gamma_env_script_wsl: str) -> str:
|
||||
script = str(gamma_env_script_wsl or "").strip()
|
||||
if not script:
|
||||
return ""
|
||||
return f". {quote_shell(script)} >/dev/null 2>&1 && "
|
||||
|
||||
|
||||
def check_pyint_environment(
|
||||
*,
|
||||
enabled: Optional[bool] = None,
|
||||
distro: Optional[str] = None,
|
||||
python_cmd: Optional[str] = None,
|
||||
pyint_home: Optional[str] = None,
|
||||
pyint_app_script: Optional[str] = None,
|
||||
template_root: Optional[str] = None,
|
||||
work_root: Optional[str] = None,
|
||||
output_root: Optional[str] = None,
|
||||
dem_root: Optional[str] = None,
|
||||
gamma_env_script: Optional[str] = None,
|
||||
smoke_test: Optional[bool] = None,
|
||||
) -> PyintEnvironmentReport:
|
||||
enabled_value = _read_bool_env("PYINT_ENABLED", False) if enabled is None else bool(enabled)
|
||||
if not enabled_value:
|
||||
return PyintEnvironmentReport(
|
||||
overall_ok=False,
|
||||
checks=[PyintCheck(name="PYINT_ENABLED", ok=False, detail="PYINT_ENABLED=false")],
|
||||
message="PyINT is disabled. Set PYINT_ENABLED=true to enable it.",
|
||||
)
|
||||
|
||||
distro_value = str(distro or _read_env("PYINT_WSL_DISTRO", settings.ISCE2_WSL_DISTRO)).strip()
|
||||
python_value = str(python_cmd or _read_env("PYINT_WSL_PYTHON", settings.ISCE2_PYTHON)).strip()
|
||||
pyint_home_wsl = to_wsl_path(str(pyint_home or _read_env("PYINT_HOME", "")))
|
||||
pyint_app_wsl = to_wsl_path(str(pyint_app_script or _read_env("PYINT_APP_SCRIPT", "")))
|
||||
template_root_wsl = to_wsl_path(str(template_root or _read_env("PYINT_TEMPLATE_ROOT", "")))
|
||||
work_root_wsl = to_wsl_path(str(work_root or _read_env("PYINT_WORK_ROOT", "")))
|
||||
output_root_wsl = to_wsl_path(str(output_root or _read_env("PYINT_OUTPUT_ROOT", "")))
|
||||
dem_root_wsl = to_wsl_path(str(dem_root or _read_env("PYINT_DEM_ROOT", "")))
|
||||
gamma_env_wsl = to_wsl_path(str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")))
|
||||
smoke_enabled = _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) if smoke_test is None else bool(smoke_test)
|
||||
precise_orbit_enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True)
|
||||
|
||||
checks: List[PyintCheck] = []
|
||||
|
||||
def add(name: str, ok: bool, detail: str = "", skipped: bool = False) -> None:
|
||||
checks.append(PyintCheck(name=name, ok=ok, detail=detail, skipped=skipped))
|
||||
|
||||
rc, out, err = run_wsl_command("echo pyint_alive", distro=distro_value, timeout=15)
|
||||
wsl_ok = rc == 0 and "pyint_alive" in out
|
||||
add("WSL distro", wsl_ok, out or err or distro_value)
|
||||
|
||||
if not wsl_ok:
|
||||
return PyintEnvironmentReport(
|
||||
overall_ok=False,
|
||||
checks=checks,
|
||||
message=f"WSL distro is unavailable: {distro_value}",
|
||||
)
|
||||
|
||||
rc, out, err = run_wsl_command(
|
||||
f"{quote_shell(python_value)} --version",
|
||||
distro=distro_value,
|
||||
timeout=15,
|
||||
)
|
||||
add("WSL Python", rc == 0, out or err or python_value)
|
||||
|
||||
if pyint_home_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -d {quote_shell(pyint_home_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("PYINT_HOME", rc == 0 and "ok" in out, pyint_home_wsl or err)
|
||||
else:
|
||||
add("PYINT_HOME", False, "PYINT_HOME is empty")
|
||||
|
||||
if pyint_app_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -f {quote_shell(pyint_app_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("pyintApp.py", rc == 0 and "ok" in out, pyint_app_wsl or err)
|
||||
else:
|
||||
add("pyintApp.py", False, "PYINT_APP_SCRIPT is empty")
|
||||
|
||||
for name, path_text in (
|
||||
("PYINT_TEMPLATE_ROOT", template_root_wsl),
|
||||
("PYINT_WORK_ROOT", work_root_wsl),
|
||||
("PYINT_OUTPUT_ROOT", output_root_wsl),
|
||||
("PYINT_DEM_ROOT", dem_root_wsl),
|
||||
):
|
||||
if not path_text:
|
||||
add(name, False, f"{name} is empty")
|
||||
continue
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -d {quote_shell(path_text)} && test -w {quote_shell(path_text)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add(name, rc == 0 and "ok" in out, path_text or err)
|
||||
|
||||
if gamma_env_wsl:
|
||||
rc, out, err = run_wsl_command(
|
||||
f"test -f {quote_shell(gamma_env_wsl)} && echo ok",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add("GAMMA env script", rc == 0 and "ok" in out, gamma_env_wsl or err)
|
||||
else:
|
||||
add("GAMMA env script", True, "Not configured; using current PATH", skipped=True)
|
||||
|
||||
gamma_prefix = _gamma_prefix(gamma_env_wsl)
|
||||
for name, command_name in (
|
||||
("GAMMA LT1 import", "LT1_import_SLC_from_zipfiles1"),
|
||||
("GAMMA geocode_back", "geocode_back"),
|
||||
):
|
||||
rc, out, err = run_wsl_command(
|
||||
gamma_prefix + f"command -v {quote_shell(command_name)}",
|
||||
distro=distro_value,
|
||||
timeout=10,
|
||||
)
|
||||
add(name, rc == 0 and bool(out.strip()), out or err or command_name)
|
||||
|
||||
helper_path = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "pyint_pipeline"
|
||||
/ "apply_lt1_precise_orbit.py"
|
||||
)
|
||||
if precise_orbit_enabled:
|
||||
add("LT1 precise orbit bridge helper", helper_path.is_file(), str(helper_path))
|
||||
else:
|
||||
add("LT1 precise orbit bridge helper", True, "Skipped", skipped=True)
|
||||
|
||||
if smoke_enabled:
|
||||
smoke_cmd = (
|
||||
f"export PYTHONPATH={quote_shell(pyint_home_wsl)}:$PYTHONPATH && "
|
||||
+ gamma_prefix
|
||||
+ f"{quote_shell(python_value)} {quote_shell(pyint_app_wsl)} -h >/dev/null"
|
||||
)
|
||||
rc, out, err = run_wsl_command(smoke_cmd, distro=distro_value, timeout=60)
|
||||
add("PyINT smoke test", rc == 0, out or err or "pyintApp.py -h")
|
||||
else:
|
||||
add("PyINT smoke test", True, "Skipped", skipped=True)
|
||||
|
||||
required_checks = [check for check in checks if not check.skipped]
|
||||
overall_ok = all(check.ok for check in required_checks)
|
||||
failed_names = [check.name for check in required_checks if not check.ok]
|
||||
message = "All PyINT checks passed." if overall_ok else f"Failed checks: {', '.join(failed_names)}"
|
||||
return PyintEnvironmentReport(overall_ok=overall_ok, checks=checks, message=message)
|
||||
@@ -54,12 +54,55 @@ def get_unpack_config() -> Dict[str, Any]:
|
||||
minimum=1,
|
||||
maximum=32,
|
||||
),
|
||||
"max_files_per_run": module.parse_int(
|
||||
env.get("UNPACK_MAX_FILES_PER_RUN"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
),
|
||||
"max_runtime_minutes": module.parse_int(
|
||||
env.get("UNPACK_MAX_RUNTIME_MINUTES"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
async def run_unpack_task(task_id: str):
|
||||
def _normalize_unpack_run_limits(raw_config: Optional[Dict[str, Any]]) -> Dict[str, int]:
|
||||
if not isinstance(raw_config, dict):
|
||||
return {}
|
||||
|
||||
module = _load_unpack_module()
|
||||
normalized: Dict[str, int] = {}
|
||||
|
||||
if raw_config.get("max_files_per_run") is not None:
|
||||
normalized["max_files_per_run"] = module.parse_int(
|
||||
raw_config.get("max_files_per_run"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
)
|
||||
if raw_config.get("max_runtime_minutes") is not None:
|
||||
normalized["max_runtime_minutes"] = module.parse_int(
|
||||
raw_config.get("max_runtime_minutes"),
|
||||
default=0,
|
||||
minimum=0,
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def build_unpack_run_config(overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
config = get_unpack_config()
|
||||
config.update(_normalize_unpack_run_limits(overrides))
|
||||
return config
|
||||
|
||||
|
||||
async def run_unpack_task(task_id: str, task_config: Optional[Dict[str, Any]] = None):
|
||||
module = _load_unpack_module()
|
||||
loop = asyncio.get_running_loop()
|
||||
config_overrides = _normalize_unpack_run_limits(task_config)
|
||||
if not config_overrides:
|
||||
task_record = await task_service.get_task(task_id)
|
||||
config_overrides = _normalize_unpack_run_limits(getattr(task_record, "params", None))
|
||||
|
||||
def _submit(coro):
|
||||
try:
|
||||
@@ -88,14 +131,19 @@ async def run_unpack_task(task_id: str):
|
||||
module.run_unpack_job,
|
||||
log_callback=log_cb,
|
||||
progress_callback=progress_cb,
|
||||
config_overrides=config_overrides or None,
|
||||
)
|
||||
|
||||
if not result:
|
||||
result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0}
|
||||
result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0, "remaining": 0, "message": "completed"}
|
||||
|
||||
summary = (
|
||||
"Unpack complete: processed {processed}, failed {failed}, skipped {skipped}"
|
||||
).format(**result)
|
||||
summary = "Unpack complete: processed {processed}, failed {failed}, skipped {skipped}".format(**result)
|
||||
remaining = int(result.get("remaining") or 0)
|
||||
if remaining > 0:
|
||||
summary = f"{summary}, remaining {remaining}"
|
||||
message_text = str(result.get("message") or "").strip()
|
||||
if message_text and message_text != "completed":
|
||||
summary = f"{summary} ({message_text})"
|
||||
await task_service.update_task(task_id, status="COMPLETED", progress=100, message=summary)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(task_id, status="FAILED", message=f"Unpack failed: {exc}")
|
||||
|
||||
Reference in New Issue
Block a user