"""PyINT input-asset resolution and materialization helpers.""" from __future__ import annotations import json import os import shutil from datetime import datetime from typing import Any, Dict, List from ..config import settings from .orbit_converter import get_source_orbit_inventory from .pyint_service import ( discover_lt1_archives, infer_scene_date_from_archives, infer_task_identity, validate_pyint_root_dir, ) VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"} VALID_ORBIT_POLICIES = {"validate_only", "require_txt", "stage_txt"} VALID_PRECISE_ORBIT_MODES = {"replace", "replace_and_validate"} def _utc_now_text() -> str: return datetime.utcnow().isoformat(timespec="seconds") + "Z" def _normalize_path(path: Any) -> str: text = str(path or "").strip().strip('"').strip("'") if not text: return "" return os.path.normpath(os.path.abspath(text)) def _copy_json_safe(value: Any) -> Any: return json.loads(json.dumps(value, ensure_ascii=False, default=str)) def _normalize_lt1_satellite(value: Any) -> str: text = str(value or "").strip().upper().replace("-", "").replace("_", "") if "LT1A" in text: return "LT1A" if "LT1B" in text: return "LT1B" if text in {"A", "LTA"}: return "LT1A" if text in {"B", "LTB"}: return "LT1B" return "" def _infer_satellite_from_archives(paths: List[str]) -> str: satellites = { satellite for path in paths for satellite in [_normalize_lt1_satellite(os.path.basename(path))] if satellite } if len(satellites) == 1: return next(iter(satellites)) return "" def _get_dem_mode() -> str: raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower() if raw_mode not in VALID_DEM_MODES: return "local_fabdem" return raw_mode def _get_orbit_policy() -> str: raw_policy = str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower() if raw_policy not in VALID_ORBIT_POLICIES: return "require_txt" return raw_policy def _get_orbit_pool_root() -> str: explicit = _normalize_path(getattr(settings, "PYINT_ORBIT_POOL_TXT", "")) if explicit: return explicit return _normalize_path(settings.ORBIT_POOL_ENVI) def get_pyint_precise_orbit_bridge_summary() -> Dict[str, Any]: mode = str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower() if mode not in VALID_PRECISE_ORBIT_MODES: mode = "replace" return { "enabled": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ENABLED", True)), "mode": mode, "strict": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_STRICT", True)), "validate_with_orb_filt": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False)), "backup": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_BACKUP", True)), "orb_filt_degree": max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5)), } def _prepared_dem_variants(value: Any) -> List[str]: text = str(value or "").strip().strip('"').strip("'") if not text: return [] normalized = _normalize_path(text) if not normalized: return [] candidates = [normalized] root, ext = os.path.splitext(normalized) if ext.lower() == ".wgs84": candidates.append(root) elif not ext: candidates.append(normalized + ".wgs84") unique: List[str] = [] seen: set[str] = set() for candidate in candidates: item = _normalize_path(candidate) if not item or item in seen: continue seen.add(item) unique.append(item) return unique def _resolve_prepared_dem_path() -> Dict[str, str]: explicit_value = getattr(settings, "PYINT_PREPARED_DEM_PATH", "") explicit_candidates = _prepared_dem_variants(explicit_value) if str(explicit_value or "").strip(): for candidate in explicit_candidates: if os.path.isfile(candidate): return { "path": candidate, "resolved_from": "explicit", } return { "path": "", "resolved_from": "explicit", } sources = [ ("isce2_dem_path", getattr(settings, "ISCE2_DEM_PATH", "")), ("idl_dinsar_dem_base_file", getattr(settings, "IDL_DINSAR_DEM_BASE_FILE", "")), ] for source_name, raw_value in sources: for candidate in _prepared_dem_variants(raw_value): if os.path.isfile(candidate): return { "path": candidate, "resolved_from": source_name, } return { "path": "", "resolved_from": "", } def _inspect_prepared_dem_path(path: Any) -> Dict[str, Any]: normalized = _normalize_path(path) if not normalized: return { "path": "", "exists": False, "kind": "", "gamma_par_path": "", "gamma_par_exists": False, "xml_path": "", "xml_exists": False, "hdr_path": "", "hdr_exists": False, "vrt_path": "", "vrt_exists": False, "open_path": "", } gamma_par_path = normalized + ".par" xml_path = normalized + ".xml" hdr_path = normalized + ".hdr" vrt_path = normalized + ".vrt" path_exists = os.path.isfile(normalized) gamma_par_exists = os.path.isfile(gamma_par_path) xml_exists = os.path.isfile(xml_path) hdr_exists = os.path.isfile(hdr_path) vrt_exists = os.path.isfile(vrt_path) kind = "" open_path = "" if path_exists and gamma_par_exists: kind = "gamma_ready" open_path = normalized elif path_exists and (xml_exists or hdr_exists or vrt_exists): kind = "source_dem" open_path = vrt_path if vrt_exists else normalized return { "path": normalized, "exists": path_exists, "kind": kind, "gamma_par_path": gamma_par_path, "gamma_par_exists": gamma_par_exists, "xml_path": xml_path, "xml_exists": xml_exists, "hdr_path": hdr_path, "hdr_exists": hdr_exists, "vrt_path": vrt_path, "vrt_exists": vrt_exists, "open_path": open_path, } def get_pyint_dem_summary() -> Dict[str, Any]: mode = _get_dem_mode() strict = bool(getattr(settings, "PYINT_DEM_STRICT", True)) cache_root = _normalize_path(settings.PYINT_DEM_ROOT) fabdem_root = _normalize_path(getattr(settings, "PYINT_FABDEM_ROOT", "")) prepared_dem_resolution = _resolve_prepared_dem_path() prepared_dem_info = _inspect_prepared_dem_path(prepared_dem_resolution.get("path")) opentopo_dem_type = str(getattr(settings, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") or "SRTMGL1").strip() or "SRTMGL1" opentopo_api_key = str(getattr(settings, "PYINT_OPENTOPO_API_KEY", "") or "").strip() warnings: List[str] = [] blockers: List[str] = [] source_root = "" source_exists = False if mode == "local_fabdem": source_root = fabdem_root source_exists = bool(source_root and os.path.isdir(source_root)) elif mode == "prepared_file": source_root = str(prepared_dem_info.get("path") or "") source_exists = bool(prepared_dem_info.get("exists")) cache_root_exists = bool(cache_root and os.path.isdir(cache_root)) if mode == "local_fabdem": if not fabdem_root: message = "未配置 PYINT_FABDEM_ROOT。" if strict: blockers.append(message) else: warnings.append(message) elif not os.path.isdir(fabdem_root): message = f"本地 FABDEM 根目录不存在: {fabdem_root}" if strict: blockers.append(message) else: warnings.append(message) elif mode == "opentopo": if not opentopo_api_key: message = "DEM 策略为 OpenTopography，但未配置 PYINT_OPENTOPO_API_KEY。" if strict: blockers.append(message) else: warnings.append(message) elif mode == "prepared_file": if not prepared_dem_info.get("path"): if prepared_dem_resolution.get("resolved_from") == "explicit": message = "PYINT_PREPARED_DEM_PATH 已配置，但目标文件不存在。" else: message = ( "未配置 PYINT_PREPARED_DEM_PATH，且未能从 ISCE2_DEM_PATH / " "IDL_DINSAR_DEM_BASE_FILE 解析现有 DEM。" ) if strict: blockers.append(message) else: warnings.append(message) elif prepared_dem_info.get("kind") not in {"gamma_ready", "source_dem"}: message = ( "现有 DEM 缺少可识别 sidecar，至少需要同名 .par，或 .xml/.hdr/.vrt 中的一个: " + str(prepared_dem_info.get("path") or "") ) if strict: blockers.append(message) else: warnings.append(message) if not cache_root: blockers.append("未配置 PYINT_DEM_ROOT。") elif not cache_root_exists: warnings.append(f"DEM 缓存目录当前不存在，运行时将尝试创建: {cache_root}") status = "ok" if blockers: status = "blocked" elif warnings: status = "warning" if mode == "local_fabdem": detail = "使用本地 FABDEM 瓦片目录，由 PyINT 在 DEMDIR 中生成运行期 DEM。" elif mode == "prepared_file": if prepared_dem_info.get("kind") == "gamma_ready": detail = "使用现有 Gamma DEM，运行时将直接注入到 PyINT 模板。" else: detail = "使用现有系统 DEM，运行时将按任务覆盖区裁剪并转换为本次任务的 Gamma DEM。" else: detail = f"使用 OpenTopography 在线 DEM 源，DEM 类型为 {opentopo_dem_type}。" return { "mode": mode, "strict": strict, "source_root": source_root, "source_exists": source_exists, "cache_root": cache_root, "cache_root_exists": cache_root_exists, "fabdem_root": fabdem_root, "prepared_dem_path": str(prepared_dem_info.get("path") or ""), "prepared_dem_resolved_from": str(prepared_dem_resolution.get("resolved_from") or ""), "prepared_dem_kind": str(prepared_dem_info.get("kind") or ""), "prepared_dem_open_path": str(prepared_dem_info.get("open_path") or ""), "prepared_dem_support": { "gamma_par_exists": bool(prepared_dem_info.get("gamma_par_exists")), "xml_exists": bool(prepared_dem_info.get("xml_exists")), "hdr_exists": bool(prepared_dem_info.get("hdr_exists")), "vrt_exists": bool(prepared_dem_info.get("vrt_exists")), }, "opentopo_dem_type": opentopo_dem_type, "opentopo_api_key_configured": bool(opentopo_api_key), "status": status, "detail": detail, "warnings": warnings, "blockers": blockers, "allow_submit": not blockers, } def _load_orbit_inventory() -> Dict[str, Any]: pool_root = _get_orbit_pool_root() if not pool_root: return { "pool_root": "", "pool_exists": False, "files": {}, "warnings": ["未配置 PYINT_ORBIT_POOL_TXT，且 ORBIT_POOL_ENVI 为空。"], } if not os.path.isdir(pool_root): return { "pool_root": pool_root, "pool_exists": False, "files": {}, "warnings": [f"轨道池目录不存在: {pool_root}"], } inventory = get_source_orbit_inventory(pool_root, recursive=True) return { "pool_root": pool_root, "pool_exists": True, "files": inventory.get("files", {}), "warnings": list(inventory.get("errors", []) or []), "duplicate_count": int(inventory.get("duplicate_count", 0) or 0), } def get_pyint_orbit_context() -> Dict[str, Any]: return _load_orbit_inventory() def _resolve_orbit_file( *, role: str, satellite: str, date_text: str, pool_root: str, orbit_files: Dict[str, Dict[str, Any]], ) -> Dict[str, Any]: satellite_text = _normalize_lt1_satellite(satellite) normalized_date = str(date_text or "").strip() expected_name = ( f"{satellite_text}_GpsData_GAS_C_{normalized_date}.txt" if satellite_text and normalized_date else "" ) result: Dict[str, Any] = { "role": role, "satellite": satellite_text, "date": normalized_date, "expected_name": expected_name, "pool_root": pool_root, "resolved": False, "path": "", "resolution_method": "", "staged_path": "", } if not satellite_text: result["error"] = f"{role} 场景未能识别 LT-1 卫星型号。" return result if not normalized_date: result["error"] = f"{role} 场景未能识别成像日期。" return result stem = os.path.splitext(expected_name)[0] item = orbit_files.get(stem) if item and os.path.isfile(item.get("path", "")): result.update( { "resolved": True, "path": _normalize_path(item["path"]), "resolution_method": "indexed_pool_scan", } ) return result direct_candidate = os.path.join(pool_root, satellite_text, expected_name) if os.path.isfile(direct_candidate): result.update( { "resolved": True, "path": _normalize_path(direct_candidate), "resolution_method": "direct_satellite_subdir", } ) return result flat_candidate = os.path.join(pool_root, expected_name) if os.path.isfile(flat_candidate): result.update( { "resolved": True, "path": _normalize_path(flat_candidate), "resolution_method": "direct_pool_root", } ) return result result["error"] = f"轨道池中缺少 {expected_name}" return result def resolve_pyint_task_input_assets( task_dir: str, *, dem_summary: Dict[str, Any] | None = None, orbit_context: Dict[str, Any] | None = None, ) -> Dict[str, Any]: task_dir = _normalize_path(task_dir) task_identity = infer_task_identity(task_dir) pair_meta = task_identity["pair_meta"] archives = discover_lt1_archives(task_dir) master_archives = list(archives.get("master", []) or []) slave_archives = list(archives.get("slave", []) or []) warnings: List[str] = [] blockers: List[str] = [] master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives) slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives) master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives) slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives) if not master_archives: blockers.append("master/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") if not slave_archives: blockers.append("slave/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") if not master_date: blockers.append("未能识别主影像日期。") if not slave_date: blockers.append("未能识别从影像日期。") orbit_policy = _get_orbit_policy() orbit_context = orbit_context or get_pyint_orbit_context() orbit_pool_root = orbit_context.get("pool_root", "") orbit_pool_exists = bool(orbit_context.get("pool_exists")) orbit_files = orbit_context.get("files", {}) or {} orbit_warnings: List[str] = [] if orbit_context.get("warnings"): orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item) master_orbit = _resolve_orbit_file( role="master", satellite=master_satellite, date_text=master_date, pool_root=orbit_pool_root, orbit_files=orbit_files, ) slave_orbit = _resolve_orbit_file( role="slave", satellite=slave_satellite, date_text=slave_date, pool_root=orbit_pool_root, orbit_files=orbit_files, ) for orbit_item in (master_orbit, slave_orbit): if orbit_item.get("resolved"): continue message = str(orbit_item.get("error") or f"{orbit_item.get('role')} 轨道缺失").strip() if orbit_policy == "validate_only": orbit_warnings.append(message) else: blockers.append(message) if not orbit_pool_root: if orbit_policy == "validate_only": orbit_warnings.append("轨道池未配置，当前仅记录警告。") else: blockers.append("轨道池未配置。") elif not orbit_pool_exists: if orbit_policy == "validate_only": orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}") else: blockers.append(f"轨道池目录不可用: {orbit_pool_root}") warnings.extend(orbit_warnings) task_source = { "task_dir": task_dir, "task_name": task_identity["task_name"], "task_alias": task_identity["task_alias"], "pair_key": task_identity["pair_key"], "master_date": master_date, "slave_date": slave_date, "master_satellite": master_satellite, "slave_satellite": slave_satellite, "archives": { "master": master_archives, "slave": slave_archives, }, } precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() orbits_summary = { "policy": orbit_policy, "pool_root": orbit_pool_root, "pool_exists": orbit_pool_exists, "master": master_orbit, "slave": slave_orbit, "resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))), "missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")), "warnings": orbit_warnings, "stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none", "precise_orbit_bridge": precise_orbit_bridge, } dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary()) allow_submit = not blockers and bool(dem_payload.get("allow_submit", True)) return { "task_name": task_identity["task_name"], "task_alias": task_identity["task_alias"], "pair_key": task_identity["pair_key"], "task_dir": task_dir, "master_date": master_date, "slave_date": slave_date, "master_satellite": master_satellite, "slave_satellite": slave_satellite, "archive_counts": { "master": len(master_archives), "slave": len(slave_archives), }, "warnings": warnings, "blockers": blockers, "allow_submit": allow_submit, "task_source": task_source, "dem": dem_payload, "orbit_resolution": { "master": master_orbit, "slave": slave_orbit, }, "input_assets": { "task_source": task_source, "dem": dem_payload, "orbits": orbits_summary, }, } def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: validation = validate_pyint_root_dir(root_dir, num_to_process) dem_summary = get_pyint_dem_summary() orbit_context = get_pyint_orbit_context() warnings: List[str] = list(dem_summary.get("warnings") or []) blockers: List[str] = list(dem_summary.get("blockers") or []) task_summaries: List[Dict[str, Any]] = [] resolved_task_count = 0 missing_task_count = 0 for task_dir in validation.get("task_dirs", []) or []: task_summary = resolve_pyint_task_input_assets( task_dir, dem_summary=dem_summary, orbit_context=orbit_context, ) task_summaries.append(task_summary) if task_summary.get("warnings"): warnings.extend( f"{task_summary['task_alias']}: {item}" for item in task_summary["warnings"] ) if task_summary.get("blockers"): blockers.extend( f"{task_summary['task_alias']}: {item}" for item in task_summary["blockers"] ) if task_summary["input_assets"]["orbits"]["missing_count"] == 0: resolved_task_count += 1 else: missing_task_count += 1 allow_submit = not blockers precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() return { "root_dir": validation["root_dir"], "mode": validation["mode"], "task_count": len(task_summaries), "selected_task_count": len(task_summaries), "allow_submit": allow_submit, "warnings": warnings, "blockers": blockers, "invalid_candidates": validation.get("invalid_candidates", []), "dem": dem_summary, "orbits": { "policy": _get_orbit_policy(), "pool_root": orbit_context.get("pool_root", ""), "pool_exists": bool(orbit_context.get("pool_exists")), "resolved_task_count": resolved_task_count, "missing_task_count": missing_task_count, "duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0), "warnings": list(orbit_context.get("warnings") or []), }, "precise_orbit_bridge": precise_orbit_bridge, "tasks": task_summaries, } def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str: blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()] if not blockers: return "" if len(blockers) <= limit: return "; ".join(blockers) return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略" def materialize_pyint_input_assets( *, task_summary: Dict[str, Any], input_assets_dir: str, project_name: str = "", ) -> Dict[str, Any]: input_assets_dir = _normalize_path(input_assets_dir) os.makedirs(input_assets_dir, exist_ok=True) record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True)) orbits_dir = os.path.join(input_assets_dir, "orbits") dem_dir = os.path.join(input_assets_dir, "dem") if record_enabled: os.makedirs(orbits_dir, exist_ok=True) os.makedirs(dem_dir, exist_ok=True) manifest = _copy_json_safe(task_summary.get("input_assets") or {}) manifest["generated_at"] = _utc_now_text() manifest["task_name"] = task_summary.get("task_name") manifest["task_alias"] = task_summary.get("task_alias") manifest["pair_key"] = task_summary.get("pair_key") manifest["task_dir"] = task_summary.get("task_dir") manifest["allow_submit"] = bool(task_summary.get("allow_submit")) manifest["warnings"] = list(task_summary.get("warnings") or []) manifest["blockers"] = list(task_summary.get("blockers") or []) dem_summary = manifest.get("dem") or {} if project_name: dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name) manifest["dem"] = dem_summary orbits_summary = manifest.get("orbits") or {} staged_count = 0 precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() should_stage_orbits = record_enabled and ( str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt" or precise_orbit_bridge.get("enabled") ) if should_stage_orbits: for role in ("master", "slave"): orbit_item = orbits_summary.get(role) or {} orbit_path = _normalize_path(orbit_item.get("path")) expected_name = str(orbit_item.get("expected_name") or "").strip() if not orbit_item.get("resolved") or not orbit_path or not expected_name: continue target_path = os.path.join(orbits_dir, expected_name) if not os.path.exists(target_path): shutil.copy2(orbit_path, target_path) orbit_item["staged_path"] = target_path orbit_item["stage_operation"] = "copied" orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy" staged_count += 1 orbits_summary[role] = orbit_item manifest["orbits"] = orbits_summary materialized = { "input_assets_dir": input_assets_dir, "record_enabled": record_enabled, "orbits_dir": orbits_dir if record_enabled else "", "dem_dir": dem_dir if record_enabled else "", "orbits_staged_count": staged_count, "task_manifest_path": "", "dem_summary_path": "", "orbit_summary_path": "", "input_assets": manifest, } if not record_enabled: return materialized task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json") dem_summary_path = os.path.join(dem_dir, "dem_summary.json") orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json") with open(task_manifest_path, "w", encoding="utf-8") as fp: json.dump(manifest, fp, ensure_ascii=False, indent=2) fp.write("\n") with open(dem_summary_path, "w", encoding="utf-8") as fp: json.dump(dem_summary, fp, ensure_ascii=False, indent=2) fp.write("\n") with open(orbit_summary_path, "w", encoding="utf-8") as fp: json.dump(orbits_summary, fp, ensure_ascii=False, indent=2) fp.write("\n") materialized.update( { "task_manifest_path": task_manifest_path, "dem_summary_path": dem_summary_path, "orbit_summary_path": orbit_summary_path, } ) return materialized