from __future__ import annotations import json import os import re from typing import Any, Dict, Iterable, List, Optional from ..config import settings from .dinsar_naming import RUN_META_FILENAME from .dinsar_result_layout_service import ( EXECUTION_MANIFEST_FILENAME, PACKAGE_MANIFEST_FILENAME, get_run_disp_asset_paths, get_run_native_output_dir, normalize_envi_run_layout, ) _ENVI_ENGINE_CODES = {"envi", "sarscape"} _LEGACY_DISP_RE = re.compile(r"^.+_rsp_disp$", re.IGNORECASE) def _normalize_path(path: str) -> str: return os.path.normpath(os.path.abspath(str(path or "").strip())) def _load_json(path: str) -> Optional[Dict[str, Any]]: try: with open(path, "r", encoding="utf-8") as fp: payload = json.load(fp) return payload if isinstance(payload, dict) else None except Exception: return None def _iter_managed_run_dirs(root_dir: str) -> Iterable[str]: normalized_root = _normalize_path(root_dir) if not os.path.isdir(normalized_root): return for pair_name in sorted(os.listdir(normalized_root)): pair_dir = os.path.join(normalized_root, pair_name) if not os.path.isdir(pair_dir): continue runs_dir = os.path.join(pair_dir, "runs") if not os.path.isdir(runs_dir): continue for run_name in sorted(os.listdir(runs_dir)): run_dir = os.path.join(runs_dir, run_name) if os.path.isdir(run_dir): yield run_dir def _read_run_payloads(run_dir: str) -> Dict[str, Dict[str, Any]]: normalized_run_dir = _normalize_path(run_dir) payloads: Dict[str, Dict[str, Any]] = {} execution_manifest = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)) if execution_manifest: payloads["execution"] = execution_manifest run_meta = _load_json(os.path.join(normalized_run_dir, RUN_META_FILENAME)) if run_meta: payloads["run_meta"] = run_meta package_manifest = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME)) if package_manifest: payloads["package"] = package_manifest return payloads def _first_text(*values: Any) -> Optional[str]: for value in values: text = str(value or "").strip() if text: return text return None def _resolve_engine_code(payloads: Dict[str, Dict[str, Any]]) -> str: package_payload = payloads.get("package") or {} package_engine = package_payload.get("engine") if isinstance(package_payload.get("engine"), dict) else {} return ( _first_text( (payloads.get("execution") or {}).get("engine_code"), (payloads.get("run_meta") or {}).get("engine_code"), package_engine.get("code"), package_payload.get("engine_code"), ) or "" ).strip().lower() def _resolve_pair_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]: package_payload = payloads.get("package") or {} package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {} return _first_text( (payloads.get("execution") or {}).get("pair_key"), (payloads.get("run_meta") or {}).get("pair_key"), package_identity.get("pair_key"), package_payload.get("pair_key"), os.path.basename(os.path.dirname(os.path.dirname(_normalize_path(run_dir)))), ) def _resolve_run_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]: package_payload = payloads.get("package") or {} package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {} return _first_text( (payloads.get("execution") or {}).get("run_key"), (payloads.get("run_meta") or {}).get("run_key"), package_identity.get("run_key"), package_payload.get("run_key"), os.path.basename(_normalize_path(run_dir)), ) def _relpath_lower(base_dir: str, path: str) -> str: return os.path.relpath(_normalize_path(path), _normalize_path(base_dir)).replace("/", os.sep).lower() def _is_standard_disp_path(run_dir: str, path: str) -> bool: disp_paths = get_run_disp_asset_paths(run_dir) normalized_path = _normalize_path(path) return normalized_path in { _normalize_path(disp_paths["primary"]), _normalize_path(disp_paths["hdr"]), _normalize_path(disp_paths["sml"]), } def _looks_like_legacy_primary(path: str) -> bool: return bool(_LEGACY_DISP_RE.match(os.path.basename(str(path or "").strip()))) def _looks_like_envi_primary(run_dir: str, path: str) -> bool: normalized_path = _normalize_path(path) if not os.path.isfile(normalized_path): return False if _is_standard_disp_path(run_dir, normalized_path): return True return _looks_like_legacy_primary(normalized_path) def _build_source_files(primary_file: str, source_files: Optional[List[str]] = None) -> List[str]: normalized_primary = _normalize_path(primary_file) candidates: List[str] = [normalized_primary] for raw_path in source_files or []: normalized = _normalize_path(raw_path) if normalized and normalized not in candidates and os.path.isfile(normalized): candidates.append(normalized) for ext in (".hdr", ".sml"): sidecar = normalized_primary + ext if os.path.isfile(sidecar) and sidecar not in candidates: candidates.append(sidecar) return candidates def _find_standardized_run_files(run_dir: str) -> Optional[Dict[str, Any]]: disp_paths = get_run_disp_asset_paths(run_dir) primary_file = disp_paths["primary"] if not os.path.isfile(primary_file): return None return { "primary_file": _normalize_path(primary_file), "source_files": _build_source_files(primary_file), "discovery": "standard_asset", } def _iter_metadata_candidates(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Iterable[Dict[str, Any]]: execution_payload = payloads.get("execution") or {} if execution_payload: primary_file = execution_payload.get("primary_file") source_files = execution_payload.get("source_files") if isinstance(execution_payload.get("source_files"), list) else None if primary_file: yield { "primary_file": primary_file, "source_files": source_files, "discovery": "execution_manifest", } package_payload = payloads.get("package") or {} source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {} if source_payload: primary_file = source_payload.get("primary_path") if primary_file: yield { "primary_file": primary_file, "source_files": None, "discovery": "package_manifest", } def _find_latest_legacy_primary(search_dir: str) -> Optional[str]: normalized_search_dir = _normalize_path(search_dir) if not os.path.isdir(normalized_search_dir): return None candidates: List[str] = [] try: with os.scandir(normalized_search_dir) as entries: for entry in entries: try: if entry.is_file(follow_symlinks=False) and _looks_like_legacy_primary(entry.name): candidates.append(entry.path) except OSError: continue except OSError: return None if not candidates: return None candidates.sort(key=lambda item: os.path.getmtime(item), reverse=True) return _normalize_path(candidates[0]) def _discover_envi_run_files(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[Dict[str, Any]]: standardized = _find_standardized_run_files(run_dir) if standardized: return standardized for candidate in _iter_metadata_candidates(run_dir, payloads): primary_file = _normalize_path(str(candidate.get("primary_file") or "").strip()) if not _looks_like_envi_primary(run_dir, primary_file): continue return { "primary_file": primary_file, "source_files": _build_source_files(primary_file, candidate.get("source_files")), "discovery": candidate["discovery"], } search_dirs = [ _normalize_path(run_dir), get_run_native_output_dir(run_dir), ] for search_dir in search_dirs: primary_file = _find_latest_legacy_primary(search_dir) if primary_file: return { "primary_file": primary_file, "source_files": _build_source_files(primary_file), "discovery": "filesystem_scan", } return None def validate_envi_run_layout(run_dir: str) -> Dict[str, Any]: normalized_run_dir = _normalize_path(run_dir) native_output_dir = get_run_native_output_dir(normalized_run_dir) disp_paths = get_run_disp_asset_paths(normalized_run_dir) issues: List[str] = [] if not os.path.isdir(normalized_run_dir): issues.append("run_dir_missing") if not os.path.isdir(native_output_dir): issues.append("native_dir_missing") if not os.path.isfile(disp_paths["primary"]): issues.append("standard_disp_missing") execution_payload = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)) or {} if execution_payload: if _normalize_path(str(execution_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]): issues.append("execution_manifest_primary_mismatch") if _normalize_path(str(execution_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir): issues.append("execution_manifest_native_mismatch") package_payload = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME)) or {} if package_payload: source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {} if _normalize_path(str(source_payload.get("primary_path") or "")) != _normalize_path(disp_paths["primary"]): issues.append("package_manifest_primary_mismatch") if _normalize_path(str(source_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir): issues.append("package_manifest_native_mismatch") assets_payload = package_payload.get("assets") if isinstance(package_payload.get("assets"), list) else [] disp_asset = next( (item for item in assets_payload if str((item or {}).get("role") or "").strip() == "disp"), None, ) expected_relative = _relpath_lower(normalized_run_dir, disp_paths["primary"]) actual_relative = _relpath_lower( normalized_run_dir, os.path.join(normalized_run_dir, str((disp_asset or {}).get("relative_path") or "")), ) if disp_asset else "" if not disp_asset or actual_relative != expected_relative: issues.append("package_manifest_disp_asset_mismatch") run_key = _resolve_run_key(normalized_run_dir, _read_run_payloads(normalized_run_dir)) current_dir = os.path.join(os.path.dirname(os.path.dirname(normalized_run_dir)), "current") matched_pointer_count = 0 if run_key and os.path.isdir(current_dir): for name in os.listdir(current_dir): if not name.lower().endswith(".json"): continue pointer_payload = _load_json(os.path.join(current_dir, name)) if not pointer_payload: continue if str(pointer_payload.get("run_key") or "").strip() != run_key: continue matched_pointer_count += 1 if _normalize_path(str(pointer_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]): issues.append(f"current_pointer_primary_mismatch:{name}") if _normalize_path(str(pointer_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir): issues.append(f"current_pointer_native_mismatch:{name}") return { "ok": len(issues) == 0, "issues": issues, "matched_current_pointer_count": matched_pointer_count, } def inspect_envi_run_layout(run_dir: str) -> Dict[str, Any]: normalized_run_dir = _normalize_path(run_dir) payloads = _read_run_payloads(normalized_run_dir) engine_code = _resolve_engine_code(payloads) files = _discover_envi_run_files(normalized_run_dir, payloads) disp_paths = get_run_disp_asset_paths(normalized_run_dir) standardized = bool(os.path.isfile(disp_paths["primary"])) native_exists = bool(os.path.isdir(get_run_native_output_dir(normalized_run_dir))) is_envi_run = bool(files) or engine_code in _ENVI_ENGINE_CODES layout_state = "unknown" if not is_envi_run: layout_state = "not_envi" elif standardized and native_exists: layout_state = "normalized" elif files and _looks_like_legacy_primary(files["primary_file"]): layout_state = "legacy" elif standardized: layout_state = "partial_normalized" validation = validate_envi_run_layout(normalized_run_dir) if is_envi_run else {"ok": True, "issues": [], "matched_current_pointer_count": 0} needs_migration = bool( is_envi_run and ( layout_state in {"legacy", "partial_normalized"} or (layout_state == "normalized" and not validation["ok"]) ) ) return { "run_dir": normalized_run_dir, "pair_key": _resolve_pair_key(normalized_run_dir, payloads), "run_key": _resolve_run_key(normalized_run_dir, payloads), "engine_code": engine_code, "is_envi_run": is_envi_run, "layout_state": layout_state, "needs_migration": needs_migration, "primary_file": (files or {}).get("primary_file"), "source_files": (files or {}).get("source_files") or [], "discovery": (files or {}).get("discovery"), "standard_disp_exists": standardized, "native_dir_exists": native_exists, "validation": validation, } class DinsarLayoutMigrationService: def inspect_managed_envi_runs( self, *, root_dir: Optional[str] = None, pair_keys: Optional[List[str]] = None, limit: Optional[int] = None, ) -> Dict[str, Any]: normalized_root = _normalize_path(root_dir or settings.DINSAR_PRODUCT_DIR) filter_pair_keys = { str(item or "").strip() for item in pair_keys or [] if str(item or "").strip() } details: List[Dict[str, Any]] = [] inspected = 0 envi_run_count = 0 legacy_count = 0 normalized_count = 0 pending_count = 0 for run_dir in _iter_managed_run_dirs(normalized_root): inspection = inspect_envi_run_layout(run_dir) if filter_pair_keys and inspection.get("pair_key") not in filter_pair_keys: continue inspected += 1 if inspection["is_envi_run"]: envi_run_count += 1 if inspection["layout_state"] == "legacy": legacy_count += 1 if inspection["layout_state"] == "normalized": normalized_count += 1 if inspection["needs_migration"]: pending_count += 1 details.append(inspection) if limit is not None and len(details) >= int(limit): break return { "root_dir": normalized_root, "inspected_run_count": inspected, "envi_run_count": envi_run_count, "legacy_run_count": legacy_count, "normalized_run_count": normalized_count, "pending_migration_count": pending_count, "details": details, } def migrate_managed_envi_runs( self, *, root_dir: Optional[str] = None, pair_keys: Optional[List[str]] = None, limit: Optional[int] = None, dry_run: bool = False, ) -> Dict[str, Any]: inspection = self.inspect_managed_envi_runs( root_dir=root_dir, pair_keys=pair_keys, limit=limit, ) details: List[Dict[str, Any]] = [] migrated_count = 0 rewritten_count = 0 skipped_count = 0 failed_count = 0 for item in inspection["details"]: if not item["is_envi_run"]: skipped_count += 1 details.append( { **item, "status": "skipped_not_envi", } ) continue if not item["primary_file"]: failed_count += 1 details.append( { **item, "status": "failed_no_primary", "error": "Unable to resolve ENVI primary displacement file.", } ) continue if not item["needs_migration"]: skipped_count += 1 details.append( { **item, "status": "skipped_already_normalized", } ) continue if dry_run: details.append( { **item, "status": "would_migrate", } ) continue try: result = normalize_envi_run_layout( item["run_dir"], primary_file=item["primary_file"], source_files=item["source_files"], rewrite_metadata=True, ) validation = validate_envi_run_layout(item["run_dir"]) status = "migrated" if not result["promoted_files"] and not result["moved_entries"]: status = "rewritten" if validation["ok"]: if status == "rewritten": rewritten_count += 1 else: migrated_count += 1 else: status = "failed_validation" failed_count += 1 details.append( { **item, **result, "status": status, "post_validation": validation, } ) except Exception as exc: failed_count += 1 details.append( { **item, "status": "failed", "error": str(exc), } ) return { "root_dir": inspection["root_dir"], "dry_run": bool(dry_run), "inspected_run_count": inspection["inspected_run_count"], "envi_run_count": inspection["envi_run_count"], "pending_migration_count": inspection["pending_migration_count"], "migrated_count": migrated_count, "rewritten_count": rewritten_count, "skipped_count": skipped_count, "failed_count": failed_count, "details": details, } dinsar_layout_migration_service = DinsarLayoutMigrationService()