Integrate GF3 SARscape flood workflow
This commit is contained in:
@@ -0,0 +1,843 @@
|
||||
"""Run GF3 SARscape production and clean native intermediate files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from ..config import settings, split_env_paths
|
||||
from .gf3_native_inventory_service import (
|
||||
NATIVE_MANIFEST_NAME,
|
||||
POLARIZATION_PRIORITY,
|
||||
scan_gf3_sarscape_native_roots,
|
||||
)
|
||||
from .gf3_standardize_service import STANDARD_MANIFEST_NAME
|
||||
|
||||
LogCallback = Callable[[str, str], None]
|
||||
ProgressCallback = Callable[[int, str], None]
|
||||
|
||||
SUPPORTED_WRAPPER_INPUT_EXTS = (".tar.gz", ".tgz", ".meta.xml")
|
||||
CLEANUP_MANIFEST_NAME = "gf3_cleanup_manifest.json"
|
||||
INTERMEDIATE_DIR_NAMES = {
|
||||
".gf3_extract",
|
||||
".gf3_extract.tmp",
|
||||
"temp",
|
||||
"tmp",
|
||||
"work",
|
||||
}
|
||||
KEEP_FILE_NAMES = {
|
||||
NATIVE_MANIFEST_NAME,
|
||||
CLEANUP_MANIFEST_NAME,
|
||||
"gf3_sarscape_cli.log",
|
||||
}
|
||||
INTERMEDIATE_SUFFIXES = (
|
||||
".par",
|
||||
".par_command",
|
||||
".trace",
|
||||
".working",
|
||||
".working_warning",
|
||||
".workinggetcornerfromslantrangeimage_dem",
|
||||
".txt",
|
||||
".list",
|
||||
".listhv",
|
||||
".listunknown",
|
||||
".shp",
|
||||
".shx",
|
||||
".dbf",
|
||||
".prj",
|
||||
)
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
return str(value or "").strip().strip('"').strip("'")
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.with_name(f".{path.name}.tmp")
|
||||
with tmp_path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
data = json.load(stream)
|
||||
return data if isinstance(data, dict) else None
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
|
||||
text = _clean_text(value) or default
|
||||
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
|
||||
return safe or default
|
||||
|
||||
|
||||
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in values or []:
|
||||
text = _clean_text(raw)
|
||||
if not text:
|
||||
continue
|
||||
path = Path(os.path.normpath(text)).resolve()
|
||||
key = str(path).lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if path.is_dir():
|
||||
roots.append(path)
|
||||
else:
|
||||
missing.append(str(path))
|
||||
return roots, missing
|
||||
|
||||
|
||||
def _resolve_config_path(raw: Any) -> Path | None:
|
||||
text = _clean_text(raw)
|
||||
if not text:
|
||||
return None
|
||||
return Path(os.path.normpath(text)).resolve()
|
||||
|
||||
|
||||
def _fallback_wrapper_exe() -> Path | None:
|
||||
candidate = (
|
||||
Path(settings.PROJECT_ROOT)
|
||||
/ ".codex_tmp"
|
||||
/ "GF3_L1A_To_L2_pipeline"
|
||||
/ "dist"
|
||||
/ "windows"
|
||||
/ "gf3wrapper.exe"
|
||||
)
|
||||
return candidate.resolve() if candidate.is_file() else None
|
||||
|
||||
|
||||
def _wrapper_exe_path(value: str | None = None) -> Path:
|
||||
configured = _resolve_config_path(value or settings.GF3_SARSCAPE_WRAPPER_EXE)
|
||||
path = configured or _fallback_wrapper_exe()
|
||||
if path is None:
|
||||
raise FileNotFoundError("GF3 SARscape wrapper is not configured.")
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"GF3 SARscape wrapper does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _dem_path(value: str | None = None) -> Path:
|
||||
path = _resolve_config_path(value or settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH)
|
||||
if path is None:
|
||||
raise FileNotFoundError("GF3 SARscape DEM path is not configured.")
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"GF3 SARscape DEM path does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _idlrt_path(value: str | None = None) -> Path | None:
|
||||
path = _resolve_config_path(value or settings.GF3_SARSCAPE_IDLRT_PATH)
|
||||
if path is None:
|
||||
return None
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"GF3 SARscape idlrt.exe does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _requested_polarizations(value: str | None = None) -> list[str]:
|
||||
raw = _clean_text(value or settings.GF3_SARSCAPE_POLARIZATIONS or "HH,HV")
|
||||
items: list[str] = []
|
||||
for token in re.split(r"[,;\s]+", raw):
|
||||
pol = token.strip().upper()
|
||||
if not pol:
|
||||
continue
|
||||
if pol not in items:
|
||||
items.append(pol)
|
||||
return items or ["HH", "HV"]
|
||||
|
||||
|
||||
def _archive_exts_for_wrapper(archive_exts: list[str] | tuple[str, ...] | None) -> list[str]:
|
||||
ordered: list[str] = []
|
||||
for raw_ext in archive_exts or []:
|
||||
ext = _clean_text(raw_ext).lower()
|
||||
if not ext:
|
||||
continue
|
||||
if not ext.startswith("."):
|
||||
ext = "." + ext
|
||||
if ext in SUPPORTED_WRAPPER_INPUT_EXTS and ext not in ordered:
|
||||
ordered.append(ext)
|
||||
if not ordered:
|
||||
ordered.extend((".tar.gz", ".tgz"))
|
||||
return sorted(ordered, key=len, reverse=True)
|
||||
|
||||
|
||||
def _input_ext(path: Path, exts: list[str]) -> str | None:
|
||||
name = path.name.lower()
|
||||
for ext in exts:
|
||||
if name.endswith(ext):
|
||||
return ext
|
||||
return None
|
||||
|
||||
|
||||
def _scene_name_from_input(path: Path) -> str:
|
||||
name = path.name
|
||||
lower_name = name.lower()
|
||||
for ext in SUPPORTED_WRAPPER_INPUT_EXTS:
|
||||
if lower_name.endswith(ext):
|
||||
return name[: -len(ext)]
|
||||
return path.stem
|
||||
|
||||
|
||||
def discover_gf3_sarscape_inputs(
|
||||
source_dirs: list[str] | tuple[str, ...] | None,
|
||||
*,
|
||||
archive_exts: list[str] | tuple[str, ...] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Find wrapper-supported GF3 source inputs in configured archive pools."""
|
||||
roots, missing_roots = _resolve_existing_dirs(source_dirs)
|
||||
exts = _archive_exts_for_wrapper(archive_exts)
|
||||
inputs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for root in roots:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
ext = _input_ext(path, exts)
|
||||
if not ext:
|
||||
continue
|
||||
resolved = path.resolve()
|
||||
key = str(resolved).lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
inputs.append(
|
||||
{
|
||||
"path": str(resolved),
|
||||
"scene_name": _scene_name_from_input(path),
|
||||
"ext": ext,
|
||||
"source_root": str(root),
|
||||
}
|
||||
)
|
||||
|
||||
inputs.sort(key=lambda item: str(item.get("path") or "").lower())
|
||||
return {
|
||||
"source_roots": [str(path) for path in roots],
|
||||
"missing_roots": missing_roots,
|
||||
"archive_exts": exts,
|
||||
"input_count": len(inputs),
|
||||
"inputs": inputs,
|
||||
}
|
||||
|
||||
|
||||
def _is_nonempty_file(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_file() and path.stat().st_size > 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _completed_geo_product(scene_dir: Path, polarization: str) -> Path | None:
|
||||
lower_pol = polarization.lower()
|
||||
try:
|
||||
entries = list(scene_dir.iterdir())
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
for path in entries:
|
||||
name = path.name.lower()
|
||||
if not name.endswith(f"_{lower_pol}_geo.sml"):
|
||||
continue
|
||||
data_file = Path(str(path)[: -len(".sml")])
|
||||
if _is_nonempty_file(path) and _is_nonempty_file(data_file):
|
||||
return data_file
|
||||
return None
|
||||
|
||||
|
||||
def _scene_complete(scene_dir: Path, polarizations: list[str]) -> bool:
|
||||
if not scene_dir.is_dir():
|
||||
return False
|
||||
return all(_completed_geo_product(scene_dir, pol) is not None for pol in polarizations)
|
||||
|
||||
|
||||
def _missing_geo_polarizations(scene_dir: Path, polarizations: list[str]) -> list[str]:
|
||||
if not scene_dir.is_dir():
|
||||
return list(polarizations)
|
||||
return [pol for pol in polarizations if _completed_geo_product(scene_dir, pol) is None]
|
||||
|
||||
|
||||
def _compact_failure_text(text: str, *, max_chars: int = 1000) -> str:
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
compact = " | ".join(lines)
|
||||
if len(compact) <= max_chars:
|
||||
return compact
|
||||
return compact[: max_chars - 3].rstrip() + "..."
|
||||
|
||||
|
||||
def _read_failure_file(path: Path) -> str | None:
|
||||
try:
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
return None
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _scene_failure_hint(scene_dir: Path) -> str | None:
|
||||
work_dir = scene_dir / "work"
|
||||
candidates = [
|
||||
work_dir / "Process.working_error",
|
||||
work_dir / "Process.trace_cerr.txt",
|
||||
scene_dir / "gf3_sarscape_cli.log",
|
||||
]
|
||||
for path in candidates:
|
||||
text = _read_failure_file(path)
|
||||
if text:
|
||||
compact = _compact_failure_text(text)
|
||||
if compact:
|
||||
return f"{path.name}: {compact}"
|
||||
|
||||
process_log = work_dir / "Process.log"
|
||||
text = _read_failure_file(process_log)
|
||||
if not text:
|
||||
return None
|
||||
important = [
|
||||
line.strip()
|
||||
for line in text.splitlines()
|
||||
if "ERROR" in line.upper() or "[EC:" in line.upper() or "PARAMETER FILE READ ERROR" in line.upper()
|
||||
]
|
||||
if important:
|
||||
return f"{process_log.name}: {_compact_failure_text(chr(10).join(important[-8:]))}"
|
||||
return None
|
||||
|
||||
|
||||
def _format_missing_output_error(scene_dir: Path, polarizations: list[str], returncode: int) -> str:
|
||||
missing = _missing_geo_polarizations(scene_dir, polarizations)
|
||||
missing_text = ",".join(missing) if missing else "unknown"
|
||||
if returncode == 0:
|
||||
base = f"wrapper returned 0 but required _geo outputs are missing: {missing_text}"
|
||||
else:
|
||||
base = f"wrapper return code {returncode}; missing _geo outputs: {missing_text}"
|
||||
hint = _scene_failure_hint(scene_dir)
|
||||
return f"{base}; {hint}" if hint else base
|
||||
|
||||
|
||||
def _emit_log(callback: LogCallback | None, level: str, message: str) -> None:
|
||||
if callback:
|
||||
callback(level, message)
|
||||
|
||||
|
||||
def _emit_progress(callback: ProgressCallback | None, progress: int, message: str) -> None:
|
||||
if callback:
|
||||
callback(max(0, min(100, int(progress))), message)
|
||||
|
||||
|
||||
def _run_wrapper_command(
|
||||
cmd: list[str],
|
||||
*,
|
||||
cwd: Path,
|
||||
env: dict[str, str],
|
||||
timeout_seconds: int | None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout_seconds if timeout_seconds and timeout_seconds > 0 else None,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _log_completed_process_output(
|
||||
completed: subprocess.CompletedProcess[str],
|
||||
*,
|
||||
log_callback: LogCallback | None,
|
||||
line_limit: int = 300,
|
||||
) -> list[str]:
|
||||
output = completed.stdout or ""
|
||||
lines = [line.rstrip() for line in output.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
clipped = False
|
||||
display_lines = lines
|
||||
if len(lines) > line_limit:
|
||||
clipped = True
|
||||
head_count = max(1, line_limit // 2)
|
||||
tail_count = max(1, line_limit - head_count)
|
||||
display_lines = lines[:head_count] + [f"... clipped {len(lines) - line_limit} wrapper log lines ..."] + lines[-tail_count:]
|
||||
|
||||
for line in display_lines:
|
||||
_emit_log(log_callback, "INFO", f"[gf3wrapper] {line}")
|
||||
if clipped:
|
||||
_emit_log(log_callback, "WARNING", f"GF3 wrapper output was clipped to {line_limit} log lines.")
|
||||
return lines[-20:]
|
||||
|
||||
|
||||
def run_gf3_sarscape_production(
|
||||
*,
|
||||
source_dirs: list[str] | None = None,
|
||||
native_root: str | None = None,
|
||||
wrapper_exe: str | None = None,
|
||||
dem_path: str | None = None,
|
||||
idlrt_path: str | None = None,
|
||||
polarizations: str | None = None,
|
||||
archive_exts: list[str] | None = None,
|
||||
max_archives_per_run: int | None = None,
|
||||
timeout_seconds: int | None = None,
|
||||
keep_extracted: bool | None = None,
|
||||
log_callback: LogCallback | None = None,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the external GF3 SARscape wrapper for pending raw archives."""
|
||||
source_dirs = source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
|
||||
native_roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
native_root_text = _clean_text(native_root or (native_roots[0] if native_roots else ""))
|
||||
if not source_dirs:
|
||||
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
if not native_root_text:
|
||||
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
native_root_path = Path(os.path.normpath(native_root_text)).resolve()
|
||||
|
||||
native_root_path.mkdir(parents=True, exist_ok=True)
|
||||
wrapper_path = _wrapper_exe_path(wrapper_exe)
|
||||
dem = _dem_path(dem_path)
|
||||
idlrt = _idlrt_path(idlrt_path)
|
||||
pols = _requested_polarizations(polarizations)
|
||||
pol_text = ",".join(pols)
|
||||
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
|
||||
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
|
||||
inputs = discovery.get("inputs") or []
|
||||
max_to_process = int(max_archives_per_run or 0)
|
||||
timeout = int(timeout_seconds or 0)
|
||||
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
|
||||
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape source roots: {source_dirs}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape native root: {native_root_path}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
|
||||
|
||||
if not inputs:
|
||||
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
|
||||
return {
|
||||
"ok": True,
|
||||
"found_count": 0,
|
||||
"processed_count": 0,
|
||||
"skipped_count": 0,
|
||||
"failed_count": 0,
|
||||
"deferred_count": 0,
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"results": [],
|
||||
}
|
||||
|
||||
runtime_dir = native_root_path / ".gf3_runtime"
|
||||
runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
config_path = runtime_dir / "gf3wrapper.json"
|
||||
env = os.environ.copy()
|
||||
if idlrt is not None:
|
||||
env["IDLRT_PATH"] = str(idlrt)
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
deferred = 0
|
||||
results: list[dict[str, Any]] = []
|
||||
total = len(inputs)
|
||||
|
||||
for idx, input_info in enumerate(inputs):
|
||||
input_path = Path(str(input_info.get("path") or "")).resolve()
|
||||
scene_name = str(input_info.get("scene_name") or _scene_name_from_input(input_path))
|
||||
scene_dir = native_root_path / scene_name
|
||||
progress = 5 + int((idx / max(total, 1)) * 60)
|
||||
_emit_progress(progress_callback, progress, f"GF3 SARscape checking {idx + 1}/{total}: {scene_name}")
|
||||
|
||||
if _scene_complete(scene_dir, pols):
|
||||
skipped += 1
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped_complete",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if max_to_process > 0 and processed + failed >= max_to_process:
|
||||
deferred += 1
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "deferred",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
cmd = [
|
||||
str(wrapper_path),
|
||||
"-config",
|
||||
str(config_path),
|
||||
"-input",
|
||||
str(input_path),
|
||||
"-output",
|
||||
str(native_root_path),
|
||||
"-dem",
|
||||
str(dem),
|
||||
"-pol",
|
||||
pol_text,
|
||||
f"-keep-extracted={str(keep).lower()}",
|
||||
]
|
||||
if idlrt is not None:
|
||||
cmd.extend(["-idlrt", str(idlrt)])
|
||||
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape processing {scene_name}: {input_path}")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
completed = _run_wrapper_command(
|
||||
cmd,
|
||||
cwd=wrapper_path.parent,
|
||||
env=env,
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
output_tail = _log_completed_process_output(completed, log_callback=log_callback)
|
||||
elapsed_seconds = round(time.monotonic() - started, 3)
|
||||
output_complete = _scene_complete(scene_dir, pols)
|
||||
if completed.returncode == 0 and output_complete:
|
||||
processed += 1
|
||||
status = "processed"
|
||||
error = None
|
||||
elif output_complete:
|
||||
processed += 1
|
||||
status = "processed_with_warning"
|
||||
error = f"wrapper return code {completed.returncode}"
|
||||
_emit_log(log_callback, "WARNING", f"GF3 wrapper returned {completed.returncode}, but output is complete: {scene_name}")
|
||||
else:
|
||||
failed += 1
|
||||
status = "failed"
|
||||
error = _format_missing_output_error(scene_dir, pols, int(completed.returncode))
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape failed for {scene_name}: {error}")
|
||||
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": status,
|
||||
"returncode": int(completed.returncode),
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"output_complete": output_complete,
|
||||
"error": error,
|
||||
"output_tail": output_tail,
|
||||
}
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
failed += 1
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape timed out for {scene_name}: {exc}")
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "failed",
|
||||
"error": f"timeout after {timeout}s",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape exception for {scene_name}: {exc}")
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
_emit_progress(progress_callback, 70, "GF3 SARscape production stage finished.")
|
||||
return {
|
||||
"ok": failed == 0,
|
||||
"found_count": total,
|
||||
"processed_count": processed,
|
||||
"skipped_count": skipped,
|
||||
"failed_count": failed,
|
||||
"deferred_count": deferred,
|
||||
"native_root": str(native_root_path),
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
def _is_relative_to(path: Path, parent: Path) -> bool:
|
||||
try:
|
||||
path.resolve().relative_to(parent.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _entry_size(path: Path) -> int:
|
||||
try:
|
||||
if path.is_file():
|
||||
return int(path.stat().st_size)
|
||||
if path.is_dir():
|
||||
total = 0
|
||||
for current, _dir_names, file_names in os.walk(path):
|
||||
for file_name in file_names:
|
||||
file_path = Path(current) / file_name
|
||||
try:
|
||||
total += int(file_path.stat().st_size)
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
except OSError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _is_final_geo_asset_file(path: Path) -> bool:
|
||||
name = path.name.lower()
|
||||
if name in KEEP_FILE_NAMES or name.endswith(".log"):
|
||||
return True
|
||||
return (
|
||||
name.endswith("_geo")
|
||||
or name.endswith("_geo.hdr")
|
||||
or name.endswith("_geo.sml")
|
||||
or name.endswith("_geo.ovr")
|
||||
or name.endswith("_geo.aux.xml")
|
||||
or name.endswith("_geo.kml")
|
||||
or name.endswith("_geo_ql.tif")
|
||||
or name.endswith("_geo_ql.kml")
|
||||
)
|
||||
|
||||
|
||||
def _is_known_intermediate_file(path: Path) -> bool:
|
||||
name = path.name.lower()
|
||||
if _is_final_geo_asset_file(path):
|
||||
return False
|
||||
if any(token in name for token in ("_slc", "_ml", "_filt")):
|
||||
return True
|
||||
if name.endswith(INTERMEDIATE_SUFFIXES):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _standard_manifest_path(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
|
||||
batch_name = _safe_slug(scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date"), default="unknown_batch")
|
||||
scene_name = _safe_slug(scene_manifest.get("scene_name"), default="unknown_scene")
|
||||
return storage_root / batch_name / scene_name / STANDARD_MANIFEST_NAME
|
||||
|
||||
|
||||
def _standard_manifest_allows_cleanup(scene_manifest: dict[str, Any], storage_root: Path) -> bool:
|
||||
manifest = _read_json(_standard_manifest_path(scene_manifest, storage_root))
|
||||
if not manifest:
|
||||
return False
|
||||
return str(manifest.get("status") or "").upper() == "DONE"
|
||||
|
||||
|
||||
def _assert_safe_delete(target: Path, scene_dir: Path, native_roots: list[Path]) -> None:
|
||||
resolved_target = target.resolve()
|
||||
resolved_scene = scene_dir.resolve()
|
||||
if resolved_target == resolved_scene:
|
||||
raise RuntimeError(f"Refusing to delete scene directory itself: {resolved_target}")
|
||||
if not _is_relative_to(resolved_target, resolved_scene):
|
||||
raise RuntimeError(f"Refusing to delete outside scene directory: {resolved_target}")
|
||||
if not any(_is_relative_to(resolved_target, root) for root in native_roots):
|
||||
raise RuntimeError(f"Refusing to delete outside GF3 native roots: {resolved_target}")
|
||||
|
||||
|
||||
def _cleanup_scene_intermediates(
|
||||
scene_manifest: dict[str, Any],
|
||||
*,
|
||||
native_roots: list[Path],
|
||||
dry_run: bool,
|
||||
) -> dict[str, Any]:
|
||||
scene_dir = Path(str(scene_manifest.get("native_dir") or "")).resolve()
|
||||
if not scene_dir.is_dir():
|
||||
return {
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped",
|
||||
"reason": "scene directory missing",
|
||||
"deleted_entries": [],
|
||||
"bytes_deleted": 0,
|
||||
}
|
||||
if not any(_is_relative_to(scene_dir, root) for root in native_roots):
|
||||
return {
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped",
|
||||
"reason": "scene directory is outside configured native roots",
|
||||
"deleted_entries": [],
|
||||
"bytes_deleted": 0,
|
||||
}
|
||||
|
||||
candidates: list[Path] = []
|
||||
for entry in sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()):
|
||||
if entry.is_dir() and entry.name.lower() in INTERMEDIATE_DIR_NAMES:
|
||||
candidates.append(entry)
|
||||
elif entry.is_file() and _is_known_intermediate_file(entry):
|
||||
candidates.append(entry)
|
||||
|
||||
deleted_entries: list[dict[str, Any]] = []
|
||||
bytes_deleted = 0
|
||||
errors: list[dict[str, str]] = []
|
||||
|
||||
for target in candidates:
|
||||
try:
|
||||
_assert_safe_delete(target, scene_dir, native_roots)
|
||||
size = _entry_size(target)
|
||||
bytes_deleted += size
|
||||
deleted_entries.append(
|
||||
{
|
||||
"path": str(target),
|
||||
"type": "directory" if target.is_dir() else "file",
|
||||
"size": size,
|
||||
}
|
||||
)
|
||||
if not dry_run:
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
else:
|
||||
target.unlink()
|
||||
except Exception as exc:
|
||||
errors.append({"path": str(target), "error": str(exc)})
|
||||
|
||||
status = "cleaned" if deleted_entries and not errors else ("error" if errors else "nothing_to_delete")
|
||||
cleanup_manifest = {
|
||||
"schema": "gf3_sarscape_cleanup.v1",
|
||||
"generated_at": _utc_now(),
|
||||
"dry_run": dry_run,
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": status,
|
||||
"bytes_deleted": bytes_deleted,
|
||||
"deleted_entries": deleted_entries,
|
||||
"errors": errors,
|
||||
"retention_policy": {
|
||||
"keep": "final *_geo native assets, quicklooks, manifests, and logs",
|
||||
"delete": "extract/temp/work directories and slc/ml/filt intermediate files",
|
||||
},
|
||||
}
|
||||
if not dry_run:
|
||||
_write_json(scene_dir / CLEANUP_MANIFEST_NAME, cleanup_manifest)
|
||||
return cleanup_manifest
|
||||
|
||||
|
||||
def cleanup_gf3_sarscape_native_pool(
|
||||
*,
|
||||
native_dirs: list[str] | None = None,
|
||||
storage_root: str | None = None,
|
||||
require_standardized: bool = True,
|
||||
dry_run: bool = False,
|
||||
max_scenes: int | None = None,
|
||||
log_callback: LogCallback | None = None,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Delete intermediate SARscape files while keeping final native _geo assets."""
|
||||
native_dirs = native_dirs if native_dirs is not None else split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
native_roots, missing_roots = _resolve_existing_dirs(native_dirs)
|
||||
if not native_roots:
|
||||
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS has no accessible directories.")
|
||||
storage = Path(os.path.normpath(storage_root or settings.GF3_STORAGE_DIRS)).resolve() if storage_root or settings.GF3_STORAGE_DIRS else None
|
||||
|
||||
inventory = scan_gf3_sarscape_native_roots([str(root) for root in native_roots], write_manifest=not dry_run)
|
||||
scenes = inventory.get("scenes") or []
|
||||
max_count = int(max_scenes or 0)
|
||||
cleaned = 0
|
||||
skipped = 0
|
||||
error_count = 0
|
||||
bytes_deleted = 0
|
||||
scene_results: list[dict[str, Any]] = []
|
||||
|
||||
for idx, scene_manifest in enumerate(scenes):
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
5 + int((idx / max(len(scenes), 1)) * 90),
|
||||
f"GF3 native cleanup checking {idx + 1}/{len(scenes)}: {scene_manifest.get('scene_name')}",
|
||||
)
|
||||
scene_status = str(scene_manifest.get("status") or "")
|
||||
if scene_status != "NATIVE_READY":
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": f"native status is {scene_status or 'UNKNOWN'}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if require_standardized:
|
||||
if storage is None:
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "GF3_STORAGE_DIRS is not configured",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if not _standard_manifest_allows_cleanup(scene_manifest, storage):
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "standard GeoTIFF manifest is not DONE",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if max_count > 0 and cleaned >= max_count:
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "max_scenes limit reached",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
result = _cleanup_scene_intermediates(scene_manifest, native_roots=native_roots, dry_run=dry_run)
|
||||
scene_results.append(result)
|
||||
bytes_deleted += int(result.get("bytes_deleted") or 0)
|
||||
if result.get("status") == "error":
|
||||
error_count += 1
|
||||
if result.get("status") in {"cleaned", "nothing_to_delete"}:
|
||||
cleaned += 1
|
||||
_emit_log(
|
||||
log_callback,
|
||||
"INFO",
|
||||
f"GF3 cleanup {result.get('status')}: {result.get('scene_name')} bytes={result.get('bytes_deleted')}",
|
||||
)
|
||||
|
||||
_emit_progress(progress_callback, 100, "GF3 native cleanup finished.")
|
||||
return {
|
||||
"ok": error_count == 0,
|
||||
"dry_run": dry_run,
|
||||
"native_roots": [str(root) for root in native_roots],
|
||||
"missing_roots": missing_roots,
|
||||
"scene_count": len(scenes),
|
||||
"cleaned_scene_count": cleaned,
|
||||
"skipped_scene_count": skipped,
|
||||
"error_scene_count": error_count,
|
||||
"bytes_deleted": bytes_deleted,
|
||||
"scenes": scene_results,
|
||||
}
|
||||
Reference in New Issue
Block a user