906 lines
32 KiB
Python
906 lines
32 KiB
Python
"""Run GF3 SARscape production and clean native intermediate files."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from ..config import settings, split_env_paths
|
|
from .gf3_native_inventory_service import (
|
|
NATIVE_MANIFEST_NAME,
|
|
POLARIZATION_PRIORITY,
|
|
scan_gf3_sarscape_native_roots,
|
|
)
|
|
from .gf3_standardize_service import STANDARD_MANIFEST_NAME
|
|
|
|
LogCallback = Callable[[str, str], None]
|
|
ProgressCallback = Callable[[int, str], None]
|
|
|
|
SUPPORTED_WRAPPER_INPUT_EXTS = (".tar.gz", ".tgz", ".meta.xml")
|
|
CLEANUP_MANIFEST_NAME = "gf3_cleanup_manifest.json"
|
|
INTERMEDIATE_DIR_NAMES = {
|
|
".gf3_extract",
|
|
".gf3_extract.tmp",
|
|
"temp",
|
|
"tmp",
|
|
"work",
|
|
}
|
|
KEEP_FILE_NAMES = {
|
|
NATIVE_MANIFEST_NAME,
|
|
CLEANUP_MANIFEST_NAME,
|
|
"gf3_sarscape_cli.log",
|
|
}
|
|
INTERMEDIATE_SUFFIXES = (
|
|
".par",
|
|
".par_command",
|
|
".trace",
|
|
".working",
|
|
".working_warning",
|
|
".workinggetcornerfromslantrangeimage_dem",
|
|
".txt",
|
|
".list",
|
|
".listhv",
|
|
".listunknown",
|
|
".shp",
|
|
".shx",
|
|
".dbf",
|
|
".prj",
|
|
)
|
|
|
|
|
|
def _utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _clean_text(value: Any) -> str:
|
|
return str(value or "").strip().strip('"').strip("'")
|
|
|
|
|
|
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp_path = path.with_name(f".{path.name}.tmp")
|
|
with tmp_path.open("w", encoding="utf-8") as stream:
|
|
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
|
os.replace(tmp_path, path)
|
|
|
|
|
|
def _read_json(path: Path) -> dict[str, Any] | None:
|
|
try:
|
|
with path.open("r", encoding="utf-8") as stream:
|
|
data = json.load(stream)
|
|
return data if isinstance(data, dict) else None
|
|
except (OSError, json.JSONDecodeError):
|
|
return None
|
|
|
|
|
|
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
|
|
text = _clean_text(value) or default
|
|
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
|
|
return safe or default
|
|
|
|
|
|
def _date_from_scene_name(scene_name: str) -> str | None:
|
|
match = re.search(r"(20\d{6})", str(scene_name or ""))
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
|
roots: list[Path] = []
|
|
missing: list[str] = []
|
|
seen: set[str] = set()
|
|
for raw in values or []:
|
|
text = _clean_text(raw)
|
|
if not text:
|
|
continue
|
|
path = Path(os.path.normpath(text)).resolve()
|
|
key = str(path).lower()
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
if path.is_dir():
|
|
roots.append(path)
|
|
else:
|
|
missing.append(str(path))
|
|
return roots, missing
|
|
|
|
|
|
def _resolve_config_path(raw: Any) -> Path | None:
|
|
text = _clean_text(raw)
|
|
if not text:
|
|
return None
|
|
return Path(os.path.normpath(text)).resolve()
|
|
|
|
|
|
def _fallback_wrapper_exe() -> Path | None:
|
|
candidate = (
|
|
Path(settings.PROJECT_ROOT)
|
|
/ "third_party"
|
|
/ "GF3_L1A_To_L2_pipeline"
|
|
/ "dist"
|
|
/ "windows"
|
|
/ "gf3wrapper.exe"
|
|
)
|
|
return candidate.resolve() if candidate.is_file() else None
|
|
|
|
|
|
def _wrapper_exe_path(value: str | None = None) -> Path:
|
|
configured = _resolve_config_path(value or settings.GF3_SARSCAPE_WRAPPER_EXE)
|
|
path = configured or _fallback_wrapper_exe()
|
|
if path is None:
|
|
raise FileNotFoundError("GF3 SARscape wrapper is not configured.")
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"GF3 SARscape wrapper does not exist: {path}")
|
|
return path
|
|
|
|
|
|
def _dem_path(value: str | None = None) -> Path:
|
|
path = _resolve_config_path(value or settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH)
|
|
if path is None:
|
|
raise FileNotFoundError("GF3 SARscape DEM path is not configured.")
|
|
if not path.exists():
|
|
raise FileNotFoundError(f"GF3 SARscape DEM path does not exist: {path}")
|
|
return path
|
|
|
|
|
|
def _idlrt_path(value: str | None = None) -> Path | None:
|
|
path = _resolve_config_path(value or settings.GF3_SARSCAPE_IDLRT_PATH)
|
|
if path is None:
|
|
return None
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"GF3 SARscape idlrt.exe does not exist: {path}")
|
|
return path
|
|
|
|
|
|
def _requested_polarizations(value: str | None = None) -> list[str]:
|
|
raw = _clean_text(value or settings.GF3_SARSCAPE_POLARIZATIONS or "HH,HV")
|
|
items: list[str] = []
|
|
for token in re.split(r"[,;\s]+", raw):
|
|
pol = token.strip().upper()
|
|
if not pol:
|
|
continue
|
|
if pol not in items:
|
|
items.append(pol)
|
|
return items or ["HH", "HV"]
|
|
|
|
|
|
def _archive_exts_for_wrapper(archive_exts: list[str] | tuple[str, ...] | None) -> list[str]:
|
|
ordered: list[str] = []
|
|
for raw_ext in archive_exts or []:
|
|
ext = _clean_text(raw_ext).lower()
|
|
if not ext:
|
|
continue
|
|
if not ext.startswith("."):
|
|
ext = "." + ext
|
|
if ext in SUPPORTED_WRAPPER_INPUT_EXTS and ext not in ordered:
|
|
ordered.append(ext)
|
|
if not ordered:
|
|
ordered.extend((".tar.gz", ".tgz"))
|
|
return sorted(ordered, key=len, reverse=True)
|
|
|
|
|
|
def _input_ext(path: Path, exts: list[str]) -> str | None:
|
|
name = path.name.lower()
|
|
for ext in exts:
|
|
if name.endswith(ext):
|
|
return ext
|
|
return None
|
|
|
|
|
|
def _scene_name_from_input(path: Path) -> str:
|
|
name = path.name
|
|
lower_name = name.lower()
|
|
for ext in SUPPORTED_WRAPPER_INPUT_EXTS:
|
|
if lower_name.endswith(ext):
|
|
return name[: -len(ext)]
|
|
return path.stem
|
|
|
|
|
|
def discover_gf3_sarscape_inputs(
|
|
source_dirs: list[str] | tuple[str, ...] | None,
|
|
*,
|
|
archive_exts: list[str] | tuple[str, ...] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Find wrapper-supported GF3 source inputs in configured archive pools."""
|
|
roots, missing_roots = _resolve_existing_dirs(source_dirs)
|
|
exts = _archive_exts_for_wrapper(archive_exts)
|
|
inputs: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
|
|
for root in roots:
|
|
for path in root.rglob("*"):
|
|
if not path.is_file():
|
|
continue
|
|
ext = _input_ext(path, exts)
|
|
if not ext:
|
|
continue
|
|
resolved = path.resolve()
|
|
key = str(resolved).lower()
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
inputs.append(
|
|
{
|
|
"path": str(resolved),
|
|
"scene_name": _scene_name_from_input(path),
|
|
"ext": ext,
|
|
"source_root": str(root),
|
|
}
|
|
)
|
|
|
|
inputs.sort(key=lambda item: str(item.get("path") or "").lower())
|
|
return {
|
|
"source_roots": [str(path) for path in roots],
|
|
"missing_roots": missing_roots,
|
|
"archive_exts": exts,
|
|
"input_count": len(inputs),
|
|
"inputs": inputs,
|
|
}
|
|
|
|
|
|
def _is_nonempty_file(path: Path) -> bool:
|
|
try:
|
|
return path.is_file() and path.stat().st_size > 0
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _completed_geo_product(scene_dir: Path, polarization: str) -> Path | None:
|
|
lower_pol = polarization.lower()
|
|
try:
|
|
entries = list(scene_dir.iterdir())
|
|
except OSError:
|
|
return None
|
|
|
|
for path in entries:
|
|
name = path.name.lower()
|
|
if not name.endswith(f"_{lower_pol}_geo.sml"):
|
|
continue
|
|
data_file = Path(str(path)[: -len(".sml")])
|
|
if _is_nonempty_file(path) and _is_nonempty_file(data_file):
|
|
return data_file
|
|
return None
|
|
|
|
|
|
def _scene_complete(scene_dir: Path, polarizations: list[str]) -> bool:
|
|
if not scene_dir.is_dir():
|
|
return False
|
|
return all(_completed_geo_product(scene_dir, pol) is not None for pol in polarizations)
|
|
|
|
|
|
def _standardized_scene_complete(storage_root: Path | None, scene_name: str, polarizations: list[str]) -> tuple[bool, Path | None, str]:
|
|
if storage_root is None:
|
|
return False, None, "storage_root_not_configured"
|
|
date_text = _date_from_scene_name(scene_name)
|
|
candidate_dirs = []
|
|
if date_text:
|
|
candidate_dirs.append(storage_root / _safe_slug(date_text) / _safe_slug(scene_name))
|
|
candidate_dirs.append(storage_root / "unknown_batch" / _safe_slug(scene_name))
|
|
|
|
for scene_dir in candidate_dirs:
|
|
manifest_path = scene_dir / STANDARD_MANIFEST_NAME
|
|
manifest = _read_json(manifest_path)
|
|
if manifest and str(manifest.get("status") or "").upper() in {"DONE", "PARTIAL"}:
|
|
assets = manifest.get("assets") or []
|
|
complete_pols = {
|
|
str(asset.get("polarization") or "").upper()
|
|
for asset in assets
|
|
if str(asset.get("status") or "").lower() in {"converted", "skipped"}
|
|
and _is_nonempty_file(Path(str(asset.get("path") or "")))
|
|
}
|
|
if all(str(pol or "").upper() in complete_pols for pol in polarizations):
|
|
return True, scene_dir, "standard_manifest_complete"
|
|
|
|
if scene_dir.is_dir():
|
|
if all(_is_nonempty_file(scene_dir / f"{str(pol).upper()}_L2.tif") for pol in polarizations):
|
|
return True, scene_dir, "standard_tifs_complete"
|
|
|
|
return False, None, "standardized_result_missing"
|
|
|
|
|
|
def _missing_geo_polarizations(scene_dir: Path, polarizations: list[str]) -> list[str]:
|
|
if not scene_dir.is_dir():
|
|
return list(polarizations)
|
|
return [pol for pol in polarizations if _completed_geo_product(scene_dir, pol) is None]
|
|
|
|
|
|
def _compact_failure_text(text: str, *, max_chars: int = 1000) -> str:
|
|
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
compact = " | ".join(lines)
|
|
if len(compact) <= max_chars:
|
|
return compact
|
|
return compact[: max_chars - 3].rstrip() + "..."
|
|
|
|
|
|
def _read_failure_file(path: Path) -> str | None:
|
|
try:
|
|
if not path.is_file() or path.stat().st_size <= 0:
|
|
return None
|
|
return path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def _scene_failure_hint(scene_dir: Path) -> str | None:
|
|
work_dir = scene_dir / "work"
|
|
candidates = [
|
|
work_dir / "Process.working_error",
|
|
work_dir / "Process.trace_cerr.txt",
|
|
scene_dir / "gf3_sarscape_cli.log",
|
|
]
|
|
for path in candidates:
|
|
text = _read_failure_file(path)
|
|
if text:
|
|
compact = _compact_failure_text(text)
|
|
if compact:
|
|
return f"{path.name}: {compact}"
|
|
|
|
process_log = work_dir / "Process.log"
|
|
text = _read_failure_file(process_log)
|
|
if not text:
|
|
return None
|
|
important = [
|
|
line.strip()
|
|
for line in text.splitlines()
|
|
if "ERROR" in line.upper() or "[EC:" in line.upper() or "PARAMETER FILE READ ERROR" in line.upper()
|
|
]
|
|
if important:
|
|
return f"{process_log.name}: {_compact_failure_text(chr(10).join(important[-8:]))}"
|
|
return None
|
|
|
|
|
|
def _format_missing_output_error(scene_dir: Path, polarizations: list[str], returncode: int) -> str:
|
|
missing = _missing_geo_polarizations(scene_dir, polarizations)
|
|
missing_text = ",".join(missing) if missing else "unknown"
|
|
if returncode == 0:
|
|
base = f"wrapper returned 0 but required _geo outputs are missing: {missing_text}"
|
|
else:
|
|
base = f"wrapper return code {returncode}; missing _geo outputs: {missing_text}"
|
|
hint = _scene_failure_hint(scene_dir)
|
|
return f"{base}; {hint}" if hint else base
|
|
|
|
|
|
def _emit_log(callback: LogCallback | None, level: str, message: str) -> None:
|
|
if callback:
|
|
callback(level, message)
|
|
|
|
|
|
def _emit_progress(callback: ProgressCallback | None, progress: int, message: str) -> None:
|
|
if callback:
|
|
callback(max(0, min(100, int(progress))), message)
|
|
|
|
|
|
def _run_wrapper_command(
|
|
cmd: list[str],
|
|
*,
|
|
cwd: Path,
|
|
env: dict[str, str],
|
|
timeout_seconds: int | None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
cmd,
|
|
cwd=str(cwd),
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
timeout=timeout_seconds if timeout_seconds and timeout_seconds > 0 else None,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def _log_completed_process_output(
|
|
completed: subprocess.CompletedProcess[str],
|
|
*,
|
|
log_callback: LogCallback | None,
|
|
line_limit: int = 300,
|
|
) -> list[str]:
|
|
output = completed.stdout or ""
|
|
lines = [line.rstrip() for line in output.splitlines() if line.strip()]
|
|
if not lines:
|
|
return []
|
|
|
|
clipped = False
|
|
display_lines = lines
|
|
if len(lines) > line_limit:
|
|
clipped = True
|
|
head_count = max(1, line_limit // 2)
|
|
tail_count = max(1, line_limit - head_count)
|
|
display_lines = lines[:head_count] + [f"... clipped {len(lines) - line_limit} wrapper log lines ..."] + lines[-tail_count:]
|
|
|
|
for line in display_lines:
|
|
_emit_log(log_callback, "INFO", f"[gf3wrapper] {line}")
|
|
if clipped:
|
|
_emit_log(log_callback, "WARNING", f"GF3 wrapper output was clipped to {line_limit} log lines.")
|
|
return lines[-20:]
|
|
|
|
|
|
def run_gf3_sarscape_production(
|
|
*,
|
|
source_dirs: list[str] | None = None,
|
|
native_root: str | None = None,
|
|
wrapper_exe: str | None = None,
|
|
dem_path: str | None = None,
|
|
idlrt_path: str | None = None,
|
|
polarizations: str | None = None,
|
|
archive_exts: list[str] | None = None,
|
|
max_archives_per_run: int | None = None,
|
|
timeout_seconds: int | None = None,
|
|
keep_extracted: bool | None = None,
|
|
log_callback: LogCallback | None = None,
|
|
progress_callback: ProgressCallback | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Run the external GF3 SARscape wrapper for pending raw archives."""
|
|
source_dirs = source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
|
|
native_roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
|
native_root_text = _clean_text(native_root or (native_roots[0] if native_roots else ""))
|
|
if not source_dirs:
|
|
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
|
if not native_root_text:
|
|
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
|
native_root_path = Path(os.path.normpath(native_root_text)).resolve()
|
|
|
|
native_root_path.mkdir(parents=True, exist_ok=True)
|
|
wrapper_path = _wrapper_exe_path(wrapper_exe)
|
|
dem = _dem_path(dem_path)
|
|
idlrt = _idlrt_path(idlrt_path)
|
|
pols = _requested_polarizations(polarizations)
|
|
pol_text = ",".join(pols)
|
|
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
|
|
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
|
|
inputs = discovery.get("inputs") or []
|
|
max_to_process = int(max_archives_per_run or 0)
|
|
timeout = int(timeout_seconds or 0)
|
|
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
|
|
storage_root = Path(os.path.normpath(settings.GF3_STORAGE_DIRS)).resolve() if settings.GF3_STORAGE_DIRS else None
|
|
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape source roots: {source_dirs}")
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape native root: {native_root_path}")
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape standardized root: {storage_root or '(not configured)'}")
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
|
|
|
|
if not inputs:
|
|
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
|
|
return {
|
|
"ok": True,
|
|
"found_count": 0,
|
|
"processed_count": 0,
|
|
"skipped_count": 0,
|
|
"failed_count": 0,
|
|
"deferred_count": 0,
|
|
"missing_roots": discovery.get("missing_roots") or [],
|
|
"results": [],
|
|
}
|
|
|
|
configured_runtime_dir = _clean_text(getattr(settings, "GF3_SARSCAPE_RUNTIME_DIR", ""))
|
|
runtime_dir = (
|
|
Path(os.path.normpath(configured_runtime_dir)).resolve()
|
|
if configured_runtime_dir
|
|
else native_root_path / ".gf3_runtime"
|
|
)
|
|
runtime_dir.mkdir(parents=True, exist_ok=True)
|
|
config_path = runtime_dir / "gf3wrapper.json"
|
|
env = os.environ.copy()
|
|
if idlrt is not None:
|
|
env["IDLRT_PATH"] = str(idlrt)
|
|
|
|
processed = 0
|
|
skipped = 0
|
|
failed = 0
|
|
deferred = 0
|
|
results: list[dict[str, Any]] = []
|
|
total = len(inputs)
|
|
|
|
for idx, input_info in enumerate(inputs):
|
|
input_path = Path(str(input_info.get("path") or "")).resolve()
|
|
scene_name = str(input_info.get("scene_name") or _scene_name_from_input(input_path))
|
|
scene_dir = native_root_path / scene_name
|
|
progress = 5 + int((idx / max(total, 1)) * 60)
|
|
_emit_progress(progress_callback, progress, f"GF3 SARscape checking {idx + 1}/{total}: {scene_name}")
|
|
|
|
standardized_complete, standardized_dir, standardized_reason = _standardized_scene_complete(storage_root, scene_name, pols)
|
|
if standardized_complete:
|
|
skipped += 1
|
|
_emit_log(
|
|
log_callback,
|
|
"INFO",
|
|
f"GF3 SARscape skipping {scene_name}: standardized result already exists ({standardized_reason})",
|
|
)
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"standardized_dir": str(standardized_dir) if standardized_dir else None,
|
|
"status": "skipped_standardized_complete",
|
|
"skip_reason": standardized_reason,
|
|
}
|
|
)
|
|
continue
|
|
|
|
if _scene_complete(scene_dir, pols):
|
|
skipped += 1
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "skipped_complete",
|
|
}
|
|
)
|
|
continue
|
|
|
|
if max_to_process > 0 and processed + failed >= max_to_process:
|
|
deferred += 1
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "deferred",
|
|
}
|
|
)
|
|
continue
|
|
|
|
cmd = [
|
|
str(wrapper_path),
|
|
"-config",
|
|
str(config_path),
|
|
"-input",
|
|
str(input_path),
|
|
"-output",
|
|
str(native_root_path),
|
|
"-dem",
|
|
str(dem),
|
|
"-pol",
|
|
pol_text,
|
|
f"-keep-extracted={str(keep).lower()}",
|
|
]
|
|
if idlrt is not None:
|
|
cmd.extend(["-idlrt", str(idlrt)])
|
|
|
|
_emit_log(log_callback, "INFO", f"GF3 SARscape processing {scene_name}: {input_path}")
|
|
started = time.monotonic()
|
|
try:
|
|
completed = _run_wrapper_command(
|
|
cmd,
|
|
cwd=wrapper_path.parent,
|
|
env=env,
|
|
timeout_seconds=timeout,
|
|
)
|
|
output_tail = _log_completed_process_output(completed, log_callback=log_callback)
|
|
elapsed_seconds = round(time.monotonic() - started, 3)
|
|
output_complete = _scene_complete(scene_dir, pols)
|
|
if completed.returncode == 0 and output_complete:
|
|
processed += 1
|
|
status = "processed"
|
|
error = None
|
|
elif output_complete:
|
|
processed += 1
|
|
status = "processed_with_warning"
|
|
error = f"wrapper return code {completed.returncode}"
|
|
_emit_log(log_callback, "WARNING", f"GF3 wrapper returned {completed.returncode}, but output is complete: {scene_name}")
|
|
else:
|
|
failed += 1
|
|
status = "failed"
|
|
error = _format_missing_output_error(scene_dir, pols, int(completed.returncode))
|
|
_emit_log(log_callback, "ERROR", f"GF3 SARscape failed for {scene_name}: {error}")
|
|
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"status": status,
|
|
"returncode": int(completed.returncode),
|
|
"elapsed_seconds": elapsed_seconds,
|
|
"output_complete": output_complete,
|
|
"error": error,
|
|
"output_tail": output_tail,
|
|
}
|
|
)
|
|
except subprocess.TimeoutExpired as exc:
|
|
failed += 1
|
|
_emit_log(log_callback, "ERROR", f"GF3 SARscape timed out for {scene_name}: {exc}")
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "failed",
|
|
"error": f"timeout after {timeout}s",
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
failed += 1
|
|
_emit_log(log_callback, "ERROR", f"GF3 SARscape exception for {scene_name}: {exc}")
|
|
results.append(
|
|
{
|
|
"scene_name": scene_name,
|
|
"input_path": str(input_path),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "failed",
|
|
"error": str(exc),
|
|
}
|
|
)
|
|
|
|
_emit_progress(progress_callback, 70, "GF3 SARscape production stage finished.")
|
|
return {
|
|
"ok": failed == 0,
|
|
"found_count": total,
|
|
"processed_count": processed,
|
|
"skipped_count": skipped,
|
|
"failed_count": failed,
|
|
"deferred_count": deferred,
|
|
"native_root": str(native_root_path),
|
|
"missing_roots": discovery.get("missing_roots") or [],
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def _is_relative_to(path: Path, parent: Path) -> bool:
|
|
try:
|
|
path.resolve().relative_to(parent.resolve())
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def _entry_size(path: Path) -> int:
|
|
try:
|
|
if path.is_file():
|
|
return int(path.stat().st_size)
|
|
if path.is_dir():
|
|
total = 0
|
|
for current, _dir_names, file_names in os.walk(path):
|
|
for file_name in file_names:
|
|
file_path = Path(current) / file_name
|
|
try:
|
|
total += int(file_path.stat().st_size)
|
|
except OSError:
|
|
continue
|
|
return total
|
|
except OSError:
|
|
return 0
|
|
return 0
|
|
|
|
|
|
def _is_final_geo_asset_file(path: Path) -> bool:
|
|
name = path.name.lower()
|
|
if name in KEEP_FILE_NAMES or name.endswith(".log"):
|
|
return True
|
|
return (
|
|
name.endswith("_geo")
|
|
or name.endswith("_geo.hdr")
|
|
or name.endswith("_geo.sml")
|
|
or name.endswith("_geo.ovr")
|
|
or name.endswith("_geo.aux.xml")
|
|
or name.endswith("_geo.kml")
|
|
or name.endswith("_geo_ql.tif")
|
|
or name.endswith("_geo_ql.kml")
|
|
)
|
|
|
|
|
|
def _is_known_intermediate_file(path: Path) -> bool:
|
|
name = path.name.lower()
|
|
if _is_final_geo_asset_file(path):
|
|
return False
|
|
if any(token in name for token in ("_slc", "_ml", "_filt")):
|
|
return True
|
|
if name.endswith(INTERMEDIATE_SUFFIXES):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _standard_manifest_path(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
|
|
batch_name = _safe_slug(scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date"), default="unknown_batch")
|
|
scene_name = _safe_slug(scene_manifest.get("scene_name"), default="unknown_scene")
|
|
return storage_root / batch_name / scene_name / STANDARD_MANIFEST_NAME
|
|
|
|
|
|
def _standard_manifest_allows_cleanup(scene_manifest: dict[str, Any], storage_root: Path) -> bool:
|
|
manifest = _read_json(_standard_manifest_path(scene_manifest, storage_root))
|
|
if not manifest:
|
|
return False
|
|
return str(manifest.get("status") or "").upper() == "DONE"
|
|
|
|
|
|
def _assert_safe_delete(target: Path, scene_dir: Path, native_roots: list[Path]) -> None:
|
|
resolved_target = target.resolve()
|
|
resolved_scene = scene_dir.resolve()
|
|
if resolved_target == resolved_scene:
|
|
raise RuntimeError(f"Refusing to delete scene directory itself: {resolved_target}")
|
|
if not _is_relative_to(resolved_target, resolved_scene):
|
|
raise RuntimeError(f"Refusing to delete outside scene directory: {resolved_target}")
|
|
if not any(_is_relative_to(resolved_target, root) for root in native_roots):
|
|
raise RuntimeError(f"Refusing to delete outside GF3 native roots: {resolved_target}")
|
|
|
|
|
|
def _cleanup_scene_intermediates(
|
|
scene_manifest: dict[str, Any],
|
|
*,
|
|
native_roots: list[Path],
|
|
dry_run: bool,
|
|
) -> dict[str, Any]:
|
|
scene_dir = Path(str(scene_manifest.get("native_dir") or "")).resolve()
|
|
if not scene_dir.is_dir():
|
|
return {
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "skipped",
|
|
"reason": "scene directory missing",
|
|
"deleted_entries": [],
|
|
"bytes_deleted": 0,
|
|
}
|
|
if not any(_is_relative_to(scene_dir, root) for root in native_roots):
|
|
return {
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"scene_dir": str(scene_dir),
|
|
"status": "skipped",
|
|
"reason": "scene directory is outside configured native roots",
|
|
"deleted_entries": [],
|
|
"bytes_deleted": 0,
|
|
}
|
|
|
|
candidates: list[Path] = []
|
|
for entry in sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()):
|
|
if entry.is_dir() and entry.name.lower() in INTERMEDIATE_DIR_NAMES:
|
|
candidates.append(entry)
|
|
elif entry.is_file() and _is_known_intermediate_file(entry):
|
|
candidates.append(entry)
|
|
|
|
deleted_entries: list[dict[str, Any]] = []
|
|
bytes_deleted = 0
|
|
errors: list[dict[str, str]] = []
|
|
|
|
for target in candidates:
|
|
try:
|
|
_assert_safe_delete(target, scene_dir, native_roots)
|
|
size = _entry_size(target)
|
|
bytes_deleted += size
|
|
deleted_entries.append(
|
|
{
|
|
"path": str(target),
|
|
"type": "directory" if target.is_dir() else "file",
|
|
"size": size,
|
|
}
|
|
)
|
|
if not dry_run:
|
|
if target.is_dir():
|
|
shutil.rmtree(target)
|
|
else:
|
|
target.unlink()
|
|
except Exception as exc:
|
|
errors.append({"path": str(target), "error": str(exc)})
|
|
|
|
status = "cleaned" if deleted_entries and not errors else ("error" if errors else "nothing_to_delete")
|
|
cleanup_manifest = {
|
|
"schema": "gf3_sarscape_cleanup.v1",
|
|
"generated_at": _utc_now(),
|
|
"dry_run": dry_run,
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"scene_dir": str(scene_dir),
|
|
"status": status,
|
|
"bytes_deleted": bytes_deleted,
|
|
"deleted_entries": deleted_entries,
|
|
"errors": errors,
|
|
"retention_policy": {
|
|
"keep": "final *_geo native assets, quicklooks, manifests, and logs",
|
|
"delete": "extract/temp/work directories and slc/ml/filt intermediate files",
|
|
},
|
|
}
|
|
if not dry_run:
|
|
_write_json(scene_dir / CLEANUP_MANIFEST_NAME, cleanup_manifest)
|
|
return cleanup_manifest
|
|
|
|
|
|
def cleanup_gf3_sarscape_native_pool(
|
|
*,
|
|
native_dirs: list[str] | None = None,
|
|
storage_root: str | None = None,
|
|
require_standardized: bool = True,
|
|
dry_run: bool = False,
|
|
max_scenes: int | None = None,
|
|
log_callback: LogCallback | None = None,
|
|
progress_callback: ProgressCallback | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Delete intermediate SARscape files while keeping final native _geo assets."""
|
|
native_dirs = native_dirs if native_dirs is not None else split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
|
native_roots, missing_roots = _resolve_existing_dirs(native_dirs)
|
|
if not native_roots:
|
|
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS has no accessible directories.")
|
|
storage = Path(os.path.normpath(storage_root or settings.GF3_STORAGE_DIRS)).resolve() if storage_root or settings.GF3_STORAGE_DIRS else None
|
|
|
|
inventory = scan_gf3_sarscape_native_roots([str(root) for root in native_roots], write_manifest=not dry_run)
|
|
scenes = inventory.get("scenes") or []
|
|
max_count = int(max_scenes or 0)
|
|
cleaned = 0
|
|
skipped = 0
|
|
error_count = 0
|
|
bytes_deleted = 0
|
|
scene_results: list[dict[str, Any]] = []
|
|
|
|
for idx, scene_manifest in enumerate(scenes):
|
|
_emit_progress(
|
|
progress_callback,
|
|
5 + int((idx / max(len(scenes), 1)) * 90),
|
|
f"GF3 native cleanup checking {idx + 1}/{len(scenes)}: {scene_manifest.get('scene_name')}",
|
|
)
|
|
scene_status = str(scene_manifest.get("status") or "")
|
|
if scene_status != "NATIVE_READY":
|
|
skipped += 1
|
|
scene_results.append(
|
|
{
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"status": "skipped",
|
|
"reason": f"native status is {scene_status or 'UNKNOWN'}",
|
|
}
|
|
)
|
|
continue
|
|
if require_standardized:
|
|
if storage is None:
|
|
skipped += 1
|
|
scene_results.append(
|
|
{
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"status": "skipped",
|
|
"reason": "GF3_STORAGE_DIRS is not configured",
|
|
}
|
|
)
|
|
continue
|
|
if not _standard_manifest_allows_cleanup(scene_manifest, storage):
|
|
skipped += 1
|
|
scene_results.append(
|
|
{
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"status": "skipped",
|
|
"reason": "standard GeoTIFF manifest is not DONE",
|
|
}
|
|
)
|
|
continue
|
|
if max_count > 0 and cleaned >= max_count:
|
|
skipped += 1
|
|
scene_results.append(
|
|
{
|
|
"scene_name": scene_manifest.get("scene_name"),
|
|
"status": "skipped",
|
|
"reason": "max_scenes limit reached",
|
|
}
|
|
)
|
|
continue
|
|
|
|
result = _cleanup_scene_intermediates(scene_manifest, native_roots=native_roots, dry_run=dry_run)
|
|
scene_results.append(result)
|
|
bytes_deleted += int(result.get("bytes_deleted") or 0)
|
|
if result.get("status") == "error":
|
|
error_count += 1
|
|
if result.get("status") in {"cleaned", "nothing_to_delete"}:
|
|
cleaned += 1
|
|
_emit_log(
|
|
log_callback,
|
|
"INFO",
|
|
f"GF3 cleanup {result.get('status')}: {result.get('scene_name')} bytes={result.get('bytes_deleted')}",
|
|
)
|
|
|
|
_emit_progress(progress_callback, 100, "GF3 native cleanup finished.")
|
|
return {
|
|
"ok": error_count == 0,
|
|
"dry_run": dry_run,
|
|
"native_roots": [str(root) for root in native_roots],
|
|
"missing_roots": missing_roots,
|
|
"scene_count": len(scenes),
|
|
"cleaned_scene_count": cleaned,
|
|
"skipped_scene_count": skipped,
|
|
"error_scene_count": error_count,
|
|
"bytes_deleted": bytes_deleted,
|
|
"scenes": scene_results,
|
|
}
|