456 lines
15 KiB
Python
456 lines
15 KiB
Python
"""Inventory GF3 SARscape native geocoded outputs.
|
|
|
|
The production server writes ENVI/SARscape native ``*_geo`` datasets. This
|
|
service treats those files as the source-of-truth evidence layer and produces a
|
|
small manifest that later conversion jobs can consume.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from ..utils import parse_gf3_l2_dirname
|
|
|
|
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
|
|
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
|
|
FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes"
|
|
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
|
SKIP_DIR_NAMES = {
|
|
".git",
|
|
FLAT_SCENE_CATALOG_DIR_NAME,
|
|
".gf3_extract",
|
|
".gf3_runtime",
|
|
".sarmap",
|
|
"__pycache__",
|
|
"temp",
|
|
"tmp",
|
|
"work",
|
|
"sarscape_work",
|
|
"GTOPO30_DIR",
|
|
"SRTM_DEM_DIR",
|
|
"TANDEMX_DEM_DIR",
|
|
}
|
|
|
|
|
|
def _utc_now() -> str:
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _safe_stat(path: Path) -> dict[str, Any] | None:
|
|
try:
|
|
stat = path.stat()
|
|
except OSError:
|
|
return None
|
|
return {
|
|
"path": str(path),
|
|
"size": int(stat.st_size),
|
|
"mtime": float(stat.st_mtime),
|
|
"mtime_ns": int(stat.st_mtime_ns),
|
|
}
|
|
|
|
|
|
def _is_nonempty_file(path: Path) -> bool:
|
|
try:
|
|
return path.is_file() and path.stat().st_size > 0
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp_path = path.with_name(f".{path.name}.tmp")
|
|
with tmp_path.open("w", encoding="utf-8") as stream:
|
|
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
|
os.replace(tmp_path, path)
|
|
|
|
|
|
def _extract_date_from_text(value: str) -> str | None:
|
|
match = re.search(r"(20\d{6})", value or "")
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _extract_product_unique_id(value: str) -> str | None:
|
|
match = re.search(r"(L\d{8,})", value or "", flags=re.IGNORECASE)
|
|
return match.group(1).upper() if match else None
|
|
|
|
|
|
def _polarization_from_geo_name(name: str) -> str | None:
|
|
upper_name = name.upper()
|
|
match = re.search(r"(?:^|[_-])(HH|HV|VH|VV)[_-]GEO$", upper_name)
|
|
if match:
|
|
return match.group(1)
|
|
tokens = [token for token in re.split(r"[_\-.]+", upper_name) if token]
|
|
for token in reversed(tokens):
|
|
if token in POLARIZATION_PRIORITY:
|
|
return token
|
|
return None
|
|
|
|
|
|
def _scene_name_from_geo_name(name: str) -> str:
|
|
text = str(name or "").strip()
|
|
match = re.match(r"^(?P<scene>.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE)
|
|
if match:
|
|
return match.group("scene")
|
|
if text.lower().endswith("_geo"):
|
|
return text[:-4]
|
|
return Path(text).stem
|
|
|
|
|
|
def _is_geo_native_data_file(path: Path) -> bool:
|
|
return path.is_file() and path.name.lower().endswith("_geo")
|
|
|
|
|
|
def _scene_batch_name(root: Path, scene_dir: Path, scene_name: str) -> str | None:
|
|
try:
|
|
rel_parts = scene_dir.relative_to(root).parts
|
|
except ValueError:
|
|
rel_parts = ()
|
|
if len(rel_parts) >= 2:
|
|
return rel_parts[0]
|
|
date = _extract_date_from_text(scene_name)
|
|
return date
|
|
|
|
|
|
def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict[str, Any]:
|
|
parsed = parse_gf3_l2_dirname(scene_name) or {}
|
|
metadata: dict[str, Any] = {
|
|
"satellite": "GF3",
|
|
"satellite_family": "GF3",
|
|
**parsed,
|
|
}
|
|
if not metadata.get("imaging_date"):
|
|
metadata["imaging_date"] = _extract_date_from_text(scene_name)
|
|
if not metadata.get("product_unique_id"):
|
|
metadata["product_unique_id"] = _extract_product_unique_id(scene_name)
|
|
|
|
polarizations = [
|
|
str(asset.get("polarization") or "").upper()
|
|
for asset in assets
|
|
if asset.get("polarization") and asset.get("polarization") != "UNKNOWN"
|
|
]
|
|
if polarizations:
|
|
metadata["polarization"] = ",".join(
|
|
pol for pol in POLARIZATION_PRIORITY if pol in set(polarizations)
|
|
) or ",".join(sorted(set(polarizations)))
|
|
metadata["product_level"] = "L2"
|
|
metadata["source_format"] = "GF3_SARSCAPE_NATIVE"
|
|
return metadata
|
|
|
|
|
|
def _native_asset_from_base(base: Path) -> dict[str, Any]:
|
|
hdr = Path(str(base) + ".hdr")
|
|
sml = Path(str(base) + ".sml")
|
|
aux_xml = Path(str(base) + ".aux.xml")
|
|
ovr = Path(str(base) + ".ovr")
|
|
kml = Path(str(base) + ".kml")
|
|
quicklook = base.with_name(base.name + "_ql.tif")
|
|
ql_kml = base.with_name(base.name + "_ql.kml")
|
|
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
|
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
|
|
|
return {
|
|
"polarization": polarization,
|
|
"role": "geo_native",
|
|
"path": str(base),
|
|
"hdr": str(hdr) if hdr.exists() else None,
|
|
"sml": str(sml) if sml.exists() else None,
|
|
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
|
"ovr": str(ovr) if ovr.exists() else None,
|
|
"quicklook": str(quicklook) if quicklook.exists() else None,
|
|
"kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None),
|
|
"complete": bool(complete),
|
|
"source": _safe_stat(base),
|
|
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
|
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
|
}
|
|
|
|
|
|
def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"polarization": str(asset.get("polarization") or "").upper(),
|
|
"path": asset.get("path"),
|
|
"complete": bool(asset.get("complete")),
|
|
"source": asset.get("source"),
|
|
"hdr": asset.get("hdr_info"),
|
|
"sml": asset.get("sml_info"),
|
|
}
|
|
|
|
|
|
def _native_fingerprint(assets: list[dict[str, Any]]) -> str:
|
|
payload = [
|
|
_asset_fingerprint_part(asset)
|
|
for asset in sorted(
|
|
assets,
|
|
key=lambda item: (
|
|
str(item.get("polarization") or ""),
|
|
str(item.get("path") or ""),
|
|
),
|
|
)
|
|
]
|
|
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
|
import hashlib
|
|
|
|
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()
|
|
|
|
|
|
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
|
assets: list[dict[str, Any]] = []
|
|
try:
|
|
entries = sorted(scene_dir.iterdir(), key=lambda item: item.name.lower())
|
|
except OSError:
|
|
return assets
|
|
|
|
for path in entries:
|
|
if not _is_geo_native_data_file(path):
|
|
continue
|
|
|
|
assets.append(_native_asset_from_base(path))
|
|
|
|
return assets
|
|
|
|
|
|
def _build_scene_manifest(
|
|
root: Path,
|
|
*,
|
|
scene_name: str,
|
|
native_dir: Path,
|
|
manifest_dir: Path,
|
|
assets: list[dict[str, Any]],
|
|
source_dir: Path,
|
|
storage_layout: str,
|
|
) -> dict[str, Any] | None:
|
|
if not assets:
|
|
return None
|
|
|
|
batch_name = _scene_batch_name(root, source_dir, scene_name)
|
|
complete_assets = [asset for asset in assets if asset.get("complete")]
|
|
complete_pols = [
|
|
pol
|
|
for pol in POLARIZATION_PRIORITY
|
|
if any(asset.get("complete") and asset.get("polarization") == pol for asset in assets)
|
|
]
|
|
other_complete_pols = sorted(
|
|
{
|
|
str(asset.get("polarization") or "")
|
|
for asset in complete_assets
|
|
if asset.get("polarization") not in POLARIZATION_PRIORITY
|
|
}
|
|
)
|
|
complete_pols.extend([pol for pol in other_complete_pols if pol])
|
|
|
|
if complete_assets and len(complete_assets) == len(assets):
|
|
status = "NATIVE_READY"
|
|
elif complete_assets:
|
|
status = "PARTIAL"
|
|
else:
|
|
status = "FAILED"
|
|
|
|
logs = []
|
|
for name in ("gf3_sarscape_cli.log",):
|
|
log_path = source_dir / name
|
|
if log_path.is_file():
|
|
logs.append(str(log_path))
|
|
try:
|
|
logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
|
except OSError:
|
|
pass
|
|
|
|
metadata = _parse_scene_metadata(scene_name, assets)
|
|
native_fingerprint = _native_fingerprint(assets)
|
|
metadata["native_fingerprint"] = native_fingerprint
|
|
manifest_path = manifest_dir / NATIVE_MANIFEST_NAME
|
|
return {
|
|
"schema": NATIVE_MANIFEST_SCHEMA,
|
|
"generated_at": _utc_now(),
|
|
"scene_name": scene_name,
|
|
"batch_name": batch_name,
|
|
"native_root": str(root),
|
|
"native_dir": str(native_dir),
|
|
"source_dir": str(source_dir),
|
|
"manifest_path": str(manifest_path),
|
|
"source_archive": None,
|
|
"storage_layout": storage_layout,
|
|
"native_fingerprint": native_fingerprint,
|
|
"status": status,
|
|
"polarizations": complete_pols,
|
|
"metadata": metadata,
|
|
"assets": assets,
|
|
"logs": logs,
|
|
}
|
|
|
|
|
|
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
|
assets = _collect_native_assets(scene_dir)
|
|
return _build_scene_manifest(
|
|
root,
|
|
scene_name=scene_dir.name,
|
|
native_dir=scene_dir,
|
|
manifest_dir=scene_dir,
|
|
assets=assets,
|
|
source_dir=scene_dir,
|
|
storage_layout="scene_dir",
|
|
)
|
|
|
|
|
|
def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]:
|
|
try:
|
|
entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower())
|
|
except OSError:
|
|
return []
|
|
|
|
grouped_assets: dict[str, list[dict[str, Any]]] = {}
|
|
for path in entries:
|
|
if not _is_geo_native_data_file(path):
|
|
continue
|
|
scene_name = _scene_name_from_geo_name(path.name)
|
|
grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path))
|
|
|
|
manifests: list[dict[str, Any]] = []
|
|
for scene_name in sorted(grouped_assets):
|
|
scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name
|
|
manifest = _build_scene_manifest(
|
|
root,
|
|
scene_name=scene_name,
|
|
native_dir=scene_catalog_dir,
|
|
manifest_dir=scene_catalog_dir,
|
|
assets=grouped_assets[scene_name],
|
|
source_dir=source_dir,
|
|
storage_layout="flat_files",
|
|
)
|
|
if manifest:
|
|
manifests.append(manifest)
|
|
return manifests
|
|
|
|
|
|
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
|
roots: list[Path] = []
|
|
missing: list[str] = []
|
|
seen: set[str] = set()
|
|
for raw in native_dirs or []:
|
|
text = str(raw or "").strip()
|
|
if not text:
|
|
continue
|
|
path = Path(os.path.normpath(text)).resolve()
|
|
key = str(path).lower()
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
if path.is_dir():
|
|
roots.append(path)
|
|
else:
|
|
missing.append(str(path))
|
|
return roots, missing
|
|
|
|
|
|
def scan_gf3_sarscape_native_roots(
|
|
native_dirs: list[str] | tuple[str, ...] | None,
|
|
*,
|
|
write_manifest: bool = True,
|
|
scene_dirs_only: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Scan configured native roots and return discovered scene manifests."""
|
|
roots, missing_roots = _normalize_roots(native_dirs)
|
|
scenes: list[dict[str, Any]] = []
|
|
seen_scene_dirs: set[str] = set()
|
|
write_errors: list[dict[str, str]] = []
|
|
|
|
def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None:
|
|
seen_scene_dirs.add(scene_key)
|
|
if write_manifest:
|
|
try:
|
|
_write_json(Path(manifest["manifest_path"]), manifest)
|
|
except OSError as exc:
|
|
write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)})
|
|
scenes.append(manifest)
|
|
|
|
for root in roots:
|
|
if scene_dirs_only:
|
|
try:
|
|
candidate_dirs = [root] + [
|
|
child
|
|
for child in sorted(root.iterdir(), key=lambda item: item.name.lower())
|
|
if child.is_dir()
|
|
and child.name not in SKIP_DIR_NAMES
|
|
and not child.name.startswith(".SARscape")
|
|
and not child.name.startswith(".gf3_")
|
|
]
|
|
except OSError:
|
|
candidate_dirs = [root]
|
|
|
|
for scene_dir in candidate_dirs:
|
|
scene_key = str(scene_dir).lower()
|
|
if scene_key in seen_scene_dirs:
|
|
continue
|
|
|
|
if not scene_dir.name.upper().startswith("GF3_"):
|
|
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
|
if flat_manifests:
|
|
for manifest in flat_manifests:
|
|
flat_key = str(manifest.get("native_dir") or "").lower()
|
|
if flat_key and flat_key not in seen_scene_dirs:
|
|
append_manifest(manifest, flat_key, scene_dir)
|
|
continue
|
|
|
|
manifest = _collect_scene_manifest(root, scene_dir)
|
|
if not manifest:
|
|
continue
|
|
|
|
append_manifest(manifest, scene_key, scene_dir)
|
|
continue
|
|
|
|
for current_dir, dir_names, _file_names in os.walk(root):
|
|
dir_names[:] = [
|
|
name
|
|
for name in dir_names
|
|
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
|
and not name.startswith(".gf3_")
|
|
]
|
|
scene_dir = Path(current_dir)
|
|
scene_key = str(scene_dir).lower()
|
|
if scene_key in seen_scene_dirs:
|
|
dir_names[:] = []
|
|
continue
|
|
|
|
if not scene_dir.name.upper().startswith("GF3_"):
|
|
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
|
if flat_manifests:
|
|
for manifest in flat_manifests:
|
|
flat_key = str(manifest.get("native_dir") or "").lower()
|
|
if flat_key and flat_key not in seen_scene_dirs:
|
|
append_manifest(manifest, flat_key, scene_dir)
|
|
continue
|
|
|
|
manifest = _collect_scene_manifest(root, scene_dir)
|
|
if not manifest:
|
|
continue
|
|
|
|
append_manifest(manifest, scene_key, scene_dir)
|
|
dir_names[:] = []
|
|
|
|
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
|
|
partial = sum(1 for scene in scenes if scene.get("status") == "PARTIAL")
|
|
failed = sum(1 for scene in scenes if scene.get("status") == "FAILED")
|
|
complete_assets = sum(
|
|
1
|
|
for scene in scenes
|
|
for asset in scene.get("assets") or []
|
|
if asset.get("complete")
|
|
)
|
|
return {
|
|
"schema": "gf3_sarscape_native_inventory.v1",
|
|
"generated_at": _utc_now(),
|
|
"native_roots": [str(path) for path in roots],
|
|
"missing_roots": missing_roots,
|
|
"scene_count": len(scenes),
|
|
"native_ready_count": native_ready,
|
|
"partial_count": partial,
|
|
"failed_count": failed,
|
|
"complete_asset_count": complete_assets,
|
|
"write_errors": write_errors,
|
|
"scenes": scenes,
|
|
}
|