Integrate GF3 SARscape flood workflow

This commit is contained in:
2026-06-02 01:25:42 +08:00
parent 16cac0e292
commit 79e08b3a47
24 changed files with 4220 additions and 84 deletions
+10 -1
View File
@@ -67,7 +67,16 @@ GF3_ARCHIVE_SOURCE_DIRS=D:\GF3_L1A_Image_Zip
GF3_ARCHIVE_EXTS=.zip,.tar,.tar.gz,.tgz
GF3_UNPACK_DELETE_ARCHIVE=true
GF3_SOURCE_DIRS=D:\GF3_L1A_Image
GF3_STORAGE_DIRS=D:\GF3_L2_Image
GF3_SARSCAPE_NATIVE_DIRS=D:\GF3_L2_ENVI_Binary_Pool
GF3_STORAGE_DIRS=D:\GF3_L2_Image_Pool
GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\.codex_tmp\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe
GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM
GF3_SARSCAPE_POLARIZATIONS=HH,HV
GF3_SARSCAPE_KEEP_EXTRACTED=true
GF3_SARSCAPE_AUTO_STANDARDIZE=true
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0
HAZARD_POINTS_DIR=D:\Code\Insar_management_system_v2\backend\Point
HAZARD_POINTS_FILENAME=Point.shp
+19
View File
@@ -207,7 +207,16 @@ class Settings(BaseSettings):
GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz"
GF3_UNPACK_DELETE_ARCHIVE: bool = True
GF3_SOURCE_DIRS: str = ""
GF3_SARSCAPE_NATIVE_DIRS: str = ""
GF3_STORAGE_DIRS: str = ""
GF3_SARSCAPE_WRAPPER_EXE: str = ""
GF3_SARSCAPE_IDLRT_PATH: str = r"C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe"
GF3_SARSCAPE_DEM_PATH: str = ""
GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV"
GF3_SARSCAPE_KEEP_EXTRACTED: bool = True
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0
MONITOR_ORBIT_DIR: str = ""
ORBIT_POOL_ENVI: str = ""
@@ -991,6 +1000,15 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(
label="GF3_SARSCAPE_DEM_PATH",
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
errors=errors,
warnings=warnings,
expect_file=True,
)
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
@@ -1024,6 +1042,7 @@ def validate_runtime_config() -> dict[str, Any]:
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS),
("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS),
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
):
values = split_env_paths(raw_value)
+6 -6
View File
@@ -210,12 +210,12 @@ class RadarData(BaseModel):
has_orbit_data: bool
orbit_file_path: Optional[str] = None
is_envi_processed: bool = False
coverage_polygon: List[Tuple[float, float]]
coverage_polygon: Optional[List[Tuple[float, float]]] = None
min_lon: float
min_lat: float
max_lon: float
max_lat: float
min_lon: Optional[float] = None
min_lat: Optional[float] = None
max_lon: Optional[float] = None
max_lat: Optional[float] = None
preview_cache_status: str = "NONE"
preview_cache_version: Optional[str] = None
preview_cache_updated_at: Optional[datetime] = None
@@ -236,7 +236,7 @@ class RadarData(BaseModel):
@computed_field
@property
def coverage_bbox(self) -> Tuple[float, float, float, float]:
def coverage_bbox(self) -> Tuple[Optional[float], Optional[float], Optional[float], Optional[float]]:
"""A computed property to provide the bbox tuple, used by existing logic."""
return (self.min_lon, self.min_lat, self.max_lon, self.max_lat)
+159
View File
@@ -43,7 +43,14 @@ class MonitorConfig(BaseModel):
dinsar_dirs: List[str] = []
gf3_archive_source_dirs: List[str] = []
gf3_source_dirs: List[str] = []
gf3_sarscape_native_dirs: List[str] = []
gf3_storage_dirs: List[str] = []
gf3_sarscape_wrapper_exe: Optional[str] = None
gf3_sarscape_idlrt_path: Optional[str] = None
gf3_sarscape_dem_path: Optional[str] = None
gf3_sarscape_polarizations: Optional[str] = None
gf3_sarscape_auto_standardize: bool = True
gf3_sarscape_clean_after_success: bool = True
# Manual-only: config is read from .env
@@ -58,6 +65,26 @@ class GF3UnpackRunRequest(BaseModel):
max_files_per_run: Optional[int] = Field(default=None, ge=0)
class GF3SarscapeSyncRequest(BaseModel):
force: bool = False
register: bool = True
class GF3SarscapeProduceRequest(BaseModel):
max_archives_per_run: Optional[int] = Field(default=None, ge=0)
auto_standardize: Optional[bool] = None
clean_after_success: Optional[bool] = None
force_standardize: bool = False
register: bool = True
cleanup_dry_run: bool = False
class GF3SarscapeCleanRequest(BaseModel):
dry_run: bool = False
require_standardized: bool = True
max_scenes: Optional[int] = Field(default=None, ge=0)
@router.post("/monitor/config")
async def update_monitor_config(config: MonitorConfig):
"""
@@ -154,6 +181,138 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-sarscape-sync", status_code=202)
async def run_gf3_sarscape_sync(
request_data: GF3SarscapeSyncRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
扫描 GF3 SARscape 原生 _geo 二进制池,转换为标准 GeoTIFF,并登记入库。
"""
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
options = request_data or GF3SarscapeSyncRequest()
task_type = "GF3_SARSCAPE_SYNC"
task_name = "GF3 SARscape 原生结果标准化"
payload = {
"native_dirs": gf3_native_dirs,
"storage_root": gf3_storage_dirs[0],
"force": bool(options.force),
"register": bool(options.register),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape 原生结果标准化任务已提交",
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-sarscape-produce", status_code=202)
async def run_gf3_sarscape_produce(
request_data: GF3SarscapeProduceRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates.
"""
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
if not settings.GF3_SARSCAPE_WRAPPER_EXE:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.")
if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH):
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
options = request_data or GF3SarscapeProduceRequest()
task_type = "GF3_SARSCAPE_PRODUCE"
task_name = "GF3 SARscape production"
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success)
payload = {
"source_dirs": gf3_archive_source_dirs,
"native_dirs": gf3_native_dirs,
"native_root": gf3_native_dirs[0],
"storage_root": gf3_storage_dirs[0],
"wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
"idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
"dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
"max_archives_per_run": int(options.max_archives_per_run or 0),
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
"auto_standardize": bool(auto_standardize),
"clean_after_success": bool(clean_after_success),
"force_standardize": bool(options.force_standardize),
"register": bool(options.register),
"cleanup_require_standardized": True,
"cleanup_dry_run": bool(options.cleanup_dry_run),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape production task submitted",
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
async def run_gf3_sarscape_clean(
request_data: GF3SarscapeCleanRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Clean SARscape intermediate files from GF3 native pool after standard GeoTIFFs exist.
"""
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
options = request_data or GF3SarscapeCleanRequest()
task_type = "GF3_SARSCAPE_CLEAN"
task_name = "GF3 SARscape native cleanup"
payload = {
"native_dirs": gf3_native_dirs,
"storage_root": gf3_storage_dirs[0],
"dry_run": bool(options.dry_run),
"require_standardized": bool(options.require_standardized),
"max_scenes": int(options.max_scenes or 0),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape cleanup task submitted",
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.get("/monitor/gf3-unpack/config")
async def get_gf3_unpack_config(admin_user: AuthUserORM = Depends(_require_admin)):
return GF3UnpackConfig(
+7
View File
@@ -21,7 +21,14 @@ MONITOR_CONFIG = {
# GF3 链路
"gf3_archive_source_dirs": split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
"gf3_sarscape_native_dirs": split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS),
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
"gf3_sarscape_wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
"gf3_sarscape_idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
"gf3_sarscape_dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
"gf3_sarscape_polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
"gf3_sarscape_auto_standardize": bool(settings.GF3_SARSCAPE_AUTO_STANDARDIZE),
"gf3_sarscape_clean_after_success": bool(settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS),
"mode": "manual",
"config_source": "env",
}
@@ -0,0 +1,308 @@
"""Inventory GF3 SARscape native geocoded outputs.
The production server writes ENVI/SARscape native ``*_geo`` datasets. This
service treats those files as the source-of-truth evidence layer and produces a
small manifest that later conversion jobs can consume.
"""
from __future__ import annotations
import json
import os
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from ..utils import parse_gf3_l2_dirname
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
SKIP_DIR_NAMES = {
".git",
".gf3_extract",
".sarmap",
"__pycache__",
"temp",
"tmp",
"work",
"sarscape_work",
"GTOPO30_DIR",
"SRTM_DEM_DIR",
"TANDEMX_DEM_DIR",
}
def _utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def _safe_stat(path: Path) -> dict[str, Any] | None:
try:
stat = path.stat()
except OSError:
return None
return {
"path": str(path),
"size": int(stat.st_size),
"mtime": float(stat.st_mtime),
"mtime_ns": int(stat.st_mtime_ns),
}
def _is_nonempty_file(path: Path) -> bool:
try:
return path.is_file() and path.stat().st_size > 0
except OSError:
return False
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = path.with_name(f".{path.name}.tmp")
with tmp_path.open("w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
os.replace(tmp_path, path)
def _extract_date_from_text(value: str) -> str | None:
match = re.search(r"(20\d{6})", value or "")
return match.group(1) if match else None
def _extract_product_unique_id(value: str) -> str | None:
match = re.search(r"(L\d{8,})", value or "", flags=re.IGNORECASE)
return match.group(1).upper() if match else None
def _polarization_from_geo_name(name: str) -> str | None:
upper_name = name.upper()
match = re.search(r"(?:^|[_-])(HH|HV|VH|VV)[_-]GEO$", upper_name)
if match:
return match.group(1)
tokens = [token for token in re.split(r"[_\-.]+", upper_name) if token]
for token in reversed(tokens):
if token in POLARIZATION_PRIORITY:
return token
return None
def _is_geo_native_data_file(path: Path) -> bool:
return path.is_file() and path.name.lower().endswith("_geo")
def _scene_batch_name(root: Path, scene_dir: Path, scene_name: str) -> str | None:
try:
rel_parts = scene_dir.relative_to(root).parts
except ValueError:
rel_parts = ()
if len(rel_parts) >= 2:
return rel_parts[0]
date = _extract_date_from_text(scene_name)
return date
def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict[str, Any]:
parsed = parse_gf3_l2_dirname(scene_name) or {}
metadata: dict[str, Any] = {
"satellite": "GF3",
"satellite_family": "GF3",
**parsed,
}
if not metadata.get("imaging_date"):
metadata["imaging_date"] = _extract_date_from_text(scene_name)
if not metadata.get("product_unique_id"):
metadata["product_unique_id"] = _extract_product_unique_id(scene_name)
polarizations = [
str(asset.get("polarization") or "").upper()
for asset in assets
if asset.get("polarization") and asset.get("polarization") != "UNKNOWN"
]
if polarizations:
metadata["polarization"] = ",".join(
pol for pol in POLARIZATION_PRIORITY if pol in set(polarizations)
) or ",".join(sorted(set(polarizations)))
metadata["product_level"] = "L2"
metadata["source_format"] = "GF3_SARSCAPE_NATIVE"
return metadata
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
assets: list[dict[str, Any]] = []
try:
entries = sorted(scene_dir.iterdir(), key=lambda item: item.name.lower())
except OSError:
return assets
for path in entries:
if not _is_geo_native_data_file(path):
continue
base = path
hdr = Path(str(base) + ".hdr")
sml = Path(str(base) + ".sml")
aux_xml = Path(str(base) + ".aux.xml")
ovr = Path(str(base) + ".ovr")
kml = Path(str(base) + ".kml")
quicklook = base.with_name(base.name + "_ql.tif")
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
asset: dict[str, Any] = {
"polarization": polarization,
"role": "geo_native",
"path": str(base),
"hdr": str(hdr) if hdr.exists() else None,
"sml": str(sml) if sml.exists() else None,
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
"ovr": str(ovr) if ovr.exists() else None,
"quicklook": str(quicklook) if quicklook.exists() else None,
"kml": str(kml) if kml.exists() else None,
"complete": bool(complete),
"source": _safe_stat(base),
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
"sml_info": _safe_stat(sml) if sml.exists() else None,
}
assets.append(asset)
return assets
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
assets = _collect_native_assets(scene_dir)
if not assets:
return None
scene_name = scene_dir.name
batch_name = _scene_batch_name(root, scene_dir, scene_name)
complete_assets = [asset for asset in assets if asset.get("complete")]
complete_pols = [
pol
for pol in POLARIZATION_PRIORITY
if any(asset.get("complete") and asset.get("polarization") == pol for asset in assets)
]
other_complete_pols = sorted(
{
str(asset.get("polarization") or "")
for asset in complete_assets
if asset.get("polarization") not in POLARIZATION_PRIORITY
}
)
complete_pols.extend([pol for pol in other_complete_pols if pol])
if complete_assets and len(complete_assets) == len(assets):
status = "NATIVE_READY"
elif complete_assets:
status = "PARTIAL"
else:
status = "FAILED"
logs = []
for name in ("gf3_sarscape_cli.log",):
log_path = scene_dir / name
if log_path.is_file():
logs.append(str(log_path))
try:
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
except OSError:
pass
metadata = _parse_scene_metadata(scene_name, assets)
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
return {
"schema": NATIVE_MANIFEST_SCHEMA,
"generated_at": _utc_now(),
"scene_name": scene_name,
"batch_name": batch_name,
"native_root": str(root),
"native_dir": str(scene_dir),
"manifest_path": str(manifest_path),
"source_archive": None,
"status": status,
"polarizations": complete_pols,
"metadata": metadata,
"assets": assets,
"logs": logs,
}
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
roots: list[Path] = []
missing: list[str] = []
seen: set[str] = set()
for raw in native_dirs or []:
text = str(raw or "").strip()
if not text:
continue
path = Path(os.path.normpath(text)).resolve()
key = str(path).lower()
if key in seen:
continue
seen.add(key)
if path.is_dir():
roots.append(path)
else:
missing.append(str(path))
return roots, missing
def scan_gf3_sarscape_native_roots(
native_dirs: list[str] | tuple[str, ...] | None,
*,
write_manifest: bool = True,
) -> dict[str, Any]:
"""Scan configured native roots and return discovered scene manifests."""
roots, missing_roots = _normalize_roots(native_dirs)
scenes: list[dict[str, Any]] = []
seen_scene_dirs: set[str] = set()
write_errors: list[dict[str, str]] = []
for root in roots:
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
]
scene_dir = Path(current_dir)
scene_key = str(scene_dir).lower()
if scene_key in seen_scene_dirs:
dir_names[:] = []
continue
manifest = _collect_scene_manifest(root, scene_dir)
if not manifest:
continue
seen_scene_dirs.add(scene_key)
if write_manifest:
try:
_write_json(Path(manifest["manifest_path"]), manifest)
except OSError as exc:
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
scenes.append(manifest)
dir_names[:] = []
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
partial = sum(1 for scene in scenes if scene.get("status") == "PARTIAL")
failed = sum(1 for scene in scenes if scene.get("status") == "FAILED")
complete_assets = sum(
1
for scene in scenes
for asset in scene.get("assets") or []
if asset.get("complete")
)
return {
"schema": "gf3_sarscape_native_inventory.v1",
"generated_at": _utc_now(),
"native_roots": [str(path) for path in roots],
"missing_roots": missing_roots,
"scene_count": len(scenes),
"native_ready_count": native_ready,
"partial_count": partial,
"failed_count": failed,
"complete_asset_count": complete_assets,
"write_errors": write_errors,
"scenes": scenes,
}
@@ -0,0 +1,843 @@
"""Run GF3 SARscape production and clean native intermediate files."""
from __future__ import annotations
import json
import os
import re
import shutil
import subprocess
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable
from ..config import settings, split_env_paths
from .gf3_native_inventory_service import (
NATIVE_MANIFEST_NAME,
POLARIZATION_PRIORITY,
scan_gf3_sarscape_native_roots,
)
from .gf3_standardize_service import STANDARD_MANIFEST_NAME
LogCallback = Callable[[str, str], None]
ProgressCallback = Callable[[int, str], None]
SUPPORTED_WRAPPER_INPUT_EXTS = (".tar.gz", ".tgz", ".meta.xml")
CLEANUP_MANIFEST_NAME = "gf3_cleanup_manifest.json"
INTERMEDIATE_DIR_NAMES = {
".gf3_extract",
".gf3_extract.tmp",
"temp",
"tmp",
"work",
}
KEEP_FILE_NAMES = {
NATIVE_MANIFEST_NAME,
CLEANUP_MANIFEST_NAME,
"gf3_sarscape_cli.log",
}
INTERMEDIATE_SUFFIXES = (
".par",
".par_command",
".trace",
".working",
".working_warning",
".workinggetcornerfromslantrangeimage_dem",
".txt",
".list",
".listhv",
".listunknown",
".shp",
".shx",
".dbf",
".prj",
)
def _utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def _clean_text(value: Any) -> str:
return str(value or "").strip().strip('"').strip("'")
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = path.with_name(f".{path.name}.tmp")
with tmp_path.open("w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
os.replace(tmp_path, path)
def _read_json(path: Path) -> dict[str, Any] | None:
try:
with path.open("r", encoding="utf-8") as stream:
data = json.load(stream)
return data if isinstance(data, dict) else None
except (OSError, json.JSONDecodeError):
return None
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
text = _clean_text(value) or default
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
return safe or default
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
roots: list[Path] = []
missing: list[str] = []
seen: set[str] = set()
for raw in values or []:
text = _clean_text(raw)
if not text:
continue
path = Path(os.path.normpath(text)).resolve()
key = str(path).lower()
if key in seen:
continue
seen.add(key)
if path.is_dir():
roots.append(path)
else:
missing.append(str(path))
return roots, missing
def _resolve_config_path(raw: Any) -> Path | None:
text = _clean_text(raw)
if not text:
return None
return Path(os.path.normpath(text)).resolve()
def _fallback_wrapper_exe() -> Path | None:
candidate = (
Path(settings.PROJECT_ROOT)
/ ".codex_tmp"
/ "GF3_L1A_To_L2_pipeline"
/ "dist"
/ "windows"
/ "gf3wrapper.exe"
)
return candidate.resolve() if candidate.is_file() else None
def _wrapper_exe_path(value: str | None = None) -> Path:
configured = _resolve_config_path(value or settings.GF3_SARSCAPE_WRAPPER_EXE)
path = configured or _fallback_wrapper_exe()
if path is None:
raise FileNotFoundError("GF3 SARscape wrapper is not configured.")
if not path.is_file():
raise FileNotFoundError(f"GF3 SARscape wrapper does not exist: {path}")
return path
def _dem_path(value: str | None = None) -> Path:
path = _resolve_config_path(value or settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH)
if path is None:
raise FileNotFoundError("GF3 SARscape DEM path is not configured.")
if not path.exists():
raise FileNotFoundError(f"GF3 SARscape DEM path does not exist: {path}")
return path
def _idlrt_path(value: str | None = None) -> Path | None:
path = _resolve_config_path(value or settings.GF3_SARSCAPE_IDLRT_PATH)
if path is None:
return None
if not path.is_file():
raise FileNotFoundError(f"GF3 SARscape idlrt.exe does not exist: {path}")
return path
def _requested_polarizations(value: str | None = None) -> list[str]:
raw = _clean_text(value or settings.GF3_SARSCAPE_POLARIZATIONS or "HH,HV")
items: list[str] = []
for token in re.split(r"[,;\s]+", raw):
pol = token.strip().upper()
if not pol:
continue
if pol not in items:
items.append(pol)
return items or ["HH", "HV"]
def _archive_exts_for_wrapper(archive_exts: list[str] | tuple[str, ...] | None) -> list[str]:
ordered: list[str] = []
for raw_ext in archive_exts or []:
ext = _clean_text(raw_ext).lower()
if not ext:
continue
if not ext.startswith("."):
ext = "." + ext
if ext in SUPPORTED_WRAPPER_INPUT_EXTS and ext not in ordered:
ordered.append(ext)
if not ordered:
ordered.extend((".tar.gz", ".tgz"))
return sorted(ordered, key=len, reverse=True)
def _input_ext(path: Path, exts: list[str]) -> str | None:
name = path.name.lower()
for ext in exts:
if name.endswith(ext):
return ext
return None
def _scene_name_from_input(path: Path) -> str:
name = path.name
lower_name = name.lower()
for ext in SUPPORTED_WRAPPER_INPUT_EXTS:
if lower_name.endswith(ext):
return name[: -len(ext)]
return path.stem
def discover_gf3_sarscape_inputs(
source_dirs: list[str] | tuple[str, ...] | None,
*,
archive_exts: list[str] | tuple[str, ...] | None = None,
) -> dict[str, Any]:
"""Find wrapper-supported GF3 source inputs in configured archive pools."""
roots, missing_roots = _resolve_existing_dirs(source_dirs)
exts = _archive_exts_for_wrapper(archive_exts)
inputs: list[dict[str, Any]] = []
seen: set[str] = set()
for root in roots:
for path in root.rglob("*"):
if not path.is_file():
continue
ext = _input_ext(path, exts)
if not ext:
continue
resolved = path.resolve()
key = str(resolved).lower()
if key in seen:
continue
seen.add(key)
inputs.append(
{
"path": str(resolved),
"scene_name": _scene_name_from_input(path),
"ext": ext,
"source_root": str(root),
}
)
inputs.sort(key=lambda item: str(item.get("path") or "").lower())
return {
"source_roots": [str(path) for path in roots],
"missing_roots": missing_roots,
"archive_exts": exts,
"input_count": len(inputs),
"inputs": inputs,
}
def _is_nonempty_file(path: Path) -> bool:
try:
return path.is_file() and path.stat().st_size > 0
except OSError:
return False
def _completed_geo_product(scene_dir: Path, polarization: str) -> Path | None:
lower_pol = polarization.lower()
try:
entries = list(scene_dir.iterdir())
except OSError:
return None
for path in entries:
name = path.name.lower()
if not name.endswith(f"_{lower_pol}_geo.sml"):
continue
data_file = Path(str(path)[: -len(".sml")])
if _is_nonempty_file(path) and _is_nonempty_file(data_file):
return data_file
return None
def _scene_complete(scene_dir: Path, polarizations: list[str]) -> bool:
if not scene_dir.is_dir():
return False
return all(_completed_geo_product(scene_dir, pol) is not None for pol in polarizations)
def _missing_geo_polarizations(scene_dir: Path, polarizations: list[str]) -> list[str]:
if not scene_dir.is_dir():
return list(polarizations)
return [pol for pol in polarizations if _completed_geo_product(scene_dir, pol) is None]
def _compact_failure_text(text: str, *, max_chars: int = 1000) -> str:
lines = [line.strip() for line in text.splitlines() if line.strip()]
compact = " | ".join(lines)
if len(compact) <= max_chars:
return compact
return compact[: max_chars - 3].rstrip() + "..."
def _read_failure_file(path: Path) -> str | None:
try:
if not path.is_file() or path.stat().st_size <= 0:
return None
return path.read_text(encoding="utf-8", errors="replace")
except OSError:
return None
def _scene_failure_hint(scene_dir: Path) -> str | None:
work_dir = scene_dir / "work"
candidates = [
work_dir / "Process.working_error",
work_dir / "Process.trace_cerr.txt",
scene_dir / "gf3_sarscape_cli.log",
]
for path in candidates:
text = _read_failure_file(path)
if text:
compact = _compact_failure_text(text)
if compact:
return f"{path.name}: {compact}"
process_log = work_dir / "Process.log"
text = _read_failure_file(process_log)
if not text:
return None
important = [
line.strip()
for line in text.splitlines()
if "ERROR" in line.upper() or "[EC:" in line.upper() or "PARAMETER FILE READ ERROR" in line.upper()
]
if important:
return f"{process_log.name}: {_compact_failure_text(chr(10).join(important[-8:]))}"
return None
def _format_missing_output_error(scene_dir: Path, polarizations: list[str], returncode: int) -> str:
missing = _missing_geo_polarizations(scene_dir, polarizations)
missing_text = ",".join(missing) if missing else "unknown"
if returncode == 0:
base = f"wrapper returned 0 but required _geo outputs are missing: {missing_text}"
else:
base = f"wrapper return code {returncode}; missing _geo outputs: {missing_text}"
hint = _scene_failure_hint(scene_dir)
return f"{base}; {hint}" if hint else base
def _emit_log(callback: LogCallback | None, level: str, message: str) -> None:
if callback:
callback(level, message)
def _emit_progress(callback: ProgressCallback | None, progress: int, message: str) -> None:
if callback:
callback(max(0, min(100, int(progress))), message)
def _run_wrapper_command(
cmd: list[str],
*,
cwd: Path,
env: dict[str, str],
timeout_seconds: int | None,
) -> subprocess.CompletedProcess[str]:
return subprocess.run(
cmd,
cwd=str(cwd),
env=env,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=timeout_seconds if timeout_seconds and timeout_seconds > 0 else None,
check=False,
)
def _log_completed_process_output(
completed: subprocess.CompletedProcess[str],
*,
log_callback: LogCallback | None,
line_limit: int = 300,
) -> list[str]:
output = completed.stdout or ""
lines = [line.rstrip() for line in output.splitlines() if line.strip()]
if not lines:
return []
clipped = False
display_lines = lines
if len(lines) > line_limit:
clipped = True
head_count = max(1, line_limit // 2)
tail_count = max(1, line_limit - head_count)
display_lines = lines[:head_count] + [f"... clipped {len(lines) - line_limit} wrapper log lines ..."] + lines[-tail_count:]
for line in display_lines:
_emit_log(log_callback, "INFO", f"[gf3wrapper] {line}")
if clipped:
_emit_log(log_callback, "WARNING", f"GF3 wrapper output was clipped to {line_limit} log lines.")
return lines[-20:]
def run_gf3_sarscape_production(
*,
source_dirs: list[str] | None = None,
native_root: str | None = None,
wrapper_exe: str | None = None,
dem_path: str | None = None,
idlrt_path: str | None = None,
polarizations: str | None = None,
archive_exts: list[str] | None = None,
max_archives_per_run: int | None = None,
timeout_seconds: int | None = None,
keep_extracted: bool | None = None,
log_callback: LogCallback | None = None,
progress_callback: ProgressCallback | None = None,
) -> dict[str, Any]:
"""Run the external GF3 SARscape wrapper for pending raw archives."""
source_dirs = source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
native_roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
native_root_text = _clean_text(native_root or (native_roots[0] if native_roots else ""))
if not source_dirs:
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
if not native_root_text:
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS is not configured.")
native_root_path = Path(os.path.normpath(native_root_text)).resolve()
native_root_path.mkdir(parents=True, exist_ok=True)
wrapper_path = _wrapper_exe_path(wrapper_exe)
dem = _dem_path(dem_path)
idlrt = _idlrt_path(idlrt_path)
pols = _requested_polarizations(polarizations)
pol_text = ",".join(pols)
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
inputs = discovery.get("inputs") or []
max_to_process = int(max_archives_per_run or 0)
timeout = int(timeout_seconds or 0)
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
_emit_log(log_callback, "INFO", f"GF3 SARscape source roots: {source_dirs}")
_emit_log(log_callback, "INFO", f"GF3 SARscape native root: {native_root_path}")
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
if not inputs:
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
return {
"ok": True,
"found_count": 0,
"processed_count": 0,
"skipped_count": 0,
"failed_count": 0,
"deferred_count": 0,
"missing_roots": discovery.get("missing_roots") or [],
"results": [],
}
runtime_dir = native_root_path / ".gf3_runtime"
runtime_dir.mkdir(parents=True, exist_ok=True)
config_path = runtime_dir / "gf3wrapper.json"
env = os.environ.copy()
if idlrt is not None:
env["IDLRT_PATH"] = str(idlrt)
processed = 0
skipped = 0
failed = 0
deferred = 0
results: list[dict[str, Any]] = []
total = len(inputs)
for idx, input_info in enumerate(inputs):
input_path = Path(str(input_info.get("path") or "")).resolve()
scene_name = str(input_info.get("scene_name") or _scene_name_from_input(input_path))
scene_dir = native_root_path / scene_name
progress = 5 + int((idx / max(total, 1)) * 60)
_emit_progress(progress_callback, progress, f"GF3 SARscape checking {idx + 1}/{total}: {scene_name}")
if _scene_complete(scene_dir, pols):
skipped += 1
results.append(
{
"scene_name": scene_name,
"input_path": str(input_path),
"scene_dir": str(scene_dir),
"status": "skipped_complete",
}
)
continue
if max_to_process > 0 and processed + failed >= max_to_process:
deferred += 1
results.append(
{
"scene_name": scene_name,
"input_path": str(input_path),
"scene_dir": str(scene_dir),
"status": "deferred",
}
)
continue
cmd = [
str(wrapper_path),
"-config",
str(config_path),
"-input",
str(input_path),
"-output",
str(native_root_path),
"-dem",
str(dem),
"-pol",
pol_text,
f"-keep-extracted={str(keep).lower()}",
]
if idlrt is not None:
cmd.extend(["-idlrt", str(idlrt)])
_emit_log(log_callback, "INFO", f"GF3 SARscape processing {scene_name}: {input_path}")
started = time.monotonic()
try:
completed = _run_wrapper_command(
cmd,
cwd=wrapper_path.parent,
env=env,
timeout_seconds=timeout,
)
output_tail = _log_completed_process_output(completed, log_callback=log_callback)
elapsed_seconds = round(time.monotonic() - started, 3)
output_complete = _scene_complete(scene_dir, pols)
if completed.returncode == 0 and output_complete:
processed += 1
status = "processed"
error = None
elif output_complete:
processed += 1
status = "processed_with_warning"
error = f"wrapper return code {completed.returncode}"
_emit_log(log_callback, "WARNING", f"GF3 wrapper returned {completed.returncode}, but output is complete: {scene_name}")
else:
failed += 1
status = "failed"
error = _format_missing_output_error(scene_dir, pols, int(completed.returncode))
_emit_log(log_callback, "ERROR", f"GF3 SARscape failed for {scene_name}: {error}")
results.append(
{
"scene_name": scene_name,
"input_path": str(input_path),
"scene_dir": str(scene_dir),
"status": status,
"returncode": int(completed.returncode),
"elapsed_seconds": elapsed_seconds,
"output_complete": output_complete,
"error": error,
"output_tail": output_tail,
}
)
except subprocess.TimeoutExpired as exc:
failed += 1
_emit_log(log_callback, "ERROR", f"GF3 SARscape timed out for {scene_name}: {exc}")
results.append(
{
"scene_name": scene_name,
"input_path": str(input_path),
"scene_dir": str(scene_dir),
"status": "failed",
"error": f"timeout after {timeout}s",
}
)
except Exception as exc:
failed += 1
_emit_log(log_callback, "ERROR", f"GF3 SARscape exception for {scene_name}: {exc}")
results.append(
{
"scene_name": scene_name,
"input_path": str(input_path),
"scene_dir": str(scene_dir),
"status": "failed",
"error": str(exc),
}
)
_emit_progress(progress_callback, 70, "GF3 SARscape production stage finished.")
return {
"ok": failed == 0,
"found_count": total,
"processed_count": processed,
"skipped_count": skipped,
"failed_count": failed,
"deferred_count": deferred,
"native_root": str(native_root_path),
"missing_roots": discovery.get("missing_roots") or [],
"results": results,
}
def _is_relative_to(path: Path, parent: Path) -> bool:
try:
path.resolve().relative_to(parent.resolve())
return True
except ValueError:
return False
def _entry_size(path: Path) -> int:
try:
if path.is_file():
return int(path.stat().st_size)
if path.is_dir():
total = 0
for current, _dir_names, file_names in os.walk(path):
for file_name in file_names:
file_path = Path(current) / file_name
try:
total += int(file_path.stat().st_size)
except OSError:
continue
return total
except OSError:
return 0
return 0
def _is_final_geo_asset_file(path: Path) -> bool:
name = path.name.lower()
if name in KEEP_FILE_NAMES or name.endswith(".log"):
return True
return (
name.endswith("_geo")
or name.endswith("_geo.hdr")
or name.endswith("_geo.sml")
or name.endswith("_geo.ovr")
or name.endswith("_geo.aux.xml")
or name.endswith("_geo.kml")
or name.endswith("_geo_ql.tif")
or name.endswith("_geo_ql.kml")
)
def _is_known_intermediate_file(path: Path) -> bool:
name = path.name.lower()
if _is_final_geo_asset_file(path):
return False
if any(token in name for token in ("_slc", "_ml", "_filt")):
return True
if name.endswith(INTERMEDIATE_SUFFIXES):
return True
return False
def _standard_manifest_path(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
batch_name = _safe_slug(scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date"), default="unknown_batch")
scene_name = _safe_slug(scene_manifest.get("scene_name"), default="unknown_scene")
return storage_root / batch_name / scene_name / STANDARD_MANIFEST_NAME
def _standard_manifest_allows_cleanup(scene_manifest: dict[str, Any], storage_root: Path) -> bool:
manifest = _read_json(_standard_manifest_path(scene_manifest, storage_root))
if not manifest:
return False
return str(manifest.get("status") or "").upper() == "DONE"
def _assert_safe_delete(target: Path, scene_dir: Path, native_roots: list[Path]) -> None:
resolved_target = target.resolve()
resolved_scene = scene_dir.resolve()
if resolved_target == resolved_scene:
raise RuntimeError(f"Refusing to delete scene directory itself: {resolved_target}")
if not _is_relative_to(resolved_target, resolved_scene):
raise RuntimeError(f"Refusing to delete outside scene directory: {resolved_target}")
if not any(_is_relative_to(resolved_target, root) for root in native_roots):
raise RuntimeError(f"Refusing to delete outside GF3 native roots: {resolved_target}")
def _cleanup_scene_intermediates(
scene_manifest: dict[str, Any],
*,
native_roots: list[Path],
dry_run: bool,
) -> dict[str, Any]:
scene_dir = Path(str(scene_manifest.get("native_dir") or "")).resolve()
if not scene_dir.is_dir():
return {
"scene_name": scene_manifest.get("scene_name"),
"scene_dir": str(scene_dir),
"status": "skipped",
"reason": "scene directory missing",
"deleted_entries": [],
"bytes_deleted": 0,
}
if not any(_is_relative_to(scene_dir, root) for root in native_roots):
return {
"scene_name": scene_manifest.get("scene_name"),
"scene_dir": str(scene_dir),
"status": "skipped",
"reason": "scene directory is outside configured native roots",
"deleted_entries": [],
"bytes_deleted": 0,
}
candidates: list[Path] = []
for entry in sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()):
if entry.is_dir() and entry.name.lower() in INTERMEDIATE_DIR_NAMES:
candidates.append(entry)
elif entry.is_file() and _is_known_intermediate_file(entry):
candidates.append(entry)
deleted_entries: list[dict[str, Any]] = []
bytes_deleted = 0
errors: list[dict[str, str]] = []
for target in candidates:
try:
_assert_safe_delete(target, scene_dir, native_roots)
size = _entry_size(target)
bytes_deleted += size
deleted_entries.append(
{
"path": str(target),
"type": "directory" if target.is_dir() else "file",
"size": size,
}
)
if not dry_run:
if target.is_dir():
shutil.rmtree(target)
else:
target.unlink()
except Exception as exc:
errors.append({"path": str(target), "error": str(exc)})
status = "cleaned" if deleted_entries and not errors else ("error" if errors else "nothing_to_delete")
cleanup_manifest = {
"schema": "gf3_sarscape_cleanup.v1",
"generated_at": _utc_now(),
"dry_run": dry_run,
"scene_name": scene_manifest.get("scene_name"),
"scene_dir": str(scene_dir),
"status": status,
"bytes_deleted": bytes_deleted,
"deleted_entries": deleted_entries,
"errors": errors,
"retention_policy": {
"keep": "final *_geo native assets, quicklooks, manifests, and logs",
"delete": "extract/temp/work directories and slc/ml/filt intermediate files",
},
}
if not dry_run:
_write_json(scene_dir / CLEANUP_MANIFEST_NAME, cleanup_manifest)
return cleanup_manifest
def cleanup_gf3_sarscape_native_pool(
*,
native_dirs: list[str] | None = None,
storage_root: str | None = None,
require_standardized: bool = True,
dry_run: bool = False,
max_scenes: int | None = None,
log_callback: LogCallback | None = None,
progress_callback: ProgressCallback | None = None,
) -> dict[str, Any]:
"""Delete intermediate SARscape files while keeping final native _geo assets."""
native_dirs = native_dirs if native_dirs is not None else split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
native_roots, missing_roots = _resolve_existing_dirs(native_dirs)
if not native_roots:
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS has no accessible directories.")
storage = Path(os.path.normpath(storage_root or settings.GF3_STORAGE_DIRS)).resolve() if storage_root or settings.GF3_STORAGE_DIRS else None
inventory = scan_gf3_sarscape_native_roots([str(root) for root in native_roots], write_manifest=not dry_run)
scenes = inventory.get("scenes") or []
max_count = int(max_scenes or 0)
cleaned = 0
skipped = 0
error_count = 0
bytes_deleted = 0
scene_results: list[dict[str, Any]] = []
for idx, scene_manifest in enumerate(scenes):
_emit_progress(
progress_callback,
5 + int((idx / max(len(scenes), 1)) * 90),
f"GF3 native cleanup checking {idx + 1}/{len(scenes)}: {scene_manifest.get('scene_name')}",
)
scene_status = str(scene_manifest.get("status") or "")
if scene_status != "NATIVE_READY":
skipped += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"status": "skipped",
"reason": f"native status is {scene_status or 'UNKNOWN'}",
}
)
continue
if require_standardized:
if storage is None:
skipped += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"status": "skipped",
"reason": "GF3_STORAGE_DIRS is not configured",
}
)
continue
if not _standard_manifest_allows_cleanup(scene_manifest, storage):
skipped += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"status": "skipped",
"reason": "standard GeoTIFF manifest is not DONE",
}
)
continue
if max_count > 0 and cleaned >= max_count:
skipped += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"status": "skipped",
"reason": "max_scenes limit reached",
}
)
continue
result = _cleanup_scene_intermediates(scene_manifest, native_roots=native_roots, dry_run=dry_run)
scene_results.append(result)
bytes_deleted += int(result.get("bytes_deleted") or 0)
if result.get("status") == "error":
error_count += 1
if result.get("status") in {"cleaned", "nothing_to_delete"}:
cleaned += 1
_emit_log(
log_callback,
"INFO",
f"GF3 cleanup {result.get('status')}: {result.get('scene_name')} bytes={result.get('bytes_deleted')}",
)
_emit_progress(progress_callback, 100, "GF3 native cleanup finished.")
return {
"ok": error_count == 0,
"dry_run": dry_run,
"native_roots": [str(root) for root in native_roots],
"missing_roots": missing_roots,
"scene_count": len(scenes),
"cleaned_scene_count": cleaned,
"skipped_scene_count": skipped,
"error_scene_count": error_count,
"bytes_deleted": bytes_deleted,
"scenes": scene_results,
}
@@ -0,0 +1,927 @@
"""Convert GF3 SARscape native geocoded outputs to platform GeoTIFFs."""
from __future__ import annotations
import asyncio
import hashlib
import json
import math
import os
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from geoalchemy2.shape import from_shape
from shapely.geometry import Polygon
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
from .data_service import extract_geotiff_bounds
from .gf3_native_inventory_service import (
NATIVE_MANIFEST_NAME,
POLARIZATION_PRIORITY,
scan_gf3_sarscape_native_roots,
)
from .image_service import image_service
from .sar_analysis_ready_service import register_analysis_ready_tif
STANDARD_MANIFEST_NAME = "gf3_standard_manifest.json"
STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
CONVERTER_VERSION = "v1"
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
def _utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def _path_text(value: Any) -> str:
text = str(value or "").strip()
return os.path.normpath(text) if text else ""
def _db_now() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0)
def _path_kind(path: str) -> str:
text = str(path or "").strip()
if text.startswith("\\\\"):
return "unc"
if len(text) >= 3 and text[1:3] in {":\\", ":/"} and text[0].isalpha():
return "windows"
if text.startswith("/mnt/"):
return "wsl_mount"
if text.startswith("/"):
return "posix"
return "relative"
def _source_asset_uid(path: str) -> str:
normalized = os.path.normpath(str(path or "").strip())
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
return f"source:{digest[:32]}"
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
text = str(value or "").strip()
if not text:
text = default
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
return safe or default
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp_path = path.with_name(f".{path.name}.tmp")
with tmp_path.open("w", encoding="utf-8") as stream:
json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False)
os.replace(tmp_path, path)
def _json_safe(value: Any) -> Any:
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, dict):
return {key: _json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe(item) for item in value]
return value
def _finite_float(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if math.isfinite(number) else None
def _read_json(path: Path) -> dict[str, Any] | None:
try:
with path.open("r", encoding="utf-8") as stream:
data = json.load(stream)
return data if isinstance(data, dict) else None
except (OSError, json.JSONDecodeError):
return None
def _file_fingerprint(path: Path) -> dict[str, Any] | None:
try:
stat = path.stat()
except OSError:
return None
return {
"path": str(path),
"size": int(stat.st_size),
"mtime": float(stat.st_mtime),
"mtime_ns": int(stat.st_mtime_ns),
}
def _tree_stats(path: Path) -> dict[str, Any]:
if path.is_file():
info = _file_fingerprint(path)
return {
"size_bytes": info.get("size") if info else None,
"mtime_epoch": info.get("mtime") if info else None,
}
total = 0
newest: float | None = None
try:
iterator = path.rglob("*")
for item in iterator:
try:
if not item.is_file():
continue
stat = item.stat()
except OSError:
continue
total += int(stat.st_size)
mtime = float(stat.st_mtime)
newest = mtime if newest is None else max(newest, mtime)
except OSError:
return {"size_bytes": None, "mtime_epoch": None}
return {"size_bytes": total, "mtime_epoch": newest}
async def _find_managed_root_for_path(db: AsyncSession, path: str) -> ManagedRootORM | None:
target = os.path.normcase(os.path.normpath(str(path or "")))
if not target:
return None
result = await db.execute(
select(ManagedRootORM)
.where(ManagedRootORM.enabled == True) # noqa: E712
.order_by(func.length(ManagedRootORM.path).desc())
)
for root in result.scalars().all():
root_path = os.path.normcase(os.path.normpath(str(root.path or "")))
if target == root_path or target.startswith(root_path + os.sep):
return root
return None
def _batch_name(scene_manifest: dict[str, Any]) -> str:
raw = scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date")
return _safe_slug(raw, default="unknown_batch")
def _standard_scene_dir(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
return storage_root / _batch_name(scene_manifest) / _safe_slug(scene_manifest.get("scene_name"))
def _target_tif_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
return _standard_scene_dir(scene_manifest, storage_root) / f"{pol}_L2.tif"
def _preview_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
return _standard_scene_dir(scene_manifest, storage_root) / f"preview_{pol}.png"
def _quality_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
return _standard_scene_dir(scene_manifest, storage_root) / f"quality_{pol}.json"
def _source_changed(asset: dict[str, Any], target_tif: Path, existing_asset: dict[str, Any] | None) -> bool:
source_path = Path(_path_text(asset.get("path")))
source_fp = _file_fingerprint(source_path)
if not target_tif.is_file() or target_tif.stat().st_size <= 0:
return True
if not existing_asset:
return True
if str(existing_asset.get("source_native") or "") != str(source_path):
return True
if (existing_asset.get("source_fingerprint") or {}) != source_fp:
return True
if str(existing_asset.get("converter_version") or "") != CONVERTER_VERSION:
return True
return False
def _existing_manifest_asset(standard_manifest: dict[str, Any] | None, polarization: str) -> dict[str, Any] | None:
if not standard_manifest:
return None
target_pol = str(polarization or "").upper()
for item in standard_manifest.get("assets") or []:
if str(item.get("polarization") or "").upper() == target_pol:
return item
return None
def _convert_native_asset_to_tif(asset: dict[str, Any], target_tif: Path) -> dict[str, Any]:
source_path = Path(_path_text(asset.get("path")))
if not source_path.is_file():
raise FileNotFoundError(f"GF3 native data file does not exist: {source_path}")
hdr_path = Path(_path_text(asset.get("hdr")))
if not hdr_path.is_file():
raise FileNotFoundError(f"GF3 native ENVI header does not exist: {hdr_path}")
target_tif.parent.mkdir(parents=True, exist_ok=True)
tmp_tif = target_tif.with_name(f".{target_tif.name}.tmp.tif")
if tmp_tif.exists():
tmp_tif.unlink()
try:
from osgeo import gdal
src_ds = gdal.Open(str(source_path), gdal.GA_ReadOnly)
if src_ds is None:
raise RuntimeError(f"GDAL cannot open GF3 native dataset: {source_path}")
creation_options = ["TILED=YES", "COMPRESS=DEFLATE", "BIGTIFF=IF_SAFER"]
if bool(settings.SAR_ANALYSIS_OUTPUT_COG):
creation_options.append("COPY_SRC_OVERVIEWS=YES")
translated = gdal.Translate(
str(tmp_tif),
src_ds,
format="GTiff",
creationOptions=creation_options,
)
src_ds = None
if translated is None:
raise RuntimeError(f"GDAL Translate failed for GF3 native dataset: {source_path}")
translated.FlushCache()
translated = None
except ImportError:
import rasterio
with rasterio.open(source_path) as src:
profile = src.profile.copy()
profile.update(
driver="GTiff",
tiled=True,
compress="deflate",
BIGTIFF="IF_SAFER",
)
with rasterio.open(tmp_tif, "w", **profile) as dst:
for band_idx in range(1, src.count + 1):
for _block_index, window in src.block_windows(band_idx):
dst.write(src.read(band_idx, window=window), band_idx, window=window)
dst.update_tags(**src.tags())
for band_idx in range(1, src.count + 1):
dst.update_tags(band_idx, **src.tags(band_idx))
os.replace(tmp_tif, target_tif)
return {
"path": str(target_tif),
"source_native": str(source_path),
"source_fingerprint": _file_fingerprint(source_path),
"converter_name": CONVERTER_NAME,
"converter_version": CONVERTER_VERSION,
}
def _raster_quality(path: Path) -> dict[str, Any]:
try:
import numpy as np
import rasterio
except Exception as exc:
return {"ok": False, "warning": f"rasterio unavailable: {exc}"}
with rasterio.open(path) as src:
if src.height > 2048 or src.width > 2048:
scale = min(1024 / src.width, 1024 / src.height)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
sampled = src.read(1, out_shape=(out_height, out_width), masked=True)
else:
sampled = src.read(1, masked=True)
valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)]
bounds = src.bounds
quality: dict[str, Any] = {
"ok": True,
"driver": src.driver,
"width": src.width,
"height": src.height,
"count": src.count,
"dtype": str(src.dtypes[0]) if src.dtypes else None,
"crs": src.crs.to_string() if src.crs else None,
"bounds": {
"left": bounds.left,
"bottom": bounds.bottom,
"right": bounds.right,
"top": bounds.top,
},
"transform": list(src.transform)[:6],
"nodata": _finite_float(src.nodata),
"valid_sample_count": int(valid.size),
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
}
if valid.size:
quality.update(
{
"sample_min": float(np.nanmin(valid)),
"sample_max": float(np.nanmax(valid)),
"sample_mean": float(np.nanmean(valid)),
"sample_p02": float(np.nanpercentile(valid, 2)),
"sample_p98": float(np.nanpercentile(valid, 98)),
}
)
return quality
def _build_preview_png(source: Path, target: Path) -> str | None:
try:
import numpy as np
import rasterio
from PIL import Image
except Exception:
return None
target.parent.mkdir(parents=True, exist_ok=True)
with rasterio.open(source) as src:
if src.height > 1600 or src.width > 1600:
scale = min(1600 / src.width, 1600 / src.height)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
band = src.read(1, out_shape=(out_height, out_width), masked=True)
else:
band = src.read(1, masked=True)
data = band.filled(float("nan")).astype("float32")
valid = data[np.isfinite(data)]
if valid.size:
p2, p98 = np.nanpercentile(valid, [2, 98])
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
normalized = np.where(np.isfinite(normalized), normalized, 0)
gray = (normalized * 255).astype("uint8")
else:
gray = np.zeros(data.shape, dtype="uint8")
alpha = np.where(np.isfinite(data), 255, 0).astype("uint8")
rgba = np.stack([gray, gray, gray, alpha], axis=-1)
Image.fromarray(rgba, "RGBA").save(target)
return str(target)
def _build_preview_from_quicklook(source: Path | None, target: Path) -> str | None:
if source is None or not source.is_file():
return None
try:
from PIL import Image
except Exception:
return None
target.parent.mkdir(parents=True, exist_ok=True)
try:
with Image.open(source) as img:
preview = img.copy()
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
preview.thumbnail((1600, 1600), resampling)
if preview.mode in {"1", "I", "I;16", "F"}:
preview = preview.convert("L")
elif preview.mode not in {"L", "LA", "RGB", "RGBA"}:
preview = preview.convert("RGB")
preview = image_service.make_edge_dark_transparent(preview)
preview.save(target, "PNG")
return str(target)
except Exception:
return None
def _asset_quicklook_path(asset: dict[str, Any]) -> Path | None:
text = _path_text(asset.get("quicklook"))
if not text:
return None
return Path(text)
def _points_look_like_lonlat(points: list[tuple[float, float]]) -> bool:
if not points:
return False
for lon, lat in points:
if not (math.isfinite(float(lon)) and math.isfinite(float(lat))):
return False
if not (-180.0 <= float(lon) <= 180.0 and -90.0 <= float(lat) <= 90.0):
return False
return True
def _crs_is_geographic_lonlat(crs: Any) -> bool:
if not crs:
return False
try:
if crs.to_epsg() == 4326:
return True
except Exception:
pass
try:
if bool(crs.is_geographic):
return True
except Exception:
pass
try:
wkt = str(crs.to_wkt() or "").upper()
if "GEOGCS" in wkt and ("WGS 84" in wkt or "WORLD GEODETIC" in wkt):
return True
except Exception:
pass
return False
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
polygon = extract_geotiff_bounds(str(path))
if polygon and len(polygon) >= 4:
return polygon
try:
import rasterio
from rasterio.warp import transform
with rasterio.open(path) as src:
if not src.crs:
return None
corners_xy = [
src.transform * (0, 0),
src.transform * (src.width, 0),
src.transform * (src.width, src.height),
src.transform * (0, src.height),
]
xs = [point[0] for point in corners_xy]
ys = [point[1] for point in corners_xy]
raw_points = [(float(x), float(y)) for x, y in zip(xs, ys)]
if _crs_is_geographic_lonlat(src.crs):
points = raw_points
else:
try:
lons, lats = transform(src.crs, "EPSG:4326", xs, ys)
points = [(float(lon), float(lat)) for lon, lat in zip(lons, lats)]
except Exception:
if not _points_look_like_lonlat(raw_points):
raise
points = raw_points
if not _points_look_like_lonlat(points):
return None
points.append(points[0])
return points
except Exception:
return None
return None
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
if not polygon:
return None, None
try:
shp = Polygon(polygon)
if not shp.is_valid:
shp = shp.buffer(0)
if shp.is_valid and not shp.is_empty:
return float(shp.centroid.x), float(shp.centroid.y)
except Exception:
return None, None
return None, None
def _bounds_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None, float | None, float | None]:
if not polygon:
return None, None, None, None
lons = [float(point[0]) for point in polygon]
lats = [float(point[1]) for point in polygon]
return min(lons), min(lats), max(lons), max(lats)
def _geom_from_polygon(polygon: list[tuple[float, float]] | None) -> Any | None:
if not polygon:
return None
try:
shp = Polygon(polygon)
if not shp.is_valid:
shp = shp.buffer(0)
if shp.is_valid and not shp.is_empty:
return from_shape(shp, srid=4326)
except Exception:
return None
return None
def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
by_pol = {str(asset.get("polarization") or "").upper(): asset for asset in assets}
for pol in POLARIZATION_PRIORITY:
if pol in by_pol:
return by_pol[pol]
return assets[0] if assets else None
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
metadata = dict(scene_manifest.get("metadata") or {})
metadata.update(
{
"native_dir": scene_manifest.get("native_dir"),
"native_manifest": scene_manifest.get("manifest_path"),
"standard_manifest": standard_manifest.get("manifest_path"),
"standard_dir": standard_manifest.get("standard_dir"),
"standard_assets": standard_manifest.get("assets") or [],
"analysis_engine": "gf3_sarscape",
}
)
return metadata
async def _upsert_source_product_asset(
db: AsyncSession,
scene_manifest: dict[str, Any],
standard_manifest: dict[str, Any],
) -> int | None:
standard_dir_text = _path_text(standard_manifest.get("standard_dir"))
if not standard_dir_text:
return None
standard_dir = Path(standard_dir_text)
metadata = scene_manifest.get("metadata") or {}
scene_name = scene_manifest.get("scene_name") or standard_dir.name
now = _db_now()
root = await _find_managed_root_for_path(db, standard_dir_text)
stats = await asyncio.to_thread(_tree_stats, standard_dir)
asset_metadata = _json_safe(
{
"source": "GF3 SARscape standardized L2",
"native_dir": scene_manifest.get("native_dir"),
"native_manifest": scene_manifest.get("manifest_path"),
"standard_manifest": standard_manifest.get("manifest_path"),
"standard_dir": standard_dir_text,
"standard_status": standard_manifest.get("status"),
"standard_assets": standard_manifest.get("assets") or [],
"summary": standard_manifest.get("summary") or {},
"errors": standard_manifest.get("errors") or [],
"analysis_engine": "gf3_sarscape",
}
)
data = {
"asset_uid": _source_asset_uid(standard_dir_text),
"logical_product_uid": scene_name,
"satellite_family": "GF3",
"satellite": "GF3",
"source_format": SOURCE_ASSET_FORMAT,
"product_type": metadata.get("product_type") or "SARSCAPE_L2",
"product_level": "L2",
"imaging_mode": metadata.get("imaging_mode"),
"polarization": metadata.get("polarization"),
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"orbit_direction": metadata.get("orbit_direction"),
"acquisition_start_time_utc": None,
"acquisition_stop_time_utc": None,
"imaging_date": metadata.get("imaging_date"),
"root_ref_id": root.id if root else None,
"root_path": root.path if root else str(standard_dir.parent),
"file_path": standard_dir_text,
"archive_path": scene_manifest.get("native_dir"),
"path_kind": _path_kind(standard_dir_text),
"file_name": standard_dir.name,
"file_stem": standard_dir.name,
"file_ext": "",
"size_bytes": stats.get("size_bytes"),
"mtime_epoch": stats.get("mtime_epoch"),
"checksum_status": "NOT_COMPUTED",
"parser_name": "gf3_sarscape_standard_manifest",
"parser_version": CONVERTER_VERSION,
"parse_status": "OK" if standard_manifest.get("status") == "DONE" else str(standard_manifest.get("status") or "PARTIAL"),
"parse_error": "; ".join(str(item.get("error") or item) for item in (standard_manifest.get("errors") or [])) or None,
"parsed_at": now,
"metadata_json": asset_metadata,
"is_active": True,
"missing_since": None,
"updated_at": now,
}
result = await db.execute(
select(SourceProductAssetORM).where(
or_(
SourceProductAssetORM.asset_uid == data["asset_uid"],
SourceProductAssetORM.file_path == standard_dir_text,
)
)
)
asset = result.scalars().first()
if asset is None:
asset = SourceProductAssetORM(**data)
db.add(asset)
else:
for key, value in data.items():
setattr(asset, key, value)
await db.flush()
return int(asset.id) if asset.id is not None else None
async def _upsert_radar_data(
db: AsyncSession,
scene_manifest: dict[str, Any],
standard_manifest: dict[str, Any],
source_product_ref_id: int | None = None,
) -> int | None:
assets = standard_manifest.get("assets") or []
default_asset = _select_default_asset(assets)
if not default_asset:
return None
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
metadata = scene_manifest.get("metadata") or {}
radar_metadata = _metadata_for_radar(scene_manifest, standard_manifest)
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
unique_id = f"gf3_sarscape:{scene_name}"
file_path = str(standard_manifest.get("standard_dir") or "")
data_to_upsert = {
"unique_id": unique_id,
"satellite": "GF3",
"satellite_family": "GF3",
"imaging_date": metadata.get("imaging_date"),
"imaging_mode": metadata.get("imaging_mode"),
"polarization": ",".join(
pol
for pol in POLARIZATION_PRIORITY
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
)
or metadata.get("polarization"),
"scene_center_lon": metadata.get("scene_center_lon") if metadata.get("scene_center_lon") is not None else center_lon,
"scene_center_lat": metadata.get("scene_center_lat") if metadata.get("scene_center_lat") is not None else center_lat,
"product_level": "L2",
"product_unique_id": metadata.get("product_unique_id"),
"source_format": SOURCE_ASSET_FORMAT,
"source_product_ref_id": source_product_ref_id,
"image_data_format": "GEOTIFF",
"geocoded_flag": True,
"metadata_json": radar_metadata,
"file_path": file_path,
"has_orbit_data": False,
"orbit_file_path": None,
"is_envi_processed": True,
"coverage_polygon": polygon,
"geom": geom,
"min_lon": min_lon,
"min_lat": min_lat,
"max_lon": max_lon,
"max_lat": max_lat,
}
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == file_path,
)
)
)
radar = result.scalars().first()
if radar is None:
radar = RadarDataORM(**data_to_upsert)
db.add(radar)
else:
for key, value in data_to_upsert.items():
setattr(radar, key, value)
await db.flush()
return int(radar.id) if radar.id is not None else None
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
scene = result.scalar_one_or_none()
if scene:
return scene
scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING")
db.add(scene)
await db.flush()
return scene
async def _register_analysis_ready(
db: AsyncSession,
radar_id: int,
scene_manifest: dict[str, Any],
standard_manifest: dict[str, Any],
) -> dict[str, Any] | None:
radar = await db.get(RadarDataORM, radar_id)
if not radar:
return None
assets = standard_manifest.get("assets") or []
default_asset = _select_default_asset(assets)
if not default_asset:
return None
scene = await _get_or_create_scene(db, radar_id)
return await register_analysis_ready_tif(
db=db,
scene=scene,
radar=radar,
source_tif_path=str(default_asset.get("path") or ""),
engine="gf3_sarscape",
profile=CONVERTER_NAME,
backscatter_unit="unknown",
polarization=str(default_asset.get("polarization") or "").upper() or None,
preview_source_path=str(default_asset.get("preview") or "") or None,
metadata={
"source": "GF3 SARscape native _geo",
"native_dir": scene_manifest.get("native_dir"),
"native_manifest": scene_manifest.get("manifest_path"),
"standard_manifest": standard_manifest.get("manifest_path"),
"available_polarization": [asset.get("polarization") for asset in assets],
"standard_assets": assets,
},
)
def standardize_scene_manifest(
scene_manifest: dict[str, Any],
*,
storage_root: str | Path | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Convert one native scene manifest to GeoTIFF assets."""
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
out_dir = _standard_scene_dir(scene_manifest, root)
manifest_path = out_dir / STANDARD_MANIFEST_NAME
existing_manifest = _read_json(manifest_path)
converted = 0
skipped = 0
failed = 0
output_assets: list[dict[str, Any]] = []
errors: list[dict[str, str]] = []
for asset in scene_manifest.get("assets") or []:
if not asset.get("complete"):
continue
pol = str(asset.get("polarization") or "UNKNOWN").upper()
target_tif = _target_tif_path(scene_manifest, asset, root)
existing_asset = _existing_manifest_asset(existing_manifest, pol)
try:
if force or _source_changed(asset, target_tif, existing_asset):
convert_info = _convert_native_asset_to_tif(asset, target_tif)
converted += 1
status = "converted"
else:
convert_info = {
"path": str(target_tif),
"source_native": str(Path(_path_text(asset.get("path")))),
"source_fingerprint": _file_fingerprint(Path(_path_text(asset.get("path")))),
"converter_name": CONVERTER_NAME,
"converter_version": CONVERTER_VERSION,
}
skipped += 1
status = "skipped"
quality = _raster_quality(target_tif)
quality_file = _quality_path(scene_manifest, asset, root)
_write_json(quality_file, quality)
preview_target = _preview_path(scene_manifest, asset, root)
quicklook_path = _asset_quicklook_path(asset)
preview = _build_preview_from_quicklook(quicklook_path, preview_target)
preview_source = str(quicklook_path) if preview and quicklook_path else str(target_tif)
if not preview:
preview = _build_preview_png(target_tif, preview_target)
output_assets.append(
{
"polarization": pol,
"role": "analysis_tif",
"path": str(target_tif),
"source_native": convert_info["source_native"],
"source_fingerprint": convert_info["source_fingerprint"],
"converter_name": CONVERTER_NAME,
"converter_version": CONVERTER_VERSION,
"quality": str(quality_file),
"preview": preview,
"preview_source": preview_source,
"status": status,
}
)
except Exception as exc:
failed += 1
errors.append({"polarization": pol, "source_native": str(asset.get("path") or ""), "error": str(exc)})
status = "DONE" if output_assets and failed == 0 else ("PARTIAL" if output_assets else "FAILED")
standard_manifest = {
"schema": STANDARD_MANIFEST_SCHEMA,
"generated_at": _utc_now(),
"scene_name": scene_manifest.get("scene_name"),
"batch_name": scene_manifest.get("batch_name"),
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
"native_dir": scene_manifest.get("native_dir"),
"standard_dir": str(out_dir),
"manifest_path": str(manifest_path),
"status": status,
"converter": {"name": CONVERTER_NAME, "version": CONVERTER_VERSION},
"assets": output_assets,
"summary": {
"converted": converted,
"skipped": skipped,
"failed": failed,
},
"errors": errors,
}
_write_json(manifest_path, standard_manifest)
return standard_manifest
async def standardize_gf3_sarscape_native_roots(
db: AsyncSession,
*,
native_dirs: list[str] | None = None,
storage_root: str | None = None,
force: bool = False,
register: bool = True,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Scan native roots, convert complete assets, and register standard scenes."""
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
native_dirs,
write_manifest=True,
)
scenes = inventory.get("scenes") or []
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
converted_scenes = 0
partial_scenes = 0
failed_scenes = 0
skipped_assets = 0
converted_assets = 0
failed_assets = 0
registered = 0
analysis_ready = 0
scene_results: list[dict[str, Any]] = []
total = len(ready_scenes)
for idx, scene_manifest in enumerate(ready_scenes):
if progress_callback:
pct = 10 + int((idx / max(total, 1)) * 80)
progress_callback(pct, f"标准化 GF3 SARscape 原生结果 {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
standard_manifest = await asyncio.to_thread(
standardize_scene_manifest,
scene_manifest,
storage_root=storage_root,
force=force,
)
summary = standard_manifest.get("summary") or {}
converted_assets += int(summary.get("converted") or 0)
skipped_assets += int(summary.get("skipped") or 0)
failed_assets += int(summary.get("failed") or 0)
status = standard_manifest.get("status")
if status == "DONE":
converted_scenes += 1
elif status == "PARTIAL":
partial_scenes += 1
else:
failed_scenes += 1
radar_id = None
source_asset_id = None
analysis_manifest_path = None
if register and status in {"DONE", "PARTIAL"}:
source_asset_id = await _upsert_source_product_asset(db, scene_manifest, standard_manifest)
radar_id = await _upsert_radar_data(
db,
scene_manifest,
standard_manifest,
source_product_ref_id=source_asset_id,
)
if radar_id:
registered += 1
analysis_manifest = await _register_analysis_ready(db, radar_id, scene_manifest, standard_manifest)
if analysis_manifest:
analysis_ready += 1
analysis_manifest_path = analysis_manifest.get("analysis_dir")
await db.commit()
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"native_status": scene_manifest.get("status"),
"standard_status": status,
"standard_manifest": standard_manifest.get("manifest_path"),
"source_asset_id": source_asset_id,
"radar_id": radar_id,
"analysis_manifest_path": analysis_manifest_path,
"summary": summary,
"errors": standard_manifest.get("errors") or [],
}
)
return {
"ok": failed_scenes == 0 and failed_assets == 0,
"inventory": {
key: value
for key, value in inventory.items()
if key != "scenes"
},
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": converted_scenes,
"partial_scenes": partial_scenes,
"failed_scenes": failed_scenes,
"converted_assets": converted_assets,
"skipped_assets": skipped_assets,
"failed_assets": failed_assets,
"registered": registered,
"analysis_ready": analysis_ready,
"scenes": scene_results,
}
+53
View File
@@ -879,6 +879,36 @@ def _probe_directory_status(path: str) -> Dict[str, Any]:
return payload
def _probe_file_status(path: str) -> Dict[str, Any]:
normalized = str(path or "").strip()
payload = {
"path": normalized,
"exists": False,
"accessible": False,
"error": None,
}
if not normalized:
payload["error"] = "empty path"
return payload
try:
if os.path.isfile(normalized):
payload["exists"] = True
try:
with open(normalized, "rb") as stream:
stream.read(1)
payload["accessible"] = True
except Exception as exc:
payload["error"] = str(exc)
return payload
os.stat(normalized)
payload["error"] = "path exists but is not a file"
except Exception as exc:
payload["error"] = str(exc)
return payload
async def _check_source_roots() -> Dict[str, Any]:
items = []
@@ -908,11 +938,34 @@ async def _check_source_roots() -> Dict[str, Any]:
status["role"] = "gf3_l1a_source"
items.append(status)
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_sarscape_native"
items.append(status)
for path in split_env_paths(settings.GF3_STORAGE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_l2_storage"
items.append(status)
wrapper_exe = str(settings.GF3_SARSCAPE_WRAPPER_EXE or "").strip()
if wrapper_exe:
status = _probe_file_status(wrapper_exe)
status["role"] = "gf3_sarscape_wrapper"
items.append(status)
idlrt_path = str(settings.GF3_SARSCAPE_IDLRT_PATH or "").strip()
if idlrt_path:
status = _probe_file_status(idlrt_path)
status["role"] = "gf3_sarscape_idlrt"
items.append(status)
dem_path = str(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or "").strip()
if dem_path:
status = _probe_file_status(dem_path)
status["role"] = "gf3_sarscape_dem"
items.append(status)
configured_count = len(items)
accessible_count = sum(1 for item in items if item.get("accessible"))
inaccessible_count = configured_count - accessible_count
+61 -11
View File
@@ -15,6 +15,7 @@
import os
import time
import json
from collections import deque
from typing import Tuple, Optional, Dict, Any, List
from PIL import Image
import rasterio
@@ -335,6 +336,55 @@ class ImageService:
os.makedirs(os.path.dirname(output_path), exist_ok=True)
image.save(output_path, format='WEBP', quality=quality)
@staticmethod
def make_edge_dark_transparent(
image: Image.Image,
*,
threshold: int = 6,
) -> Image.Image:
"""Make edge-connected near-black preview background transparent."""
rgba = image.convert("RGBA")
arr = np.array(rgba, dtype=np.uint8, copy=True)
if arr.ndim != 3 or arr.shape[2] < 4:
return rgba
alpha = arr[:, :, 3]
dark = (alpha > 0) & (arr[:, :, :3].max(axis=2) <= int(threshold))
if not dark.any():
return rgba
h, w = dark.shape
edge = np.zeros_like(dark, dtype=bool)
edge[0, :] = dark[0, :]
edge[h - 1, :] = dark[h - 1, :]
edge[:, 0] |= dark[:, 0]
edge[:, w - 1] |= dark[:, w - 1]
if not edge.any():
return rgba
visited = np.zeros_like(dark, dtype=bool)
ys, xs = np.where(edge)
queue = deque(zip(ys.tolist(), xs.tolist()))
visited[ys, xs] = True
while queue:
y, x = queue.popleft()
if y > 0 and dark[y - 1, x] and not visited[y - 1, x]:
visited[y - 1, x] = True
queue.append((y - 1, x))
if y + 1 < h and dark[y + 1, x] and not visited[y + 1, x]:
visited[y + 1, x] = True
queue.append((y + 1, x))
if x > 0 and dark[y, x - 1] and not visited[y, x - 1]:
visited[y, x - 1] = True
queue.append((y, x - 1))
if x + 1 < w and dark[y, x + 1] and not visited[y, x + 1]:
visited[y, x + 1] = True
queue.append((y, x + 1))
arr[:, :, 3][visited] = 0
return Image.fromarray(arr, "RGBA")
@staticmethod
def create_cached_image(
file_path: str,
@@ -497,12 +547,12 @@ class ImageService:
@staticmethod
def _warp_preview_to_geo_bbox(
source_rgb: np.ndarray,
source_rgba: np.ndarray,
inverse_h: np.ndarray,
bbox: Tuple[float, float, float, float],
out_size: Tuple[int, int],
) -> np.ndarray:
src_h, src_w = source_rgb.shape[:2]
src_h, src_w = source_rgba.shape[:2]
out_w, out_h = out_size
min_lon, min_lat, max_lon, max_lat = bbox
lon_span = max_lon - min_lon
@@ -557,7 +607,7 @@ class ImageService:
du = (u_valid - x0).astype(np.float32)
dv = (v_valid - y0).astype(np.float32)
src_float = source_rgb.astype(np.float32, copy=False)
src_float = source_rgba.astype(np.float32, copy=False)
s00 = src_float[y0, x0]
s10 = src_float[y0, x1]
s01 = src_float[y1, x0]
@@ -568,15 +618,14 @@ class ImageService:
+ s01 * (1 - du)[:, None] * dv[:, None]
+ s11 * du[:, None] * dv[:, None]
)
rgb = np.clip(samples, 0, 255).astype(np.uint8)
rgba = np.clip(samples, 0, 255).astype(np.uint8)
else:
nearest_x = np.clip(np.round(u_valid).astype(np.int32), 0, src_w - 1)
nearest_y = np.clip(np.round(v_valid).astype(np.int32), 0, src_h - 1)
rgb = source_rgb[nearest_y, nearest_x]
rgba = source_rgba[nearest_y, nearest_x]
flat = output.reshape(-1, 4)
flat[valid_idx, :3] = rgb
flat[valid_idx, 3] = 255
flat[valid_idx] = rgba
return output
@staticmethod
@@ -608,9 +657,10 @@ class ImageService:
return False, "invalid_bbox"
with Image.open(source_image_path) as image:
source_rgb = np.asarray(image.convert("RGB"), dtype=np.uint8)
source = ImageService.make_edge_dark_transparent(image)
source_rgba = np.asarray(source, dtype=np.uint8)
src_h, src_w = source_rgb.shape[:2]
src_h, src_w = source_rgba.shape[:2]
if src_h < 1 or src_w < 1:
return False, "invalid_source_image_size"
@@ -633,7 +683,7 @@ class ImageService:
return False, "homography_invert_failed"
warped_rgba = ImageService._warp_preview_to_geo_bbox(
source_rgb=source_rgb,
source_rgba=source_rgba,
inverse_h=inverse_h,
bbox=bbox,
out_size=out_size,
@@ -672,7 +722,7 @@ class ImageService:
)
with Image.open(source_image_path) as image:
image = image.convert("RGB")
image = ImageService.make_edge_dark_transparent(image)
image.thumbnail(max_size, Image.Resampling.LANCZOS)
ImageService.save_image_as_webp(image, cache_path, quality=82)
return True
+334 -1
View File
@@ -86,6 +86,9 @@ JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION"
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
@@ -3455,7 +3458,8 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
raise ValueError("水体检测缺少输入路径 input_path")
output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}"
output_dir = os.path.join(os.path.dirname(input_path), output_name)
output_root = settings.WATER_RESULTS_DIR or os.path.join(settings.BACKEND_DIR, "water_results")
output_dir = os.path.join(output_root, output_name)
os.makedirs(output_dir, exist_ok=True)
await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...")
@@ -3496,6 +3500,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
det.threshold_value = result.get("threshold_value")
det.metadata_json = {
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
"value_transform": result.get("value_transform"),
"job_id": job.job_id,
}
else:
@@ -3518,6 +3523,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
mirror.task_id = job.task_id
mirror.metadata_json = {
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
"value_transform": result.get("value_transform"),
"legacy_detection_id": int(record_id),
"job_id": job.job_id,
}
@@ -3781,6 +3787,330 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
)
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
if not job.task_id:
raise ValueError("GF3_SARSCAPE_SYNC requires task_id for progress tracking.")
payload = job.payload or {}
native_dirs = payload.get("native_dirs") or []
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
if not native_dirs:
raise ValueError("GF3_SARSCAPE_SYNC: native_dirs is empty")
if not storage_root:
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
loop = asyncio.get_running_loop()
def _progress_cb(progress: int, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.update_task(job.task_id, progress=progress, message=message),
loop,
)
def _swallow_progress_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape] progress callback failed: %s", exc)
future.add_done_callback(_swallow_progress_error)
except RuntimeError:
return
try:
async with AsyncSessionLocal() as db:
result = await standardize_gf3_sarscape_native_roots(
db,
native_dirs=native_dirs,
storage_root=storage_root,
force=bool(payload.get("force", False)),
register=bool(payload.get("register", True)),
progress_callback=_progress_cb,
)
except Exception as exc:
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=f"GF3 SARscape 标准化失败: {exc}",
)
raise
message = (
"GF3 SARscape 标准化完成: "
f"发现 {int(result.get('scene_count') or 0)} 景, "
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)}"
)
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
from .gf3_sarscape_production_service import (
cleanup_gf3_sarscape_native_pool,
run_gf3_sarscape_production,
)
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
if not job.task_id:
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
payload = job.payload or {}
source_dirs = payload.get("source_dirs") or []
native_dirs = payload.get("native_dirs") or []
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
if not source_dirs:
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
if not native_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
if not storage_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
loop = asyncio.get_running_loop()
def _progress_cb(progress: int, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.update_task(job.task_id, progress=progress, message=message),
loop,
)
def _swallow_progress_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
future.add_done_callback(_swallow_progress_error)
except RuntimeError:
return
def _log_cb(level: str, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.add_log(job.task_id, level, message),
loop,
)
def _swallow_log_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
future.add_done_callback(_swallow_log_error)
except RuntimeError:
return
async def _production_keepalive() -> None:
progress = 8
while True:
await asyncio.sleep(60)
progress = min(68, progress + 1)
await task_service.update_task(
job.task_id,
progress=progress,
message="GF3 SARscape production is still running...",
)
try:
production_task = asyncio.create_task(
asyncio.to_thread(
run_gf3_sarscape_production,
source_dirs=source_dirs,
native_root=native_root,
wrapper_exe=payload.get("wrapper_exe"),
dem_path=payload.get("dem_path"),
idlrt_path=payload.get("idlrt_path"),
polarizations=payload.get("polarizations"),
archive_exts=payload.get("archive_exts") or [],
max_archives_per_run=payload.get("max_archives_per_run"),
timeout_seconds=payload.get("timeout_seconds"),
keep_extracted=payload.get("keep_extracted"),
log_callback=_log_cb,
progress_callback=_progress_cb,
)
)
keepalive_task = asyncio.create_task(_production_keepalive())
try:
production_result = await production_task
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
standardize_result: Dict[str, Any] = {}
if bool(payload.get("auto_standardize", True)):
await task_service.update_task(
job.task_id,
progress=72,
message="GF3 SARscape production finished; standardizing native _geo outputs...",
)
async with AsyncSessionLocal() as db:
standardize_result = await standardize_gf3_sarscape_native_roots(
db,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
force=bool(payload.get("force_standardize", False)),
register=bool(payload.get("register", True)),
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
)
cleanup_result: Dict[str, Any] = {}
production_ok = int(production_result.get("failed_count") or 0) == 0
standardize_ok = (
not standardize_result
or (
int(standardize_result.get("failed_assets") or 0) == 0
and int(standardize_result.get("failed_scenes") or 0) == 0
)
)
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
await task_service.update_task(
job.task_id,
progress=90,
message="Cleaning GF3 SARscape intermediate files...",
)
cleanup_result = await asyncio.to_thread(
cleanup_gf3_sarscape_native_pool,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
dry_run=bool(payload.get("cleanup_dry_run", False)),
max_scenes=payload.get("cleanup_max_scenes"),
log_callback=_log_cb,
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
)
elif bool(payload.get("clean_after_success", True)):
_log_cb(
"WARNING",
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
)
except Exception as exc:
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=f"GF3 SARscape production failed: {exc}",
)
raise
failed_count = int(production_result.get("failed_count") or 0)
failed_assets = int(standardize_result.get("failed_assets") or 0)
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
message = (
"GF3 SARscape production chain finished: "
f"found={int(production_result.get('found_count') or 0)}, "
f"produced={int(production_result.get('processed_count') or 0)}, "
f"skipped={int(production_result.get('skipped_count') or 0)}, "
f"failed={failed_count}, "
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
f"registered={int(standardize_result.get('registered') or 0)}, "
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
)
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
"""Clean GF3 SARscape intermediate files from native pool."""
from .gf3_sarscape_production_service import cleanup_gf3_sarscape_native_pool
if not job.task_id:
raise ValueError("GF3_SARSCAPE_CLEAN requires task_id for progress tracking.")
payload = job.payload or {}
native_dirs = payload.get("native_dirs") or []
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
if not native_dirs:
raise ValueError("GF3_SARSCAPE_CLEAN: native_dirs is empty")
await task_service.start_task(job.task_id, message="GF3 SARscape native cleanup starting...")
loop = asyncio.get_running_loop()
def _progress_cb(progress: int, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.update_task(job.task_id, progress=progress, message=message),
loop,
)
def _swallow_progress_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Clean] progress callback failed: %s", exc)
future.add_done_callback(_swallow_progress_error)
except RuntimeError:
return
def _log_cb(level: str, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.add_log(job.task_id, level, message),
loop,
)
def _swallow_log_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Clean] log callback failed: %s", exc)
future.add_done_callback(_swallow_log_error)
except RuntimeError:
return
try:
result = await asyncio.to_thread(
cleanup_gf3_sarscape_native_pool,
native_dirs=native_dirs,
storage_root=storage_root,
require_standardized=bool(payload.get("require_standardized", True)),
dry_run=bool(payload.get("dry_run", False)),
max_scenes=payload.get("max_scenes"),
log_callback=_log_cb,
progress_callback=_progress_cb,
)
except Exception as exc:
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=f"GF3 SARscape native cleanup failed: {exc}",
)
raise
status = "FAILED" if int(result.get("error_scene_count") or 0) else "COMPLETED"
message = (
"GF3 SARscape native cleanup finished: "
f"scenes={int(result.get('scene_count') or 0)}, "
f"cleaned={int(result.get('cleaned_scene_count') or 0)}, "
f"skipped={int(result.get('skipped_scene_count') or 0)}, "
f"errors={int(result.get('error_scene_count') or 0)}, "
f"bytes={int(result.get('bytes_deleted') or 0)}, "
f"dry_run={bool(result.get('dry_run'))}"
)
await task_service.update_task(job.task_id, status=status, progress=100, message=message)
async def _handle_publish_dinsar_products_clean(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("PUBLISH_DINSAR_PRODUCTS requires task_id for progress tracking.")
@@ -4671,6 +5001,9 @@ _HANDLERS = {
JOB_TYPE_GF3_PROCESS: _handle_gf3_process,
JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack,
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
JOB_TYPE_SBAS_INTERFEROGRAMS: _handle_sbas_interferograms,
@@ -262,6 +262,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="scene_directory",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_SARSCAPE_NATIVE_DIRS",
paths=split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS),
root_role="source_pool_gf3_sarscape_native",
display_prefix="GF3 SARscape Native Pool",
scan_mode="scene_directory",
)
)
specs.extend(
_iter_single_root_specs(
env_var="SAR_ANALYSIS_READY_ROOT",
@@ -20,6 +20,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..models import RadarDataORM, SARSceneGeoORM
from ..utils import normalize_satellite_family
from .image_service import image_service
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
_POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
@@ -82,7 +83,25 @@ def scene_analysis_dir(
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False)
def _json_safe(value: Any) -> Any:
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, dict):
return {key: _json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe(item) for item in value]
return value
def _finite_float(value: Any) -> float | None:
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if math.isfinite(number) else None
def _link_or_copy(source: Path, target: Path) -> str:
@@ -171,7 +190,7 @@ def _raster_quality(path: Path) -> dict[str, Any]:
"top": bounds.top,
},
"transform": list(transform)[:6],
"nodata": src.nodata,
"nodata": _finite_float(src.nodata),
"valid_sample_count": int(valid.size),
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
}
@@ -231,6 +250,7 @@ def _build_preview_png(source: Path, target: Path) -> str | None:
if valid.size:
p2, p98 = np.nanpercentile(valid, [2, 98])
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
normalized = np.where(np.isfinite(normalized), normalized, 0)
gray = (normalized * 255).astype("uint8")
else:
gray = np.zeros(data.shape, dtype="uint8")
@@ -240,6 +260,31 @@ def _build_preview_png(source: Path, target: Path) -> str | None:
return str(target)
def _build_preview_from_existing(source: Path | None, target: Path) -> str | None:
if source is None or not source.is_file():
return None
try:
from PIL import Image
except Exception:
return None
target.parent.mkdir(parents=True, exist_ok=True)
try:
with Image.open(source) as img:
preview = img.copy()
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
preview.thumbnail((1600, 1600), resampling)
if preview.mode in {"1", "I", "I;16", "F"}:
preview = preview.convert("L")
elif preview.mode not in {"L", "LA", "RGB", "RGBA"}:
preview = preview.convert("RGB")
preview = image_service.make_edge_dark_transparent(preview)
preview.save(target, "PNG")
return str(target)
except Exception:
return None
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
scene = result.scalar_one_or_none()
@@ -262,6 +307,7 @@ async def register_analysis_ready_tif(
backscatter_unit: str,
polarization: str | None = None,
metadata: dict[str, Any] | None = None,
preview_source_path: str | None = None,
copy_mode: str = "link_or_copy",
) -> dict[str, Any]:
source = Path(os.path.normpath(str(source_tif_path or "").strip()))
@@ -283,6 +329,9 @@ async def register_analysis_ready_tif(
transfer = _link_or_copy(source, target_tif)
quality = _raster_quality(target_tif)
preview_source = Path(os.path.normpath(preview_source_path)) if preview_source_path else None
preview_path = _build_preview_from_existing(preview_source, out_dir / "preview.png")
if not preview_path:
preview_path = _build_preview_png(target_tif, out_dir / "preview.png")
manifest = {
"scene_id": scene.id,
@@ -296,6 +345,7 @@ async def register_analysis_ready_tif(
"backscatter_unit": backscatter_unit,
"polarization": polarization,
"transfer": transfer,
"preview_source_path": str(preview_source) if preview_path and preview_source else None,
"metadata": metadata or {},
"quality": quality,
}
@@ -309,14 +359,9 @@ async def register_analysis_ready_tif(
scene.analysis_engine = engine
scene.analysis_profile = profile
scene.analysis_backscatter_unit = backscatter_unit
nodata_value = quality.get("nodata")
scene.analysis_nodata_value = (
float(nodata_value)
if nodata_value is not None
else float(settings.SAR_ANALYSIS_NODATA_VALUE)
)
scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")}
scene.analysis_quality_json = quality
scene.analysis_nodata_value = _finite_float(quality.get("nodata")) or float(settings.SAR_ANALYSIS_NODATA_VALUE)
scene.analysis_metadata_json = _json_safe({**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")})
scene.analysis_quality_json = _json_safe(quality)
scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m
scene.status = "DONE"
scene.error_msg = None
+159 -21
View File
@@ -10,6 +10,7 @@ import logging
import math
import os
import struct
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
import numpy as np
@@ -21,6 +22,65 @@ logger = logging.getLogger(__name__)
# SRTM HGT helpers
# ---------------------------------------------------------------------------
def _otsu_threshold(values: np.ndarray, bins: int = 512) -> float:
"""Compute Otsu's threshold without depending on scikit-image."""
finite_values = values[np.isfinite(values)]
if finite_values.size == 0:
raise ValueError("No finite pixels available for Otsu threshold")
min_value = float(np.min(finite_values))
max_value = float(np.max(finite_values))
if min_value == max_value:
return min_value
counts, edges = np.histogram(finite_values, bins=bins, range=(min_value, max_value))
centers = (edges[:-1] + edges[1:]) / 2.0
total = float(counts.sum())
if total <= 0:
return min_value
weight_background = np.cumsum(counts).astype(np.float64)
weight_foreground = total - weight_background
cumulative_mean = np.cumsum(counts * centers)
total_mean = cumulative_mean[-1]
valid = (weight_background > 0) & (weight_foreground > 0)
between = np.zeros_like(centers, dtype=np.float64)
mean_background = np.zeros_like(centers, dtype=np.float64)
mean_foreground = np.zeros_like(centers, dtype=np.float64)
mean_background[valid] = cumulative_mean[valid] / weight_background[valid]
mean_foreground[valid] = (total_mean - cumulative_mean[valid]) / weight_foreground[valid]
between[valid] = (
weight_background[valid]
* weight_foreground[valid]
* (mean_background[valid] - mean_foreground[valid]) ** 2
)
return float(centers[int(np.argmax(between))])
def _disk_structure(radius: int) -> np.ndarray:
y, x = np.ogrid[-radius: radius + 1, -radius: radius + 1]
return (x * x + y * y) <= radius * radius
def _prepare_detection_image(img: np.ndarray, valid: np.ndarray) -> tuple[np.ndarray, np.ndarray, str]:
"""Normalize SAR values for water thresholding and keep the original mask."""
working = img.astype(np.float32, copy=True)
working[~valid] = np.nan
valid_values = working[valid]
if valid_values.size == 0:
return working, valid, "raw"
min_value = float(np.nanmin(valid_values))
p99_value = float(np.nanpercentile(valid_values, 99))
max_value = float(np.nanmax(valid_values))
if min_value >= 0.0 and (p99_value > 1.0 or max_value > 5.0):
positive_valid = valid & (working > 0)
converted = np.full_like(working, np.nan, dtype=np.float32)
converted[positive_valid] = 10.0 * np.log10(np.maximum(working[positive_valid], 1e-12))
return converted, positive_valid, "linear_to_db"
return working, valid, "raw"
def _read_hgt(filepath: str) -> np.ndarray:
"""Read a single SRTM .hgt file. Auto-detect SRTM1 (3601) vs SRTM3 (1201)."""
file_size = os.path.getsize(filepath)
@@ -96,6 +156,70 @@ def _load_srtm3_dem(
return mosaic, dem_bounds
def _candidate_dem_paths(dem_path: str) -> list[str]:
text = str(dem_path or "").strip()
if not text:
return []
path = Path(text)
candidates: list[Path] = []
for suffix in (".vrt", ".wgs84.vrt", ".tif", ".tiff", ".img", ".wgs84"):
candidate = Path(text + suffix)
if candidate.exists() and candidate not in candidates:
candidates.append(candidate)
if path.is_file() and path not in candidates:
candidates.append(path)
if path.is_dir():
for pattern in ("*.vrt", "*.tif", "*.tiff", "*.img"):
for candidate in path.glob(pattern):
if candidate not in candidates:
candidates.append(candidate)
return [str(candidate) for candidate in candidates]
def _load_raster_dem(
bounds: Tuple[float, float, float, float],
dem_path: str,
out_shape: tuple[int, int],
) -> Optional[np.ndarray]:
"""Read a DEM raster subset and resample it to the SAR image grid size."""
import rasterio
from rasterio.enums import Resampling
from rasterio.windows import from_bounds
height, width = out_shape
for candidate in _candidate_dem_paths(dem_path):
try:
with rasterio.open(candidate) as src:
if src.crs and not src.crs.is_geographic:
continue
dem_bounds = src.bounds
min_lon, min_lat, max_lon, max_lat = bounds
if (
max_lon <= dem_bounds.left
or min_lon >= dem_bounds.right
or max_lat <= dem_bounds.bottom
or min_lat >= dem_bounds.top
):
continue
window = from_bounds(min_lon, min_lat, max_lon, max_lat, transform=src.transform)
data = src.read(
1,
window=window,
out_shape=(height, width),
boundless=True,
fill_value=np.nan,
resampling=Resampling.bilinear,
).astype(np.float32)
nodata = src.nodata
if nodata is not None and np.isfinite(nodata):
data[data == np.float32(nodata)] = np.nan
logger.info("[WaterDetect] DEM raster loaded: %s", candidate)
return data
except Exception as exc:
logger.warning("[WaterDetect] DEM raster candidate skipped: %s (%s)", candidate, exc)
return None
# ---------------------------------------------------------------------------
# Core detection
# ---------------------------------------------------------------------------
@@ -119,14 +243,12 @@ def run_water_detection(
Returns dict with keys: ok, output_path, water_area_km2, water_pixel_count, otsu_threshold_db
"""
import rasterio
from scipy import ndimage
from scipy.ndimage import median_filter, gaussian_filter, label, zoom
from skimage.filters import threshold_otsu
from skimage.morphology import disk, binary_dilation, binary_erosion
from skimage.measure import regionprops
from ..config import settings
dem_dir = settings.SRTM_DEM_DIR
dem_path = settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or settings.SRTM_DEM_DIR
os.makedirs(output_dir, exist_ok=True)
logger.info("[WaterDetect] Reading input: %s", geo_tiff_path)
@@ -136,6 +258,7 @@ def run_water_detection(
img = src.read(1).astype(np.float32)
transform = src.transform
crs = src.crs
nodata = src.nodata
height, width = img.shape
pixel_size_x = transform.a # degrees per pixel (x)
pixel_size_y = transform.e # degrees per pixel (y, negative)
@@ -155,17 +278,23 @@ def run_water_detection(
# Step 5: Valid mask
valid = np.isfinite(img) & (img != 0)
if nodata is not None and np.isfinite(nodata):
valid &= img != np.float32(nodata)
if np.count_nonzero(valid) < 100:
return {"ok": False, "error": "Too few valid pixels in input image"}
detection_img, valid, value_transform = _prepare_detection_image(img, valid)
logger.info("[WaterDetect] Value transform: %s", value_transform)
# Step 6: Otsu threshold on valid pixels
valid_pixels = img[valid]
thresh = threshold_otsu(valid_pixels)
valid_pixels = detection_img[valid]
thresh = _otsu_threshold(valid_pixels)
logger.info("[WaterDetect] Otsu threshold: %.4f", thresh)
# Step 7-8: Median + Gaussian filtering
filtered = median_filter(img, size=3)
filtered_input = np.where(valid, detection_img, np.nanmedian(valid_pixels)).astype(np.float32)
filtered = median_filter(filtered_input, size=3)
filtered = gaussian_filter(filtered, sigma=1.0)
# Step 9: Initial water mask
@@ -173,8 +302,11 @@ def run_water_detection(
# Step 3-4: Load and resample DEM (if available)
dem_applied = False
if dem_dir and os.path.isdir(dem_dir):
dem, dem_bounds = _load_srtm3_dem(bounds, dem_dir)
if dem_path and (os.path.isdir(dem_path) or os.path.isfile(dem_path)):
dem_resampled = _load_raster_dem(bounds, dem_path, (height, width))
dem, dem_bounds = (None, None)
if dem_resampled is None and os.path.isdir(dem_path):
dem, dem_bounds = _load_srtm3_dem(bounds, dem_path)
if dem is not None and dem_bounds is not None:
# Resample DEM to image resolution
zoom_y = height / dem.shape[0]
@@ -183,6 +315,7 @@ def run_water_detection(
# Clip to match image shape exactly
dem_resampled = dem_resampled[:height, :width]
if dem_resampled is not None:
# Step 10: DEM height constraint (0m <= DEM <= 1000m)
dem_valid = np.isfinite(dem_resampled)
height_mask = dem_valid & (dem_resampled >= 0) & (dem_resampled <= 1000)
@@ -204,14 +337,14 @@ def run_water_detection(
else:
logger.warning("[WaterDetect] DEM not available, skipping DEM constraints")
else:
logger.warning("[WaterDetect] SRTM_DEM_DIR not configured, skipping DEM constraints")
logger.warning("[WaterDetect] DEM path not configured, skipping DEM constraints")
# Step 12: Morphological processing — disk(5) dilate→erode→dilate→erode
selem = disk(5)
water = binary_dilation(water, selem)
water = binary_erosion(water, selem)
water = binary_dilation(water, selem)
water = binary_erosion(water, selem)
selem = _disk_structure(5)
water = ndimage.binary_dilation(water, structure=selem)
water = ndimage.binary_erosion(water, structure=selem)
water = ndimage.binary_dilation(water, structure=selem)
water = ndimage.binary_erosion(water, structure=selem)
# Step 13: Connected component filtering
labeled, num_features = label(water)
@@ -220,18 +353,22 @@ def run_water_detection(
pixel_area_m2 = abs(pixel_size_x) * 111320 * abs(pixel_size_y) * 111320
min_pixels_by_area = max(1, int(3000 / max(pixel_area_m2, 1)))
props = regionprops(labeled)
areas = [p.area for p in props]
if areas:
areas = ndimage.sum(
np.ones_like(labeled, dtype=np.uint8),
labeled,
index=np.arange(1, num_features + 1),
)
areas = np.asarray(areas, dtype=np.float64)
if areas.size:
median_area = float(np.median(areas))
min_area = max(min_pixels_by_area, int(median_area))
else:
min_area = min_pixels_by_area
# Remove small components
for prop in props:
if prop.area < min_area:
water[labeled == prop.label] = False
small_labels = np.where(areas < min_area)[0] + 1
if small_labels.size:
water[np.isin(labeled, small_labels)] = False
logger.info("[WaterDetect] Connected component filter: min_area=%d pixels, kept %d/%d components",
min_area, np.count_nonzero(np.unique(labeled[water])), num_features)
@@ -265,4 +402,5 @@ def run_water_detection(
"water_area_km2": round(water_area, 4),
"water_pixel_count": water_pixel_count,
"otsu_threshold_db": round(float(thresh), 4),
"value_transform": value_transform,
}
@@ -0,0 +1,511 @@
# 洪涝灾害分析模块工程交接文档
日期:2026-06-02
面向对象:算法工程师、后端工程师
范围:新洪涝灾害分析模块 `/flood/*`,不包含旧兼容 `/water/*` 页面和接口。
## 1. 当前定位
系统现在把洪涝分析拆成两层:
1. 工程层:负责数据入库、场景标准化、任务队列、状态更新、预览上图、套合分析、产品登记。
2. 算法层:只负责从标准化 SAR GeoTIFF 生成水体/洪涝分类栅格,并返回面积、像元数、阈值、模型信息等元数据。
后续算法优化应尽量只替换或新增 processor,不要绕开现有 `SARSceneGeoORM``WaterExtractionORM``FloodDetectionORM` 和任务队列。
## 2. 关键代码入口
| 职责 | 文件 |
| --- | --- |
| 洪涝 API 路由 | `backend/app/routers/flood.py` |
| 洪涝业务编排 | `backend/app/services/flood_analysis_service.py` |
| 后台任务执行 | `backend/app/services/job_handlers.py` |
| analysis-ready GeoTIFF 注册 | `backend/app/services/sar_analysis_ready_service.py` |
| 当前水体提取算法 | `backend/app/services/water_detect_service.py` |
| 水体 processor 包装 | `backend/app/services/water_extraction_service.py` |
| 当前洪涝变化检测算法 | `backend/app/services/flood_detection_service.py` |
| 洪涝矢量化与套合 | `backend/app/services/flood_overlay_service.py` |
| 洪涝产品登记 | `backend/app/services/flood_product_service.py` |
| 前端工作台 | `frontend/src/FloodAnalysisWorkspace.jsx` |
| 前端 API | `frontend/src/api/flood.js` |
`/water/*` 路由仍在,但只作为历史兼容,不作为新算法接入目标。
## 3. 数据模型
### 3.1 SARSceneGeoORM
表:`sar_scene_geo`
这是算法输入场景表。每条记录对应一景已标准化的 SAR 分析影像。
关键字段:
| 字段 | 含义 |
| --- | --- |
| `radar_data_id` | 关联源影像 `radar_data.id` |
| `analysis_tif_path` | 算法统一输入,单波段 GeoTIFF |
| `analysis_dir` | analysis-ready 目录 |
| `analysis_preview_path` | 场景预览 PNG |
| `analysis_engine` | 标准化引擎,如 `gf3_sarscape``gf3_gdal``lt_gamma` |
| `analysis_profile` | 标准化 profile |
| `analysis_backscatter_unit` | 后向散射单位,如 `sigma0_db``unknown` |
| `analysis_quality_json` | 栅格尺寸、范围、nodata、采样统计 |
| `pixel_size_m` | 近似像元大小 |
| `status` | `PENDING/RUNNING/DONE/FAILED` |
当前 GF3 SARscape 链路会把原生 `_geo` ENVI 二进制转换为 `D:\GF3_L2_Image_Pool` 下的 GeoTIFF,并注册到这里。
### 3.2 WaterExtractionORM
表:`water_extractions`
用于单景水体提取。
关键字段:
| 字段 | 含义 |
| --- | --- |
| `scene_id` | 输入场景 |
| `processor` | 算法名称,当前默认 `otsu` |
| `input_path` | 实际输入 GeoTIFF |
| `output_path` | 输出水体掩膜 GeoTIFF |
| `preview_path` | 预留,目前预览按需渲染 |
| `vector_path` | 预留,用于未来水体矢量 |
| `water_area_km2` | 水体面积 |
| `water_pixel_count` | 水体像元数 |
| `threshold_value` | 阈值或模型置信阈值 |
| `metadata_json` | 算法元数据 |
| `status/error_msg/task_id` | 任务状态 |
### 3.3 FloodDetectionORM
表:`flood_detections`
用于灾前/灾后两景洪涝变化检测。
关键字段:
| 字段 | 含义 |
| --- | --- |
| `pre_scene_id` | 灾前场景 |
| `post_scene_id` | 灾后场景 |
| `output_dir` | 输出目录,默认 `WATER_RESULTS_DIR/flood_{id}` |
| `classified_path` | 分类结果 GeoTIFF |
| `flood_area_km2` | 新增洪涝面积 |
| `stable_water_area_km2` | 稳定水体面积 |
| `status/error_msg` | 任务状态 |
分类图当前约定:
| 值 | 类别 | 前端颜色 |
| ---: | --- | --- |
| 0 | nodata | 透明 |
| 1 | stable_water | 蓝色 |
| 2 | flood | 红色 |
| 3 | high_backscatter | 橙色 |
| 4 | non_water | 灰色 |
## 4. 现有业务流程
### 4.1 单景水体提取
流程:
```text
前端选择 SARSceneGeo
-> POST /flood/water-extractions { scene_id }
-> 创建 WaterExtractionORM(PENDING)
-> 创建 SystemJob: WATER_DETECT
-> job_handlers._handle_water_detect
-> water_extraction_service.run_otsu_water_extraction
-> water_detect_service.run_water_detection
-> 写 water_mask.tif
-> 更新 WaterExtractionORM 为 DONE/FAILED
-> 前端 GET /flood/water-extractions/{id}/preview 上图
```
当前输出目录:
```text
WATER_RESULTS_DIR/
water_extraction_{id}/
water_mask.tif
```
当前算法状态:
- Otsu 阈值;
- GF3 线性强度自动转 `10*log10`
- 支持 COPDEM/SRTM 类 DEM 栅格约束;
- 中值滤波、高斯滤波、形态学、连通域过滤;
- 可作为 baseline,不适合作为最终高精度算法。
### 4.2 灾前/灾后洪涝检测
流程:
```text
前端输入灾害日期 + 行政区 AOI
-> POST /flood/disaster-pairs/search
-> 后端按时间窗、AOI 覆盖率、重叠率推荐 pre/post 配对
-> POST /flood/detections { pre_scene_id, post_scene_id, refine }
-> 创建 FloodDetectionORM(PENDING)
-> 创建 SystemJob: FLOOD_DETECTION
-> job_handlers._handle_flood_detection
-> flood_detection_service.run_geotiff_flood_detection
-> 写 classified.tif/flood_mask.tif/stable_water_mask.tif/metadata.json
-> 更新 FloodDetectionORM
-> 前端加载 pre/post/classified 图层
```
当前输出目录:
```text
WATER_RESULTS_DIR/
flood_{id}/
classified.tif
flood_mask.tif
stable_water_mask.tif
metadata.json
```
当前算法状态:
- 灾前、灾后分别 Otsu
- 灾后水体且灾前非水体判为 flood;
- 灾前灾后均水体判为 stable_water
- 可选 `refine` 做简单形态学清理;
- 支持灾前重投影到灾后网格;
- 还未接入更强的 GF3 双极化分类、深度学习或弱监督模型。
### 4.3 套合分析
流程:
```text
FloodDetection DONE
-> POST /flood/detections/{id}/overlay
-> classified.tif 中 value=2 的 flood 区域矢量化
-> 与灾害点、DInSAR 产品、行政区 AOI 套合
-> 写 flood_detection_{id}_overlay.geojson
-> 新增 FloodOverlayORM
```
输出:
```text
WATER_RESULTS_DIR/
flood_overlays/
flood_detection_{id}_overlay.geojson
```
### 4.4 产品登记
流程:
```text
FloodDetection DONE
-> POST /flood/detections/{id}/products
-> 创建 FloodProductORM
-> GET /flood/products 或 /flood/results 查询
```
当前只是轻量登记,没有完整归档包导出。
## 5. analysis-ready 输入契约
算法工程师应以 `SARSceneGeoORM.analysis_tif_path` 为唯一标准输入。
输入约定:
| 项 | 要求 |
| --- | --- |
| 格式 | 单波段 GeoTIFF |
| 坐标 | 有 CRS,推荐 EPSG:4326 或投影坐标 |
| transform | 必须正确 |
| nodata | 支持 `NaN` 或明确 nodata |
| 单位 | 可能是 dB,也可能是线性强度,需读 `analysis_backscatter_unit` 或自行稳健判断 |
| 文件大小 | GF3 单极化可达几千万像元 |
现有 GF3 SARscape 标准化结果大致为:
- 数据来自 ENVI/SARscape `_geo`
- 转为 GeoTIFF 后注册;
- `analysis_backscatter_unit` 当前可能为 `unknown`
- 实际数值可能是线性强度,需做 dB 转换。
## 6. 算法 processor 输出契约
### 6.1 水体提取 processor
建议新增统一接口:
```python
def run_xxx_water_extraction(
*,
input_path: str,
output_dir: str,
job_id: str | None = None,
options: dict | None = None,
) -> dict:
...
```
返回:
```python
{
"ok": True,
"processor": "gf3_rf_v1",
"output_path": ".../water_mask.tif",
"water_area_km2": 123.45,
"water_pixel_count": 123456,
"threshold_value": 0.62,
"metadata": {
"model_version": "...",
"features": ["hh_db", "hv_db", "ratio", "slope"],
"confidence_path": ".../water_probability.tif"
}
}
```
最低要求:
- `output_path` 是 GeoTIFF
- 水体像元值为 `255``1`,背景为 `0`
- CRS/transform 与输入一致;
- nodata 推荐为 `0`
- 面积统计要与输出一致。
### 6.2 洪涝检测 processor
建议接口:
```python
def run_xxx_flood_detection(
*,
pre_tif_path: str,
post_tif_path: str,
output_dir: str,
job_id: str | None = None,
refine: bool = False,
options: dict | None = None,
) -> dict:
...
```
返回:
```python
{
"ok": True,
"processor": "gf3_change_rf_v1",
"classified_path": ".../classified.tif",
"flood_mask_path": ".../flood_mask.tif",
"stable_water_mask_path": ".../stable_water_mask.tif",
"metadata_path": ".../metadata.json",
"flood_area_km2": 12.34,
"stable_water_area_km2": 56.78,
"flood_pixel_count": 12345,
"stable_water_pixel_count": 67890,
"metadata": {
"model_version": "...",
"pre_scene_reprojected": True
}
}
```
`classified.tif` 必须遵守第 3.3 节的分类值,否则前端预览和套合分析会失效。
## 7. 推荐算法路线
### 7.1 短期:GF3 快速分类器
目标:替换当前单阈值水体提取,减少误判。
建议 processor 名称:
- `gf3_rf_v1`
- `gf3_lgbm_v1`
输入:
- 优先支持 GF3 HH/HV 双极化;
- 如果系统当前只注册单极化 `analysis_ready.tif`,工程侧需要补充“同一产品多极化查找”能力,或算法先支持单极化。
特征建议:
- `HH_db`
- `HV_db`
- `HH-HV`
- `HH/HV ratio`
- 局部均值、方差、纹理;
- DEM 高程、坡度;
- 可选永久水体、河网、土地覆盖先验。
样本策略:
- 第一版可用弱监督样本:永久水体为正样本,远离水系/坡度较大/高后向散射区域为负样本;
- 后续在系统内加入人工修正样本导出;
- 不建议用当前 Otsu 结果直接当唯一标签。
### 7.2 中期:GF3 深度学习推理
目标:面向洪涝产品的高质量识别。
可参考:
- Sen2GF3FloodsGF3 洪水数据集和 PyTorch 代码;
- FCN/UNet++/DeepLabV3+/SegFormer
- 支持 patch 推理和边缘重叠融合。
工程要求:
- 模型权重必须版本化;
- processor 输出必须仍是标准 GeoTIFF
- 推理可以 GPU 加速,但不能阻塞任务队列主进程;
- 大图必须 tile 化,避免一次性占满显存/内存。
### 7.3 保留 baseline
当前 `otsu` 应保留为:
- 快速预览;
- 无模型时兜底;
- 算法对比 baseline。
不建议作为最终默认高质量结果。
## 8. 工程侧下一步
### 8.1 后端 processor 注册
建议在 `water_extraction_service.py` 增加 processor 分发:
```python
def run_water_extraction(processor: str, **kwargs):
if processor == "otsu":
return run_otsu_water_extraction(**kwargs)
if processor == "gf3_rf_v1":
return run_gf3_rf_water_extraction(**kwargs)
...
```
`FloodWaterExtractionRequest` 需要增加:
```python
processor: str = "otsu"
options: dict | None = None
```
然后 `submit_water_extraction` 把 processor/options 写入 `WaterExtractionORM` 和 job payload。
### 8.2 多极化场景组织
目前 `SARSceneGeoORM``RadarDataORM` 基本是一景一条。GF3 标准化链路可能存在 HH/HV 两个 GeoTIFF,但洪涝算法输入仍是单 `analysis_tif_path`
如果算法需要 HH/HV,应补一个工程能力:
-`analysis_metadata_json` 中记录同产品全部极化 GeoTIFF;
- 或新增 `SARSceneBandORM`/`analysis_assets` 表;
- 或在 processor 内根据当前路径和命名规则寻找同目录同产品 HV/HH。
建议先采用 metadata 方案,改动最小。
### 8.3 水体结果矢量化
当前水体提取只按需返回 PNG 预览,没有持久化矢量。
建议新增:
- `water_vector_path` GeoJSON
- `confidence_path` 概率图;
- `preview_path` 持久 PNG
- 后端接口支持水体结果矢量上图。
### 8.4 质量评估字段
建议 `metadata_json` 至少写入:
- `processor`
- `model_version`
- `input_paths`
- `features`
- `threshold`
- `confidence_stats`
- `valid_pixel_count`
- `water_ratio`
- `runtime_seconds`
- `warnings`
### 8.5 前端入口
当前前端有水体提取按钮,但没有 processor 选择。
建议增加:
- 水体提取 processor 下拉框;
- `快速 Otsu / GF3 RF / 深度学习`
- 结果行显示 processor 和模型版本;
- 可选显示置信度图层。
## 9. 算法开发边界
算法工程师只需要保证:
1. 能读取输入 GeoTIFF
2. 能输出符合契约的 GeoTIFF
3. 返回标准 dict
4. 大图处理不会把内存/显存打爆;
5. 错误抛出清晰异常或返回 `{"ok": False, "error": "..."}`
算法工程师不需要处理:
- 前端;
- 任务队列;
- 用户权限;
- 数据库事务;
- 资产扫描;
- 上图预览;
- 套合分析;
- 产品登记。
## 10. 当前风险和已知问题
1. 当前 Otsu 水体提取误判较多,只能作为 baseline。
2. GF3 双极化没有形成正式算法输入契约。
3. `analysis_backscatter_unit` 对 GF3 SARscape 输出仍可能是 `unknown`
4. 洪涝变化检测仍是双阈值差分,复杂地物下误判会明显。
5. 水体和洪涝结果缺少质量评价和置信度图层。
6. 产品登记还不是完整归档包。
7.`/water/*` 和新 `/flood/*` 共存,后续需要逐步收敛到 `/flood/*`
## 11. 建议交付里程碑
### M1GF3 RF 水体 processor
- 输入单景 GF3 HH/HV 或单极化;
- 输出 `water_mask.tif`
- 写入模型元数据;
- 接入 `/flood/water-extractions`
### M2:多极化输入契约
- 工程侧让 processor 能稳定拿到 HH/HV
- 文档化极化路径和 metadata;
- 前端显示使用的极化。
### M3:洪涝变化 processor
- 输入灾前/灾后;
- 输出标准 `classified.tif`
- 与现有套合和产品链路兼容。
### M4:深度学习推理
- 支持 tile 推理;
- 支持模型版本;
- 输出概率图和二值图;
- 与 RF/baseline 可切换。
@@ -0,0 +1,476 @@
# GF3 SARscape Native To GeoTIFF Design
更新日期:2026-05-30
## 1. 结论
GF3 生产链路后续采用“生产解耦、系统标准化”的模型:
```text
GF3 原始压缩包池
-> 生产服务器使用 ENVI / IDL Runtime / SARscape 稳定生产
-> 只保留 SARscape 最终 _geo 原生结果组
-> 系统扫描原生结果池
-> 后台转换为标准 GeoTIFF
-> 入库、预览、洪涝分析和后续业务只消费 GeoTIFF
```
系统不直接把 SARscape `.sml` 或无后缀二进制作为业务算法输入。`.sml``.hdr` 和无后缀主数据属于原生证据层;`GeoTIFF` 属于平台消费层。
## 2. 目录约定
推荐继续沿用现场已有目录语义,并新增一个 ENVI/SARscape 原生池。
```env
GF3_ARCHIVE_SOURCE_DIRS=D:\GF3_Image_Pool_Zip
GF3_SARSCAPE_NATIVE_DIRS=D:\GF3_L2_ENVI_Binary_Pool
GF3_STORAGE_DIRS=D:\GF3_L2_Image_Pool
SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready
```
目录职责:
| 目录 | 职责 | 系统是否直接分析 |
| --- | --- | --- |
| `GF3_ARCHIVE_SOURCE_DIRS` | 原始 GF3 L1A `.tar.gz` 池 | 否 |
| `GF3_SARSCAPE_NATIVE_DIRS` | SARscape `_geo` 原生结果池 | 否 |
| `GF3_STORAGE_DIRS` | GF3 标准 GeoTIFF 池 | 是 |
| `SAR_ANALYSIS_READY_ROOT` | 洪涝/水体分析级统一输入 | 是 |
生产服务器可以不部署完整管理系统。只要把完成后的 `_geo` 原生结果组放入 `GF3_SARSCAPE_NATIVE_DIRS`,管理系统就可以扫描、转换和入库。
## 3. 原生池结构
原生池以批次日期或人工批次号分组。单景目录名尽量保持 GF3 原始产品名。
```text
D:\GF3_L2_ENVI_Binary_Pool
20260514
GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478
GF3_MH1_FSII_..._hh_geo
GF3_MH1_FSII_..._hh_geo.hdr
GF3_MH1_FSII_..._hh_geo.sml
GF3_MH1_FSII_..._hh_geo.ovr
GF3_MH1_FSII_..._hh_geo.aux.xml
GF3_MH1_FSII_..._hh_geo_ql.tif
GF3_MH1_FSII_..._hh_geo.kml
GF3_MH1_FSII_..._hv_geo
GF3_MH1_FSII_..._hv_geo.hdr
GF3_MH1_FSII_..._hv_geo.sml
GF3_MH1_FSII_..._hv_geo.ovr
GF3_MH1_FSII_..._hv_geo.aux.xml
GF3_MH1_FSII_..._hv_geo_ql.tif
GF3_MH1_FSII_..._hv_geo.kml
gf3_sarscape_cli.log
```
### 3.1 必保文件
每个极化至少保留:
```text
*_geo
*_geo.hdr
*_geo.sml
```
建议同时保留:
```text
*_geo.ovr
*_geo.aux.xml
*_geo_ql.tif
*_geo.kml
gf3_sarscape_cli.log
```
说明:
- 无后缀 `*_geo` 是 SARscape 主数据。
- `.hdr` 是 ENVI/GDAL 读取二进制数据的关键 sidecar。
- `.sml` 是 SARscape 追溯和完成判定的关键 sidecar。
- `*_geo_ql.tif` 只能作为快视或预览参考,不作为科学分析主输入。
### 3.2 可清理文件
生产结束并确认 `_geo` 结果完整后,可以清理:
```text
.gf3_extract
temp
SLC 中间产物
*_ml*
*_filt*
```
如果需要完整复现 SARscape 处理过程,应额外保留 `work` 中的参数 XML、trace 和日志;否则可将 `work` 作为可选审计资料归档。
## 4. 标准 GeoTIFF 池结构
系统从原生池转换后写入 `GF3_STORAGE_DIRS`
```text
D:\GF3_L2_Image_Pool
20260514
GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478
HH_L2.tif
HV_L2.tif
preview_HH.png
preview_HV.png
gf3_standard_manifest.json
quality_HH.json
quality_HV.json
```
平台后续只从该目录或 `SAR_ANALYSIS_READY_ROOT` 消费 GeoTIFF,不直接读取 SARscape 原生目录。
## 5. 扫描与转换流程
推荐把“扫描”和“转换”都放在后台任务中执行,避免普通扫描接口长时间阻塞。
```text
用户触发 GF3 扫描
-> 扫描 GF3_SARSCAPE_NATIVE_DIRS
-> 识别 scene / polarization / _geo 完整性
-> 对待转换项创建或执行 GF3_NATIVE_TO_TIF 任务
-> 转换到 GF3_STORAGE_DIRS
-> 写 gf3_standard_manifest.json
-> 登记 radar_data / sar_scene_geo
-> 生成预览图和 quality.json
```
### 5.1 完整性判定
一个极化的原生 `_geo` 结果完整条件:
```text
*_geo 存在且非空
*_geo.hdr 存在且非空
*_geo.sml 存在且非空
```
如果存在 `.aux.xml``.ovr``*_geo_ql.tif``.kml`,登记为辅助资产。
一个 scene 的状态:
| 状态 | 条件 |
| --- | --- |
| `DONE` | 请求极化全部具备完整 `_geo` 结果,且 GeoTIFF 转换成功 |
| `NATIVE_READY` | 原生 `_geo` 完整,但 GeoTIFF 尚未转换 |
| `PARTIAL` | 只完成部分极化 |
| `FAILED` | 原生结果不完整或转换失败 |
### 5.2 增量跳过规则
转换任务应根据 manifest 判断是否需要重跑:
```text
source path
source size
source mtime
converter version
target tif exists
```
当上述信息未变化时,跳过转换。
如果 `_geo` 原生文件被替换、修改时间变化、转换器版本变化或目标 tif 缺失,应重新转换。
## 6. 转换策略
优先使用 GDAL/rasterio 读取 ENVI header 或 SARscape sidecar。
输入优先级:
```text
1. *_geo + *_geo.hdr
2. 可被 GDAL 识别的 *_geo.sml
3. 其他明确可读的 SARscape/ENVI sidecar
```
输出要求:
```text
GeoTIFF
单极化单文件
尽量保留地理参考、nodata、数据类型和投影
默认输出 HH_L2.tif / HV_L2.tif
```
如果转换失败,不应把 quicklook tif 冒充为分析级 tif。应记录:
```text
analysis_ready_status=FAILED
error_message=<转换错误>
source_native_status=NATIVE_READY
```
## 7. 数据库登记
### 7.1 `radar_data`
每个 GF3 scene 至少登记一条 `radar_data`
```text
satellite=GF3
satellite_family=GF3
source_format=GF3_SARSCAPE_NATIVE
product_level=L2
file_path=<GF3_STORAGE_DIRS 下的 scene 标准目录>
metadata_json.native_dir=<GF3_SARSCAPE_NATIVE_DIRS 下的 scene 原生目录>
metadata_json.standard_manifest=<gf3_standard_manifest.json>
```
应从 scene 名解析:
```text
imaging_date
imaging_mode
polarization
scene_center_lon
scene_center_lat
product_unique_id
```
如果 GeoTIFF 可读,应同步:
```text
min_lon / min_lat / max_lon / max_lat
coverage_polygon
geom
```
### 7.2 `sar_scene_geo`
每个可分析 scene 记录:
```text
analysis_engine=gf3_sarscape
analysis_profile=gf3_sarscape_geo_to_tif
analysis_tif_path=<HH_L2.tif 或默认极化 tif>
analysis_dir=<SAR_ANALYSIS_READY_ROOT 下目录>
analysis_preview_path=<preview.png>
analysis_backscatter_unit=sigma0_linear 或 unknown
analysis_metadata_json.native_dir=<原生目录>
analysis_metadata_json.native_assets=<原生资产列表>
analysis_quality_json=<quality.json 内容>
status=DONE
```
如果需要同时保留 HH 和 HV 两个可分析产品,建议长期扩展为资产表或 scene-pol 级记录;短期可以选择默认极化写入 `sar_scene_geo.analysis_tif_path`,并在 metadata 中登记全部极化 tif。
## 8. Manifest 契约
### 8.1 原生 manifest
`gf3_native_manifest.json` 写入原生 scene 目录或系统索引目录:
```json
{
"schema": "gf3_sarscape_native.v1",
"scene_name": "GF3_MH1_FSII_...",
"native_dir": "D:\\GF3_L2_ENVI_Binary_Pool\\20260514\\GF3_MH1_FSII_...",
"source_archive": "D:\\GF3_Image_Pool_Zip\\20260514\\GF3_MH1_FSII_....tar.gz",
"polarizations": ["HH", "HV"],
"status": "NATIVE_READY",
"assets": [
{
"polarization": "HH",
"role": "geo_native",
"path": "..._hh_geo",
"hdr": "..._hh_geo.hdr",
"sml": "..._hh_geo.sml",
"quicklook": "..._hh_geo_ql.tif"
}
],
"logs": ["gf3_sarscape_cli.log"]
}
```
### 8.2 标准 manifest
`gf3_standard_manifest.json` 写入标准 GeoTIFF scene 目录:
```json
{
"schema": "gf3_standard_geotiff.v1",
"scene_name": "GF3_MH1_FSII_...",
"native_manifest": "D:\\GF3_L2_ENVI_Binary_Pool\\...\\gf3_native_manifest.json",
"standard_dir": "D:\\GF3_L2_Image_Pool\\20260514\\GF3_MH1_FSII_...",
"status": "DONE",
"converter": {
"name": "gf3_sarscape_geo_to_tif",
"version": "v1"
},
"assets": [
{
"polarization": "HH",
"role": "analysis_tif",
"path": "HH_L2.tif",
"source_native": "..._hh_geo",
"quality": "quality_HH.json",
"preview": "preview_HH.png"
}
]
}
```
## 9. 前端与操作入口
短期不新增复杂页面,沿用“数据管理 / 归档预处理”里的 GF3 操作区:
```text
GF3 解包
GF3 SARscape 原生扫描
GF3 原生转 GeoTIFF
扫描 GF3 标准结果
```
也可以先合并为一个按钮:
```text
扫描 GF3
```
后台自动完成:
```text
native scan -> convert missing tif -> register standard result
```
任务日志必须显示:
```text
发现 scene 数
NATIVE_READY 数
转换成功数
转换失败数
跳过数
失败原因
```
## 10. 实施顺序
### Phase 1:设计与配置
- 新增本文档。
- 新增 `.env.example` 中的 `GF3_SARSCAPE_NATIVE_DIRS`
- 保留 `GF3_STORAGE_DIRS` 作为标准 GeoTIFF 池。
### Phase 2:原生扫描
- 新增 `gf3_native_inventory_service.py`
- 扫描 `_geo` 原生结果组。
- 生成 native manifest。
- 不做转换、不入业务分析。
### Phase 3GeoTIFF 标准化
- 新增 `gf3_standardize_service.py`
- 将 `_geo` 原生结果转换为 `HH_L2.tif` / `HV_L2.tif`
- 生成 preview 和 quality。
- 写 standard manifest。
### Phase 4:入库与洪涝接入
- 登记 `radar_data`
- 登记或更新 `sar_scene_geo`
- `/flood/preprocess` 对 GF3 优先复用已标准化 GeoTIFF。
### Phase 5:清理策略
- 增加 native pool 检查报告。
- 增加可选中间文件清理建议,但系统不主动删除生产机文件。
- 后续如需自动清理,应只清理系统明确生成的临时文件。
## 11. 当前约束
- 不把 `*_geo_ql.tif` 当作分析级产品。
- 不让洪涝、水体、地图预览直接依赖 `.sml`
- 不要求生产服务器部署管理系统。
- 不在扫描请求同步执行长时间转换,应使用后台任务。
- 不删除用户生产目录中的文件,除非后续新增明确的、受控的清理任务。
## 12. 与旧 GF3 GDAL 路线关系
现有 `gf3_service.py` 的 Python/GDAL L1A -> L2 路线可以保留为 fallback 或实验处理器:
```text
gf3_gdal
```
新 SARscape 原生池路线作为正式现场路线:
```text
gf3_sarscape
```
两条路线最终都必须收敛到:
```text
GF3_STORAGE_DIRS / SAR_ANALYSIS_READY_ROOT 中的标准 GeoTIFF
```
因此后续业务模块只关心标准 GeoTIFF,不关心上游来自 SARscape、GDAL、GAMMA 或其他处理器。
## 13. 2026-05-30 首轮落地
首轮代码已按本文档的主路径实现最小闭环:
- 新增 `GF3_SARSCAPE_NATIVE_DIRS` 配置,作为 SARscape/ENVI 原生 `_geo` 二进制池。
- 新增 `gf3_native_inventory_service.py`,扫描 `*_geo + *_geo.hdr + *_geo.sml` 并写 `gf3_native_manifest.json`
- 新增 `gf3_standardize_service.py`,将完整原生结果转换到 `GF3_STORAGE_DIRS` 下的 `HH_L2.tif` / `HV_L2.tif`,并写 `gf3_standard_manifest.json``quality_*.json``preview_*.png`
- 新增后台任务 `GF3_SARSCAPE_SYNC` 和接口 `POST /api/monitor/gf3-sarscape-sync`
- 数据监控面板新增 `GF3 SARscape 入库` 按钮。
- 转换成功后登记 `radar_data`,并通过 `sar_analysis_ready_service` 登记 `sar_scene_geo`,使洪涝/水体模块可以继续消费标准 GeoTIFF。
当前实现仍遵守约束:
- 不把 `*_geo_ql.tif` 当作分析级输入。
- 不要求生产服务器部署管理系统。
- 转换优先使用 GDAL Python 绑定;当前环境没有 `osgeo` 时走 rasterio 兜底。
## 14. 2026-05-30 生产链路接入
在首轮“原生结果入库”基础上,系统进一步接入 GF3 SARscape wrapper
```text
GF3_ARCHIVE_SOURCE_DIRS
-> gf3wrapper.exe / IDL Runtime / SARscape
-> GF3_SARSCAPE_NATIVE_DIRS
-> GF3_SARSCAPE_SYNC 标准化
-> GF3_STORAGE_DIRS
-> 雷达数据扫描、预览、洪涝/水体业务
```
新增配置:
```env
GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\.codex_tmp\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe
GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
GF3_SARSCAPE_DEM_PATH=D:\DEM\GMTED2010.jp2
GF3_SARSCAPE_POLARIZATIONS=HH,HV
GF3_SARSCAPE_KEEP_EXTRACTED=true
GF3_SARSCAPE_AUTO_STANDARDIZE=true
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0
```
新增后台任务:
| 任务 | 接口 | 用途 |
| --- | --- | --- |
| `GF3_SARSCAPE_PRODUCE` | `POST /api/monitor/gf3-sarscape-produce` | 从原始 `.tar.gz/.tgz` 触发 SARscape 生产,随后自动标准化、入库、清理 |
| `GF3_SARSCAPE_SYNC` | `POST /api/monitor/gf3-sarscape-sync` | 仅扫描已有 `_geo` 原生结果并转 GeoTIFF 入库 |
| `GF3_SARSCAPE_CLEAN` | `POST /api/monitor/gf3-sarscape-clean` | 手动清理原生池中间数据 |
清理策略:
- 只处理 `GF3_SARSCAPE_NATIVE_DIRS` 内的场景目录。
- 默认要求 `GF3_STORAGE_DIRS/<batch>/<scene>/gf3_standard_manifest.json` 状态为 `DONE` 后才清理。
- 保留最终原生 `_geo` 主数据、`.hdr/.sml/.ovr/.aux.xml``*_geo_ql.tif/.kml`、日志和 manifest。
- 删除 `.gf3_extract``temp``work`,以及根目录中的 `*_slc*``*_ml*``*_filt*``.par/.trace/.working/.list` 等中间文件。
- 每景写 `gf3_cleanup_manifest.json`,记录删除条目和释放字节数。
- 不删除 `GF3_ARCHIVE_SOURCE_DIRS` 中的原始压缩包,也不删除 `GF3_STORAGE_DIRS` 中的标准 GeoTIFF。
这样 `D:\GF3_L2_ENVI_Binary_Pool` 只长期保存可追溯的最终 `_geo` 原生结果组,中间过程文件在标准化完成后自动释放空间。
+7 -1
View File
@@ -1,6 +1,6 @@
# 文档索引
最后更新:2026-05-28
最后更新:2026-05-30
本页是当前有效文档入口。没有列在本页的历史设计、实验记录和过程文档不再作为当前系统事实依据。
@@ -51,9 +51,15 @@
- [FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md](FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md)
洪涝模块 GeoTIFF 化与 Gamma 前处理方向。
- [GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md](GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md)
GF3 SARscape 原生 `_geo` 二进制池、GeoTIFF 标准化、入库和洪涝接入设计。
- [FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md](FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md)
洪涝灾害分析工作台、产品包和矢量套合边界。
- [FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md](FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md)
洪涝/水体算法接入现状、processor 输出契约和工程交接路线。
## 安全
- [SECURITY_AUDIT_2026-03-12.md](SECURITY_AUDIT_2026-03-12.md)
+16 -2
View File
@@ -1358,12 +1358,27 @@ function App() {
}
}, []);
const updateRadarPreviewVisibility = useCallback((item, shouldBeVisible) => {
const updateRadarPreviewVisibility = useCallback(async (item, shouldBeVisible) => {
if (!item || !mapRef.current) return;
const itemId = item.id;
const layer = radarPreviewLayersRef.current[itemId];
if (shouldBeVisible) {
const refreshedStatus = await fetchRadarPreviewStatus(itemId, { silent: true });
if (refreshedStatus) {
item = {
...item,
previewStatus: normalizePreviewStatus(refreshedStatus.status),
previewFallbackInUse: !!refreshedStatus.fallback_in_use,
previewHasGeoCache: !!refreshedStatus.has_geo_cache,
previewHasRawCache: !!refreshedStatus.has_raw_cache,
previewSourceFound: !!refreshedStatus.source_found,
previewMessage: refreshedStatus.message || '',
previewError: refreshedStatus.error || '',
previewCacheKey: refreshedStatus.cache_updated_at || item.previewCacheKey || `${Date.now()}-${itemId}`,
};
}
if (layer) {
if (!mapRef.current.hasLayer(layer)) {
layer.addTo(mapRef.current);
@@ -1393,7 +1408,6 @@ function App() {
radarPreviewLayersRef.current[itemId] = previewLayer;
previewLayer.addTo(mapRef.current);
fetchRadarPreviewStatus(itemId, { silent: true });
} else if (layer) {
layer.remove();
delete radarPreviewLayersRef.current[itemId];
+1
View File
@@ -132,6 +132,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) {
<option value="all">全部卫星族</option>
<option value="S1">Sentinel-1</option>
<option value="LT1">LT-1</option>
<option value="GF3">GF3</option>
</select>
<button type="button" onClick={() => refresh()} disabled={loading}>刷新</button>
<button type="button" onClick={handleScan} disabled={readOnly || scanLoading}>扫描资产</button>
+149 -1
View File
@@ -9,7 +9,14 @@ const DEFAULT_MONITOR_CONFIG = {
dinsar_dirs: [],
gf3_archive_source_dirs: [],
gf3_source_dirs: [],
gf3_sarscape_native_dirs: [],
gf3_storage_dirs: [],
gf3_sarscape_wrapper_exe: '',
gf3_sarscape_idlrt_path: '',
gf3_sarscape_dem_path: '',
gf3_sarscape_polarizations: 'HH,HV',
gf3_sarscape_auto_standardize: true,
gf3_sarscape_clean_after_success: true,
s1_source_dirs: [],
s1_storage_dirs: [],
s1_orbit_dirs: [],
@@ -68,6 +75,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const [s1Message, setS1Message] = useState('');
const [gf3UnpackLoading, setGf3UnpackLoading] = useState(false);
const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false);
const [gf3SarscapeProduceLoading, setGf3SarscapeProduceLoading] = useState(false);
const [gf3SarscapeSyncLoading, setGf3SarscapeSyncLoading] = useState(false);
const [gf3SarscapeCleanLoading, setGf3SarscapeCleanLoading] = useState(false);
const [gf3ScanLoading, setGf3ScanLoading] = useState(false);
const [gf3Message, setGf3Message] = useState('');
const logEndRef = useRef(null);
@@ -87,7 +97,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
);
const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1');
const gf3ActiveTask = displayActiveTasks.find((task) =>
['GF3_UNPACK', 'GF3_BATCH_PROCESS'].includes(task.task_type)
['GF3_UNPACK', 'GF3_BATCH_PROCESS', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN'].includes(task.task_type)
);
useEffect(() => {
@@ -120,6 +130,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
s1_orbit_dirs: toArray(data?.s1_orbit_dirs),
gf3_archive_source_dirs: toArray(data?.gf3_archive_source_dirs),
gf3_source_dirs: toArray(data?.gf3_source_dirs),
gf3_sarscape_native_dirs: toArray(data?.gf3_sarscape_native_dirs),
gf3_storage_dirs: toArray(data?.gf3_storage_dirs),
});
setConfigLoaded(true);
@@ -289,7 +300,10 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0;
const hasGf3ArchiveSourceDirs = config.gf3_archive_source_dirs.length > 0;
const hasGf3SourceDirs = config.gf3_source_dirs.length > 0;
const hasGf3SarscapeNativeDirs = config.gf3_sarscape_native_dirs.length > 0;
const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0;
const hasGf3SarscapeWrapper = typeof config.gf3_sarscape_wrapper_exe === 'string' && config.gf3_sarscape_wrapper_exe.trim() !== '';
const hasGf3SarscapeDem = typeof config.gf3_sarscape_dem_path === 'string' && config.gf3_sarscape_dem_path.trim() !== '';
const canRunRadar = !readOnly && configLoaded && hasRadarDirs;
const canRunOrbit = !readOnly && configLoaded && hasOrbitDir;
@@ -299,6 +313,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs;
const canRunGf3Unpack = !readOnly && configLoaded && hasGf3ArchiveSourceDirs && hasGf3SourceDirs;
const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs;
const canRunGf3SarscapeProduce = !readOnly && configLoaded && hasGf3ArchiveSourceDirs && hasGf3SarscapeNativeDirs && hasGf3StorageDirs && hasGf3SarscapeWrapper && hasGf3SarscapeDem;
const canRunGf3SarscapeSync = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs;
const canRunGf3SarscapeClean = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs;
const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0;
const handleS1Run = async () => {
@@ -452,6 +469,111 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
}
};
const handleGf3SarscapeProduce = async () => {
if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发 GF3 SARscape 生产。');
return;
}
setGf3SarscapeProduceLoading(true);
setGf3Message('GF3 SARscape 生产链路启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-produce`, {
method: 'POST',
credentials: 'include',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({}),
});
const data = await parseJsonSafe(res, {});
if (res.ok) {
setGf3Message(data.message || 'GF3 SARscape 生产任务已启动');
if (onTaskStart) {
onTaskStart(data.task_id, 'GF3 SARscape 生产链路已启动。', {
nonBlocking: true,
taskType: 'GF3_SARSCAPE_PRODUCE',
});
}
} else {
setGf3Message(`失败:${data.detail || '未知错误'}`);
}
} catch (err) {
setGf3Message(`失败:${err.message || '未知错误'}`);
} finally {
setGf3SarscapeProduceLoading(false);
}
};
const handleGf3SarscapeSync = async () => {
if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发 GF3 SARscape 标准化。');
return;
}
setGf3SarscapeSyncLoading(true);
setGf3Message('GF3 SARscape 原生结果标准化启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-sync`, {
method: 'POST',
credentials: 'include',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({}),
});
const data = await parseJsonSafe(res, {});
if (res.ok) {
setGf3Message(data.message || 'GF3 SARscape 标准化任务已启动');
if (onTaskStart) {
onTaskStart(data.task_id, 'GF3 SARscape 原生结果标准化已启动。', {
nonBlocking: true,
taskType: 'GF3_SARSCAPE_SYNC',
});
}
} else {
setGf3Message(`失败:${data.detail || '未知错误'}`);
}
} catch (err) {
setGf3Message(`失败:${err.message || '未知错误'}`);
} finally {
setGf3SarscapeSyncLoading(false);
}
};
const handleGf3SarscapeClean = async () => {
if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发 GF3 SARscape 清理。');
return;
}
setGf3SarscapeCleanLoading(true);
setGf3Message('GF3 SARscape 中间数据清理启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-clean`, {
method: 'POST',
credentials: 'include',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ dry_run: false, require_standardized: true }),
});
const data = await parseJsonSafe(res, {});
if (res.ok) {
setGf3Message(data.message || 'GF3 SARscape 清理任务已启动');
if (onTaskStart) {
onTaskStart(data.task_id, 'GF3 SARscape 中间数据清理已启动。', {
nonBlocking: true,
taskType: 'GF3_SARSCAPE_CLEAN',
});
}
} else {
setGf3Message(`失败:${data.detail || '未知错误'}`);
}
} catch (err) {
setGf3Message(`失败:${err.message || '未知错误'}`);
} finally {
setGf3SarscapeCleanLoading(false);
}
};
const handleOpenUnpackDialog = () => {
if (readOnly) {
setUnpackMessage('当前账户为只读模式,无法触发解包任务。');
@@ -676,6 +798,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
<div style={rowStyle}><span style={labelStyle}>S1 精轨</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_orbit_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 压缩包</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_archive_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 原生</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_sarscape_native_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>D-InSAR 结果</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.dinsar_dirs)}</span></div>
</div>
@@ -769,7 +892,11 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
<div style={{ ...gridStyle, marginBottom: '8px' }}>
<div style={rowStyle}><span style={labelStyle}>压缩包来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_archive_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>L1A 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>SARscape 原生</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_sarscape_native_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>L2 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>Wrapper</span><span style={{ wordBreak: 'break-all' }}>{config.gf3_sarscape_wrapper_exe || '未配置'}</span></div>
<div style={rowStyle}><span style={labelStyle}>SARscape DEM</span><span style={{ wordBreak: 'break-all' }}>{config.gf3_sarscape_dem_path || '未配置'}</span></div>
<div style={rowStyle}><span style={labelStyle}>极化</span><span>{config.gf3_sarscape_polarizations || 'HH,HV'}</span></div>
</div>
<div style={{ display: 'flex', gap: '10px', flexWrap: 'wrap' }}>
<button
@@ -786,6 +913,27 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
>
{gf3ProcessLoading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 预处理')}
</button>
<button
onClick={handleGf3SarscapeProduce}
disabled={gf3SarscapeProduceLoading || readOnly || !canRunGf3SarscapeProduce}
style={actionBtnStyle(gf3SarscapeProduceLoading, readOnly || !canRunGf3SarscapeProduce)}
>
{gf3SarscapeProduceLoading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 SARscape 生产')}
</button>
<button
onClick={handleGf3SarscapeSync}
disabled={gf3SarscapeSyncLoading || readOnly || !canRunGf3SarscapeSync}
style={actionBtnStyle(gf3SarscapeSyncLoading, readOnly || !canRunGf3SarscapeSync)}
>
{gf3SarscapeSyncLoading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 SARscape 入库')}
</button>
<button
onClick={handleGf3SarscapeClean}
disabled={gf3SarscapeCleanLoading || readOnly || !canRunGf3SarscapeClean}
style={actionBtnStyle(gf3SarscapeCleanLoading, readOnly || !canRunGf3SarscapeClean)}
>
{gf3SarscapeCleanLoading ? '运行中...' : (readOnly ? '只读模式' : '清理 GF3 中间')}
</button>
<button
onClick={handleGf3Scan}
disabled={gf3ScanLoading || readOnly || !canRunGf3Scan}
@@ -26,6 +26,12 @@ const getTaskTypeLabel = (taskType) => {
return 'GF3 解包';
case 'GF3_BATCH_PROCESS':
return 'GF3 预处理';
case 'GF3_SARSCAPE_PRODUCE':
return 'GF3 SARscape 生产';
case 'GF3_SARSCAPE_SYNC':
return 'GF3 SARscape 入库';
case 'GF3_SARSCAPE_CLEAN':
return 'GF3 中间清理';
case 'SCAN_ASSET_INVENTORY':
return '资产库存扫描';
case 'IDL_IMPORT':
+23 -1
View File
@@ -11,7 +11,7 @@ import { normalizePagePayload } from '../utils/appHelpers';
import { normalizeTaskStatus } from '../utils/appUiHelpers';
import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants';
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
export default function useDinsarOperations({
onCleanupDinsarLayers,
@@ -235,6 +235,28 @@ export default function useDinsarOperations({
} else if (taskStatus === 'FAILED') {
addLog('error', `GF3 解包失败: ${taskInfo.message || '未知错误'}`);
}
} else if (taskInfo.task_type === 'GF3_SARSCAPE_PRODUCE') {
if (taskStatus === 'COMPLETED') {
addLog('success', taskInfo.message || 'GF3 SARscape 生产链完成。');
addLog('info', '正在同步 GF3 生产后的数据视图...');
void syncRadarViewsAfterUnpack();
} else if (taskStatus === 'FAILED') {
addLog('error', `GF3 SARscape 生产失败: ${taskInfo.message || '未知错误'}`);
}
} else if (taskInfo.task_type === 'GF3_SARSCAPE_SYNC') {
if (taskStatus === 'COMPLETED') {
addLog('success', taskInfo.message || 'GF3 SARscape 标准化完成。');
addLog('info', '正在同步 GF3 标准化后的数据视图...');
void syncRadarViewsAfterUnpack();
} else if (taskStatus === 'FAILED') {
addLog('error', `GF3 SARscape 标准化失败: ${taskInfo.message || '未知错误'}`);
}
} else if (taskInfo.task_type === 'GF3_SARSCAPE_CLEAN') {
if (taskStatus === 'COMPLETED') {
addLog('success', taskInfo.message || 'GF3 SARscape 中间数据清理完成。');
} else if (taskStatus === 'FAILED') {
addLog('error', `GF3 SARscape 中间数据清理失败: ${taskInfo.message || '未知错误'}`);
}
}
};
+1 -1
View File
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react';
import apiClient from '../api/client';
import { normalizeTaskStatus } from '../utils/appUiHelpers';
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => (
NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase())
+79 -27
View File
@@ -72,6 +72,11 @@ export default function useRadarSearch({
);
};
const getSatelliteFallbackForGroup = (groupKey) => {
const group = SATELLITE_GROUPS.find((item) => item.key === groupKey);
return Array.isArray(group?.prefixes) && group.prefixes.length > 0 ? group.prefixes[0] : '';
};
const fetchRadarImagingDates = useCallback(async () => {
try {
const response = await apiClient.get('/radar-data/imaging-dates');
@@ -132,33 +137,6 @@ export default function useRadarSearch({
}
}, [setRadarSearchOptions, setRadarSearchOptionsLoading]);
const changeSatelliteGroup = useCallback((groupKey) => {
setSelectedSatelliteGroup(groupKey);
// Clear sub-filters that may be invalid for the new satellite group
setRadarSearchDraft((prev) => ({
...prev,
satellite: '',
imaging_mode: '',
polarization: '',
satellite_mode: '',
receiving_station: '',
orbit_circle: '',
acquisition_time_utc: '',
product_type: '',
product_level: '',
product_unique_id: '',
orbit_direction: '',
}));
if (groupKey === 'all') {
fetchRadarSearchOptions([]);
} else {
const matched = getSatellitesForGroup(groupKey);
if (matched.length > 0) {
fetchRadarSearchOptions(matched);
}
}
}, [setSelectedSatelliteGroup, setRadarSearchDraft, fetchRadarSearchOptions]);
const processAndSetAllData = useCallback((data) => {
const nameCounts = {};
const dataWithDisplayNames = data.map(item => {
@@ -294,6 +272,8 @@ export default function useRadarSearch({
const matched = getSatellitesForGroup(selectedSatelliteGroup);
if (matched.length > 0) {
draftWithSatelliteGroup.satellite = matched.join(',');
} else {
draftWithSatelliteGroup.satellite = getSatelliteFallbackForGroup(selectedSatelliteGroup);
}
}
const normalizedCriteria = normalizeRadarSearchCriteria(draftWithSatelliteGroup, RADAR_SEARCH_DEFAULTS);
@@ -354,6 +334,78 @@ export default function useRadarSearch({
clearRadarSearchResults, fetchAllData,
]);
const changeSatelliteGroup = useCallback(async (groupKey) => {
const nextGroupKey = groupKey || 'all';
setSelectedSatelliteGroup(nextGroupKey);
const clearedDraft = {
...radarSearchDraft,
satellite: '',
imaging_mode: '',
polarization: '',
satellite_mode: '',
receiving_station: '',
orbit_circle: '',
acquisition_time_utc: '',
product_type: '',
product_level: '',
product_unique_id: '',
orbit_direction: '',
};
setRadarSearchDraft(clearedDraft);
let matched = [];
let satelliteFilter = '';
if (nextGroupKey === 'all') {
fetchRadarSearchOptions([]);
} else {
matched = getSatellitesForGroup(nextGroupKey);
satelliteFilter = matched.length > 0
? matched.join(',')
: getSatelliteFallbackForGroup(nextGroupKey);
if (matched.length > 0) {
fetchRadarSearchOptions(matched);
}
}
const normalizedCriteria = normalizeRadarSearchCriteria(
{
...clearedDraft,
satellite: nextGroupKey === 'all' ? '' : satelliteFilter,
},
RADAR_SEARCH_DEFAULTS
);
const requestId = radarSearchRequestSeqRef.current + 1;
radarSearchRequestSeqRef.current = requestId;
setRadarSearchApplied(normalizedCriteria);
setRadarSearchAppliedAoiMode('none');
setRadarSearchAppliedRegionTreeId('');
setRadarSearchAoiMode('none');
setRadarSearchAoiToken('');
setHasRadarSearched(true);
setIsLoading(true);
clearRadarSearchResults({ limit: radarPagination.limit });
try {
await fetchAllData({
limit: radarPagination.limit,
offset: 0,
criteria: normalizedCriteria,
aoiMode: 'none',
regionTreeId: '',
files: null,
aoiToken: '',
requestId,
});
} finally {
setIsLoading(false);
}
}, [
radarSearchDraft, radarPagination.limit, radarSearchRequestSeqRef,
setSelectedSatelliteGroup, setRadarSearchDraft, setRadarSearchApplied,
setRadarSearchAppliedAoiMode, setRadarSearchAppliedRegionTreeId,
setRadarSearchAoiMode, setRadarSearchAoiToken, setHasRadarSearched,
setIsLoading, clearRadarSearchResults, fetchAllData, fetchRadarSearchOptions,
]);
const resetRadarSearch = useCallback(() => {
setRadarSearchDraft(RADAR_SEARCH_DEFAULTS);
setRadarSearchApplied(RADAR_SEARCH_DEFAULTS);