From 79e08b3a479095d3cf411a2f7534c3792c6f7a58 Mon Sep 17 00:00:00 2001 From: Harmon Date: Tue, 2 Jun 2026 01:25:42 +0800 Subject: [PATCH] Integrate GF3 SARscape flood workflow --- .env.example | 11 +- backend/app/config.py | 19 + backend/app/models/schemas.py | 12 +- backend/app/routers/monitor.py | 159 +++ backend/app/scheduler.py | 7 + .../services/gf3_native_inventory_service.py | 308 ++++++ .../gf3_sarscape_production_service.py | 843 ++++++++++++++++ .../app/services/gf3_standardize_service.py | 927 ++++++++++++++++++ backend/app/services/health_service.py | 53 + backend/app/services/image_service.py | 72 +- backend/app/services/job_handlers.py | 335 ++++++- backend/app/services/root_registry_service.py | 9 + .../services/sar_analysis_ready_service.py | 67 +- backend/app/services/water_detect_service.py | 180 +++- ..._ALGORITHM_ENGINEERING_HANDOFF_20260602.md | 511 ++++++++++ ...SCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md | 476 +++++++++ docs/INDEX.md | 8 +- frontend/src/App.jsx | 18 +- frontend/src/AssetInventoryPanel.jsx | 1 + frontend/src/DataMonitorPanel.jsx | 150 ++- .../src/components/ActiveTasksOverlay.jsx | 6 + frontend/src/hooks/useDinsarOperations.js | 24 +- frontend/src/hooks/useGlobalTaskControl.js | 2 +- frontend/src/hooks/useRadarSearch.js | 106 +- 24 files changed, 4220 insertions(+), 84 deletions(-) create mode 100644 backend/app/services/gf3_native_inventory_service.py create mode 100644 backend/app/services/gf3_sarscape_production_service.py create mode 100644 backend/app/services/gf3_standardize_service.py create mode 100644 docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md create mode 100644 docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md diff --git a/.env.example b/.env.example index aab4b21..25f26d5 100644 --- a/.env.example +++ b/.env.example @@ -67,7 +67,16 @@ GF3_ARCHIVE_SOURCE_DIRS=D:\GF3_L1A_Image_Zip GF3_ARCHIVE_EXTS=.zip,.tar,.tar.gz,.tgz GF3_UNPACK_DELETE_ARCHIVE=true GF3_SOURCE_DIRS=D:\GF3_L1A_Image -GF3_STORAGE_DIRS=D:\GF3_L2_Image +GF3_SARSCAPE_NATIVE_DIRS=D:\GF3_L2_ENVI_Binary_Pool +GF3_STORAGE_DIRS=D:\GF3_L2_Image_Pool +GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\.codex_tmp\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe +GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe +GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM +GF3_SARSCAPE_POLARIZATIONS=HH,HV +GF3_SARSCAPE_KEEP_EXTRACTED=true +GF3_SARSCAPE_AUTO_STANDARDIZE=true +GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true +GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 HAZARD_POINTS_DIR=D:\Code\Insar_management_system_v2\backend\Point HAZARD_POINTS_FILENAME=Point.shp diff --git a/backend/app/config.py b/backend/app/config.py index 8990197..be4d655 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -207,7 +207,16 @@ class Settings(BaseSettings): GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz" GF3_UNPACK_DELETE_ARCHIVE: bool = True GF3_SOURCE_DIRS: str = "" + GF3_SARSCAPE_NATIVE_DIRS: str = "" GF3_STORAGE_DIRS: str = "" + GF3_SARSCAPE_WRAPPER_EXE: str = "" + GF3_SARSCAPE_IDLRT_PATH: str = r"C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe" + GF3_SARSCAPE_DEM_PATH: str = "" + GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV" + GF3_SARSCAPE_KEEP_EXTRACTED: bool = True + GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True + GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True + GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0 MONITOR_ORBIT_DIR: str = "" ORBIT_POOL_ENVI: str = "" @@ -991,6 +1000,15 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True) _check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True) + _check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True) + _check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True) + _check_path( + label="GF3_SARSCAPE_DEM_PATH", + value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH), + errors=errors, + warnings=warnings, + expect_file=True, + ) _check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False) @@ -1024,6 +1042,7 @@ def validate_runtime_config() -> dict[str, Any]: ("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS), ("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS), ("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS), + ("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS), ("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS), ): values = split_env_paths(raw_value) diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index ac8a51a..b7b8312 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -210,12 +210,12 @@ class RadarData(BaseModel): has_orbit_data: bool orbit_file_path: Optional[str] = None is_envi_processed: bool = False - coverage_polygon: List[Tuple[float, float]] + coverage_polygon: Optional[List[Tuple[float, float]]] = None - min_lon: float - min_lat: float - max_lon: float - max_lat: float + min_lon: Optional[float] = None + min_lat: Optional[float] = None + max_lon: Optional[float] = None + max_lat: Optional[float] = None preview_cache_status: str = "NONE" preview_cache_version: Optional[str] = None preview_cache_updated_at: Optional[datetime] = None @@ -236,7 +236,7 @@ class RadarData(BaseModel): @computed_field @property - def coverage_bbox(self) -> Tuple[float, float, float, float]: + def coverage_bbox(self) -> Tuple[Optional[float], Optional[float], Optional[float], Optional[float]]: """A computed property to provide the bbox tuple, used by existing logic.""" return (self.min_lon, self.min_lat, self.max_lon, self.max_lat) diff --git a/backend/app/routers/monitor.py b/backend/app/routers/monitor.py index eca5cf3..a9f5653 100644 --- a/backend/app/routers/monitor.py +++ b/backend/app/routers/monitor.py @@ -43,7 +43,14 @@ class MonitorConfig(BaseModel): dinsar_dirs: List[str] = [] gf3_archive_source_dirs: List[str] = [] gf3_source_dirs: List[str] = [] + gf3_sarscape_native_dirs: List[str] = [] gf3_storage_dirs: List[str] = [] + gf3_sarscape_wrapper_exe: Optional[str] = None + gf3_sarscape_idlrt_path: Optional[str] = None + gf3_sarscape_dem_path: Optional[str] = None + gf3_sarscape_polarizations: Optional[str] = None + gf3_sarscape_auto_standardize: bool = True + gf3_sarscape_clean_after_success: bool = True # Manual-only: config is read from .env @@ -58,6 +65,26 @@ class GF3UnpackRunRequest(BaseModel): max_files_per_run: Optional[int] = Field(default=None, ge=0) +class GF3SarscapeSyncRequest(BaseModel): + force: bool = False + register: bool = True + + +class GF3SarscapeProduceRequest(BaseModel): + max_archives_per_run: Optional[int] = Field(default=None, ge=0) + auto_standardize: Optional[bool] = None + clean_after_success: Optional[bool] = None + force_standardize: bool = False + register: bool = True + cleanup_dry_run: bool = False + + +class GF3SarscapeCleanRequest(BaseModel): + dry_run: bool = False + require_standardized: bool = True + max_scenes: Optional[int] = Field(default=None, ge=0) + + @router.post("/monitor/config") async def update_monitor_config(config: MonitorConfig): """ @@ -154,6 +181,138 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin raise HTTPException(status_code=409, detail=str(e)) +@router.post("/monitor/gf3-sarscape-sync", status_code=202) +async def run_gf3_sarscape_sync( + request_data: GF3SarscapeSyncRequest | None = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + """ + 扫描 GF3 SARscape 原生 _geo 二进制池，转换为标准 GeoTIFF，并登记入库。 + """ + gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] + gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or [] + if not gf3_native_dirs: + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") + if not gf3_storage_dirs: + raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.") + + options = request_data or GF3SarscapeSyncRequest() + task_type = "GF3_SARSCAPE_SYNC" + task_name = "GF3 SARscape 原生结果标准化" + payload = { + "native_dirs": gf3_native_dirs, + "storage_root": gf3_storage_dirs[0], + "force": bool(options.force), + "register": bool(options.register), + } + + try: + task_id = await task_service.create_task(task_type, task_name, params=payload) + await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) + return { + "message": "GF3 SARscape 原生结果标准化任务已提交", + "task_id": task_id, + } + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + +@router.post("/monitor/gf3-sarscape-produce", status_code=202) +async def run_gf3_sarscape_produce( + request_data: GF3SarscapeProduceRequest | None = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + """ + Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates. + """ + gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] + gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] + gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or [] + if not gf3_archive_source_dirs: + raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.") + if not gf3_native_dirs: + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") + if not gf3_storage_dirs: + raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.") + if not settings.GF3_SARSCAPE_WRAPPER_EXE: + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.") + if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH): + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.") + + options = request_data or GF3SarscapeProduceRequest() + task_type = "GF3_SARSCAPE_PRODUCE" + task_name = "GF3 SARscape production" + auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize) + clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success) + payload = { + "source_dirs": gf3_archive_source_dirs, + "native_dirs": gf3_native_dirs, + "native_root": gf3_native_dirs[0], + "storage_root": gf3_storage_dirs[0], + "wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE, + "idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH, + "dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH, + "polarizations": settings.GF3_SARSCAPE_POLARIZATIONS, + "archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS), + "max_archives_per_run": int(options.max_archives_per_run or 0), + "timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0), + "keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED), + "auto_standardize": bool(auto_standardize), + "clean_after_success": bool(clean_after_success), + "force_standardize": bool(options.force_standardize), + "register": bool(options.register), + "cleanup_require_standardized": True, + "cleanup_dry_run": bool(options.cleanup_dry_run), + } + + try: + task_id = await task_service.create_task(task_type, task_name, params=payload) + await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) + return { + "message": "GF3 SARscape production task submitted", + "task_id": task_id, + } + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + +@router.post("/monitor/gf3-sarscape-clean", status_code=202) +async def run_gf3_sarscape_clean( + request_data: GF3SarscapeCleanRequest | None = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + """ + Clean SARscape intermediate files from GF3 native pool after standard GeoTIFFs exist. + """ + gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] + gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or [] + if not gf3_native_dirs: + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") + if not gf3_storage_dirs: + raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.") + + options = request_data or GF3SarscapeCleanRequest() + task_type = "GF3_SARSCAPE_CLEAN" + task_name = "GF3 SARscape native cleanup" + payload = { + "native_dirs": gf3_native_dirs, + "storage_root": gf3_storage_dirs[0], + "dry_run": bool(options.dry_run), + "require_standardized": bool(options.require_standardized), + "max_scenes": int(options.max_scenes or 0), + } + + try: + task_id = await task_service.create_task(task_type, task_name, params=payload) + await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) + return { + "message": "GF3 SARscape cleanup task submitted", + "task_id": task_id, + } + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + @router.get("/monitor/gf3-unpack/config") async def get_gf3_unpack_config(admin_user: AuthUserORM = Depends(_require_admin)): return GF3UnpackConfig( diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index f09ba52..250898a 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -21,7 +21,14 @@ MONITOR_CONFIG = { # GF3 链路 "gf3_archive_source_dirs": split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS), "gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS), + "gf3_sarscape_native_dirs": split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), "gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS), + "gf3_sarscape_wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE, + "gf3_sarscape_idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH, + "gf3_sarscape_dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH, + "gf3_sarscape_polarizations": settings.GF3_SARSCAPE_POLARIZATIONS, + "gf3_sarscape_auto_standardize": bool(settings.GF3_SARSCAPE_AUTO_STANDARDIZE), + "gf3_sarscape_clean_after_success": bool(settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS), "mode": "manual", "config_source": "env", } diff --git a/backend/app/services/gf3_native_inventory_service.py b/backend/app/services/gf3_native_inventory_service.py new file mode 100644 index 0000000..f440739 --- /dev/null +++ b/backend/app/services/gf3_native_inventory_service.py @@ -0,0 +1,308 @@ +"""Inventory GF3 SARscape native geocoded outputs. + +The production server writes ENVI/SARscape native ``*_geo`` datasets. This +service treats those files as the source-of-truth evidence layer and produces a +small manifest that later conversion jobs can consume. +""" +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ..utils import parse_gf3_l2_dirname + +NATIVE_MANIFEST_NAME = "gf3_native_manifest.json" +NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1" +POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH") +SKIP_DIR_NAMES = { + ".git", + ".gf3_extract", + ".sarmap", + "__pycache__", + "temp", + "tmp", + "work", + "sarscape_work", + "GTOPO30_DIR", + "SRTM_DEM_DIR", + "TANDEMX_DEM_DIR", +} + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _safe_stat(path: Path) -> dict[str, Any] | None: + try: + stat = path.stat() + except OSError: + return None + return { + "path": str(path), + "size": int(stat.st_size), + "mtime": float(stat.st_mtime), + "mtime_ns": int(stat.st_mtime_ns), + } + + +def _is_nonempty_file(path: Path) -> bool: + try: + return path.is_file() and path.stat().st_size > 0 + except OSError: + return False + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_name(f".{path.name}.tmp") + with tmp_path.open("w", encoding="utf-8") as stream: + json.dump(payload, stream, ensure_ascii=False, indent=2, default=str) + os.replace(tmp_path, path) + + +def _extract_date_from_text(value: str) -> str | None: + match = re.search(r"(20\d{6})", value or "") + return match.group(1) if match else None + + +def _extract_product_unique_id(value: str) -> str | None: + match = re.search(r"(L\d{8,})", value or "", flags=re.IGNORECASE) + return match.group(1).upper() if match else None + + +def _polarization_from_geo_name(name: str) -> str | None: + upper_name = name.upper() + match = re.search(r"(?:^|[_-])(HH|HV|VH|VV)[_-]GEO$", upper_name) + if match: + return match.group(1) + tokens = [token for token in re.split(r"[_\-.]+", upper_name) if token] + for token in reversed(tokens): + if token in POLARIZATION_PRIORITY: + return token + return None + + +def _is_geo_native_data_file(path: Path) -> bool: + return path.is_file() and path.name.lower().endswith("_geo") + + +def _scene_batch_name(root: Path, scene_dir: Path, scene_name: str) -> str | None: + try: + rel_parts = scene_dir.relative_to(root).parts + except ValueError: + rel_parts = () + if len(rel_parts) >= 2: + return rel_parts[0] + date = _extract_date_from_text(scene_name) + return date + + +def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict[str, Any]: + parsed = parse_gf3_l2_dirname(scene_name) or {} + metadata: dict[str, Any] = { + "satellite": "GF3", + "satellite_family": "GF3", + **parsed, + } + if not metadata.get("imaging_date"): + metadata["imaging_date"] = _extract_date_from_text(scene_name) + if not metadata.get("product_unique_id"): + metadata["product_unique_id"] = _extract_product_unique_id(scene_name) + + polarizations = [ + str(asset.get("polarization") or "").upper() + for asset in assets + if asset.get("polarization") and asset.get("polarization") != "UNKNOWN" + ] + if polarizations: + metadata["polarization"] = ",".join( + pol for pol in POLARIZATION_PRIORITY if pol in set(polarizations) + ) or ",".join(sorted(set(polarizations))) + metadata["product_level"] = "L2" + metadata["source_format"] = "GF3_SARSCAPE_NATIVE" + return metadata + + +def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]: + assets: list[dict[str, Any]] = [] + try: + entries = sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()) + except OSError: + return assets + + for path in entries: + if not _is_geo_native_data_file(path): + continue + + base = path + hdr = Path(str(base) + ".hdr") + sml = Path(str(base) + ".sml") + aux_xml = Path(str(base) + ".aux.xml") + ovr = Path(str(base) + ".ovr") + kml = Path(str(base) + ".kml") + quicklook = base.with_name(base.name + "_ql.tif") + polarization = _polarization_from_geo_name(base.name) or "UNKNOWN" + complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml) + + asset: dict[str, Any] = { + "polarization": polarization, + "role": "geo_native", + "path": str(base), + "hdr": str(hdr) if hdr.exists() else None, + "sml": str(sml) if sml.exists() else None, + "aux_xml": str(aux_xml) if aux_xml.exists() else None, + "ovr": str(ovr) if ovr.exists() else None, + "quicklook": str(quicklook) if quicklook.exists() else None, + "kml": str(kml) if kml.exists() else None, + "complete": bool(complete), + "source": _safe_stat(base), + "hdr_info": _safe_stat(hdr) if hdr.exists() else None, + "sml_info": _safe_stat(sml) if sml.exists() else None, + } + assets.append(asset) + + return assets + + +def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None: + assets = _collect_native_assets(scene_dir) + if not assets: + return None + + scene_name = scene_dir.name + batch_name = _scene_batch_name(root, scene_dir, scene_name) + complete_assets = [asset for asset in assets if asset.get("complete")] + complete_pols = [ + pol + for pol in POLARIZATION_PRIORITY + if any(asset.get("complete") and asset.get("polarization") == pol for asset in assets) + ] + other_complete_pols = sorted( + { + str(asset.get("polarization") or "") + for asset in complete_assets + if asset.get("polarization") not in POLARIZATION_PRIORITY + } + ) + complete_pols.extend([pol for pol in other_complete_pols if pol]) + + if complete_assets and len(complete_assets) == len(assets): + status = "NATIVE_READY" + elif complete_assets: + status = "PARTIAL" + else: + status = "FAILED" + + logs = [] + for name in ("gf3_sarscape_cli.log",): + log_path = scene_dir / name + if log_path.is_file(): + logs.append(str(log_path)) + try: + logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs) + except OSError: + pass + + metadata = _parse_scene_metadata(scene_name, assets) + manifest_path = scene_dir / NATIVE_MANIFEST_NAME + return { + "schema": NATIVE_MANIFEST_SCHEMA, + "generated_at": _utc_now(), + "scene_name": scene_name, + "batch_name": batch_name, + "native_root": str(root), + "native_dir": str(scene_dir), + "manifest_path": str(manifest_path), + "source_archive": None, + "status": status, + "polarizations": complete_pols, + "metadata": metadata, + "assets": assets, + "logs": logs, + } + + +def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]: + roots: list[Path] = [] + missing: list[str] = [] + seen: set[str] = set() + for raw in native_dirs or []: + text = str(raw or "").strip() + if not text: + continue + path = Path(os.path.normpath(text)).resolve() + key = str(path).lower() + if key in seen: + continue + seen.add(key) + if path.is_dir(): + roots.append(path) + else: + missing.append(str(path)) + return roots, missing + + +def scan_gf3_sarscape_native_roots( + native_dirs: list[str] | tuple[str, ...] | None, + *, + write_manifest: bool = True, +) -> dict[str, Any]: + """Scan configured native roots and return discovered scene manifests.""" + roots, missing_roots = _normalize_roots(native_dirs) + scenes: list[dict[str, Any]] = [] + seen_scene_dirs: set[str] = set() + write_errors: list[dict[str, str]] = [] + + for root in roots: + for current_dir, dir_names, _file_names in os.walk(root): + dir_names[:] = [ + name + for name in dir_names + if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape") + ] + scene_dir = Path(current_dir) + scene_key = str(scene_dir).lower() + if scene_key in seen_scene_dirs: + dir_names[:] = [] + continue + + manifest = _collect_scene_manifest(root, scene_dir) + if not manifest: + continue + + seen_scene_dirs.add(scene_key) + if write_manifest: + try: + _write_json(Path(manifest["manifest_path"]), manifest) + except OSError as exc: + write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)}) + scenes.append(manifest) + dir_names[:] = [] + + native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY") + partial = sum(1 for scene in scenes if scene.get("status") == "PARTIAL") + failed = sum(1 for scene in scenes if scene.get("status") == "FAILED") + complete_assets = sum( + 1 + for scene in scenes + for asset in scene.get("assets") or [] + if asset.get("complete") + ) + return { + "schema": "gf3_sarscape_native_inventory.v1", + "generated_at": _utc_now(), + "native_roots": [str(path) for path in roots], + "missing_roots": missing_roots, + "scene_count": len(scenes), + "native_ready_count": native_ready, + "partial_count": partial, + "failed_count": failed, + "complete_asset_count": complete_assets, + "write_errors": write_errors, + "scenes": scenes, + } diff --git a/backend/app/services/gf3_sarscape_production_service.py b/backend/app/services/gf3_sarscape_production_service.py new file mode 100644 index 0000000..1641b73 --- /dev/null +++ b/backend/app/services/gf3_sarscape_production_service.py @@ -0,0 +1,843 @@ +"""Run GF3 SARscape production and clean native intermediate files.""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from ..config import settings, split_env_paths +from .gf3_native_inventory_service import ( + NATIVE_MANIFEST_NAME, + POLARIZATION_PRIORITY, + scan_gf3_sarscape_native_roots, +) +from .gf3_standardize_service import STANDARD_MANIFEST_NAME + +LogCallback = Callable[[str, str], None] +ProgressCallback = Callable[[int, str], None] + +SUPPORTED_WRAPPER_INPUT_EXTS = (".tar.gz", ".tgz", ".meta.xml") +CLEANUP_MANIFEST_NAME = "gf3_cleanup_manifest.json" +INTERMEDIATE_DIR_NAMES = { + ".gf3_extract", + ".gf3_extract.tmp", + "temp", + "tmp", + "work", +} +KEEP_FILE_NAMES = { + NATIVE_MANIFEST_NAME, + CLEANUP_MANIFEST_NAME, + "gf3_sarscape_cli.log", +} +INTERMEDIATE_SUFFIXES = ( + ".par", + ".par_command", + ".trace", + ".working", + ".working_warning", + ".workinggetcornerfromslantrangeimage_dem", + ".txt", + ".list", + ".listhv", + ".listunknown", + ".shp", + ".shx", + ".dbf", + ".prj", +) + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _clean_text(value: Any) -> str: + return str(value or "").strip().strip('"').strip("'") + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_name(f".{path.name}.tmp") + with tmp_path.open("w", encoding="utf-8") as stream: + json.dump(payload, stream, ensure_ascii=False, indent=2, default=str) + os.replace(tmp_path, path) + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + with path.open("r", encoding="utf-8") as stream: + data = json.load(stream) + return data if isinstance(data, dict) else None + except (OSError, json.JSONDecodeError): + return None + + +def _safe_slug(value: Any, *, default: str = "unknown") -> str: + text = _clean_text(value) or default + safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-") + return safe or default + + +def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]: + roots: list[Path] = [] + missing: list[str] = [] + seen: set[str] = set() + for raw in values or []: + text = _clean_text(raw) + if not text: + continue + path = Path(os.path.normpath(text)).resolve() + key = str(path).lower() + if key in seen: + continue + seen.add(key) + if path.is_dir(): + roots.append(path) + else: + missing.append(str(path)) + return roots, missing + + +def _resolve_config_path(raw: Any) -> Path | None: + text = _clean_text(raw) + if not text: + return None + return Path(os.path.normpath(text)).resolve() + + +def _fallback_wrapper_exe() -> Path | None: + candidate = ( + Path(settings.PROJECT_ROOT) + / ".codex_tmp" + / "GF3_L1A_To_L2_pipeline" + / "dist" + / "windows" + / "gf3wrapper.exe" + ) + return candidate.resolve() if candidate.is_file() else None + + +def _wrapper_exe_path(value: str | None = None) -> Path: + configured = _resolve_config_path(value or settings.GF3_SARSCAPE_WRAPPER_EXE) + path = configured or _fallback_wrapper_exe() + if path is None: + raise FileNotFoundError("GF3 SARscape wrapper is not configured.") + if not path.is_file(): + raise FileNotFoundError(f"GF3 SARscape wrapper does not exist: {path}") + return path + + +def _dem_path(value: str | None = None) -> Path: + path = _resolve_config_path(value or settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH) + if path is None: + raise FileNotFoundError("GF3 SARscape DEM path is not configured.") + if not path.exists(): + raise FileNotFoundError(f"GF3 SARscape DEM path does not exist: {path}") + return path + + +def _idlrt_path(value: str | None = None) -> Path | None: + path = _resolve_config_path(value or settings.GF3_SARSCAPE_IDLRT_PATH) + if path is None: + return None + if not path.is_file(): + raise FileNotFoundError(f"GF3 SARscape idlrt.exe does not exist: {path}") + return path + + +def _requested_polarizations(value: str | None = None) -> list[str]: + raw = _clean_text(value or settings.GF3_SARSCAPE_POLARIZATIONS or "HH,HV") + items: list[str] = [] + for token in re.split(r"[,;\s]+", raw): + pol = token.strip().upper() + if not pol: + continue + if pol not in items: + items.append(pol) + return items or ["HH", "HV"] + + +def _archive_exts_for_wrapper(archive_exts: list[str] | tuple[str, ...] | None) -> list[str]: + ordered: list[str] = [] + for raw_ext in archive_exts or []: + ext = _clean_text(raw_ext).lower() + if not ext: + continue + if not ext.startswith("."): + ext = "." + ext + if ext in SUPPORTED_WRAPPER_INPUT_EXTS and ext not in ordered: + ordered.append(ext) + if not ordered: + ordered.extend((".tar.gz", ".tgz")) + return sorted(ordered, key=len, reverse=True) + + +def _input_ext(path: Path, exts: list[str]) -> str | None: + name = path.name.lower() + for ext in exts: + if name.endswith(ext): + return ext + return None + + +def _scene_name_from_input(path: Path) -> str: + name = path.name + lower_name = name.lower() + for ext in SUPPORTED_WRAPPER_INPUT_EXTS: + if lower_name.endswith(ext): + return name[: -len(ext)] + return path.stem + + +def discover_gf3_sarscape_inputs( + source_dirs: list[str] | tuple[str, ...] | None, + *, + archive_exts: list[str] | tuple[str, ...] | None = None, +) -> dict[str, Any]: + """Find wrapper-supported GF3 source inputs in configured archive pools.""" + roots, missing_roots = _resolve_existing_dirs(source_dirs) + exts = _archive_exts_for_wrapper(archive_exts) + inputs: list[dict[str, Any]] = [] + seen: set[str] = set() + + for root in roots: + for path in root.rglob("*"): + if not path.is_file(): + continue + ext = _input_ext(path, exts) + if not ext: + continue + resolved = path.resolve() + key = str(resolved).lower() + if key in seen: + continue + seen.add(key) + inputs.append( + { + "path": str(resolved), + "scene_name": _scene_name_from_input(path), + "ext": ext, + "source_root": str(root), + } + ) + + inputs.sort(key=lambda item: str(item.get("path") or "").lower()) + return { + "source_roots": [str(path) for path in roots], + "missing_roots": missing_roots, + "archive_exts": exts, + "input_count": len(inputs), + "inputs": inputs, + } + + +def _is_nonempty_file(path: Path) -> bool: + try: + return path.is_file() and path.stat().st_size > 0 + except OSError: + return False + + +def _completed_geo_product(scene_dir: Path, polarization: str) -> Path | None: + lower_pol = polarization.lower() + try: + entries = list(scene_dir.iterdir()) + except OSError: + return None + + for path in entries: + name = path.name.lower() + if not name.endswith(f"_{lower_pol}_geo.sml"): + continue + data_file = Path(str(path)[: -len(".sml")]) + if _is_nonempty_file(path) and _is_nonempty_file(data_file): + return data_file + return None + + +def _scene_complete(scene_dir: Path, polarizations: list[str]) -> bool: + if not scene_dir.is_dir(): + return False + return all(_completed_geo_product(scene_dir, pol) is not None for pol in polarizations) + + +def _missing_geo_polarizations(scene_dir: Path, polarizations: list[str]) -> list[str]: + if not scene_dir.is_dir(): + return list(polarizations) + return [pol for pol in polarizations if _completed_geo_product(scene_dir, pol) is None] + + +def _compact_failure_text(text: str, *, max_chars: int = 1000) -> str: + lines = [line.strip() for line in text.splitlines() if line.strip()] + compact = " | ".join(lines) + if len(compact) <= max_chars: + return compact + return compact[: max_chars - 3].rstrip() + "..." + + +def _read_failure_file(path: Path) -> str | None: + try: + if not path.is_file() or path.stat().st_size <= 0: + return None + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def _scene_failure_hint(scene_dir: Path) -> str | None: + work_dir = scene_dir / "work" + candidates = [ + work_dir / "Process.working_error", + work_dir / "Process.trace_cerr.txt", + scene_dir / "gf3_sarscape_cli.log", + ] + for path in candidates: + text = _read_failure_file(path) + if text: + compact = _compact_failure_text(text) + if compact: + return f"{path.name}: {compact}" + + process_log = work_dir / "Process.log" + text = _read_failure_file(process_log) + if not text: + return None + important = [ + line.strip() + for line in text.splitlines() + if "ERROR" in line.upper() or "[EC:" in line.upper() or "PARAMETER FILE READ ERROR" in line.upper() + ] + if important: + return f"{process_log.name}: {_compact_failure_text(chr(10).join(important[-8:]))}" + return None + + +def _format_missing_output_error(scene_dir: Path, polarizations: list[str], returncode: int) -> str: + missing = _missing_geo_polarizations(scene_dir, polarizations) + missing_text = ",".join(missing) if missing else "unknown" + if returncode == 0: + base = f"wrapper returned 0 but required _geo outputs are missing: {missing_text}" + else: + base = f"wrapper return code {returncode}; missing _geo outputs: {missing_text}" + hint = _scene_failure_hint(scene_dir) + return f"{base}; {hint}" if hint else base + + +def _emit_log(callback: LogCallback | None, level: str, message: str) -> None: + if callback: + callback(level, message) + + +def _emit_progress(callback: ProgressCallback | None, progress: int, message: str) -> None: + if callback: + callback(max(0, min(100, int(progress))), message) + + +def _run_wrapper_command( + cmd: list[str], + *, + cwd: Path, + env: dict[str, str], + timeout_seconds: int | None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + cmd, + cwd=str(cwd), + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout_seconds if timeout_seconds and timeout_seconds > 0 else None, + check=False, + ) + + +def _log_completed_process_output( + completed: subprocess.CompletedProcess[str], + *, + log_callback: LogCallback | None, + line_limit: int = 300, +) -> list[str]: + output = completed.stdout or "" + lines = [line.rstrip() for line in output.splitlines() if line.strip()] + if not lines: + return [] + + clipped = False + display_lines = lines + if len(lines) > line_limit: + clipped = True + head_count = max(1, line_limit // 2) + tail_count = max(1, line_limit - head_count) + display_lines = lines[:head_count] + [f"... clipped {len(lines) - line_limit} wrapper log lines ..."] + lines[-tail_count:] + + for line in display_lines: + _emit_log(log_callback, "INFO", f"[gf3wrapper] {line}") + if clipped: + _emit_log(log_callback, "WARNING", f"GF3 wrapper output was clipped to {line_limit} log lines.") + return lines[-20:] + + +def run_gf3_sarscape_production( + *, + source_dirs: list[str] | None = None, + native_root: str | None = None, + wrapper_exe: str | None = None, + dem_path: str | None = None, + idlrt_path: str | None = None, + polarizations: str | None = None, + archive_exts: list[str] | None = None, + max_archives_per_run: int | None = None, + timeout_seconds: int | None = None, + keep_extracted: bool | None = None, + log_callback: LogCallback | None = None, + progress_callback: ProgressCallback | None = None, +) -> dict[str, Any]: + """Run the external GF3 SARscape wrapper for pending raw archives.""" + source_dirs = source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS) + native_roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS) + native_root_text = _clean_text(native_root or (native_roots[0] if native_roots else "")) + if not source_dirs: + raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.") + if not native_root_text: + raise ValueError("GF3_SARSCAPE_NATIVE_DIRS is not configured.") + native_root_path = Path(os.path.normpath(native_root_text)).resolve() + + native_root_path.mkdir(parents=True, exist_ok=True) + wrapper_path = _wrapper_exe_path(wrapper_exe) + dem = _dem_path(dem_path) + idlrt = _idlrt_path(idlrt_path) + pols = _requested_polarizations(polarizations) + pol_text = ",".join(pols) + ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS) + discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config) + inputs = discovery.get("inputs") or [] + max_to_process = int(max_archives_per_run or 0) + timeout = int(timeout_seconds or 0) + keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted) + + _emit_log(log_callback, "INFO", f"GF3 SARscape source roots: {source_dirs}") + _emit_log(log_callback, "INFO", f"GF3 SARscape native root: {native_root_path}") + _emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}") + _emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}") + _emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}") + + if not inputs: + _emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.") + return { + "ok": True, + "found_count": 0, + "processed_count": 0, + "skipped_count": 0, + "failed_count": 0, + "deferred_count": 0, + "missing_roots": discovery.get("missing_roots") or [], + "results": [], + } + + runtime_dir = native_root_path / ".gf3_runtime" + runtime_dir.mkdir(parents=True, exist_ok=True) + config_path = runtime_dir / "gf3wrapper.json" + env = os.environ.copy() + if idlrt is not None: + env["IDLRT_PATH"] = str(idlrt) + + processed = 0 + skipped = 0 + failed = 0 + deferred = 0 + results: list[dict[str, Any]] = [] + total = len(inputs) + + for idx, input_info in enumerate(inputs): + input_path = Path(str(input_info.get("path") or "")).resolve() + scene_name = str(input_info.get("scene_name") or _scene_name_from_input(input_path)) + scene_dir = native_root_path / scene_name + progress = 5 + int((idx / max(total, 1)) * 60) + _emit_progress(progress_callback, progress, f"GF3 SARscape checking {idx + 1}/{total}: {scene_name}") + + if _scene_complete(scene_dir, pols): + skipped += 1 + results.append( + { + "scene_name": scene_name, + "input_path": str(input_path), + "scene_dir": str(scene_dir), + "status": "skipped_complete", + } + ) + continue + + if max_to_process > 0 and processed + failed >= max_to_process: + deferred += 1 + results.append( + { + "scene_name": scene_name, + "input_path": str(input_path), + "scene_dir": str(scene_dir), + "status": "deferred", + } + ) + continue + + cmd = [ + str(wrapper_path), + "-config", + str(config_path), + "-input", + str(input_path), + "-output", + str(native_root_path), + "-dem", + str(dem), + "-pol", + pol_text, + f"-keep-extracted={str(keep).lower()}", + ] + if idlrt is not None: + cmd.extend(["-idlrt", str(idlrt)]) + + _emit_log(log_callback, "INFO", f"GF3 SARscape processing {scene_name}: {input_path}") + started = time.monotonic() + try: + completed = _run_wrapper_command( + cmd, + cwd=wrapper_path.parent, + env=env, + timeout_seconds=timeout, + ) + output_tail = _log_completed_process_output(completed, log_callback=log_callback) + elapsed_seconds = round(time.monotonic() - started, 3) + output_complete = _scene_complete(scene_dir, pols) + if completed.returncode == 0 and output_complete: + processed += 1 + status = "processed" + error = None + elif output_complete: + processed += 1 + status = "processed_with_warning" + error = f"wrapper return code {completed.returncode}" + _emit_log(log_callback, "WARNING", f"GF3 wrapper returned {completed.returncode}, but output is complete: {scene_name}") + else: + failed += 1 + status = "failed" + error = _format_missing_output_error(scene_dir, pols, int(completed.returncode)) + _emit_log(log_callback, "ERROR", f"GF3 SARscape failed for {scene_name}: {error}") + + results.append( + { + "scene_name": scene_name, + "input_path": str(input_path), + "scene_dir": str(scene_dir), + "status": status, + "returncode": int(completed.returncode), + "elapsed_seconds": elapsed_seconds, + "output_complete": output_complete, + "error": error, + "output_tail": output_tail, + } + ) + except subprocess.TimeoutExpired as exc: + failed += 1 + _emit_log(log_callback, "ERROR", f"GF3 SARscape timed out for {scene_name}: {exc}") + results.append( + { + "scene_name": scene_name, + "input_path": str(input_path), + "scene_dir": str(scene_dir), + "status": "failed", + "error": f"timeout after {timeout}s", + } + ) + except Exception as exc: + failed += 1 + _emit_log(log_callback, "ERROR", f"GF3 SARscape exception for {scene_name}: {exc}") + results.append( + { + "scene_name": scene_name, + "input_path": str(input_path), + "scene_dir": str(scene_dir), + "status": "failed", + "error": str(exc), + } + ) + + _emit_progress(progress_callback, 70, "GF3 SARscape production stage finished.") + return { + "ok": failed == 0, + "found_count": total, + "processed_count": processed, + "skipped_count": skipped, + "failed_count": failed, + "deferred_count": deferred, + "native_root": str(native_root_path), + "missing_roots": discovery.get("missing_roots") or [], + "results": results, + } + + +def _is_relative_to(path: Path, parent: Path) -> bool: + try: + path.resolve().relative_to(parent.resolve()) + return True + except ValueError: + return False + + +def _entry_size(path: Path) -> int: + try: + if path.is_file(): + return int(path.stat().st_size) + if path.is_dir(): + total = 0 + for current, _dir_names, file_names in os.walk(path): + for file_name in file_names: + file_path = Path(current) / file_name + try: + total += int(file_path.stat().st_size) + except OSError: + continue + return total + except OSError: + return 0 + return 0 + + +def _is_final_geo_asset_file(path: Path) -> bool: + name = path.name.lower() + if name in KEEP_FILE_NAMES or name.endswith(".log"): + return True + return ( + name.endswith("_geo") + or name.endswith("_geo.hdr") + or name.endswith("_geo.sml") + or name.endswith("_geo.ovr") + or name.endswith("_geo.aux.xml") + or name.endswith("_geo.kml") + or name.endswith("_geo_ql.tif") + or name.endswith("_geo_ql.kml") + ) + + +def _is_known_intermediate_file(path: Path) -> bool: + name = path.name.lower() + if _is_final_geo_asset_file(path): + return False + if any(token in name for token in ("_slc", "_ml", "_filt")): + return True + if name.endswith(INTERMEDIATE_SUFFIXES): + return True + return False + + +def _standard_manifest_path(scene_manifest: dict[str, Any], storage_root: Path) -> Path: + batch_name = _safe_slug(scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date"), default="unknown_batch") + scene_name = _safe_slug(scene_manifest.get("scene_name"), default="unknown_scene") + return storage_root / batch_name / scene_name / STANDARD_MANIFEST_NAME + + +def _standard_manifest_allows_cleanup(scene_manifest: dict[str, Any], storage_root: Path) -> bool: + manifest = _read_json(_standard_manifest_path(scene_manifest, storage_root)) + if not manifest: + return False + return str(manifest.get("status") or "").upper() == "DONE" + + +def _assert_safe_delete(target: Path, scene_dir: Path, native_roots: list[Path]) -> None: + resolved_target = target.resolve() + resolved_scene = scene_dir.resolve() + if resolved_target == resolved_scene: + raise RuntimeError(f"Refusing to delete scene directory itself: {resolved_target}") + if not _is_relative_to(resolved_target, resolved_scene): + raise RuntimeError(f"Refusing to delete outside scene directory: {resolved_target}") + if not any(_is_relative_to(resolved_target, root) for root in native_roots): + raise RuntimeError(f"Refusing to delete outside GF3 native roots: {resolved_target}") + + +def _cleanup_scene_intermediates( + scene_manifest: dict[str, Any], + *, + native_roots: list[Path], + dry_run: bool, +) -> dict[str, Any]: + scene_dir = Path(str(scene_manifest.get("native_dir") or "")).resolve() + if not scene_dir.is_dir(): + return { + "scene_name": scene_manifest.get("scene_name"), + "scene_dir": str(scene_dir), + "status": "skipped", + "reason": "scene directory missing", + "deleted_entries": [], + "bytes_deleted": 0, + } + if not any(_is_relative_to(scene_dir, root) for root in native_roots): + return { + "scene_name": scene_manifest.get("scene_name"), + "scene_dir": str(scene_dir), + "status": "skipped", + "reason": "scene directory is outside configured native roots", + "deleted_entries": [], + "bytes_deleted": 0, + } + + candidates: list[Path] = [] + for entry in sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()): + if entry.is_dir() and entry.name.lower() in INTERMEDIATE_DIR_NAMES: + candidates.append(entry) + elif entry.is_file() and _is_known_intermediate_file(entry): + candidates.append(entry) + + deleted_entries: list[dict[str, Any]] = [] + bytes_deleted = 0 + errors: list[dict[str, str]] = [] + + for target in candidates: + try: + _assert_safe_delete(target, scene_dir, native_roots) + size = _entry_size(target) + bytes_deleted += size + deleted_entries.append( + { + "path": str(target), + "type": "directory" if target.is_dir() else "file", + "size": size, + } + ) + if not dry_run: + if target.is_dir(): + shutil.rmtree(target) + else: + target.unlink() + except Exception as exc: + errors.append({"path": str(target), "error": str(exc)}) + + status = "cleaned" if deleted_entries and not errors else ("error" if errors else "nothing_to_delete") + cleanup_manifest = { + "schema": "gf3_sarscape_cleanup.v1", + "generated_at": _utc_now(), + "dry_run": dry_run, + "scene_name": scene_manifest.get("scene_name"), + "scene_dir": str(scene_dir), + "status": status, + "bytes_deleted": bytes_deleted, + "deleted_entries": deleted_entries, + "errors": errors, + "retention_policy": { + "keep": "final *_geo native assets, quicklooks, manifests, and logs", + "delete": "extract/temp/work directories and slc/ml/filt intermediate files", + }, + } + if not dry_run: + _write_json(scene_dir / CLEANUP_MANIFEST_NAME, cleanup_manifest) + return cleanup_manifest + + +def cleanup_gf3_sarscape_native_pool( + *, + native_dirs: list[str] | None = None, + storage_root: str | None = None, + require_standardized: bool = True, + dry_run: bool = False, + max_scenes: int | None = None, + log_callback: LogCallback | None = None, + progress_callback: ProgressCallback | None = None, +) -> dict[str, Any]: + """Delete intermediate SARscape files while keeping final native _geo assets.""" + native_dirs = native_dirs if native_dirs is not None else split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS) + native_roots, missing_roots = _resolve_existing_dirs(native_dirs) + if not native_roots: + raise ValueError("GF3_SARSCAPE_NATIVE_DIRS has no accessible directories.") + storage = Path(os.path.normpath(storage_root or settings.GF3_STORAGE_DIRS)).resolve() if storage_root or settings.GF3_STORAGE_DIRS else None + + inventory = scan_gf3_sarscape_native_roots([str(root) for root in native_roots], write_manifest=not dry_run) + scenes = inventory.get("scenes") or [] + max_count = int(max_scenes or 0) + cleaned = 0 + skipped = 0 + error_count = 0 + bytes_deleted = 0 + scene_results: list[dict[str, Any]] = [] + + for idx, scene_manifest in enumerate(scenes): + _emit_progress( + progress_callback, + 5 + int((idx / max(len(scenes), 1)) * 90), + f"GF3 native cleanup checking {idx + 1}/{len(scenes)}: {scene_manifest.get('scene_name')}", + ) + scene_status = str(scene_manifest.get("status") or "") + if scene_status != "NATIVE_READY": + skipped += 1 + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "status": "skipped", + "reason": f"native status is {scene_status or 'UNKNOWN'}", + } + ) + continue + if require_standardized: + if storage is None: + skipped += 1 + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "status": "skipped", + "reason": "GF3_STORAGE_DIRS is not configured", + } + ) + continue + if not _standard_manifest_allows_cleanup(scene_manifest, storage): + skipped += 1 + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "status": "skipped", + "reason": "standard GeoTIFF manifest is not DONE", + } + ) + continue + if max_count > 0 and cleaned >= max_count: + skipped += 1 + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "status": "skipped", + "reason": "max_scenes limit reached", + } + ) + continue + + result = _cleanup_scene_intermediates(scene_manifest, native_roots=native_roots, dry_run=dry_run) + scene_results.append(result) + bytes_deleted += int(result.get("bytes_deleted") or 0) + if result.get("status") == "error": + error_count += 1 + if result.get("status") in {"cleaned", "nothing_to_delete"}: + cleaned += 1 + _emit_log( + log_callback, + "INFO", + f"GF3 cleanup {result.get('status')}: {result.get('scene_name')} bytes={result.get('bytes_deleted')}", + ) + + _emit_progress(progress_callback, 100, "GF3 native cleanup finished.") + return { + "ok": error_count == 0, + "dry_run": dry_run, + "native_roots": [str(root) for root in native_roots], + "missing_roots": missing_roots, + "scene_count": len(scenes), + "cleaned_scene_count": cleaned, + "skipped_scene_count": skipped, + "error_scene_count": error_count, + "bytes_deleted": bytes_deleted, + "scenes": scene_results, + } diff --git a/backend/app/services/gf3_standardize_service.py b/backend/app/services/gf3_standardize_service.py new file mode 100644 index 0000000..9e8aa5c --- /dev/null +++ b/backend/app/services/gf3_standardize_service.py @@ -0,0 +1,927 @@ +"""Convert GF3 SARscape native geocoded outputs to platform GeoTIFFs.""" +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from geoalchemy2.shape import from_shape +from shapely.geometry import Polygon +from sqlalchemy import func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..config import settings +from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM +from .data_service import extract_geotiff_bounds +from .gf3_native_inventory_service import ( + NATIVE_MANIFEST_NAME, + POLARIZATION_PRIORITY, + scan_gf3_sarscape_native_roots, +) +from .image_service import image_service +from .sar_analysis_ready_service import register_analysis_ready_tif + +STANDARD_MANIFEST_NAME = "gf3_standard_manifest.json" +STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1" +CONVERTER_NAME = "gf3_sarscape_geo_to_tif" +CONVERTER_VERSION = "v1" +SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _path_text(value: Any) -> str: + text = str(value or "").strip() + return os.path.normpath(text) if text else "" + + +def _db_now() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0) + + +def _path_kind(path: str) -> str: + text = str(path or "").strip() + if text.startswith("\\\\"): + return "unc" + if len(text) >= 3 and text[1:3] in {":\\", ":/"} and text[0].isalpha(): + return "windows" + if text.startswith("/mnt/"): + return "wsl_mount" + if text.startswith("/"): + return "posix" + return "relative" + + +def _source_asset_uid(path: str) -> str: + normalized = os.path.normpath(str(path or "").strip()) + digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest() + return f"source:{digest[:32]}" + + +def _safe_slug(value: Any, *, default: str = "unknown") -> str: + text = str(value or "").strip() + if not text: + text = default + safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-") + return safe or default + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_name(f".{path.name}.tmp") + with tmp_path.open("w", encoding="utf-8") as stream: + json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False) + os.replace(tmp_path, path) + + +def _json_safe(value: Any) -> Any: + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return value + + +def _finite_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + with path.open("r", encoding="utf-8") as stream: + data = json.load(stream) + return data if isinstance(data, dict) else None + except (OSError, json.JSONDecodeError): + return None + + +def _file_fingerprint(path: Path) -> dict[str, Any] | None: + try: + stat = path.stat() + except OSError: + return None + return { + "path": str(path), + "size": int(stat.st_size), + "mtime": float(stat.st_mtime), + "mtime_ns": int(stat.st_mtime_ns), + } + + +def _tree_stats(path: Path) -> dict[str, Any]: + if path.is_file(): + info = _file_fingerprint(path) + return { + "size_bytes": info.get("size") if info else None, + "mtime_epoch": info.get("mtime") if info else None, + } + + total = 0 + newest: float | None = None + try: + iterator = path.rglob("*") + for item in iterator: + try: + if not item.is_file(): + continue + stat = item.stat() + except OSError: + continue + total += int(stat.st_size) + mtime = float(stat.st_mtime) + newest = mtime if newest is None else max(newest, mtime) + except OSError: + return {"size_bytes": None, "mtime_epoch": None} + return {"size_bytes": total, "mtime_epoch": newest} + + +async def _find_managed_root_for_path(db: AsyncSession, path: str) -> ManagedRootORM | None: + target = os.path.normcase(os.path.normpath(str(path or ""))) + if not target: + return None + result = await db.execute( + select(ManagedRootORM) + .where(ManagedRootORM.enabled == True) # noqa: E712 + .order_by(func.length(ManagedRootORM.path).desc()) + ) + for root in result.scalars().all(): + root_path = os.path.normcase(os.path.normpath(str(root.path or ""))) + if target == root_path or target.startswith(root_path + os.sep): + return root + return None + + +def _batch_name(scene_manifest: dict[str, Any]) -> str: + raw = scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date") + return _safe_slug(raw, default="unknown_batch") + + +def _standard_scene_dir(scene_manifest: dict[str, Any], storage_root: Path) -> Path: + return storage_root / _batch_name(scene_manifest) / _safe_slug(scene_manifest.get("scene_name")) + + +def _target_tif_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path: + pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper() + return _standard_scene_dir(scene_manifest, storage_root) / f"{pol}_L2.tif" + + +def _preview_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path: + pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper() + return _standard_scene_dir(scene_manifest, storage_root) / f"preview_{pol}.png" + + +def _quality_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path: + pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper() + return _standard_scene_dir(scene_manifest, storage_root) / f"quality_{pol}.json" + + +def _source_changed(asset: dict[str, Any], target_tif: Path, existing_asset: dict[str, Any] | None) -> bool: + source_path = Path(_path_text(asset.get("path"))) + source_fp = _file_fingerprint(source_path) + if not target_tif.is_file() or target_tif.stat().st_size <= 0: + return True + if not existing_asset: + return True + if str(existing_asset.get("source_native") or "") != str(source_path): + return True + if (existing_asset.get("source_fingerprint") or {}) != source_fp: + return True + if str(existing_asset.get("converter_version") or "") != CONVERTER_VERSION: + return True + return False + + +def _existing_manifest_asset(standard_manifest: dict[str, Any] | None, polarization: str) -> dict[str, Any] | None: + if not standard_manifest: + return None + target_pol = str(polarization or "").upper() + for item in standard_manifest.get("assets") or []: + if str(item.get("polarization") or "").upper() == target_pol: + return item + return None + + +def _convert_native_asset_to_tif(asset: dict[str, Any], target_tif: Path) -> dict[str, Any]: + source_path = Path(_path_text(asset.get("path"))) + if not source_path.is_file(): + raise FileNotFoundError(f"GF3 native data file does not exist: {source_path}") + hdr_path = Path(_path_text(asset.get("hdr"))) + if not hdr_path.is_file(): + raise FileNotFoundError(f"GF3 native ENVI header does not exist: {hdr_path}") + + target_tif.parent.mkdir(parents=True, exist_ok=True) + tmp_tif = target_tif.with_name(f".{target_tif.name}.tmp.tif") + if tmp_tif.exists(): + tmp_tif.unlink() + + try: + from osgeo import gdal + + src_ds = gdal.Open(str(source_path), gdal.GA_ReadOnly) + if src_ds is None: + raise RuntimeError(f"GDAL cannot open GF3 native dataset: {source_path}") + + creation_options = ["TILED=YES", "COMPRESS=DEFLATE", "BIGTIFF=IF_SAFER"] + if bool(settings.SAR_ANALYSIS_OUTPUT_COG): + creation_options.append("COPY_SRC_OVERVIEWS=YES") + translated = gdal.Translate( + str(tmp_tif), + src_ds, + format="GTiff", + creationOptions=creation_options, + ) + src_ds = None + if translated is None: + raise RuntimeError(f"GDAL Translate failed for GF3 native dataset: {source_path}") + translated.FlushCache() + translated = None + except ImportError: + import rasterio + + with rasterio.open(source_path) as src: + profile = src.profile.copy() + profile.update( + driver="GTiff", + tiled=True, + compress="deflate", + BIGTIFF="IF_SAFER", + ) + with rasterio.open(tmp_tif, "w", **profile) as dst: + for band_idx in range(1, src.count + 1): + for _block_index, window in src.block_windows(band_idx): + dst.write(src.read(band_idx, window=window), band_idx, window=window) + dst.update_tags(**src.tags()) + for band_idx in range(1, src.count + 1): + dst.update_tags(band_idx, **src.tags(band_idx)) + + os.replace(tmp_tif, target_tif) + + return { + "path": str(target_tif), + "source_native": str(source_path), + "source_fingerprint": _file_fingerprint(source_path), + "converter_name": CONVERTER_NAME, + "converter_version": CONVERTER_VERSION, + } + + +def _raster_quality(path: Path) -> dict[str, Any]: + try: + import numpy as np + import rasterio + except Exception as exc: + return {"ok": False, "warning": f"rasterio unavailable: {exc}"} + + with rasterio.open(path) as src: + if src.height > 2048 or src.width > 2048: + scale = min(1024 / src.width, 1024 / src.height) + out_width = max(1, int(src.width * scale)) + out_height = max(1, int(src.height * scale)) + sampled = src.read(1, out_shape=(out_height, out_width), masked=True) + else: + sampled = src.read(1, masked=True) + + valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)] + bounds = src.bounds + quality: dict[str, Any] = { + "ok": True, + "driver": src.driver, + "width": src.width, + "height": src.height, + "count": src.count, + "dtype": str(src.dtypes[0]) if src.dtypes else None, + "crs": src.crs.to_string() if src.crs else None, + "bounds": { + "left": bounds.left, + "bottom": bounds.bottom, + "right": bounds.right, + "top": bounds.top, + }, + "transform": list(src.transform)[:6], + "nodata": _finite_float(src.nodata), + "valid_sample_count": int(valid.size), + "valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0, + } + if valid.size: + quality.update( + { + "sample_min": float(np.nanmin(valid)), + "sample_max": float(np.nanmax(valid)), + "sample_mean": float(np.nanmean(valid)), + "sample_p02": float(np.nanpercentile(valid, 2)), + "sample_p98": float(np.nanpercentile(valid, 98)), + } + ) + return quality + + +def _build_preview_png(source: Path, target: Path) -> str | None: + try: + import numpy as np + import rasterio + from PIL import Image + except Exception: + return None + + target.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open(source) as src: + if src.height > 1600 or src.width > 1600: + scale = min(1600 / src.width, 1600 / src.height) + out_width = max(1, int(src.width * scale)) + out_height = max(1, int(src.height * scale)) + band = src.read(1, out_shape=(out_height, out_width), masked=True) + else: + band = src.read(1, masked=True) + data = band.filled(float("nan")).astype("float32") + + valid = data[np.isfinite(data)] + if valid.size: + p2, p98 = np.nanpercentile(valid, [2, 98]) + normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1) + normalized = np.where(np.isfinite(normalized), normalized, 0) + gray = (normalized * 255).astype("uint8") + else: + gray = np.zeros(data.shape, dtype="uint8") + alpha = np.where(np.isfinite(data), 255, 0).astype("uint8") + rgba = np.stack([gray, gray, gray, alpha], axis=-1) + Image.fromarray(rgba, "RGBA").save(target) + return str(target) + + +def _build_preview_from_quicklook(source: Path | None, target: Path) -> str | None: + if source is None or not source.is_file(): + return None + try: + from PIL import Image + except Exception: + return None + + target.parent.mkdir(parents=True, exist_ok=True) + try: + with Image.open(source) as img: + preview = img.copy() + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + preview.thumbnail((1600, 1600), resampling) + if preview.mode in {"1", "I", "I;16", "F"}: + preview = preview.convert("L") + elif preview.mode not in {"L", "LA", "RGB", "RGBA"}: + preview = preview.convert("RGB") + preview = image_service.make_edge_dark_transparent(preview) + preview.save(target, "PNG") + return str(target) + except Exception: + return None + + +def _asset_quicklook_path(asset: dict[str, Any]) -> Path | None: + text = _path_text(asset.get("quicklook")) + if not text: + return None + return Path(text) + + +def _points_look_like_lonlat(points: list[tuple[float, float]]) -> bool: + if not points: + return False + for lon, lat in points: + if not (math.isfinite(float(lon)) and math.isfinite(float(lat))): + return False + if not (-180.0 <= float(lon) <= 180.0 and -90.0 <= float(lat) <= 90.0): + return False + return True + + +def _crs_is_geographic_lonlat(crs: Any) -> bool: + if not crs: + return False + try: + if crs.to_epsg() == 4326: + return True + except Exception: + pass + try: + if bool(crs.is_geographic): + return True + except Exception: + pass + try: + wkt = str(crs.to_wkt() or "").upper() + if "GEOGCS" in wkt and ("WGS 84" in wkt or "WORLD GEODETIC" in wkt): + return True + except Exception: + pass + return False + + +def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None: + polygon = extract_geotiff_bounds(str(path)) + if polygon and len(polygon) >= 4: + return polygon + try: + import rasterio + from rasterio.warp import transform + + with rasterio.open(path) as src: + if not src.crs: + return None + corners_xy = [ + src.transform * (0, 0), + src.transform * (src.width, 0), + src.transform * (src.width, src.height), + src.transform * (0, src.height), + ] + xs = [point[0] for point in corners_xy] + ys = [point[1] for point in corners_xy] + raw_points = [(float(x), float(y)) for x, y in zip(xs, ys)] + if _crs_is_geographic_lonlat(src.crs): + points = raw_points + else: + try: + lons, lats = transform(src.crs, "EPSG:4326", xs, ys) + points = [(float(lon), float(lat)) for lon, lat in zip(lons, lats)] + except Exception: + if not _points_look_like_lonlat(raw_points): + raise + points = raw_points + if not _points_look_like_lonlat(points): + return None + points.append(points[0]) + return points + except Exception: + return None + return None + + +def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]: + if not polygon: + return None, None + try: + shp = Polygon(polygon) + if not shp.is_valid: + shp = shp.buffer(0) + if shp.is_valid and not shp.is_empty: + return float(shp.centroid.x), float(shp.centroid.y) + except Exception: + return None, None + return None, None + + +def _bounds_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None, float | None, float | None]: + if not polygon: + return None, None, None, None + lons = [float(point[0]) for point in polygon] + lats = [float(point[1]) for point in polygon] + return min(lons), min(lats), max(lons), max(lats) + + +def _geom_from_polygon(polygon: list[tuple[float, float]] | None) -> Any | None: + if not polygon: + return None + try: + shp = Polygon(polygon) + if not shp.is_valid: + shp = shp.buffer(0) + if shp.is_valid and not shp.is_empty: + return from_shape(shp, srid=4326) + except Exception: + return None + return None + + +def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None: + by_pol = {str(asset.get("polarization") or "").upper(): asset for asset in assets} + for pol in POLARIZATION_PRIORITY: + if pol in by_pol: + return by_pol[pol] + return assets[0] if assets else None + + +def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]: + metadata = dict(scene_manifest.get("metadata") or {}) + metadata.update( + { + "native_dir": scene_manifest.get("native_dir"), + "native_manifest": scene_manifest.get("manifest_path"), + "standard_manifest": standard_manifest.get("manifest_path"), + "standard_dir": standard_manifest.get("standard_dir"), + "standard_assets": standard_manifest.get("assets") or [], + "analysis_engine": "gf3_sarscape", + } + ) + return metadata + + +async def _upsert_source_product_asset( + db: AsyncSession, + scene_manifest: dict[str, Any], + standard_manifest: dict[str, Any], +) -> int | None: + standard_dir_text = _path_text(standard_manifest.get("standard_dir")) + if not standard_dir_text: + return None + standard_dir = Path(standard_dir_text) + metadata = scene_manifest.get("metadata") or {} + scene_name = scene_manifest.get("scene_name") or standard_dir.name + now = _db_now() + root = await _find_managed_root_for_path(db, standard_dir_text) + stats = await asyncio.to_thread(_tree_stats, standard_dir) + asset_metadata = _json_safe( + { + "source": "GF3 SARscape standardized L2", + "native_dir": scene_manifest.get("native_dir"), + "native_manifest": scene_manifest.get("manifest_path"), + "standard_manifest": standard_manifest.get("manifest_path"), + "standard_dir": standard_dir_text, + "standard_status": standard_manifest.get("status"), + "standard_assets": standard_manifest.get("assets") or [], + "summary": standard_manifest.get("summary") or {}, + "errors": standard_manifest.get("errors") or [], + "analysis_engine": "gf3_sarscape", + } + ) + data = { + "asset_uid": _source_asset_uid(standard_dir_text), + "logical_product_uid": scene_name, + "satellite_family": "GF3", + "satellite": "GF3", + "source_format": SOURCE_ASSET_FORMAT, + "product_type": metadata.get("product_type") or "SARSCAPE_L2", + "product_level": "L2", + "imaging_mode": metadata.get("imaging_mode"), + "polarization": metadata.get("polarization"), + "absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"), + "relative_orbit": metadata.get("relative_orbit"), + "orbit_direction": metadata.get("orbit_direction"), + "acquisition_start_time_utc": None, + "acquisition_stop_time_utc": None, + "imaging_date": metadata.get("imaging_date"), + "root_ref_id": root.id if root else None, + "root_path": root.path if root else str(standard_dir.parent), + "file_path": standard_dir_text, + "archive_path": scene_manifest.get("native_dir"), + "path_kind": _path_kind(standard_dir_text), + "file_name": standard_dir.name, + "file_stem": standard_dir.name, + "file_ext": "", + "size_bytes": stats.get("size_bytes"), + "mtime_epoch": stats.get("mtime_epoch"), + "checksum_status": "NOT_COMPUTED", + "parser_name": "gf3_sarscape_standard_manifest", + "parser_version": CONVERTER_VERSION, + "parse_status": "OK" if standard_manifest.get("status") == "DONE" else str(standard_manifest.get("status") or "PARTIAL"), + "parse_error": "; ".join(str(item.get("error") or item) for item in (standard_manifest.get("errors") or [])) or None, + "parsed_at": now, + "metadata_json": asset_metadata, + "is_active": True, + "missing_since": None, + "updated_at": now, + } + + result = await db.execute( + select(SourceProductAssetORM).where( + or_( + SourceProductAssetORM.asset_uid == data["asset_uid"], + SourceProductAssetORM.file_path == standard_dir_text, + ) + ) + ) + asset = result.scalars().first() + if asset is None: + asset = SourceProductAssetORM(**data) + db.add(asset) + else: + for key, value in data.items(): + setattr(asset, key, value) + await db.flush() + return int(asset.id) if asset.id is not None else None + + +async def _upsert_radar_data( + db: AsyncSession, + scene_manifest: dict[str, Any], + standard_manifest: dict[str, Any], + source_product_ref_id: int | None = None, +) -> int | None: + assets = standard_manifest.get("assets") or [] + default_asset = _select_default_asset(assets) + if not default_asset: + return None + + polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path")))) + min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon) + center_lon, center_lat = _scene_center_from_polygon(polygon) + geom = _geom_from_polygon(polygon) + metadata = scene_manifest.get("metadata") or {} + radar_metadata = _metadata_for_radar(scene_manifest, standard_manifest) + scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name + unique_id = f"gf3_sarscape:{scene_name}" + file_path = str(standard_manifest.get("standard_dir") or "") + + data_to_upsert = { + "unique_id": unique_id, + "satellite": "GF3", + "satellite_family": "GF3", + "imaging_date": metadata.get("imaging_date"), + "imaging_mode": metadata.get("imaging_mode"), + "polarization": ",".join( + pol + for pol in POLARIZATION_PRIORITY + if any(str(asset.get("polarization") or "").upper() == pol for asset in assets) + ) + or metadata.get("polarization"), + "scene_center_lon": metadata.get("scene_center_lon") if metadata.get("scene_center_lon") is not None else center_lon, + "scene_center_lat": metadata.get("scene_center_lat") if metadata.get("scene_center_lat") is not None else center_lat, + "product_level": "L2", + "product_unique_id": metadata.get("product_unique_id"), + "source_format": SOURCE_ASSET_FORMAT, + "source_product_ref_id": source_product_ref_id, + "image_data_format": "GEOTIFF", + "geocoded_flag": True, + "metadata_json": radar_metadata, + "file_path": file_path, + "has_orbit_data": False, + "orbit_file_path": None, + "is_envi_processed": True, + "coverage_polygon": polygon, + "geom": geom, + "min_lon": min_lon, + "min_lat": min_lat, + "max_lon": max_lon, + "max_lat": max_lat, + } + + result = await db.execute( + select(RadarDataORM).where( + or_( + RadarDataORM.unique_id == unique_id, + RadarDataORM.file_path == file_path, + ) + ) + ) + radar = result.scalars().first() + if radar is None: + radar = RadarDataORM(**data_to_upsert) + db.add(radar) + else: + for key, value in data_to_upsert.items(): + setattr(radar, key, value) + await db.flush() + return int(radar.id) if radar.id is not None else None + + +async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM: + result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id)) + scene = result.scalar_one_or_none() + if scene: + return scene + scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING") + db.add(scene) + await db.flush() + return scene + + +async def _register_analysis_ready( + db: AsyncSession, + radar_id: int, + scene_manifest: dict[str, Any], + standard_manifest: dict[str, Any], +) -> dict[str, Any] | None: + radar = await db.get(RadarDataORM, radar_id) + if not radar: + return None + assets = standard_manifest.get("assets") or [] + default_asset = _select_default_asset(assets) + if not default_asset: + return None + scene = await _get_or_create_scene(db, radar_id) + return await register_analysis_ready_tif( + db=db, + scene=scene, + radar=radar, + source_tif_path=str(default_asset.get("path") or ""), + engine="gf3_sarscape", + profile=CONVERTER_NAME, + backscatter_unit="unknown", + polarization=str(default_asset.get("polarization") or "").upper() or None, + preview_source_path=str(default_asset.get("preview") or "") or None, + metadata={ + "source": "GF3 SARscape native _geo", + "native_dir": scene_manifest.get("native_dir"), + "native_manifest": scene_manifest.get("manifest_path"), + "standard_manifest": standard_manifest.get("manifest_path"), + "available_polarization": [asset.get("polarization") for asset in assets], + "standard_assets": assets, + }, + ) + + +def standardize_scene_manifest( + scene_manifest: dict[str, Any], + *, + storage_root: str | Path | None = None, + force: bool = False, +) -> dict[str, Any]: + """Convert one native scene manifest to GeoTIFF assets.""" + root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve() + out_dir = _standard_scene_dir(scene_manifest, root) + manifest_path = out_dir / STANDARD_MANIFEST_NAME + existing_manifest = _read_json(manifest_path) + + converted = 0 + skipped = 0 + failed = 0 + output_assets: list[dict[str, Any]] = [] + errors: list[dict[str, str]] = [] + + for asset in scene_manifest.get("assets") or []: + if not asset.get("complete"): + continue + pol = str(asset.get("polarization") or "UNKNOWN").upper() + target_tif = _target_tif_path(scene_manifest, asset, root) + existing_asset = _existing_manifest_asset(existing_manifest, pol) + + try: + if force or _source_changed(asset, target_tif, existing_asset): + convert_info = _convert_native_asset_to_tif(asset, target_tif) + converted += 1 + status = "converted" + else: + convert_info = { + "path": str(target_tif), + "source_native": str(Path(_path_text(asset.get("path")))), + "source_fingerprint": _file_fingerprint(Path(_path_text(asset.get("path")))), + "converter_name": CONVERTER_NAME, + "converter_version": CONVERTER_VERSION, + } + skipped += 1 + status = "skipped" + + quality = _raster_quality(target_tif) + quality_file = _quality_path(scene_manifest, asset, root) + _write_json(quality_file, quality) + preview_target = _preview_path(scene_manifest, asset, root) + quicklook_path = _asset_quicklook_path(asset) + preview = _build_preview_from_quicklook(quicklook_path, preview_target) + preview_source = str(quicklook_path) if preview and quicklook_path else str(target_tif) + if not preview: + preview = _build_preview_png(target_tif, preview_target) + output_assets.append( + { + "polarization": pol, + "role": "analysis_tif", + "path": str(target_tif), + "source_native": convert_info["source_native"], + "source_fingerprint": convert_info["source_fingerprint"], + "converter_name": CONVERTER_NAME, + "converter_version": CONVERTER_VERSION, + "quality": str(quality_file), + "preview": preview, + "preview_source": preview_source, + "status": status, + } + ) + except Exception as exc: + failed += 1 + errors.append({"polarization": pol, "source_native": str(asset.get("path") or ""), "error": str(exc)}) + + status = "DONE" if output_assets and failed == 0 else ("PARTIAL" if output_assets else "FAILED") + standard_manifest = { + "schema": STANDARD_MANIFEST_SCHEMA, + "generated_at": _utc_now(), + "scene_name": scene_manifest.get("scene_name"), + "batch_name": scene_manifest.get("batch_name"), + "native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME), + "native_dir": scene_manifest.get("native_dir"), + "standard_dir": str(out_dir), + "manifest_path": str(manifest_path), + "status": status, + "converter": {"name": CONVERTER_NAME, "version": CONVERTER_VERSION}, + "assets": output_assets, + "summary": { + "converted": converted, + "skipped": skipped, + "failed": failed, + }, + "errors": errors, + } + _write_json(manifest_path, standard_manifest) + return standard_manifest + + +async def standardize_gf3_sarscape_native_roots( + db: AsyncSession, + *, + native_dirs: list[str] | None = None, + storage_root: str | None = None, + force: bool = False, + register: bool = True, + progress_callback: Any | None = None, +) -> dict[str, Any]: + """Scan native roots, convert complete assets, and register standard scenes.""" + inventory = await asyncio.to_thread( + scan_gf3_sarscape_native_roots, + native_dirs, + write_manifest=True, + ) + scenes = inventory.get("scenes") or [] + ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}] + + converted_scenes = 0 + partial_scenes = 0 + failed_scenes = 0 + skipped_assets = 0 + converted_assets = 0 + failed_assets = 0 + registered = 0 + analysis_ready = 0 + scene_results: list[dict[str, Any]] = [] + + total = len(ready_scenes) + for idx, scene_manifest in enumerate(ready_scenes): + if progress_callback: + pct = 10 + int((idx / max(total, 1)) * 80) + progress_callback(pct, f"标准化 GF3 SARscape 原生结果 {idx + 1}/{total}: {scene_manifest.get('scene_name')}") + + standard_manifest = await asyncio.to_thread( + standardize_scene_manifest, + scene_manifest, + storage_root=storage_root, + force=force, + ) + summary = standard_manifest.get("summary") or {} + converted_assets += int(summary.get("converted") or 0) + skipped_assets += int(summary.get("skipped") or 0) + failed_assets += int(summary.get("failed") or 0) + status = standard_manifest.get("status") + if status == "DONE": + converted_scenes += 1 + elif status == "PARTIAL": + partial_scenes += 1 + else: + failed_scenes += 1 + + radar_id = None + source_asset_id = None + analysis_manifest_path = None + if register and status in {"DONE", "PARTIAL"}: + source_asset_id = await _upsert_source_product_asset(db, scene_manifest, standard_manifest) + radar_id = await _upsert_radar_data( + db, + scene_manifest, + standard_manifest, + source_product_ref_id=source_asset_id, + ) + if radar_id: + registered += 1 + analysis_manifest = await _register_analysis_ready(db, radar_id, scene_manifest, standard_manifest) + if analysis_manifest: + analysis_ready += 1 + analysis_manifest_path = analysis_manifest.get("analysis_dir") + await db.commit() + + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "native_status": scene_manifest.get("status"), + "standard_status": status, + "standard_manifest": standard_manifest.get("manifest_path"), + "source_asset_id": source_asset_id, + "radar_id": radar_id, + "analysis_manifest_path": analysis_manifest_path, + "summary": summary, + "errors": standard_manifest.get("errors") or [], + } + ) + + return { + "ok": failed_scenes == 0 and failed_assets == 0, + "inventory": { + key: value + for key, value in inventory.items() + if key != "scenes" + }, + "scene_count": len(scenes), + "ready_scene_count": len(ready_scenes), + "converted_scenes": converted_scenes, + "partial_scenes": partial_scenes, + "failed_scenes": failed_scenes, + "converted_assets": converted_assets, + "skipped_assets": skipped_assets, + "failed_assets": failed_assets, + "registered": registered, + "analysis_ready": analysis_ready, + "scenes": scene_results, + } diff --git a/backend/app/services/health_service.py b/backend/app/services/health_service.py index c8455ec..de0d5e0 100644 --- a/backend/app/services/health_service.py +++ b/backend/app/services/health_service.py @@ -879,6 +879,36 @@ def _probe_directory_status(path: str) -> Dict[str, Any]: return payload +def _probe_file_status(path: str) -> Dict[str, Any]: + normalized = str(path or "").strip() + payload = { + "path": normalized, + "exists": False, + "accessible": False, + "error": None, + } + if not normalized: + payload["error"] = "empty path" + return payload + + try: + if os.path.isfile(normalized): + payload["exists"] = True + try: + with open(normalized, "rb") as stream: + stream.read(1) + payload["accessible"] = True + except Exception as exc: + payload["error"] = str(exc) + return payload + + os.stat(normalized) + payload["error"] = "path exists but is not a file" + except Exception as exc: + payload["error"] = str(exc) + return payload + + async def _check_source_roots() -> Dict[str, Any]: items = [] @@ -908,11 +938,34 @@ async def _check_source_roots() -> Dict[str, Any]: status["role"] = "gf3_l1a_source" items.append(status) + for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS): + status = _probe_directory_status(path) + status["role"] = "gf3_sarscape_native" + items.append(status) + for path in split_env_paths(settings.GF3_STORAGE_DIRS): status = _probe_directory_status(path) status["role"] = "gf3_l2_storage" items.append(status) + wrapper_exe = str(settings.GF3_SARSCAPE_WRAPPER_EXE or "").strip() + if wrapper_exe: + status = _probe_file_status(wrapper_exe) + status["role"] = "gf3_sarscape_wrapper" + items.append(status) + + idlrt_path = str(settings.GF3_SARSCAPE_IDLRT_PATH or "").strip() + if idlrt_path: + status = _probe_file_status(idlrt_path) + status["role"] = "gf3_sarscape_idlrt" + items.append(status) + + dem_path = str(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or "").strip() + if dem_path: + status = _probe_file_status(dem_path) + status["role"] = "gf3_sarscape_dem" + items.append(status) + configured_count = len(items) accessible_count = sum(1 for item in items if item.get("accessible")) inaccessible_count = configured_count - accessible_count diff --git a/backend/app/services/image_service.py b/backend/app/services/image_service.py index f5619c1..2e697c6 100644 --- a/backend/app/services/image_service.py +++ b/backend/app/services/image_service.py @@ -15,6 +15,7 @@ import os import time import json +from collections import deque from typing import Tuple, Optional, Dict, Any, List from PIL import Image import rasterio @@ -334,6 +335,55 @@ class ImageService: """ os.makedirs(os.path.dirname(output_path), exist_ok=True) image.save(output_path, format='WEBP', quality=quality) + + @staticmethod + def make_edge_dark_transparent( + image: Image.Image, + *, + threshold: int = 6, + ) -> Image.Image: + """Make edge-connected near-black preview background transparent.""" + rgba = image.convert("RGBA") + arr = np.array(rgba, dtype=np.uint8, copy=True) + if arr.ndim != 3 or arr.shape[2] < 4: + return rgba + + alpha = arr[:, :, 3] + dark = (alpha > 0) & (arr[:, :, :3].max(axis=2) <= int(threshold)) + if not dark.any(): + return rgba + + h, w = dark.shape + edge = np.zeros_like(dark, dtype=bool) + edge[0, :] = dark[0, :] + edge[h - 1, :] = dark[h - 1, :] + edge[:, 0] |= dark[:, 0] + edge[:, w - 1] |= dark[:, w - 1] + if not edge.any(): + return rgba + + visited = np.zeros_like(dark, dtype=bool) + ys, xs = np.where(edge) + queue = deque(zip(ys.tolist(), xs.tolist())) + visited[ys, xs] = True + + while queue: + y, x = queue.popleft() + if y > 0 and dark[y - 1, x] and not visited[y - 1, x]: + visited[y - 1, x] = True + queue.append((y - 1, x)) + if y + 1 < h and dark[y + 1, x] and not visited[y + 1, x]: + visited[y + 1, x] = True + queue.append((y + 1, x)) + if x > 0 and dark[y, x - 1] and not visited[y, x - 1]: + visited[y, x - 1] = True + queue.append((y, x - 1)) + if x + 1 < w and dark[y, x + 1] and not visited[y, x + 1]: + visited[y, x + 1] = True + queue.append((y, x + 1)) + + arr[:, :, 3][visited] = 0 + return Image.fromarray(arr, "RGBA") @staticmethod def create_cached_image( @@ -497,12 +547,12 @@ class ImageService: @staticmethod def _warp_preview_to_geo_bbox( - source_rgb: np.ndarray, + source_rgba: np.ndarray, inverse_h: np.ndarray, bbox: Tuple[float, float, float, float], out_size: Tuple[int, int], ) -> np.ndarray: - src_h, src_w = source_rgb.shape[:2] + src_h, src_w = source_rgba.shape[:2] out_w, out_h = out_size min_lon, min_lat, max_lon, max_lat = bbox lon_span = max_lon - min_lon @@ -557,7 +607,7 @@ class ImageService: du = (u_valid - x0).astype(np.float32) dv = (v_valid - y0).astype(np.float32) - src_float = source_rgb.astype(np.float32, copy=False) + src_float = source_rgba.astype(np.float32, copy=False) s00 = src_float[y0, x0] s10 = src_float[y0, x1] s01 = src_float[y1, x0] @@ -568,15 +618,14 @@ class ImageService: + s01 * (1 - du)[:, None] * dv[:, None] + s11 * du[:, None] * dv[:, None] ) - rgb = np.clip(samples, 0, 255).astype(np.uint8) + rgba = np.clip(samples, 0, 255).astype(np.uint8) else: nearest_x = np.clip(np.round(u_valid).astype(np.int32), 0, src_w - 1) nearest_y = np.clip(np.round(v_valid).astype(np.int32), 0, src_h - 1) - rgb = source_rgb[nearest_y, nearest_x] + rgba = source_rgba[nearest_y, nearest_x] flat = output.reshape(-1, 4) - flat[valid_idx, :3] = rgb - flat[valid_idx, 3] = 255 + flat[valid_idx] = rgba return output @staticmethod @@ -608,9 +657,10 @@ class ImageService: return False, "invalid_bbox" with Image.open(source_image_path) as image: - source_rgb = np.asarray(image.convert("RGB"), dtype=np.uint8) + source = ImageService.make_edge_dark_transparent(image) + source_rgba = np.asarray(source, dtype=np.uint8) - src_h, src_w = source_rgb.shape[:2] + src_h, src_w = source_rgba.shape[:2] if src_h < 1 or src_w < 1: return False, "invalid_source_image_size" @@ -633,7 +683,7 @@ class ImageService: return False, "homography_invert_failed" warped_rgba = ImageService._warp_preview_to_geo_bbox( - source_rgb=source_rgb, + source_rgba=source_rgba, inverse_h=inverse_h, bbox=bbox, out_size=out_size, @@ -672,7 +722,7 @@ class ImageService: ) with Image.open(source_image_path) as image: - image = image.convert("RGB") + image = ImageService.make_edge_dark_transparent(image) image.thumbnail(max_size, Image.Resampling.LANCZOS) ImageService.save_image_as_webp(image, cache_path, quality=82) return True diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 6678c0b..900e71c 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -86,6 +86,9 @@ JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION" JOB_TYPE_GF3_PROCESS = "GF3_PROCESS" JOB_TYPE_GF3_UNPACK = "GF3_UNPACK" JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS" +JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE" +JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC" +JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN" JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" JOB_TYPE_PYINT_RUN = "PYINT_RUN" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" @@ -3455,7 +3458,8 @@ async def _handle_water_detect(job: SystemJobORM) -> None: raise ValueError("水体检测缺少输入路径 input_path") output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}" - output_dir = os.path.join(os.path.dirname(input_path), output_name) + output_root = settings.WATER_RESULTS_DIR or os.path.join(settings.BACKEND_DIR, "water_results") + output_dir = os.path.join(output_root, output_name) os.makedirs(output_dir, exist_ok=True) await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...") @@ -3496,6 +3500,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None: det.threshold_value = result.get("threshold_value") det.metadata_json = { "legacy_otsu_threshold_db": result.get("otsu_threshold_db"), + "value_transform": result.get("value_transform"), "job_id": job.job_id, } else: @@ -3518,6 +3523,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None: mirror.task_id = job.task_id mirror.metadata_json = { "legacy_otsu_threshold_db": result.get("otsu_threshold_db"), + "value_transform": result.get("value_transform"), "legacy_detection_id": int(record_id), "job_id": job.job_id, } @@ -3781,6 +3787,330 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None: ) +async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None: + """Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them.""" + from .gf3_standardize_service import standardize_gf3_sarscape_native_roots + + if not job.task_id: + raise ValueError("GF3_SARSCAPE_SYNC requires task_id for progress tracking.") + + payload = job.payload or {} + native_dirs = payload.get("native_dirs") or [] + storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS + if not native_dirs: + raise ValueError("GF3_SARSCAPE_SYNC: native_dirs is empty") + if not storage_root: + raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty") + + await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...") + + loop = asyncio.get_running_loop() + + def _progress_cb(progress: int, message: str) -> None: + try: + future = asyncio.run_coroutine_threadsafe( + task_service.update_task(job.task_id, progress=progress, message=message), + loop, + ) + def _swallow_progress_error(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 SARscape] progress callback failed: %s", exc) + + future.add_done_callback(_swallow_progress_error) + except RuntimeError: + return + + try: + async with AsyncSessionLocal() as db: + result = await standardize_gf3_sarscape_native_roots( + db, + native_dirs=native_dirs, + storage_root=storage_root, + force=bool(payload.get("force", False)), + register=bool(payload.get("register", True)), + progress_callback=_progress_cb, + ) + except Exception as exc: + await task_service.update_task( + job.task_id, + status="FAILED", + progress=100, + message=f"GF3 SARscape 标准化失败: {exc}", + ) + raise + + message = ( + "GF3 SARscape 标准化完成: " + f"发现 {int(result.get('scene_count') or 0)} 景, " + f"可转换 {int(result.get('ready_scene_count') or 0)} 景, " + f"转换 {int(result.get('converted_scenes') or 0)} 景, " + f"部分 {int(result.get('partial_scenes') or 0)} 景, " + f"失败 {int(result.get('failed_scenes') or 0)} 景, " + f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, " + f"跳过 {int(result.get('skipped_assets') or 0)} 个, " + f"入库 {int(result.get('registered') or 0)} 景" + ) + await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message) + + +async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None: + """Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain.""" + from .gf3_sarscape_production_service import ( + cleanup_gf3_sarscape_native_pool, + run_gf3_sarscape_production, + ) + from .gf3_standardize_service import standardize_gf3_sarscape_native_roots + + if not job.task_id: + raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.") + + payload = job.payload or {} + source_dirs = payload.get("source_dirs") or [] + native_dirs = payload.get("native_dirs") or [] + storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS + native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "") + if not source_dirs: + raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty") + if not native_root: + raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty") + if not storage_root: + raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty") + + await task_service.start_task(job.task_id, message="GF3 SARscape production starting...") + loop = asyncio.get_running_loop() + + def _progress_cb(progress: int, message: str) -> None: + try: + future = asyncio.run_coroutine_threadsafe( + task_service.update_task(job.task_id, progress=progress, message=message), + loop, + ) + + def _swallow_progress_error(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc) + + future.add_done_callback(_swallow_progress_error) + except RuntimeError: + return + + def _log_cb(level: str, message: str) -> None: + try: + future = asyncio.run_coroutine_threadsafe( + task_service.add_log(job.task_id, level, message), + loop, + ) + + def _swallow_log_error(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc) + + future.add_done_callback(_swallow_log_error) + except RuntimeError: + return + + async def _production_keepalive() -> None: + progress = 8 + while True: + await asyncio.sleep(60) + progress = min(68, progress + 1) + await task_service.update_task( + job.task_id, + progress=progress, + message="GF3 SARscape production is still running...", + ) + + try: + production_task = asyncio.create_task( + asyncio.to_thread( + run_gf3_sarscape_production, + source_dirs=source_dirs, + native_root=native_root, + wrapper_exe=payload.get("wrapper_exe"), + dem_path=payload.get("dem_path"), + idlrt_path=payload.get("idlrt_path"), + polarizations=payload.get("polarizations"), + archive_exts=payload.get("archive_exts") or [], + max_archives_per_run=payload.get("max_archives_per_run"), + timeout_seconds=payload.get("timeout_seconds"), + keep_extracted=payload.get("keep_extracted"), + log_callback=_log_cb, + progress_callback=_progress_cb, + ) + ) + keepalive_task = asyncio.create_task(_production_keepalive()) + try: + production_result = await production_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + standardize_result: Dict[str, Any] = {} + if bool(payload.get("auto_standardize", True)): + await task_service.update_task( + job.task_id, + progress=72, + message="GF3 SARscape production finished; standardizing native _geo outputs...", + ) + async with AsyncSessionLocal() as db: + standardize_result = await standardize_gf3_sarscape_native_roots( + db, + native_dirs=native_dirs or [native_root], + storage_root=storage_root, + force=bool(payload.get("force_standardize", False)), + register=bool(payload.get("register", True)), + progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg), + ) + + cleanup_result: Dict[str, Any] = {} + production_ok = int(production_result.get("failed_count") or 0) == 0 + standardize_ok = ( + not standardize_result + or ( + int(standardize_result.get("failed_assets") or 0) == 0 + and int(standardize_result.get("failed_scenes") or 0) == 0 + ) + ) + if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok: + await task_service.update_task( + job.task_id, + progress=90, + message="Cleaning GF3 SARscape intermediate files...", + ) + cleanup_result = await asyncio.to_thread( + cleanup_gf3_sarscape_native_pool, + native_dirs=native_dirs or [native_root], + storage_root=storage_root, + require_standardized=bool(payload.get("cleanup_require_standardized", True)), + dry_run=bool(payload.get("cleanup_dry_run", False)), + max_scenes=payload.get("cleanup_max_scenes"), + log_callback=_log_cb, + progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg), + ) + elif bool(payload.get("clean_after_success", True)): + _log_cb( + "WARNING", + "GF3 SARscape automatic cleanup skipped because production or standardization had failures.", + ) + except Exception as exc: + await task_service.update_task( + job.task_id, + status="FAILED", + progress=100, + message=f"GF3 SARscape production failed: {exc}", + ) + raise + + failed_count = int(production_result.get("failed_count") or 0) + failed_assets = int(standardize_result.get("failed_assets") or 0) + cleanup_errors = int(cleanup_result.get("error_scene_count") or 0) + final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED" + message = ( + "GF3 SARscape production chain finished: " + f"found={int(production_result.get('found_count') or 0)}, " + f"produced={int(production_result.get('processed_count') or 0)}, " + f"skipped={int(production_result.get('skipped_count') or 0)}, " + f"failed={failed_count}, " + f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, " + f"registered={int(standardize_result.get('registered') or 0)}, " + f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, " + f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}" + ) + await task_service.update_task(job.task_id, status=final_status, progress=100, message=message) + + +async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None: + """Clean GF3 SARscape intermediate files from native pool.""" + from .gf3_sarscape_production_service import cleanup_gf3_sarscape_native_pool + + if not job.task_id: + raise ValueError("GF3_SARSCAPE_CLEAN requires task_id for progress tracking.") + + payload = job.payload or {} + native_dirs = payload.get("native_dirs") or [] + storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS + if not native_dirs: + raise ValueError("GF3_SARSCAPE_CLEAN: native_dirs is empty") + + await task_service.start_task(job.task_id, message="GF3 SARscape native cleanup starting...") + loop = asyncio.get_running_loop() + + def _progress_cb(progress: int, message: str) -> None: + try: + future = asyncio.run_coroutine_threadsafe( + task_service.update_task(job.task_id, progress=progress, message=message), + loop, + ) + + def _swallow_progress_error(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 SARscape Clean] progress callback failed: %s", exc) + + future.add_done_callback(_swallow_progress_error) + except RuntimeError: + return + + def _log_cb(level: str, message: str) -> None: + try: + future = asyncio.run_coroutine_threadsafe( + task_service.add_log(job.task_id, level, message), + loop, + ) + + def _swallow_log_error(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 SARscape Clean] log callback failed: %s", exc) + + future.add_done_callback(_swallow_log_error) + except RuntimeError: + return + + try: + result = await asyncio.to_thread( + cleanup_gf3_sarscape_native_pool, + native_dirs=native_dirs, + storage_root=storage_root, + require_standardized=bool(payload.get("require_standardized", True)), + dry_run=bool(payload.get("dry_run", False)), + max_scenes=payload.get("max_scenes"), + log_callback=_log_cb, + progress_callback=_progress_cb, + ) + except Exception as exc: + await task_service.update_task( + job.task_id, + status="FAILED", + progress=100, + message=f"GF3 SARscape native cleanup failed: {exc}", + ) + raise + + status = "FAILED" if int(result.get("error_scene_count") or 0) else "COMPLETED" + message = ( + "GF3 SARscape native cleanup finished: " + f"scenes={int(result.get('scene_count') or 0)}, " + f"cleaned={int(result.get('cleaned_scene_count') or 0)}, " + f"skipped={int(result.get('skipped_scene_count') or 0)}, " + f"errors={int(result.get('error_scene_count') or 0)}, " + f"bytes={int(result.get('bytes_deleted') or 0)}, " + f"dry_run={bool(result.get('dry_run'))}" + ) + await task_service.update_task(job.task_id, status=status, progress=100, message=message) + + async def _handle_publish_dinsar_products_clean(job: SystemJobORM) -> None: if not job.task_id: raise ValueError("PUBLISH_DINSAR_PRODUCTS requires task_id for progress tracking.") @@ -4671,6 +5001,9 @@ _HANDLERS = { JOB_TYPE_GF3_PROCESS: _handle_gf3_process, JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack, JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process, + JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce, + JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync, + JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean, JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration, JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem, JOB_TYPE_SBAS_INTERFEROGRAMS: _handle_sbas_interferograms, diff --git a/backend/app/services/root_registry_service.py b/backend/app/services/root_registry_service.py index 7d65d34..e2d9949 100644 --- a/backend/app/services/root_registry_service.py +++ b/backend/app/services/root_registry_service.py @@ -262,6 +262,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="scene_directory", ) ) + specs.extend( + _iter_multi_root_specs( + env_var="GF3_SARSCAPE_NATIVE_DIRS", + paths=split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), + root_role="source_pool_gf3_sarscape_native", + display_prefix="GF3 SARscape Native Pool", + scan_mode="scene_directory", + ) + ) specs.extend( _iter_single_root_specs( env_var="SAR_ANALYSIS_READY_ROOT", diff --git a/backend/app/services/sar_analysis_ready_service.py b/backend/app/services/sar_analysis_ready_service.py index 2889667..aff70dd 100644 --- a/backend/app/services/sar_analysis_ready_service.py +++ b/backend/app/services/sar_analysis_ready_service.py @@ -20,6 +20,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..config import settings from ..models import RadarDataORM, SARSceneGeoORM from ..utils import normalize_satellite_family +from .image_service import image_service _SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+") _POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH") @@ -82,7 +83,25 @@ def scene_analysis_dir( def _write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as stream: - json.dump(payload, stream, ensure_ascii=False, indent=2, default=str) + json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False) + + +def _json_safe(value: Any) -> Any: + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(item) for item in value] + return value + + +def _finite_float(value: Any) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None def _link_or_copy(source: Path, target: Path) -> str: @@ -171,7 +190,7 @@ def _raster_quality(path: Path) -> dict[str, Any]: "top": bounds.top, }, "transform": list(transform)[:6], - "nodata": src.nodata, + "nodata": _finite_float(src.nodata), "valid_sample_count": int(valid.size), "valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0, } @@ -231,6 +250,7 @@ def _build_preview_png(source: Path, target: Path) -> str | None: if valid.size: p2, p98 = np.nanpercentile(valid, [2, 98]) normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1) + normalized = np.where(np.isfinite(normalized), normalized, 0) gray = (normalized * 255).astype("uint8") else: gray = np.zeros(data.shape, dtype="uint8") @@ -240,6 +260,31 @@ def _build_preview_png(source: Path, target: Path) -> str | None: return str(target) +def _build_preview_from_existing(source: Path | None, target: Path) -> str | None: + if source is None or not source.is_file(): + return None + try: + from PIL import Image + except Exception: + return None + + target.parent.mkdir(parents=True, exist_ok=True) + try: + with Image.open(source) as img: + preview = img.copy() + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + preview.thumbnail((1600, 1600), resampling) + if preview.mode in {"1", "I", "I;16", "F"}: + preview = preview.convert("L") + elif preview.mode not in {"L", "LA", "RGB", "RGBA"}: + preview = preview.convert("RGB") + preview = image_service.make_edge_dark_transparent(preview) + preview.save(target, "PNG") + return str(target) + except Exception: + return None + + async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM: result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id)) scene = result.scalar_one_or_none() @@ -262,6 +307,7 @@ async def register_analysis_ready_tif( backscatter_unit: str, polarization: str | None = None, metadata: dict[str, Any] | None = None, + preview_source_path: str | None = None, copy_mode: str = "link_or_copy", ) -> dict[str, Any]: source = Path(os.path.normpath(str(source_tif_path or "").strip())) @@ -283,7 +329,10 @@ async def register_analysis_ready_tif( transfer = _link_or_copy(source, target_tif) quality = _raster_quality(target_tif) - preview_path = _build_preview_png(target_tif, out_dir / "preview.png") + preview_source = Path(os.path.normpath(preview_source_path)) if preview_source_path else None + preview_path = _build_preview_from_existing(preview_source, out_dir / "preview.png") + if not preview_path: + preview_path = _build_preview_png(target_tif, out_dir / "preview.png") manifest = { "scene_id": scene.id, "radar_data_id": scene.radar_data_id, @@ -296,6 +345,7 @@ async def register_analysis_ready_tif( "backscatter_unit": backscatter_unit, "polarization": polarization, "transfer": transfer, + "preview_source_path": str(preview_source) if preview_path and preview_source else None, "metadata": metadata or {}, "quality": quality, } @@ -309,14 +359,9 @@ async def register_analysis_ready_tif( scene.analysis_engine = engine scene.analysis_profile = profile scene.analysis_backscatter_unit = backscatter_unit - nodata_value = quality.get("nodata") - scene.analysis_nodata_value = ( - float(nodata_value) - if nodata_value is not None - else float(settings.SAR_ANALYSIS_NODATA_VALUE) - ) - scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")} - scene.analysis_quality_json = quality + scene.analysis_nodata_value = _finite_float(quality.get("nodata")) or float(settings.SAR_ANALYSIS_NODATA_VALUE) + scene.analysis_metadata_json = _json_safe({**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")}) + scene.analysis_quality_json = _json_safe(quality) scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m scene.status = "DONE" scene.error_msg = None diff --git a/backend/app/services/water_detect_service.py b/backend/app/services/water_detect_service.py index 64a8d38..dc12894 100644 --- a/backend/app/services/water_detect_service.py +++ b/backend/app/services/water_detect_service.py @@ -10,6 +10,7 @@ import logging import math import os import struct +from pathlib import Path from typing import Any, Dict, Optional, Tuple import numpy as np @@ -21,6 +22,65 @@ logger = logging.getLogger(__name__) # SRTM HGT helpers # --------------------------------------------------------------------------- +def _otsu_threshold(values: np.ndarray, bins: int = 512) -> float: + """Compute Otsu's threshold without depending on scikit-image.""" + finite_values = values[np.isfinite(values)] + if finite_values.size == 0: + raise ValueError("No finite pixels available for Otsu threshold") + + min_value = float(np.min(finite_values)) + max_value = float(np.max(finite_values)) + if min_value == max_value: + return min_value + + counts, edges = np.histogram(finite_values, bins=bins, range=(min_value, max_value)) + centers = (edges[:-1] + edges[1:]) / 2.0 + total = float(counts.sum()) + if total <= 0: + return min_value + + weight_background = np.cumsum(counts).astype(np.float64) + weight_foreground = total - weight_background + cumulative_mean = np.cumsum(counts * centers) + total_mean = cumulative_mean[-1] + + valid = (weight_background > 0) & (weight_foreground > 0) + between = np.zeros_like(centers, dtype=np.float64) + mean_background = np.zeros_like(centers, dtype=np.float64) + mean_foreground = np.zeros_like(centers, dtype=np.float64) + mean_background[valid] = cumulative_mean[valid] / weight_background[valid] + mean_foreground[valid] = (total_mean - cumulative_mean[valid]) / weight_foreground[valid] + between[valid] = ( + weight_background[valid] + * weight_foreground[valid] + * (mean_background[valid] - mean_foreground[valid]) ** 2 + ) + return float(centers[int(np.argmax(between))]) + + +def _disk_structure(radius: int) -> np.ndarray: + y, x = np.ogrid[-radius: radius + 1, -radius: radius + 1] + return (x * x + y * y) <= radius * radius + + +def _prepare_detection_image(img: np.ndarray, valid: np.ndarray) -> tuple[np.ndarray, np.ndarray, str]: + """Normalize SAR values for water thresholding and keep the original mask.""" + working = img.astype(np.float32, copy=True) + working[~valid] = np.nan + valid_values = working[valid] + if valid_values.size == 0: + return working, valid, "raw" + + min_value = float(np.nanmin(valid_values)) + p99_value = float(np.nanpercentile(valid_values, 99)) + max_value = float(np.nanmax(valid_values)) + if min_value >= 0.0 and (p99_value > 1.0 or max_value > 5.0): + positive_valid = valid & (working > 0) + converted = np.full_like(working, np.nan, dtype=np.float32) + converted[positive_valid] = 10.0 * np.log10(np.maximum(working[positive_valid], 1e-12)) + return converted, positive_valid, "linear_to_db" + return working, valid, "raw" + def _read_hgt(filepath: str) -> np.ndarray: """Read a single SRTM .hgt file. Auto-detect SRTM1 (3601) vs SRTM3 (1201).""" file_size = os.path.getsize(filepath) @@ -96,6 +156,70 @@ def _load_srtm3_dem( return mosaic, dem_bounds +def _candidate_dem_paths(dem_path: str) -> list[str]: + text = str(dem_path or "").strip() + if not text: + return [] + path = Path(text) + candidates: list[Path] = [] + for suffix in (".vrt", ".wgs84.vrt", ".tif", ".tiff", ".img", ".wgs84"): + candidate = Path(text + suffix) + if candidate.exists() and candidate not in candidates: + candidates.append(candidate) + if path.is_file() and path not in candidates: + candidates.append(path) + if path.is_dir(): + for pattern in ("*.vrt", "*.tif", "*.tiff", "*.img"): + for candidate in path.glob(pattern): + if candidate not in candidates: + candidates.append(candidate) + return [str(candidate) for candidate in candidates] + + +def _load_raster_dem( + bounds: Tuple[float, float, float, float], + dem_path: str, + out_shape: tuple[int, int], +) -> Optional[np.ndarray]: + """Read a DEM raster subset and resample it to the SAR image grid size.""" + import rasterio + from rasterio.enums import Resampling + from rasterio.windows import from_bounds + + height, width = out_shape + for candidate in _candidate_dem_paths(dem_path): + try: + with rasterio.open(candidate) as src: + if src.crs and not src.crs.is_geographic: + continue + dem_bounds = src.bounds + min_lon, min_lat, max_lon, max_lat = bounds + if ( + max_lon <= dem_bounds.left + or min_lon >= dem_bounds.right + or max_lat <= dem_bounds.bottom + or min_lat >= dem_bounds.top + ): + continue + window = from_bounds(min_lon, min_lat, max_lon, max_lat, transform=src.transform) + data = src.read( + 1, + window=window, + out_shape=(height, width), + boundless=True, + fill_value=np.nan, + resampling=Resampling.bilinear, + ).astype(np.float32) + nodata = src.nodata + if nodata is not None and np.isfinite(nodata): + data[data == np.float32(nodata)] = np.nan + logger.info("[WaterDetect] DEM raster loaded: %s", candidate) + return data + except Exception as exc: + logger.warning("[WaterDetect] DEM raster candidate skipped: %s (%s)", candidate, exc) + return None + + # --------------------------------------------------------------------------- # Core detection # --------------------------------------------------------------------------- @@ -119,14 +243,12 @@ def run_water_detection( Returns dict with keys: ok, output_path, water_area_km2, water_pixel_count, otsu_threshold_db """ import rasterio + from scipy import ndimage from scipy.ndimage import median_filter, gaussian_filter, label, zoom - from skimage.filters import threshold_otsu - from skimage.morphology import disk, binary_dilation, binary_erosion - from skimage.measure import regionprops from ..config import settings - dem_dir = settings.SRTM_DEM_DIR + dem_path = settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or settings.SRTM_DEM_DIR os.makedirs(output_dir, exist_ok=True) logger.info("[WaterDetect] Reading input: %s", geo_tiff_path) @@ -136,6 +258,7 @@ def run_water_detection( img = src.read(1).astype(np.float32) transform = src.transform crs = src.crs + nodata = src.nodata height, width = img.shape pixel_size_x = transform.a # degrees per pixel (x) pixel_size_y = transform.e # degrees per pixel (y, negative) @@ -155,17 +278,23 @@ def run_water_detection( # Step 5: Valid mask valid = np.isfinite(img) & (img != 0) + if nodata is not None and np.isfinite(nodata): + valid &= img != np.float32(nodata) if np.count_nonzero(valid) < 100: return {"ok": False, "error": "Too few valid pixels in input image"} + detection_img, valid, value_transform = _prepare_detection_image(img, valid) + logger.info("[WaterDetect] Value transform: %s", value_transform) + # Step 6: Otsu threshold on valid pixels - valid_pixels = img[valid] - thresh = threshold_otsu(valid_pixels) + valid_pixels = detection_img[valid] + thresh = _otsu_threshold(valid_pixels) logger.info("[WaterDetect] Otsu threshold: %.4f", thresh) # Step 7-8: Median + Gaussian filtering - filtered = median_filter(img, size=3) + filtered_input = np.where(valid, detection_img, np.nanmedian(valid_pixels)).astype(np.float32) + filtered = median_filter(filtered_input, size=3) filtered = gaussian_filter(filtered, sigma=1.0) # Step 9: Initial water mask @@ -173,8 +302,11 @@ def run_water_detection( # Step 3-4: Load and resample DEM (if available) dem_applied = False - if dem_dir and os.path.isdir(dem_dir): - dem, dem_bounds = _load_srtm3_dem(bounds, dem_dir) + if dem_path and (os.path.isdir(dem_path) or os.path.isfile(dem_path)): + dem_resampled = _load_raster_dem(bounds, dem_path, (height, width)) + dem, dem_bounds = (None, None) + if dem_resampled is None and os.path.isdir(dem_path): + dem, dem_bounds = _load_srtm3_dem(bounds, dem_path) if dem is not None and dem_bounds is not None: # Resample DEM to image resolution zoom_y = height / dem.shape[0] @@ -183,6 +315,7 @@ def run_water_detection( # Clip to match image shape exactly dem_resampled = dem_resampled[:height, :width] + if dem_resampled is not None: # Step 10: DEM height constraint (0m <= DEM <= 1000m) dem_valid = np.isfinite(dem_resampled) height_mask = dem_valid & (dem_resampled >= 0) & (dem_resampled <= 1000) @@ -204,14 +337,14 @@ def run_water_detection( else: logger.warning("[WaterDetect] DEM not available, skipping DEM constraints") else: - logger.warning("[WaterDetect] SRTM_DEM_DIR not configured, skipping DEM constraints") + logger.warning("[WaterDetect] DEM path not configured, skipping DEM constraints") # Step 12: Morphological processing — disk(5) dilate→erode→dilate→erode - selem = disk(5) - water = binary_dilation(water, selem) - water = binary_erosion(water, selem) - water = binary_dilation(water, selem) - water = binary_erosion(water, selem) + selem = _disk_structure(5) + water = ndimage.binary_dilation(water, structure=selem) + water = ndimage.binary_erosion(water, structure=selem) + water = ndimage.binary_dilation(water, structure=selem) + water = ndimage.binary_erosion(water, structure=selem) # Step 13: Connected component filtering labeled, num_features = label(water) @@ -220,18 +353,22 @@ def run_water_detection( pixel_area_m2 = abs(pixel_size_x) * 111320 * abs(pixel_size_y) * 111320 min_pixels_by_area = max(1, int(3000 / max(pixel_area_m2, 1))) - props = regionprops(labeled) - areas = [p.area for p in props] - if areas: + areas = ndimage.sum( + np.ones_like(labeled, dtype=np.uint8), + labeled, + index=np.arange(1, num_features + 1), + ) + areas = np.asarray(areas, dtype=np.float64) + if areas.size: median_area = float(np.median(areas)) min_area = max(min_pixels_by_area, int(median_area)) else: min_area = min_pixels_by_area # Remove small components - for prop in props: - if prop.area < min_area: - water[labeled == prop.label] = False + small_labels = np.where(areas < min_area)[0] + 1 + if small_labels.size: + water[np.isin(labeled, small_labels)] = False logger.info("[WaterDetect] Connected component filter: min_area=%d pixels, kept %d/%d components", min_area, np.count_nonzero(np.unique(labeled[water])), num_features) @@ -265,4 +402,5 @@ def run_water_detection( "water_area_km2": round(water_area, 4), "water_pixel_count": water_pixel_count, "otsu_threshold_db": round(float(thresh), 4), + "value_transform": value_transform, } diff --git a/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md b/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md new file mode 100644 index 0000000..f377939 --- /dev/null +++ b/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md @@ -0,0 +1,511 @@ +# 洪涝灾害分析模块工程交接文档 + +日期：2026-06-02 +面向对象：算法工程师、后端工程师 +范围：新洪涝灾害分析模块 `/flood/*`，不包含旧兼容 `/water/*` 页面和接口。 + +## 1. 当前定位 + +系统现在把洪涝分析拆成两层： + +1. 工程层：负责数据入库、场景标准化、任务队列、状态更新、预览上图、套合分析、产品登记。 +2. 算法层：只负责从标准化 SAR GeoTIFF 生成水体/洪涝分类栅格，并返回面积、像元数、阈值、模型信息等元数据。 + +后续算法优化应尽量只替换或新增 processor，不要绕开现有 `SARSceneGeoORM`、`WaterExtractionORM`、`FloodDetectionORM` 和任务队列。 + +## 2. 关键代码入口 + +| 职责 | 文件 | +| --- | --- | +| 洪涝 API 路由 | `backend/app/routers/flood.py` | +| 洪涝业务编排 | `backend/app/services/flood_analysis_service.py` | +| 后台任务执行 | `backend/app/services/job_handlers.py` | +| analysis-ready GeoTIFF 注册 | `backend/app/services/sar_analysis_ready_service.py` | +| 当前水体提取算法 | `backend/app/services/water_detect_service.py` | +| 水体 processor 包装 | `backend/app/services/water_extraction_service.py` | +| 当前洪涝变化检测算法 | `backend/app/services/flood_detection_service.py` | +| 洪涝矢量化与套合 | `backend/app/services/flood_overlay_service.py` | +| 洪涝产品登记 | `backend/app/services/flood_product_service.py` | +| 前端工作台 | `frontend/src/FloodAnalysisWorkspace.jsx` | +| 前端 API | `frontend/src/api/flood.js` | + +旧 `/water/*` 路由仍在，但只作为历史兼容，不作为新算法接入目标。 + +## 3. 数据模型 + +### 3.1 SARSceneGeoORM + +表：`sar_scene_geo` + +这是算法输入场景表。每条记录对应一景已标准化的 SAR 分析影像。 + +关键字段： + +| 字段 | 含义 | +| --- | --- | +| `radar_data_id` | 关联源影像 `radar_data.id` | +| `analysis_tif_path` | 算法统一输入，单波段 GeoTIFF | +| `analysis_dir` | analysis-ready 目录 | +| `analysis_preview_path` | 场景预览 PNG | +| `analysis_engine` | 标准化引擎，如 `gf3_sarscape`、`gf3_gdal`、`lt_gamma` | +| `analysis_profile` | 标准化 profile | +| `analysis_backscatter_unit` | 后向散射单位，如 `sigma0_db`、`unknown` | +| `analysis_quality_json` | 栅格尺寸、范围、nodata、采样统计 | +| `pixel_size_m` | 近似像元大小 | +| `status` | `PENDING/RUNNING/DONE/FAILED` | + +当前 GF3 SARscape 链路会把原生 `_geo` ENVI 二进制转换为 `D:\GF3_L2_Image_Pool` 下的 GeoTIFF，并注册到这里。 + +### 3.2 WaterExtractionORM + +表：`water_extractions` + +用于单景水体提取。 + +关键字段： + +| 字段 | 含义 | +| --- | --- | +| `scene_id` | 输入场景 | +| `processor` | 算法名称，当前默认 `otsu` | +| `input_path` | 实际输入 GeoTIFF | +| `output_path` | 输出水体掩膜 GeoTIFF | +| `preview_path` | 预留，目前预览按需渲染 | +| `vector_path` | 预留，用于未来水体矢量 | +| `water_area_km2` | 水体面积 | +| `water_pixel_count` | 水体像元数 | +| `threshold_value` | 阈值或模型置信阈值 | +| `metadata_json` | 算法元数据 | +| `status/error_msg/task_id` | 任务状态 | + +### 3.3 FloodDetectionORM + +表：`flood_detections` + +用于灾前/灾后两景洪涝变化检测。 + +关键字段： + +| 字段 | 含义 | +| --- | --- | +| `pre_scene_id` | 灾前场景 | +| `post_scene_id` | 灾后场景 | +| `output_dir` | 输出目录，默认 `WATER_RESULTS_DIR/flood_{id}` | +| `classified_path` | 分类结果 GeoTIFF | +| `flood_area_km2` | 新增洪涝面积 | +| `stable_water_area_km2` | 稳定水体面积 | +| `status/error_msg` | 任务状态 | + +分类图当前约定： + +| 值 | 类别 | 前端颜色 | +| ---: | --- | --- | +| 0 | nodata | 透明 | +| 1 | stable_water | 蓝色 | +| 2 | flood | 红色 | +| 3 | high_backscatter | 橙色 | +| 4 | non_water | 灰色 | + +## 4. 现有业务流程 + +### 4.1 单景水体提取 + +流程： + +```text +前端选择 SARSceneGeo +-> POST /flood/water-extractions { scene_id } +-> 创建 WaterExtractionORM(PENDING) +-> 创建 SystemJob: WATER_DETECT +-> job_handlers._handle_water_detect +-> water_extraction_service.run_otsu_water_extraction +-> water_detect_service.run_water_detection +-> 写 water_mask.tif +-> 更新 WaterExtractionORM 为 DONE/FAILED +-> 前端 GET /flood/water-extractions/{id}/preview 上图 +``` + +当前输出目录： + +```text +WATER_RESULTS_DIR/ + water_extraction_{id}/ + water_mask.tif +``` + +当前算法状态： + +- Otsu 阈值； +- GF3 线性强度自动转 `10*log10`； +- 支持 COPDEM/SRTM 类 DEM 栅格约束； +- 中值滤波、高斯滤波、形态学、连通域过滤； +- 可作为 baseline，不适合作为最终高精度算法。 + +### 4.2 灾前/灾后洪涝检测 + +流程： + +```text +前端输入灾害日期 + 行政区 AOI +-> POST /flood/disaster-pairs/search +-> 后端按时间窗、AOI 覆盖率、重叠率推荐 pre/post 配对 +-> POST /flood/detections { pre_scene_id, post_scene_id, refine } +-> 创建 FloodDetectionORM(PENDING) +-> 创建 SystemJob: FLOOD_DETECTION +-> job_handlers._handle_flood_detection +-> flood_detection_service.run_geotiff_flood_detection +-> 写 classified.tif/flood_mask.tif/stable_water_mask.tif/metadata.json +-> 更新 FloodDetectionORM +-> 前端加载 pre/post/classified 图层 +``` + +当前输出目录： + +```text +WATER_RESULTS_DIR/ + flood_{id}/ + classified.tif + flood_mask.tif + stable_water_mask.tif + metadata.json +``` + +当前算法状态： + +- 灾前、灾后分别 Otsu； +- 灾后水体且灾前非水体判为 flood； +- 灾前灾后均水体判为 stable_water； +- 可选 `refine` 做简单形态学清理； +- 支持灾前重投影到灾后网格； +- 还未接入更强的 GF3 双极化分类、深度学习或弱监督模型。 + +### 4.3 套合分析 + +流程： + +```text +FloodDetection DONE +-> POST /flood/detections/{id}/overlay +-> classified.tif 中 value=2 的 flood 区域矢量化 +-> 与灾害点、DInSAR 产品、行政区 AOI 套合 +-> 写 flood_detection_{id}_overlay.geojson +-> 新增 FloodOverlayORM +``` + +输出： + +```text +WATER_RESULTS_DIR/ + flood_overlays/ + flood_detection_{id}_overlay.geojson +``` + +### 4.4 产品登记 + +流程： + +```text +FloodDetection DONE +-> POST /flood/detections/{id}/products +-> 创建 FloodProductORM +-> GET /flood/products 或 /flood/results 查询 +``` + +当前只是轻量登记，没有完整归档包导出。 + +## 5. analysis-ready 输入契约 + +算法工程师应以 `SARSceneGeoORM.analysis_tif_path` 为唯一标准输入。 + +输入约定： + +| 项 | 要求 | +| --- | --- | +| 格式 | 单波段 GeoTIFF | +| 坐标 | 有 CRS，推荐 EPSG:4326 或投影坐标 | +| transform | 必须正确 | +| nodata | 支持 `NaN` 或明确 nodata | +| 单位 | 可能是 dB，也可能是线性强度，需读 `analysis_backscatter_unit` 或自行稳健判断 | +| 文件大小 | GF3 单极化可达几千万像元 | + +现有 GF3 SARscape 标准化结果大致为： + +- 数据来自 ENVI/SARscape `_geo`； +- 转为 GeoTIFF 后注册； +- `analysis_backscatter_unit` 当前可能为 `unknown`； +- 实际数值可能是线性强度，需做 dB 转换。 + +## 6. 算法 processor 输出契约 + +### 6.1 水体提取 processor + +建议新增统一接口： + +```python +def run_xxx_water_extraction( + *, + input_path: str, + output_dir: str, + job_id: str | None = None, + options: dict | None = None, +) -> dict: + ... +``` + +返回： + +```python +{ + "ok": True, + "processor": "gf3_rf_v1", + "output_path": ".../water_mask.tif", + "water_area_km2": 123.45, + "water_pixel_count": 123456, + "threshold_value": 0.62, + "metadata": { + "model_version": "...", + "features": ["hh_db", "hv_db", "ratio", "slope"], + "confidence_path": ".../water_probability.tif" + } +} +``` + +最低要求： + +- `output_path` 是 GeoTIFF； +- 水体像元值为 `255` 或 `1`，背景为 `0`； +- CRS/transform 与输入一致； +- nodata 推荐为 `0`； +- 面积统计要与输出一致。 + +### 6.2 洪涝检测 processor + +建议接口： + +```python +def run_xxx_flood_detection( + *, + pre_tif_path: str, + post_tif_path: str, + output_dir: str, + job_id: str | None = None, + refine: bool = False, + options: dict | None = None, +) -> dict: + ... +``` + +返回： + +```python +{ + "ok": True, + "processor": "gf3_change_rf_v1", + "classified_path": ".../classified.tif", + "flood_mask_path": ".../flood_mask.tif", + "stable_water_mask_path": ".../stable_water_mask.tif", + "metadata_path": ".../metadata.json", + "flood_area_km2": 12.34, + "stable_water_area_km2": 56.78, + "flood_pixel_count": 12345, + "stable_water_pixel_count": 67890, + "metadata": { + "model_version": "...", + "pre_scene_reprojected": True + } +} +``` + +`classified.tif` 必须遵守第 3.3 节的分类值，否则前端预览和套合分析会失效。 + +## 7. 推荐算法路线 + +### 7.1 短期：GF3 快速分类器 + +目标：替换当前单阈值水体提取，减少误判。 + +建议 processor 名称： + +- `gf3_rf_v1` +- `gf3_lgbm_v1` + +输入： + +- 优先支持 GF3 HH/HV 双极化； +- 如果系统当前只注册单极化 `analysis_ready.tif`，工程侧需要补充“同一产品多极化查找”能力，或算法先支持单极化。 + +特征建议： + +- `HH_db` +- `HV_db` +- `HH-HV` +- `HH/HV ratio` +- 局部均值、方差、纹理； +- DEM 高程、坡度； +- 可选永久水体、河网、土地覆盖先验。 + +样本策略： + +- 第一版可用弱监督样本：永久水体为正样本，远离水系/坡度较大/高后向散射区域为负样本； +- 后续在系统内加入人工修正样本导出； +- 不建议用当前 Otsu 结果直接当唯一标签。 + +### 7.2 中期：GF3 深度学习推理 + +目标：面向洪涝产品的高质量识别。 + +可参考： + +- Sen2GF3Floods：GF3 洪水数据集和 PyTorch 代码； +- FCN/UNet++/DeepLabV3+/SegFormer； +- 支持 patch 推理和边缘重叠融合。 + +工程要求： + +- 模型权重必须版本化； +- processor 输出必须仍是标准 GeoTIFF； +- 推理可以 GPU 加速，但不能阻塞任务队列主进程； +- 大图必须 tile 化，避免一次性占满显存/内存。 + +### 7.3 保留 baseline + +当前 `otsu` 应保留为： + +- 快速预览； +- 无模型时兜底； +- 算法对比 baseline。 + +不建议作为最终默认高质量结果。 + +## 8. 工程侧下一步 + +### 8.1 后端 processor 注册 + +建议在 `water_extraction_service.py` 增加 processor 分发： + +```python +def run_water_extraction(processor: str, **kwargs): + if processor == "otsu": + return run_otsu_water_extraction(**kwargs) + if processor == "gf3_rf_v1": + return run_gf3_rf_water_extraction(**kwargs) + ... +``` + +`FloodWaterExtractionRequest` 需要增加： + +```python +processor: str = "otsu" +options: dict | None = None +``` + +然后 `submit_water_extraction` 把 processor/options 写入 `WaterExtractionORM` 和 job payload。 + +### 8.2 多极化场景组织 + +目前 `SARSceneGeoORM` 与 `RadarDataORM` 基本是一景一条。GF3 标准化链路可能存在 HH/HV 两个 GeoTIFF，但洪涝算法输入仍是单 `analysis_tif_path`。 + +如果算法需要 HH/HV，应补一个工程能力： + +- 在 `analysis_metadata_json` 中记录同产品全部极化 GeoTIFF； +- 或新增 `SARSceneBandORM`/`analysis_assets` 表； +- 或在 processor 内根据当前路径和命名规则寻找同目录同产品 HV/HH。 + +建议先采用 metadata 方案，改动最小。 + +### 8.3 水体结果矢量化 + +当前水体提取只按需返回 PNG 预览，没有持久化矢量。 + +建议新增： + +- `water_vector_path` GeoJSON； +- `confidence_path` 概率图； +- `preview_path` 持久 PNG； +- 后端接口支持水体结果矢量上图。 + +### 8.4 质量评估字段 + +建议 `metadata_json` 至少写入： + +- `processor` +- `model_version` +- `input_paths` +- `features` +- `threshold` +- `confidence_stats` +- `valid_pixel_count` +- `water_ratio` +- `runtime_seconds` +- `warnings` + +### 8.5 前端入口 + +当前前端有水体提取按钮，但没有 processor 选择。 + +建议增加： + +- 水体提取 processor 下拉框； +- `快速 Otsu / GF3 RF / 深度学习`； +- 结果行显示 processor 和模型版本； +- 可选显示置信度图层。 + +## 9. 算法开发边界 + +算法工程师只需要保证： + +1. 能读取输入 GeoTIFF； +2. 能输出符合契约的 GeoTIFF； +3. 返回标准 dict； +4. 大图处理不会把内存/显存打爆； +5. 错误抛出清晰异常或返回 `{"ok": False, "error": "..."}`。 + +算法工程师不需要处理： + +- 前端； +- 任务队列； +- 用户权限； +- 数据库事务； +- 资产扫描； +- 上图预览； +- 套合分析； +- 产品登记。 + +## 10. 当前风险和已知问题 + +1. 当前 Otsu 水体提取误判较多，只能作为 baseline。 +2. GF3 双极化没有形成正式算法输入契约。 +3. `analysis_backscatter_unit` 对 GF3 SARscape 输出仍可能是 `unknown`。 +4. 洪涝变化检测仍是双阈值差分，复杂地物下误判会明显。 +5. 水体和洪涝结果缺少质量评价和置信度图层。 +6. 产品登记还不是完整归档包。 +7. 旧 `/water/*` 和新 `/flood/*` 共存，后续需要逐步收敛到 `/flood/*`。 + +## 11. 建议交付里程碑 + +### M1：GF3 RF 水体 processor + +- 输入单景 GF3 HH/HV 或单极化； +- 输出 `water_mask.tif`； +- 写入模型元数据； +- 接入 `/flood/water-extractions`。 + +### M2：多极化输入契约 + +- 工程侧让 processor 能稳定拿到 HH/HV； +- 文档化极化路径和 metadata； +- 前端显示使用的极化。 + +### M3：洪涝变化 processor + +- 输入灾前/灾后； +- 输出标准 `classified.tif`； +- 与现有套合和产品链路兼容。 + +### M4：深度学习推理 + +- 支持 tile 推理； +- 支持模型版本； +- 输出概率图和二值图； +- 与 RF/baseline 可切换。 + diff --git a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md new file mode 100644 index 0000000..720912d --- /dev/null +++ b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md @@ -0,0 +1,476 @@ +# GF3 SARscape Native To GeoTIFF Design + +更新日期：2026-05-30 + +## 1. 结论 + +GF3 生产链路后续采用“生产解耦、系统标准化”的模型： + +```text +GF3 原始压缩包池 + -> 生产服务器使用 ENVI / IDL Runtime / SARscape 稳定生产 + -> 只保留 SARscape 最终 _geo 原生结果组 + -> 系统扫描原生结果池 + -> 后台转换为标准 GeoTIFF + -> 入库、预览、洪涝分析和后续业务只消费 GeoTIFF +``` + +系统不直接把 SARscape `.sml` 或无后缀二进制作为业务算法输入。`.sml`、`.hdr` 和无后缀主数据属于原生证据层；`GeoTIFF` 属于平台消费层。 + +## 2. 目录约定 + +推荐继续沿用现场已有目录语义，并新增一个 ENVI/SARscape 原生池。 + +```env +GF3_ARCHIVE_SOURCE_DIRS=D:\GF3_Image_Pool_Zip +GF3_SARSCAPE_NATIVE_DIRS=D:\GF3_L2_ENVI_Binary_Pool +GF3_STORAGE_DIRS=D:\GF3_L2_Image_Pool +SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready +``` + +目录职责： + +| 目录 | 职责 | 系统是否直接分析 | +| --- | --- | --- | +| `GF3_ARCHIVE_SOURCE_DIRS` | 原始 GF3 L1A `.tar.gz` 池 | 否 | +| `GF3_SARSCAPE_NATIVE_DIRS` | SARscape `_geo` 原生结果池 | 否 | +| `GF3_STORAGE_DIRS` | GF3 标准 GeoTIFF 池 | 是 | +| `SAR_ANALYSIS_READY_ROOT` | 洪涝/水体分析级统一输入 | 是 | + +生产服务器可以不部署完整管理系统。只要把完成后的 `_geo` 原生结果组放入 `GF3_SARSCAPE_NATIVE_DIRS`，管理系统就可以扫描、转换和入库。 + +## 3. 原生池结构 + +原生池以批次日期或人工批次号分组。单景目录名尽量保持 GF3 原始产品名。 + +```text +D:\GF3_L2_ENVI_Binary_Pool + 20260514 + GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478 + GF3_MH1_FSII_..._hh_geo + GF3_MH1_FSII_..._hh_geo.hdr + GF3_MH1_FSII_..._hh_geo.sml + GF3_MH1_FSII_..._hh_geo.ovr + GF3_MH1_FSII_..._hh_geo.aux.xml + GF3_MH1_FSII_..._hh_geo_ql.tif + GF3_MH1_FSII_..._hh_geo.kml + GF3_MH1_FSII_..._hv_geo + GF3_MH1_FSII_..._hv_geo.hdr + GF3_MH1_FSII_..._hv_geo.sml + GF3_MH1_FSII_..._hv_geo.ovr + GF3_MH1_FSII_..._hv_geo.aux.xml + GF3_MH1_FSII_..._hv_geo_ql.tif + GF3_MH1_FSII_..._hv_geo.kml + gf3_sarscape_cli.log +``` + +### 3.1 必保文件 + +每个极化至少保留： + +```text +*_geo +*_geo.hdr +*_geo.sml +``` + +建议同时保留： + +```text +*_geo.ovr +*_geo.aux.xml +*_geo_ql.tif +*_geo.kml +gf3_sarscape_cli.log +``` + +说明： + +- 无后缀 `*_geo` 是 SARscape 主数据。 +- `.hdr` 是 ENVI/GDAL 读取二进制数据的关键 sidecar。 +- `.sml` 是 SARscape 追溯和完成判定的关键 sidecar。 +- `*_geo_ql.tif` 只能作为快视或预览参考，不作为科学分析主输入。 + +### 3.2 可清理文件 + +生产结束并确认 `_geo` 结果完整后，可以清理： + +```text +.gf3_extract +temp +SLC 中间产物 +*_ml* +*_filt* +``` + +如果需要完整复现 SARscape 处理过程，应额外保留 `work` 中的参数 XML、trace 和日志；否则可将 `work` 作为可选审计资料归档。 + +## 4. 标准 GeoTIFF 池结构 + +系统从原生池转换后写入 `GF3_STORAGE_DIRS`。 + +```text +D:\GF3_L2_Image_Pool + 20260514 + GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478 + HH_L2.tif + HV_L2.tif + preview_HH.png + preview_HV.png + gf3_standard_manifest.json + quality_HH.json + quality_HV.json +``` + +平台后续只从该目录或 `SAR_ANALYSIS_READY_ROOT` 消费 GeoTIFF，不直接读取 SARscape 原生目录。 + +## 5. 扫描与转换流程 + +推荐把“扫描”和“转换”都放在后台任务中执行，避免普通扫描接口长时间阻塞。 + +```text +用户触发 GF3 扫描 + -> 扫描 GF3_SARSCAPE_NATIVE_DIRS + -> 识别 scene / polarization / _geo 完整性 + -> 对待转换项创建或执行 GF3_NATIVE_TO_TIF 任务 + -> 转换到 GF3_STORAGE_DIRS + -> 写 gf3_standard_manifest.json + -> 登记 radar_data / sar_scene_geo + -> 生成预览图和 quality.json +``` + +### 5.1 完整性判定 + +一个极化的原生 `_geo` 结果完整条件： + +```text +*_geo 存在且非空 +*_geo.hdr 存在且非空 +*_geo.sml 存在且非空 +``` + +如果存在 `.aux.xml`、`.ovr`、`*_geo_ql.tif`、`.kml`，登记为辅助资产。 + +一个 scene 的状态： + +| 状态 | 条件 | +| --- | --- | +| `DONE` | 请求极化全部具备完整 `_geo` 结果，且 GeoTIFF 转换成功 | +| `NATIVE_READY` | 原生 `_geo` 完整，但 GeoTIFF 尚未转换 | +| `PARTIAL` | 只完成部分极化 | +| `FAILED` | 原生结果不完整或转换失败 | + +### 5.2 增量跳过规则 + +转换任务应根据 manifest 判断是否需要重跑： + +```text +source path +source size +source mtime +converter version +target tif exists +``` + +当上述信息未变化时，跳过转换。 + +如果 `_geo` 原生文件被替换、修改时间变化、转换器版本变化或目标 tif 缺失，应重新转换。 + +## 6. 转换策略 + +优先使用 GDAL/rasterio 读取 ENVI header 或 SARscape sidecar。 + +输入优先级： + +```text +1. *_geo + *_geo.hdr +2. 可被 GDAL 识别的 *_geo.sml +3. 其他明确可读的 SARscape/ENVI sidecar +``` + +输出要求： + +```text +GeoTIFF +单极化单文件 +尽量保留地理参考、nodata、数据类型和投影 +默认输出 HH_L2.tif / HV_L2.tif +``` + +如果转换失败，不应把 quicklook tif 冒充为分析级 tif。应记录： + +```text +analysis_ready_status=FAILED +error_message=<转换错误> +source_native_status=NATIVE_READY +``` + +## 7. 数据库登记 + +### 7.1 `radar_data` + +每个 GF3 scene 至少登记一条 `radar_data`： + +```text +satellite=GF3 +satellite_family=GF3 +source_format=GF3_SARSCAPE_NATIVE +product_level=L2 +file_path= +metadata_json.native_dir= +metadata_json.standard_manifest= +``` + +应从 scene 名解析： + +```text +imaging_date +imaging_mode +polarization +scene_center_lon +scene_center_lat +product_unique_id +``` + +如果 GeoTIFF 可读，应同步： + +```text +min_lon / min_lat / max_lon / max_lat +coverage_polygon +geom +``` + +### 7.2 `sar_scene_geo` + +每个可分析 scene 记录： + +```text +analysis_engine=gf3_sarscape +analysis_profile=gf3_sarscape_geo_to_tif +analysis_tif_path= +analysis_dir= +analysis_preview_path= +analysis_backscatter_unit=sigma0_linear 或 unknown +analysis_metadata_json.native_dir=<原生目录> +analysis_metadata_json.native_assets=<原生资产列表> +analysis_quality_json= +status=DONE +``` + +如果需要同时保留 HH 和 HV 两个可分析产品，建议长期扩展为资产表或 scene-pol 级记录；短期可以选择默认极化写入 `sar_scene_geo.analysis_tif_path`，并在 metadata 中登记全部极化 tif。 + +## 8. Manifest 契约 + +### 8.1 原生 manifest + +`gf3_native_manifest.json` 写入原生 scene 目录或系统索引目录： + +```json +{ + "schema": "gf3_sarscape_native.v1", + "scene_name": "GF3_MH1_FSII_...", + "native_dir": "D:\\GF3_L2_ENVI_Binary_Pool\\20260514\\GF3_MH1_FSII_...", + "source_archive": "D:\\GF3_Image_Pool_Zip\\20260514\\GF3_MH1_FSII_....tar.gz", + "polarizations": ["HH", "HV"], + "status": "NATIVE_READY", + "assets": [ + { + "polarization": "HH", + "role": "geo_native", + "path": "..._hh_geo", + "hdr": "..._hh_geo.hdr", + "sml": "..._hh_geo.sml", + "quicklook": "..._hh_geo_ql.tif" + } + ], + "logs": ["gf3_sarscape_cli.log"] +} +``` + +### 8.2 标准 manifest + +`gf3_standard_manifest.json` 写入标准 GeoTIFF scene 目录： + +```json +{ + "schema": "gf3_standard_geotiff.v1", + "scene_name": "GF3_MH1_FSII_...", + "native_manifest": "D:\\GF3_L2_ENVI_Binary_Pool\\...\\gf3_native_manifest.json", + "standard_dir": "D:\\GF3_L2_Image_Pool\\20260514\\GF3_MH1_FSII_...", + "status": "DONE", + "converter": { + "name": "gf3_sarscape_geo_to_tif", + "version": "v1" + }, + "assets": [ + { + "polarization": "HH", + "role": "analysis_tif", + "path": "HH_L2.tif", + "source_native": "..._hh_geo", + "quality": "quality_HH.json", + "preview": "preview_HH.png" + } + ] +} +``` + +## 9. 前端与操作入口 + +短期不新增复杂页面，沿用“数据管理 / 归档预处理”里的 GF3 操作区： + +```text +GF3 解包 +GF3 SARscape 原生扫描 +GF3 原生转 GeoTIFF +扫描 GF3 标准结果 +``` + +也可以先合并为一个按钮： + +```text +扫描 GF3 +``` + +后台自动完成： + +```text +native scan -> convert missing tif -> register standard result +``` + +任务日志必须显示： + +```text +发现 scene 数 +NATIVE_READY 数 +转换成功数 +转换失败数 +跳过数 +失败原因 +``` + +## 10. 实施顺序 + +### Phase 1：设计与配置 + +- 新增本文档。 +- 新增 `.env.example` 中的 `GF3_SARSCAPE_NATIVE_DIRS`。 +- 保留 `GF3_STORAGE_DIRS` 作为标准 GeoTIFF 池。 + +### Phase 2：原生扫描 + +- 新增 `gf3_native_inventory_service.py`。 +- 扫描 `_geo` 原生结果组。 +- 生成 native manifest。 +- 不做转换、不入业务分析。 + +### Phase 3：GeoTIFF 标准化 + +- 新增 `gf3_standardize_service.py`。 +- 将 `_geo` 原生结果转换为 `HH_L2.tif` / `HV_L2.tif`。 +- 生成 preview 和 quality。 +- 写 standard manifest。 + +### Phase 4：入库与洪涝接入 + +- 登记 `radar_data`。 +- 登记或更新 `sar_scene_geo`。 +- `/flood/preprocess` 对 GF3 优先复用已标准化 GeoTIFF。 + +### Phase 5：清理策略 + +- 增加 native pool 检查报告。 +- 增加可选中间文件清理建议，但系统不主动删除生产机文件。 +- 后续如需自动清理，应只清理系统明确生成的临时文件。 + +## 11. 当前约束 + +- 不把 `*_geo_ql.tif` 当作分析级产品。 +- 不让洪涝、水体、地图预览直接依赖 `.sml`。 +- 不要求生产服务器部署管理系统。 +- 不在扫描请求同步执行长时间转换，应使用后台任务。 +- 不删除用户生产目录中的文件，除非后续新增明确的、受控的清理任务。 + +## 12. 与旧 GF3 GDAL 路线关系 + +现有 `gf3_service.py` 的 Python/GDAL L1A -> L2 路线可以保留为 fallback 或实验处理器： + +```text +gf3_gdal +``` + +新 SARscape 原生池路线作为正式现场路线： + +```text +gf3_sarscape +``` + +两条路线最终都必须收敛到： + +```text +GF3_STORAGE_DIRS / SAR_ANALYSIS_READY_ROOT 中的标准 GeoTIFF +``` + +因此后续业务模块只关心标准 GeoTIFF，不关心上游来自 SARscape、GDAL、GAMMA 或其他处理器。 + +## 13. 2026-05-30 首轮落地 + +首轮代码已按本文档的主路径实现最小闭环： + +- 新增 `GF3_SARSCAPE_NATIVE_DIRS` 配置，作为 SARscape/ENVI 原生 `_geo` 二进制池。 +- 新增 `gf3_native_inventory_service.py`，扫描 `*_geo + *_geo.hdr + *_geo.sml` 并写 `gf3_native_manifest.json`。 +- 新增 `gf3_standardize_service.py`，将完整原生结果转换到 `GF3_STORAGE_DIRS` 下的 `HH_L2.tif` / `HV_L2.tif`，并写 `gf3_standard_manifest.json`、`quality_*.json`、`preview_*.png`。 +- 新增后台任务 `GF3_SARSCAPE_SYNC` 和接口 `POST /api/monitor/gf3-sarscape-sync`。 +- 数据监控面板新增 `GF3 SARscape 入库` 按钮。 +- 转换成功后登记 `radar_data`，并通过 `sar_analysis_ready_service` 登记 `sar_scene_geo`，使洪涝/水体模块可以继续消费标准 GeoTIFF。 + +当前实现仍遵守约束： + +- 不把 `*_geo_ql.tif` 当作分析级输入。 +- 不要求生产服务器部署管理系统。 +- 转换优先使用 GDAL Python 绑定；当前环境没有 `osgeo` 时走 rasterio 兜底。 + +## 14. 2026-05-30 生产链路接入 + +在首轮“原生结果入库”基础上，系统进一步接入 GF3 SARscape wrapper： + +```text +GF3_ARCHIVE_SOURCE_DIRS + -> gf3wrapper.exe / IDL Runtime / SARscape + -> GF3_SARSCAPE_NATIVE_DIRS + -> GF3_SARSCAPE_SYNC 标准化 + -> GF3_STORAGE_DIRS + -> 雷达数据扫描、预览、洪涝/水体业务 +``` + +新增配置： + +```env +GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\.codex_tmp\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe +GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe +GF3_SARSCAPE_DEM_PATH=D:\DEM\GMTED2010.jp2 +GF3_SARSCAPE_POLARIZATIONS=HH,HV +GF3_SARSCAPE_KEEP_EXTRACTED=true +GF3_SARSCAPE_AUTO_STANDARDIZE=true +GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true +GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 +``` + +新增后台任务： + +| 任务 | 接口 | 用途 | +| --- | --- | --- | +| `GF3_SARSCAPE_PRODUCE` | `POST /api/monitor/gf3-sarscape-produce` | 从原始 `.tar.gz/.tgz` 触发 SARscape 生产，随后自动标准化、入库、清理 | +| `GF3_SARSCAPE_SYNC` | `POST /api/monitor/gf3-sarscape-sync` | 仅扫描已有 `_geo` 原生结果并转 GeoTIFF 入库 | +| `GF3_SARSCAPE_CLEAN` | `POST /api/monitor/gf3-sarscape-clean` | 手动清理原生池中间数据 | + +清理策略： + +- 只处理 `GF3_SARSCAPE_NATIVE_DIRS` 内的场景目录。 +- 默认要求 `GF3_STORAGE_DIRS///gf3_standard_manifest.json` 状态为 `DONE` 后才清理。 +- 保留最终原生 `_geo` 主数据、`.hdr/.sml/.ovr/.aux.xml`、`*_geo_ql.tif/.kml`、日志和 manifest。 +- 删除 `.gf3_extract`、`temp`、`work`，以及根目录中的 `*_slc*`、`*_ml*`、`*_filt*`、`.par/.trace/.working/.list` 等中间文件。 +- 每景写 `gf3_cleanup_manifest.json`，记录删除条目和释放字节数。 +- 不删除 `GF3_ARCHIVE_SOURCE_DIRS` 中的原始压缩包，也不删除 `GF3_STORAGE_DIRS` 中的标准 GeoTIFF。 + +这样 `D:\GF3_L2_ENVI_Binary_Pool` 只长期保存可追溯的最终 `_geo` 原生结果组，中间过程文件在标准化完成后自动释放空间。 diff --git a/docs/INDEX.md b/docs/INDEX.md index 8ec1707..d641739 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -1,6 +1,6 @@ # 文档索引 -最后更新：2026-05-28 +最后更新：2026-05-30 本页是当前有效文档入口。没有列在本页的历史设计、实验记录和过程文档不再作为当前系统事实依据。 @@ -51,9 +51,15 @@ - [FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md](FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md) 洪涝模块 GeoTIFF 化与 Gamma 前处理方向。 +- [GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md](GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md) + GF3 SARscape 原生 `_geo` 二进制池、GeoTIFF 标准化、入库和洪涝接入设计。 + - [FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md](FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md) 洪涝灾害分析工作台、产品包和矢量套合边界。 +- [FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md](FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md) + 洪涝/水体算法接入现状、processor 输出契约和工程交接路线。 + ## 安全 - [SECURITY_AUDIT_2026-03-12.md](SECURITY_AUDIT_2026-03-12.md) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index d0d833e..a587e2c 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1358,12 +1358,27 @@ function App() { } }, []); - const updateRadarPreviewVisibility = useCallback((item, shouldBeVisible) => { + const updateRadarPreviewVisibility = useCallback(async (item, shouldBeVisible) => { if (!item || !mapRef.current) return; const itemId = item.id; const layer = radarPreviewLayersRef.current[itemId]; if (shouldBeVisible) { + const refreshedStatus = await fetchRadarPreviewStatus(itemId, { silent: true }); + if (refreshedStatus) { + item = { + ...item, + previewStatus: normalizePreviewStatus(refreshedStatus.status), + previewFallbackInUse: !!refreshedStatus.fallback_in_use, + previewHasGeoCache: !!refreshedStatus.has_geo_cache, + previewHasRawCache: !!refreshedStatus.has_raw_cache, + previewSourceFound: !!refreshedStatus.source_found, + previewMessage: refreshedStatus.message || '', + previewError: refreshedStatus.error || '', + previewCacheKey: refreshedStatus.cache_updated_at || item.previewCacheKey || `${Date.now()}-${itemId}`, + }; + } + if (layer) { if (!mapRef.current.hasLayer(layer)) { layer.addTo(mapRef.current); @@ -1393,7 +1408,6 @@ function App() { radarPreviewLayersRef.current[itemId] = previewLayer; previewLayer.addTo(mapRef.current); - fetchRadarPreviewStatus(itemId, { silent: true }); } else if (layer) { layer.remove(); delete radarPreviewLayersRef.current[itemId]; diff --git a/frontend/src/AssetInventoryPanel.jsx b/frontend/src/AssetInventoryPanel.jsx index df3e067..07cdaa1 100644 --- a/frontend/src/AssetInventoryPanel.jsx +++ b/frontend/src/AssetInventoryPanel.jsx @@ -132,6 +132,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { + diff --git a/frontend/src/DataMonitorPanel.jsx b/frontend/src/DataMonitorPanel.jsx index c1cab23..3f29e3d 100644 --- a/frontend/src/DataMonitorPanel.jsx +++ b/frontend/src/DataMonitorPanel.jsx @@ -9,7 +9,14 @@ const DEFAULT_MONITOR_CONFIG = { dinsar_dirs: [], gf3_archive_source_dirs: [], gf3_source_dirs: [], + gf3_sarscape_native_dirs: [], gf3_storage_dirs: [], + gf3_sarscape_wrapper_exe: '', + gf3_sarscape_idlrt_path: '', + gf3_sarscape_dem_path: '', + gf3_sarscape_polarizations: 'HH,HV', + gf3_sarscape_auto_standardize: true, + gf3_sarscape_clean_after_success: true, s1_source_dirs: [], s1_storage_dirs: [], s1_orbit_dirs: [], @@ -68,6 +75,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [s1Message, setS1Message] = useState(''); const [gf3UnpackLoading, setGf3UnpackLoading] = useState(false); const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false); + const [gf3SarscapeProduceLoading, setGf3SarscapeProduceLoading] = useState(false); + const [gf3SarscapeSyncLoading, setGf3SarscapeSyncLoading] = useState(false); + const [gf3SarscapeCleanLoading, setGf3SarscapeCleanLoading] = useState(false); const [gf3ScanLoading, setGf3ScanLoading] = useState(false); const [gf3Message, setGf3Message] = useState(''); const logEndRef = useRef(null); @@ -87,7 +97,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled ); const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1'); const gf3ActiveTask = displayActiveTasks.find((task) => - ['GF3_UNPACK', 'GF3_BATCH_PROCESS'].includes(task.task_type) + ['GF3_UNPACK', 'GF3_BATCH_PROCESS', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN'].includes(task.task_type) ); useEffect(() => { @@ -120,6 +130,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled s1_orbit_dirs: toArray(data?.s1_orbit_dirs), gf3_archive_source_dirs: toArray(data?.gf3_archive_source_dirs), gf3_source_dirs: toArray(data?.gf3_source_dirs), + gf3_sarscape_native_dirs: toArray(data?.gf3_sarscape_native_dirs), gf3_storage_dirs: toArray(data?.gf3_storage_dirs), }); setConfigLoaded(true); @@ -289,7 +300,10 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0; const hasGf3ArchiveSourceDirs = config.gf3_archive_source_dirs.length > 0; const hasGf3SourceDirs = config.gf3_source_dirs.length > 0; + const hasGf3SarscapeNativeDirs = config.gf3_sarscape_native_dirs.length > 0; const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0; + const hasGf3SarscapeWrapper = typeof config.gf3_sarscape_wrapper_exe === 'string' && config.gf3_sarscape_wrapper_exe.trim() !== ''; + const hasGf3SarscapeDem = typeof config.gf3_sarscape_dem_path === 'string' && config.gf3_sarscape_dem_path.trim() !== ''; const canRunRadar = !readOnly && configLoaded && hasRadarDirs; const canRunOrbit = !readOnly && configLoaded && hasOrbitDir; @@ -299,6 +313,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs; const canRunGf3Unpack = !readOnly && configLoaded && hasGf3ArchiveSourceDirs && hasGf3SourceDirs; const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs; + const canRunGf3SarscapeProduce = !readOnly && configLoaded && hasGf3ArchiveSourceDirs && hasGf3SarscapeNativeDirs && hasGf3StorageDirs && hasGf3SarscapeWrapper && hasGf3SarscapeDem; + const canRunGf3SarscapeSync = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs; + const canRunGf3SarscapeClean = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs; const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; const handleS1Run = async () => { @@ -452,6 +469,111 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; + const handleGf3SarscapeProduce = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发 GF3 SARscape 生产。'); + return; + } + setGf3SarscapeProduceLoading(true); + setGf3Message('GF3 SARscape 生产链路启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-produce`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({}), + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setGf3Message(data.message || 'GF3 SARscape 生产任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, 'GF3 SARscape 生产链路已启动。', { + nonBlocking: true, + taskType: 'GF3_SARSCAPE_PRODUCE', + }); + } + } else { + setGf3Message(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setGf3Message(`失败：${err.message || '未知错误'}`); + } finally { + setGf3SarscapeProduceLoading(false); + } + }; + + const handleGf3SarscapeSync = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发 GF3 SARscape 标准化。'); + return; + } + setGf3SarscapeSyncLoading(true); + setGf3Message('GF3 SARscape 原生结果标准化启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-sync`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({}), + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setGf3Message(data.message || 'GF3 SARscape 标准化任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, 'GF3 SARscape 原生结果标准化已启动。', { + nonBlocking: true, + taskType: 'GF3_SARSCAPE_SYNC', + }); + } + } else { + setGf3Message(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setGf3Message(`失败：${err.message || '未知错误'}`); + } finally { + setGf3SarscapeSyncLoading(false); + } + }; + + const handleGf3SarscapeClean = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发 GF3 SARscape 清理。'); + return; + } + setGf3SarscapeCleanLoading(true); + setGf3Message('GF3 SARscape 中间数据清理启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-clean`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ dry_run: false, require_standardized: true }), + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setGf3Message(data.message || 'GF3 SARscape 清理任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, 'GF3 SARscape 中间数据清理已启动。', { + nonBlocking: true, + taskType: 'GF3_SARSCAPE_CLEAN', + }); + } + } else { + setGf3Message(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setGf3Message(`失败：${err.message || '未知错误'}`); + } finally { + setGf3SarscapeCleanLoading(false); + } + }; + const handleOpenUnpackDialog = () => { if (readOnly) { setUnpackMessage('当前账户为只读模式，无法触发解包任务。'); @@ -676,6 +798,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
S1 精轨{formatList(config.s1_orbit_dirs)}
GF3 压缩包{formatList(config.gf3_archive_source_dirs)}
GF3 来源{formatList(config.gf3_source_dirs)}
+
GF3 原生{formatList(config.gf3_sarscape_native_dirs)}
GF3 存储{formatList(config.gf3_storage_dirs)}
D-InSAR 结果{formatList(config.dinsar_dirs)}
@@ -769,7 +892,11 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
压缩包来源{formatList(config.gf3_archive_source_dirs)}
L1A 来源{formatList(config.gf3_source_dirs)}
+
SARscape 原生{formatList(config.gf3_sarscape_native_dirs)}
L2 存储{formatList(config.gf3_storage_dirs)}
+
Wrapper{config.gf3_sarscape_wrapper_exe || '未配置'}
+
SARscape DEM{config.gf3_sarscape_dem_path || '未配置'}
+
极化{config.gf3_sarscape_polarizations || 'HH,HV'}
+ + +