Integrate GF3 SARscape flood workflow
This commit is contained in:
@@ -207,7 +207,16 @@ class Settings(BaseSettings):
|
||||
GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz"
|
||||
GF3_UNPACK_DELETE_ARCHIVE: bool = True
|
||||
GF3_SOURCE_DIRS: str = ""
|
||||
GF3_SARSCAPE_NATIVE_DIRS: str = ""
|
||||
GF3_STORAGE_DIRS: str = ""
|
||||
GF3_SARSCAPE_WRAPPER_EXE: str = ""
|
||||
GF3_SARSCAPE_IDLRT_PATH: str = r"C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe"
|
||||
GF3_SARSCAPE_DEM_PATH: str = ""
|
||||
GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV"
|
||||
GF3_SARSCAPE_KEEP_EXTRACTED: bool = True
|
||||
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True
|
||||
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True
|
||||
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0
|
||||
|
||||
MONITOR_ORBIT_DIR: str = ""
|
||||
ORBIT_POOL_ENVI: str = ""
|
||||
@@ -991,6 +1000,15 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
_check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(
|
||||
label="GF3_SARSCAPE_DEM_PATH",
|
||||
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
)
|
||||
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
@@ -1024,6 +1042,7 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
|
||||
("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS),
|
||||
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
|
||||
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
|
||||
):
|
||||
values = split_env_paths(raw_value)
|
||||
|
||||
@@ -210,12 +210,12 @@ class RadarData(BaseModel):
|
||||
has_orbit_data: bool
|
||||
orbit_file_path: Optional[str] = None
|
||||
is_envi_processed: bool = False
|
||||
coverage_polygon: List[Tuple[float, float]]
|
||||
coverage_polygon: Optional[List[Tuple[float, float]]] = None
|
||||
|
||||
min_lon: float
|
||||
min_lat: float
|
||||
max_lon: float
|
||||
max_lat: float
|
||||
min_lon: Optional[float] = None
|
||||
min_lat: Optional[float] = None
|
||||
max_lon: Optional[float] = None
|
||||
max_lat: Optional[float] = None
|
||||
preview_cache_status: str = "NONE"
|
||||
preview_cache_version: Optional[str] = None
|
||||
preview_cache_updated_at: Optional[datetime] = None
|
||||
@@ -236,7 +236,7 @@ class RadarData(BaseModel):
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def coverage_bbox(self) -> Tuple[float, float, float, float]:
|
||||
def coverage_bbox(self) -> Tuple[Optional[float], Optional[float], Optional[float], Optional[float]]:
|
||||
"""A computed property to provide the bbox tuple, used by existing logic."""
|
||||
return (self.min_lon, self.min_lat, self.max_lon, self.max_lat)
|
||||
|
||||
|
||||
@@ -43,7 +43,14 @@ class MonitorConfig(BaseModel):
|
||||
dinsar_dirs: List[str] = []
|
||||
gf3_archive_source_dirs: List[str] = []
|
||||
gf3_source_dirs: List[str] = []
|
||||
gf3_sarscape_native_dirs: List[str] = []
|
||||
gf3_storage_dirs: List[str] = []
|
||||
gf3_sarscape_wrapper_exe: Optional[str] = None
|
||||
gf3_sarscape_idlrt_path: Optional[str] = None
|
||||
gf3_sarscape_dem_path: Optional[str] = None
|
||||
gf3_sarscape_polarizations: Optional[str] = None
|
||||
gf3_sarscape_auto_standardize: bool = True
|
||||
gf3_sarscape_clean_after_success: bool = True
|
||||
# Manual-only: config is read from .env
|
||||
|
||||
|
||||
@@ -58,6 +65,26 @@ class GF3UnpackRunRequest(BaseModel):
|
||||
max_files_per_run: Optional[int] = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class GF3SarscapeSyncRequest(BaseModel):
|
||||
force: bool = False
|
||||
register: bool = True
|
||||
|
||||
|
||||
class GF3SarscapeProduceRequest(BaseModel):
|
||||
max_archives_per_run: Optional[int] = Field(default=None, ge=0)
|
||||
auto_standardize: Optional[bool] = None
|
||||
clean_after_success: Optional[bool] = None
|
||||
force_standardize: bool = False
|
||||
register: bool = True
|
||||
cleanup_dry_run: bool = False
|
||||
|
||||
|
||||
class GF3SarscapeCleanRequest(BaseModel):
|
||||
dry_run: bool = False
|
||||
require_standardized: bool = True
|
||||
max_scenes: Optional[int] = Field(default=None, ge=0)
|
||||
|
||||
|
||||
@router.post("/monitor/config")
|
||||
async def update_monitor_config(config: MonitorConfig):
|
||||
"""
|
||||
@@ -154,6 +181,138 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-sync", status_code=202)
|
||||
async def run_gf3_sarscape_sync(
|
||||
request_data: GF3SarscapeSyncRequest | None = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
扫描 GF3 SARscape 原生 _geo 二进制池，转换为标准 GeoTIFF，并登记入库。
|
||||
"""
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
if not gf3_storage_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeSyncRequest()
|
||||
task_type = "GF3_SARSCAPE_SYNC"
|
||||
task_name = "GF3 SARscape 原生结果标准化"
|
||||
payload = {
|
||||
"native_dirs": gf3_native_dirs,
|
||||
"storage_root": gf3_storage_dirs[0],
|
||||
"force": bool(options.force),
|
||||
"register": bool(options.register),
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 SARscape 原生结果标准化任务已提交",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-produce", status_code=202)
|
||||
async def run_gf3_sarscape_produce(
|
||||
request_data: GF3SarscapeProduceRequest | None = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates.
|
||||
"""
|
||||
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
|
||||
if not gf3_archive_source_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
if not gf3_storage_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
|
||||
if not settings.GF3_SARSCAPE_WRAPPER_EXE:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.")
|
||||
if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH):
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeProduceRequest()
|
||||
task_type = "GF3_SARSCAPE_PRODUCE"
|
||||
task_name = "GF3 SARscape production"
|
||||
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
|
||||
clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success)
|
||||
payload = {
|
||||
"source_dirs": gf3_archive_source_dirs,
|
||||
"native_dirs": gf3_native_dirs,
|
||||
"native_root": gf3_native_dirs[0],
|
||||
"storage_root": gf3_storage_dirs[0],
|
||||
"wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
|
||||
"idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
|
||||
"dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
|
||||
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
|
||||
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
|
||||
"max_archives_per_run": int(options.max_archives_per_run or 0),
|
||||
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
|
||||
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
|
||||
"auto_standardize": bool(auto_standardize),
|
||||
"clean_after_success": bool(clean_after_success),
|
||||
"force_standardize": bool(options.force_standardize),
|
||||
"register": bool(options.register),
|
||||
"cleanup_require_standardized": True,
|
||||
"cleanup_dry_run": bool(options.cleanup_dry_run),
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 SARscape production task submitted",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
|
||||
async def run_gf3_sarscape_clean(
|
||||
request_data: GF3SarscapeCleanRequest | None = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
Clean SARscape intermediate files from GF3 native pool after standard GeoTIFFs exist.
|
||||
"""
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
if not gf3_storage_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeCleanRequest()
|
||||
task_type = "GF3_SARSCAPE_CLEAN"
|
||||
task_name = "GF3 SARscape native cleanup"
|
||||
payload = {
|
||||
"native_dirs": gf3_native_dirs,
|
||||
"storage_root": gf3_storage_dirs[0],
|
||||
"dry_run": bool(options.dry_run),
|
||||
"require_standardized": bool(options.require_standardized),
|
||||
"max_scenes": int(options.max_scenes or 0),
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 SARscape cleanup task submitted",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/monitor/gf3-unpack/config")
|
||||
async def get_gf3_unpack_config(admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
return GF3UnpackConfig(
|
||||
|
||||
@@ -21,7 +21,14 @@ MONITOR_CONFIG = {
|
||||
# GF3 链路
|
||||
"gf3_archive_source_dirs": split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
|
||||
"gf3_sarscape_native_dirs": split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS),
|
||||
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
|
||||
"gf3_sarscape_wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
|
||||
"gf3_sarscape_idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
|
||||
"gf3_sarscape_dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
|
||||
"gf3_sarscape_polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
|
||||
"gf3_sarscape_auto_standardize": bool(settings.GF3_SARSCAPE_AUTO_STANDARDIZE),
|
||||
"gf3_sarscape_clean_after_success": bool(settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS),
|
||||
"mode": "manual",
|
||||
"config_source": "env",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
"""Inventory GF3 SARscape native geocoded outputs.
|
||||
|
||||
The production server writes ENVI/SARscape native ``*_geo`` datasets. This
|
||||
service treats those files as the source-of-truth evidence layer and produces a
|
||||
small manifest that later conversion jobs can consume.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..utils import parse_gf3_l2_dirname
|
||||
|
||||
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
|
||||
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
|
||||
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
||||
SKIP_DIR_NAMES = {
|
||||
".git",
|
||||
".gf3_extract",
|
||||
".sarmap",
|
||||
"__pycache__",
|
||||
"temp",
|
||||
"tmp",
|
||||
"work",
|
||||
"sarscape_work",
|
||||
"GTOPO30_DIR",
|
||||
"SRTM_DEM_DIR",
|
||||
"TANDEMX_DEM_DIR",
|
||||
}
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _safe_stat(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
return None
|
||||
return {
|
||||
"path": str(path),
|
||||
"size": int(stat.st_size),
|
||||
"mtime": float(stat.st_mtime),
|
||||
"mtime_ns": int(stat.st_mtime_ns),
|
||||
}
|
||||
|
||||
|
||||
def _is_nonempty_file(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_file() and path.stat().st_size > 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.with_name(f".{path.name}.tmp")
|
||||
with tmp_path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
|
||||
def _extract_date_from_text(value: str) -> str | None:
|
||||
match = re.search(r"(20\d{6})", value or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _extract_product_unique_id(value: str) -> str | None:
|
||||
match = re.search(r"(L\d{8,})", value or "", flags=re.IGNORECASE)
|
||||
return match.group(1).upper() if match else None
|
||||
|
||||
|
||||
def _polarization_from_geo_name(name: str) -> str | None:
|
||||
upper_name = name.upper()
|
||||
match = re.search(r"(?:^|[_-])(HH|HV|VH|VV)[_-]GEO$", upper_name)
|
||||
if match:
|
||||
return match.group(1)
|
||||
tokens = [token for token in re.split(r"[_\-.]+", upper_name) if token]
|
||||
for token in reversed(tokens):
|
||||
if token in POLARIZATION_PRIORITY:
|
||||
return token
|
||||
return None
|
||||
|
||||
|
||||
def _is_geo_native_data_file(path: Path) -> bool:
|
||||
return path.is_file() and path.name.lower().endswith("_geo")
|
||||
|
||||
|
||||
def _scene_batch_name(root: Path, scene_dir: Path, scene_name: str) -> str | None:
|
||||
try:
|
||||
rel_parts = scene_dir.relative_to(root).parts
|
||||
except ValueError:
|
||||
rel_parts = ()
|
||||
if len(rel_parts) >= 2:
|
||||
return rel_parts[0]
|
||||
date = _extract_date_from_text(scene_name)
|
||||
return date
|
||||
|
||||
|
||||
def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
parsed = parse_gf3_l2_dirname(scene_name) or {}
|
||||
metadata: dict[str, Any] = {
|
||||
"satellite": "GF3",
|
||||
"satellite_family": "GF3",
|
||||
**parsed,
|
||||
}
|
||||
if not metadata.get("imaging_date"):
|
||||
metadata["imaging_date"] = _extract_date_from_text(scene_name)
|
||||
if not metadata.get("product_unique_id"):
|
||||
metadata["product_unique_id"] = _extract_product_unique_id(scene_name)
|
||||
|
||||
polarizations = [
|
||||
str(asset.get("polarization") or "").upper()
|
||||
for asset in assets
|
||||
if asset.get("polarization") and asset.get("polarization") != "UNKNOWN"
|
||||
]
|
||||
if polarizations:
|
||||
metadata["polarization"] = ",".join(
|
||||
pol for pol in POLARIZATION_PRIORITY if pol in set(polarizations)
|
||||
) or ",".join(sorted(set(polarizations)))
|
||||
metadata["product_level"] = "L2"
|
||||
metadata["source_format"] = "GF3_SARSCAPE_NATIVE"
|
||||
return metadata
|
||||
|
||||
|
||||
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
||||
assets: list[dict[str, Any]] = []
|
||||
try:
|
||||
entries = sorted(scene_dir.iterdir(), key=lambda item: item.name.lower())
|
||||
except OSError:
|
||||
return assets
|
||||
|
||||
for path in entries:
|
||||
if not _is_geo_native_data_file(path):
|
||||
continue
|
||||
|
||||
base = path
|
||||
hdr = Path(str(base) + ".hdr")
|
||||
sml = Path(str(base) + ".sml")
|
||||
aux_xml = Path(str(base) + ".aux.xml")
|
||||
ovr = Path(str(base) + ".ovr")
|
||||
kml = Path(str(base) + ".kml")
|
||||
quicklook = base.with_name(base.name + "_ql.tif")
|
||||
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
||||
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
||||
|
||||
asset: dict[str, Any] = {
|
||||
"polarization": polarization,
|
||||
"role": "geo_native",
|
||||
"path": str(base),
|
||||
"hdr": str(hdr) if hdr.exists() else None,
|
||||
"sml": str(sml) if sml.exists() else None,
|
||||
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
||||
"ovr": str(ovr) if ovr.exists() else None,
|
||||
"quicklook": str(quicklook) if quicklook.exists() else None,
|
||||
"kml": str(kml) if kml.exists() else None,
|
||||
"complete": bool(complete),
|
||||
"source": _safe_stat(base),
|
||||
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
||||
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
||||
}
|
||||
assets.append(asset)
|
||||
|
||||
return assets
|
||||
|
||||
|
||||
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
||||
assets = _collect_native_assets(scene_dir)
|
||||
if not assets:
|
||||
return None
|
||||
|
||||
scene_name = scene_dir.name
|
||||
batch_name = _scene_batch_name(root, scene_dir, scene_name)
|
||||
complete_assets = [asset for asset in assets if asset.get("complete")]
|
||||
complete_pols = [
|
||||
pol
|
||||
for pol in POLARIZATION_PRIORITY
|
||||
if any(asset.get("complete") and asset.get("polarization") == pol for asset in assets)
|
||||
]
|
||||
other_complete_pols = sorted(
|
||||
{
|
||||
str(asset.get("polarization") or "")
|
||||
for asset in complete_assets
|
||||
if asset.get("polarization") not in POLARIZATION_PRIORITY
|
||||
}
|
||||
)
|
||||
complete_pols.extend([pol for pol in other_complete_pols if pol])
|
||||
|
||||
if complete_assets and len(complete_assets) == len(assets):
|
||||
status = "NATIVE_READY"
|
||||
elif complete_assets:
|
||||
status = "PARTIAL"
|
||||
else:
|
||||
status = "FAILED"
|
||||
|
||||
logs = []
|
||||
for name in ("gf3_sarscape_cli.log",):
|
||||
log_path = scene_dir / name
|
||||
if log_path.is_file():
|
||||
logs.append(str(log_path))
|
||||
try:
|
||||
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
metadata = _parse_scene_metadata(scene_name, assets)
|
||||
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
|
||||
return {
|
||||
"schema": NATIVE_MANIFEST_SCHEMA,
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_name,
|
||||
"batch_name": batch_name,
|
||||
"native_root": str(root),
|
||||
"native_dir": str(scene_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"source_archive": None,
|
||||
"status": status,
|
||||
"polarizations": complete_pols,
|
||||
"metadata": metadata,
|
||||
"assets": assets,
|
||||
"logs": logs,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in native_dirs or []:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
path = Path(os.path.normpath(text)).resolve()
|
||||
key = str(path).lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if path.is_dir():
|
||||
roots.append(path)
|
||||
else:
|
||||
missing.append(str(path))
|
||||
return roots, missing
|
||||
|
||||
|
||||
def scan_gf3_sarscape_native_roots(
|
||||
native_dirs: list[str] | tuple[str, ...] | None,
|
||||
*,
|
||||
write_manifest: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan configured native roots and return discovered scene manifests."""
|
||||
roots, missing_roots = _normalize_roots(native_dirs)
|
||||
scenes: list[dict[str, Any]] = []
|
||||
seen_scene_dirs: set[str] = set()
|
||||
write_errors: list[dict[str, str]] = []
|
||||
|
||||
for root in roots:
|
||||
for current_dir, dir_names, _file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
||||
]
|
||||
scene_dir = Path(current_dir)
|
||||
scene_key = str(scene_dir).lower()
|
||||
if scene_key in seen_scene_dirs:
|
||||
dir_names[:] = []
|
||||
continue
|
||||
|
||||
manifest = _collect_scene_manifest(root, scene_dir)
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
seen_scene_dirs.add(scene_key)
|
||||
if write_manifest:
|
||||
try:
|
||||
_write_json(Path(manifest["manifest_path"]), manifest)
|
||||
except OSError as exc:
|
||||
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
|
||||
scenes.append(manifest)
|
||||
dir_names[:] = []
|
||||
|
||||
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
|
||||
partial = sum(1 for scene in scenes if scene.get("status") == "PARTIAL")
|
||||
failed = sum(1 for scene in scenes if scene.get("status") == "FAILED")
|
||||
complete_assets = sum(
|
||||
1
|
||||
for scene in scenes
|
||||
for asset in scene.get("assets") or []
|
||||
if asset.get("complete")
|
||||
)
|
||||
return {
|
||||
"schema": "gf3_sarscape_native_inventory.v1",
|
||||
"generated_at": _utc_now(),
|
||||
"native_roots": [str(path) for path in roots],
|
||||
"missing_roots": missing_roots,
|
||||
"scene_count": len(scenes),
|
||||
"native_ready_count": native_ready,
|
||||
"partial_count": partial,
|
||||
"failed_count": failed,
|
||||
"complete_asset_count": complete_assets,
|
||||
"write_errors": write_errors,
|
||||
"scenes": scenes,
|
||||
}
|
||||
@@ -0,0 +1,843 @@
|
||||
"""Run GF3 SARscape production and clean native intermediate files."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from ..config import settings, split_env_paths
|
||||
from .gf3_native_inventory_service import (
|
||||
NATIVE_MANIFEST_NAME,
|
||||
POLARIZATION_PRIORITY,
|
||||
scan_gf3_sarscape_native_roots,
|
||||
)
|
||||
from .gf3_standardize_service import STANDARD_MANIFEST_NAME
|
||||
|
||||
LogCallback = Callable[[str, str], None]
|
||||
ProgressCallback = Callable[[int, str], None]
|
||||
|
||||
SUPPORTED_WRAPPER_INPUT_EXTS = (".tar.gz", ".tgz", ".meta.xml")
|
||||
CLEANUP_MANIFEST_NAME = "gf3_cleanup_manifest.json"
|
||||
INTERMEDIATE_DIR_NAMES = {
|
||||
".gf3_extract",
|
||||
".gf3_extract.tmp",
|
||||
"temp",
|
||||
"tmp",
|
||||
"work",
|
||||
}
|
||||
KEEP_FILE_NAMES = {
|
||||
NATIVE_MANIFEST_NAME,
|
||||
CLEANUP_MANIFEST_NAME,
|
||||
"gf3_sarscape_cli.log",
|
||||
}
|
||||
INTERMEDIATE_SUFFIXES = (
|
||||
".par",
|
||||
".par_command",
|
||||
".trace",
|
||||
".working",
|
||||
".working_warning",
|
||||
".workinggetcornerfromslantrangeimage_dem",
|
||||
".txt",
|
||||
".list",
|
||||
".listhv",
|
||||
".listunknown",
|
||||
".shp",
|
||||
".shx",
|
||||
".dbf",
|
||||
".prj",
|
||||
)
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
return str(value or "").strip().strip('"').strip("'")
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.with_name(f".{path.name}.tmp")
|
||||
with tmp_path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
data = json.load(stream)
|
||||
return data if isinstance(data, dict) else None
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
|
||||
text = _clean_text(value) or default
|
||||
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
|
||||
return safe or default
|
||||
|
||||
|
||||
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in values or []:
|
||||
text = _clean_text(raw)
|
||||
if not text:
|
||||
continue
|
||||
path = Path(os.path.normpath(text)).resolve()
|
||||
key = str(path).lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if path.is_dir():
|
||||
roots.append(path)
|
||||
else:
|
||||
missing.append(str(path))
|
||||
return roots, missing
|
||||
|
||||
|
||||
def _resolve_config_path(raw: Any) -> Path | None:
|
||||
text = _clean_text(raw)
|
||||
if not text:
|
||||
return None
|
||||
return Path(os.path.normpath(text)).resolve()
|
||||
|
||||
|
||||
def _fallback_wrapper_exe() -> Path | None:
|
||||
candidate = (
|
||||
Path(settings.PROJECT_ROOT)
|
||||
/ ".codex_tmp"
|
||||
/ "GF3_L1A_To_L2_pipeline"
|
||||
/ "dist"
|
||||
/ "windows"
|
||||
/ "gf3wrapper.exe"
|
||||
)
|
||||
return candidate.resolve() if candidate.is_file() else None
|
||||
|
||||
|
||||
def _wrapper_exe_path(value: str | None = None) -> Path:
|
||||
configured = _resolve_config_path(value or settings.GF3_SARSCAPE_WRAPPER_EXE)
|
||||
path = configured or _fallback_wrapper_exe()
|
||||
if path is None:
|
||||
raise FileNotFoundError("GF3 SARscape wrapper is not configured.")
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"GF3 SARscape wrapper does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _dem_path(value: str | None = None) -> Path:
|
||||
path = _resolve_config_path(value or settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH)
|
||||
if path is None:
|
||||
raise FileNotFoundError("GF3 SARscape DEM path is not configured.")
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"GF3 SARscape DEM path does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _idlrt_path(value: str | None = None) -> Path | None:
|
||||
path = _resolve_config_path(value or settings.GF3_SARSCAPE_IDLRT_PATH)
|
||||
if path is None:
|
||||
return None
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"GF3 SARscape idlrt.exe does not exist: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _requested_polarizations(value: str | None = None) -> list[str]:
|
||||
raw = _clean_text(value or settings.GF3_SARSCAPE_POLARIZATIONS or "HH,HV")
|
||||
items: list[str] = []
|
||||
for token in re.split(r"[,;\s]+", raw):
|
||||
pol = token.strip().upper()
|
||||
if not pol:
|
||||
continue
|
||||
if pol not in items:
|
||||
items.append(pol)
|
||||
return items or ["HH", "HV"]
|
||||
|
||||
|
||||
def _archive_exts_for_wrapper(archive_exts: list[str] | tuple[str, ...] | None) -> list[str]:
|
||||
ordered: list[str] = []
|
||||
for raw_ext in archive_exts or []:
|
||||
ext = _clean_text(raw_ext).lower()
|
||||
if not ext:
|
||||
continue
|
||||
if not ext.startswith("."):
|
||||
ext = "." + ext
|
||||
if ext in SUPPORTED_WRAPPER_INPUT_EXTS and ext not in ordered:
|
||||
ordered.append(ext)
|
||||
if not ordered:
|
||||
ordered.extend((".tar.gz", ".tgz"))
|
||||
return sorted(ordered, key=len, reverse=True)
|
||||
|
||||
|
||||
def _input_ext(path: Path, exts: list[str]) -> str | None:
|
||||
name = path.name.lower()
|
||||
for ext in exts:
|
||||
if name.endswith(ext):
|
||||
return ext
|
||||
return None
|
||||
|
||||
|
||||
def _scene_name_from_input(path: Path) -> str:
|
||||
name = path.name
|
||||
lower_name = name.lower()
|
||||
for ext in SUPPORTED_WRAPPER_INPUT_EXTS:
|
||||
if lower_name.endswith(ext):
|
||||
return name[: -len(ext)]
|
||||
return path.stem
|
||||
|
||||
|
||||
def discover_gf3_sarscape_inputs(
|
||||
source_dirs: list[str] | tuple[str, ...] | None,
|
||||
*,
|
||||
archive_exts: list[str] | tuple[str, ...] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Find wrapper-supported GF3 source inputs in configured archive pools."""
|
||||
roots, missing_roots = _resolve_existing_dirs(source_dirs)
|
||||
exts = _archive_exts_for_wrapper(archive_exts)
|
||||
inputs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for root in roots:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
ext = _input_ext(path, exts)
|
||||
if not ext:
|
||||
continue
|
||||
resolved = path.resolve()
|
||||
key = str(resolved).lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
inputs.append(
|
||||
{
|
||||
"path": str(resolved),
|
||||
"scene_name": _scene_name_from_input(path),
|
||||
"ext": ext,
|
||||
"source_root": str(root),
|
||||
}
|
||||
)
|
||||
|
||||
inputs.sort(key=lambda item: str(item.get("path") or "").lower())
|
||||
return {
|
||||
"source_roots": [str(path) for path in roots],
|
||||
"missing_roots": missing_roots,
|
||||
"archive_exts": exts,
|
||||
"input_count": len(inputs),
|
||||
"inputs": inputs,
|
||||
}
|
||||
|
||||
|
||||
def _is_nonempty_file(path: Path) -> bool:
|
||||
try:
|
||||
return path.is_file() and path.stat().st_size > 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _completed_geo_product(scene_dir: Path, polarization: str) -> Path | None:
|
||||
lower_pol = polarization.lower()
|
||||
try:
|
||||
entries = list(scene_dir.iterdir())
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
for path in entries:
|
||||
name = path.name.lower()
|
||||
if not name.endswith(f"_{lower_pol}_geo.sml"):
|
||||
continue
|
||||
data_file = Path(str(path)[: -len(".sml")])
|
||||
if _is_nonempty_file(path) and _is_nonempty_file(data_file):
|
||||
return data_file
|
||||
return None
|
||||
|
||||
|
||||
def _scene_complete(scene_dir: Path, polarizations: list[str]) -> bool:
|
||||
if not scene_dir.is_dir():
|
||||
return False
|
||||
return all(_completed_geo_product(scene_dir, pol) is not None for pol in polarizations)
|
||||
|
||||
|
||||
def _missing_geo_polarizations(scene_dir: Path, polarizations: list[str]) -> list[str]:
|
||||
if not scene_dir.is_dir():
|
||||
return list(polarizations)
|
||||
return [pol for pol in polarizations if _completed_geo_product(scene_dir, pol) is None]
|
||||
|
||||
|
||||
def _compact_failure_text(text: str, *, max_chars: int = 1000) -> str:
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
compact = " | ".join(lines)
|
||||
if len(compact) <= max_chars:
|
||||
return compact
|
||||
return compact[: max_chars - 3].rstrip() + "..."
|
||||
|
||||
|
||||
def _read_failure_file(path: Path) -> str | None:
|
||||
try:
|
||||
if not path.is_file() or path.stat().st_size <= 0:
|
||||
return None
|
||||
return path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _scene_failure_hint(scene_dir: Path) -> str | None:
|
||||
work_dir = scene_dir / "work"
|
||||
candidates = [
|
||||
work_dir / "Process.working_error",
|
||||
work_dir / "Process.trace_cerr.txt",
|
||||
scene_dir / "gf3_sarscape_cli.log",
|
||||
]
|
||||
for path in candidates:
|
||||
text = _read_failure_file(path)
|
||||
if text:
|
||||
compact = _compact_failure_text(text)
|
||||
if compact:
|
||||
return f"{path.name}: {compact}"
|
||||
|
||||
process_log = work_dir / "Process.log"
|
||||
text = _read_failure_file(process_log)
|
||||
if not text:
|
||||
return None
|
||||
important = [
|
||||
line.strip()
|
||||
for line in text.splitlines()
|
||||
if "ERROR" in line.upper() or "[EC:" in line.upper() or "PARAMETER FILE READ ERROR" in line.upper()
|
||||
]
|
||||
if important:
|
||||
return f"{process_log.name}: {_compact_failure_text(chr(10).join(important[-8:]))}"
|
||||
return None
|
||||
|
||||
|
||||
def _format_missing_output_error(scene_dir: Path, polarizations: list[str], returncode: int) -> str:
|
||||
missing = _missing_geo_polarizations(scene_dir, polarizations)
|
||||
missing_text = ",".join(missing) if missing else "unknown"
|
||||
if returncode == 0:
|
||||
base = f"wrapper returned 0 but required _geo outputs are missing: {missing_text}"
|
||||
else:
|
||||
base = f"wrapper return code {returncode}; missing _geo outputs: {missing_text}"
|
||||
hint = _scene_failure_hint(scene_dir)
|
||||
return f"{base}; {hint}" if hint else base
|
||||
|
||||
|
||||
def _emit_log(callback: LogCallback | None, level: str, message: str) -> None:
|
||||
if callback:
|
||||
callback(level, message)
|
||||
|
||||
|
||||
def _emit_progress(callback: ProgressCallback | None, progress: int, message: str) -> None:
|
||||
if callback:
|
||||
callback(max(0, min(100, int(progress))), message)
|
||||
|
||||
|
||||
def _run_wrapper_command(
|
||||
cmd: list[str],
|
||||
*,
|
||||
cwd: Path,
|
||||
env: dict[str, str],
|
||||
timeout_seconds: int | None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout_seconds if timeout_seconds and timeout_seconds > 0 else None,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _log_completed_process_output(
|
||||
completed: subprocess.CompletedProcess[str],
|
||||
*,
|
||||
log_callback: LogCallback | None,
|
||||
line_limit: int = 300,
|
||||
) -> list[str]:
|
||||
output = completed.stdout or ""
|
||||
lines = [line.rstrip() for line in output.splitlines() if line.strip()]
|
||||
if not lines:
|
||||
return []
|
||||
|
||||
clipped = False
|
||||
display_lines = lines
|
||||
if len(lines) > line_limit:
|
||||
clipped = True
|
||||
head_count = max(1, line_limit // 2)
|
||||
tail_count = max(1, line_limit - head_count)
|
||||
display_lines = lines[:head_count] + [f"... clipped {len(lines) - line_limit} wrapper log lines ..."] + lines[-tail_count:]
|
||||
|
||||
for line in display_lines:
|
||||
_emit_log(log_callback, "INFO", f"[gf3wrapper] {line}")
|
||||
if clipped:
|
||||
_emit_log(log_callback, "WARNING", f"GF3 wrapper output was clipped to {line_limit} log lines.")
|
||||
return lines[-20:]
|
||||
|
||||
|
||||
def run_gf3_sarscape_production(
|
||||
*,
|
||||
source_dirs: list[str] | None = None,
|
||||
native_root: str | None = None,
|
||||
wrapper_exe: str | None = None,
|
||||
dem_path: str | None = None,
|
||||
idlrt_path: str | None = None,
|
||||
polarizations: str | None = None,
|
||||
archive_exts: list[str] | None = None,
|
||||
max_archives_per_run: int | None = None,
|
||||
timeout_seconds: int | None = None,
|
||||
keep_extracted: bool | None = None,
|
||||
log_callback: LogCallback | None = None,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the external GF3 SARscape wrapper for pending raw archives."""
|
||||
source_dirs = source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
|
||||
native_roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
native_root_text = _clean_text(native_root or (native_roots[0] if native_roots else ""))
|
||||
if not source_dirs:
|
||||
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
if not native_root_text:
|
||||
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
native_root_path = Path(os.path.normpath(native_root_text)).resolve()
|
||||
|
||||
native_root_path.mkdir(parents=True, exist_ok=True)
|
||||
wrapper_path = _wrapper_exe_path(wrapper_exe)
|
||||
dem = _dem_path(dem_path)
|
||||
idlrt = _idlrt_path(idlrt_path)
|
||||
pols = _requested_polarizations(polarizations)
|
||||
pol_text = ",".join(pols)
|
||||
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
|
||||
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
|
||||
inputs = discovery.get("inputs") or []
|
||||
max_to_process = int(max_archives_per_run or 0)
|
||||
timeout = int(timeout_seconds or 0)
|
||||
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
|
||||
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape source roots: {source_dirs}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape native root: {native_root_path}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
|
||||
|
||||
if not inputs:
|
||||
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
|
||||
return {
|
||||
"ok": True,
|
||||
"found_count": 0,
|
||||
"processed_count": 0,
|
||||
"skipped_count": 0,
|
||||
"failed_count": 0,
|
||||
"deferred_count": 0,
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"results": [],
|
||||
}
|
||||
|
||||
runtime_dir = native_root_path / ".gf3_runtime"
|
||||
runtime_dir.mkdir(parents=True, exist_ok=True)
|
||||
config_path = runtime_dir / "gf3wrapper.json"
|
||||
env = os.environ.copy()
|
||||
if idlrt is not None:
|
||||
env["IDLRT_PATH"] = str(idlrt)
|
||||
|
||||
processed = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
deferred = 0
|
||||
results: list[dict[str, Any]] = []
|
||||
total = len(inputs)
|
||||
|
||||
for idx, input_info in enumerate(inputs):
|
||||
input_path = Path(str(input_info.get("path") or "")).resolve()
|
||||
scene_name = str(input_info.get("scene_name") or _scene_name_from_input(input_path))
|
||||
scene_dir = native_root_path / scene_name
|
||||
progress = 5 + int((idx / max(total, 1)) * 60)
|
||||
_emit_progress(progress_callback, progress, f"GF3 SARscape checking {idx + 1}/{total}: {scene_name}")
|
||||
|
||||
if _scene_complete(scene_dir, pols):
|
||||
skipped += 1
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped_complete",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if max_to_process > 0 and processed + failed >= max_to_process:
|
||||
deferred += 1
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "deferred",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
cmd = [
|
||||
str(wrapper_path),
|
||||
"-config",
|
||||
str(config_path),
|
||||
"-input",
|
||||
str(input_path),
|
||||
"-output",
|
||||
str(native_root_path),
|
||||
"-dem",
|
||||
str(dem),
|
||||
"-pol",
|
||||
pol_text,
|
||||
f"-keep-extracted={str(keep).lower()}",
|
||||
]
|
||||
if idlrt is not None:
|
||||
cmd.extend(["-idlrt", str(idlrt)])
|
||||
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape processing {scene_name}: {input_path}")
|
||||
started = time.monotonic()
|
||||
try:
|
||||
completed = _run_wrapper_command(
|
||||
cmd,
|
||||
cwd=wrapper_path.parent,
|
||||
env=env,
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
output_tail = _log_completed_process_output(completed, log_callback=log_callback)
|
||||
elapsed_seconds = round(time.monotonic() - started, 3)
|
||||
output_complete = _scene_complete(scene_dir, pols)
|
||||
if completed.returncode == 0 and output_complete:
|
||||
processed += 1
|
||||
status = "processed"
|
||||
error = None
|
||||
elif output_complete:
|
||||
processed += 1
|
||||
status = "processed_with_warning"
|
||||
error = f"wrapper return code {completed.returncode}"
|
||||
_emit_log(log_callback, "WARNING", f"GF3 wrapper returned {completed.returncode}, but output is complete: {scene_name}")
|
||||
else:
|
||||
failed += 1
|
||||
status = "failed"
|
||||
error = _format_missing_output_error(scene_dir, pols, int(completed.returncode))
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape failed for {scene_name}: {error}")
|
||||
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": status,
|
||||
"returncode": int(completed.returncode),
|
||||
"elapsed_seconds": elapsed_seconds,
|
||||
"output_complete": output_complete,
|
||||
"error": error,
|
||||
"output_tail": output_tail,
|
||||
}
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
failed += 1
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape timed out for {scene_name}: {exc}")
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "failed",
|
||||
"error": f"timeout after {timeout}s",
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
_emit_log(log_callback, "ERROR", f"GF3 SARscape exception for {scene_name}: {exc}")
|
||||
results.append(
|
||||
{
|
||||
"scene_name": scene_name,
|
||||
"input_path": str(input_path),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
_emit_progress(progress_callback, 70, "GF3 SARscape production stage finished.")
|
||||
return {
|
||||
"ok": failed == 0,
|
||||
"found_count": total,
|
||||
"processed_count": processed,
|
||||
"skipped_count": skipped,
|
||||
"failed_count": failed,
|
||||
"deferred_count": deferred,
|
||||
"native_root": str(native_root_path),
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
def _is_relative_to(path: Path, parent: Path) -> bool:
|
||||
try:
|
||||
path.resolve().relative_to(parent.resolve())
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _entry_size(path: Path) -> int:
|
||||
try:
|
||||
if path.is_file():
|
||||
return int(path.stat().st_size)
|
||||
if path.is_dir():
|
||||
total = 0
|
||||
for current, _dir_names, file_names in os.walk(path):
|
||||
for file_name in file_names:
|
||||
file_path = Path(current) / file_name
|
||||
try:
|
||||
total += int(file_path.stat().st_size)
|
||||
except OSError:
|
||||
continue
|
||||
return total
|
||||
except OSError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def _is_final_geo_asset_file(path: Path) -> bool:
|
||||
name = path.name.lower()
|
||||
if name in KEEP_FILE_NAMES or name.endswith(".log"):
|
||||
return True
|
||||
return (
|
||||
name.endswith("_geo")
|
||||
or name.endswith("_geo.hdr")
|
||||
or name.endswith("_geo.sml")
|
||||
or name.endswith("_geo.ovr")
|
||||
or name.endswith("_geo.aux.xml")
|
||||
or name.endswith("_geo.kml")
|
||||
or name.endswith("_geo_ql.tif")
|
||||
or name.endswith("_geo_ql.kml")
|
||||
)
|
||||
|
||||
|
||||
def _is_known_intermediate_file(path: Path) -> bool:
|
||||
name = path.name.lower()
|
||||
if _is_final_geo_asset_file(path):
|
||||
return False
|
||||
if any(token in name for token in ("_slc", "_ml", "_filt")):
|
||||
return True
|
||||
if name.endswith(INTERMEDIATE_SUFFIXES):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _standard_manifest_path(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
|
||||
batch_name = _safe_slug(scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date"), default="unknown_batch")
|
||||
scene_name = _safe_slug(scene_manifest.get("scene_name"), default="unknown_scene")
|
||||
return storage_root / batch_name / scene_name / STANDARD_MANIFEST_NAME
|
||||
|
||||
|
||||
def _standard_manifest_allows_cleanup(scene_manifest: dict[str, Any], storage_root: Path) -> bool:
|
||||
manifest = _read_json(_standard_manifest_path(scene_manifest, storage_root))
|
||||
if not manifest:
|
||||
return False
|
||||
return str(manifest.get("status") or "").upper() == "DONE"
|
||||
|
||||
|
||||
def _assert_safe_delete(target: Path, scene_dir: Path, native_roots: list[Path]) -> None:
|
||||
resolved_target = target.resolve()
|
||||
resolved_scene = scene_dir.resolve()
|
||||
if resolved_target == resolved_scene:
|
||||
raise RuntimeError(f"Refusing to delete scene directory itself: {resolved_target}")
|
||||
if not _is_relative_to(resolved_target, resolved_scene):
|
||||
raise RuntimeError(f"Refusing to delete outside scene directory: {resolved_target}")
|
||||
if not any(_is_relative_to(resolved_target, root) for root in native_roots):
|
||||
raise RuntimeError(f"Refusing to delete outside GF3 native roots: {resolved_target}")
|
||||
|
||||
|
||||
def _cleanup_scene_intermediates(
|
||||
scene_manifest: dict[str, Any],
|
||||
*,
|
||||
native_roots: list[Path],
|
||||
dry_run: bool,
|
||||
) -> dict[str, Any]:
|
||||
scene_dir = Path(str(scene_manifest.get("native_dir") or "")).resolve()
|
||||
if not scene_dir.is_dir():
|
||||
return {
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped",
|
||||
"reason": "scene directory missing",
|
||||
"deleted_entries": [],
|
||||
"bytes_deleted": 0,
|
||||
}
|
||||
if not any(_is_relative_to(scene_dir, root) for root in native_roots):
|
||||
return {
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": "skipped",
|
||||
"reason": "scene directory is outside configured native roots",
|
||||
"deleted_entries": [],
|
||||
"bytes_deleted": 0,
|
||||
}
|
||||
|
||||
candidates: list[Path] = []
|
||||
for entry in sorted(scene_dir.iterdir(), key=lambda item: item.name.lower()):
|
||||
if entry.is_dir() and entry.name.lower() in INTERMEDIATE_DIR_NAMES:
|
||||
candidates.append(entry)
|
||||
elif entry.is_file() and _is_known_intermediate_file(entry):
|
||||
candidates.append(entry)
|
||||
|
||||
deleted_entries: list[dict[str, Any]] = []
|
||||
bytes_deleted = 0
|
||||
errors: list[dict[str, str]] = []
|
||||
|
||||
for target in candidates:
|
||||
try:
|
||||
_assert_safe_delete(target, scene_dir, native_roots)
|
||||
size = _entry_size(target)
|
||||
bytes_deleted += size
|
||||
deleted_entries.append(
|
||||
{
|
||||
"path": str(target),
|
||||
"type": "directory" if target.is_dir() else "file",
|
||||
"size": size,
|
||||
}
|
||||
)
|
||||
if not dry_run:
|
||||
if target.is_dir():
|
||||
shutil.rmtree(target)
|
||||
else:
|
||||
target.unlink()
|
||||
except Exception as exc:
|
||||
errors.append({"path": str(target), "error": str(exc)})
|
||||
|
||||
status = "cleaned" if deleted_entries and not errors else ("error" if errors else "nothing_to_delete")
|
||||
cleanup_manifest = {
|
||||
"schema": "gf3_sarscape_cleanup.v1",
|
||||
"generated_at": _utc_now(),
|
||||
"dry_run": dry_run,
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"scene_dir": str(scene_dir),
|
||||
"status": status,
|
||||
"bytes_deleted": bytes_deleted,
|
||||
"deleted_entries": deleted_entries,
|
||||
"errors": errors,
|
||||
"retention_policy": {
|
||||
"keep": "final *_geo native assets, quicklooks, manifests, and logs",
|
||||
"delete": "extract/temp/work directories and slc/ml/filt intermediate files",
|
||||
},
|
||||
}
|
||||
if not dry_run:
|
||||
_write_json(scene_dir / CLEANUP_MANIFEST_NAME, cleanup_manifest)
|
||||
return cleanup_manifest
|
||||
|
||||
|
||||
def cleanup_gf3_sarscape_native_pool(
|
||||
*,
|
||||
native_dirs: list[str] | None = None,
|
||||
storage_root: str | None = None,
|
||||
require_standardized: bool = True,
|
||||
dry_run: bool = False,
|
||||
max_scenes: int | None = None,
|
||||
log_callback: LogCallback | None = None,
|
||||
progress_callback: ProgressCallback | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Delete intermediate SARscape files while keeping final native _geo assets."""
|
||||
native_dirs = native_dirs if native_dirs is not None else split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
native_roots, missing_roots = _resolve_existing_dirs(native_dirs)
|
||||
if not native_roots:
|
||||
raise ValueError("GF3_SARSCAPE_NATIVE_DIRS has no accessible directories.")
|
||||
storage = Path(os.path.normpath(storage_root or settings.GF3_STORAGE_DIRS)).resolve() if storage_root or settings.GF3_STORAGE_DIRS else None
|
||||
|
||||
inventory = scan_gf3_sarscape_native_roots([str(root) for root in native_roots], write_manifest=not dry_run)
|
||||
scenes = inventory.get("scenes") or []
|
||||
max_count = int(max_scenes or 0)
|
||||
cleaned = 0
|
||||
skipped = 0
|
||||
error_count = 0
|
||||
bytes_deleted = 0
|
||||
scene_results: list[dict[str, Any]] = []
|
||||
|
||||
for idx, scene_manifest in enumerate(scenes):
|
||||
_emit_progress(
|
||||
progress_callback,
|
||||
5 + int((idx / max(len(scenes), 1)) * 90),
|
||||
f"GF3 native cleanup checking {idx + 1}/{len(scenes)}: {scene_manifest.get('scene_name')}",
|
||||
)
|
||||
scene_status = str(scene_manifest.get("status") or "")
|
||||
if scene_status != "NATIVE_READY":
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": f"native status is {scene_status or 'UNKNOWN'}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if require_standardized:
|
||||
if storage is None:
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "GF3_STORAGE_DIRS is not configured",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if not _standard_manifest_allows_cleanup(scene_manifest, storage):
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "standard GeoTIFF manifest is not DONE",
|
||||
}
|
||||
)
|
||||
continue
|
||||
if max_count > 0 and cleaned >= max_count:
|
||||
skipped += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"status": "skipped",
|
||||
"reason": "max_scenes limit reached",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
result = _cleanup_scene_intermediates(scene_manifest, native_roots=native_roots, dry_run=dry_run)
|
||||
scene_results.append(result)
|
||||
bytes_deleted += int(result.get("bytes_deleted") or 0)
|
||||
if result.get("status") == "error":
|
||||
error_count += 1
|
||||
if result.get("status") in {"cleaned", "nothing_to_delete"}:
|
||||
cleaned += 1
|
||||
_emit_log(
|
||||
log_callback,
|
||||
"INFO",
|
||||
f"GF3 cleanup {result.get('status')}: {result.get('scene_name')} bytes={result.get('bytes_deleted')}",
|
||||
)
|
||||
|
||||
_emit_progress(progress_callback, 100, "GF3 native cleanup finished.")
|
||||
return {
|
||||
"ok": error_count == 0,
|
||||
"dry_run": dry_run,
|
||||
"native_roots": [str(root) for root in native_roots],
|
||||
"missing_roots": missing_roots,
|
||||
"scene_count": len(scenes),
|
||||
"cleaned_scene_count": cleaned,
|
||||
"skipped_scene_count": skipped,
|
||||
"error_scene_count": error_count,
|
||||
"bytes_deleted": bytes_deleted,
|
||||
"scenes": scene_results,
|
||||
}
|
||||
@@ -0,0 +1,927 @@
|
||||
"""Convert GF3 SARscape native geocoded outputs to platform GeoTIFFs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import Polygon
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import settings
|
||||
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
|
||||
from .data_service import extract_geotiff_bounds
|
||||
from .gf3_native_inventory_service import (
|
||||
NATIVE_MANIFEST_NAME,
|
||||
POLARIZATION_PRIORITY,
|
||||
scan_gf3_sarscape_native_roots,
|
||||
)
|
||||
from .image_service import image_service
|
||||
from .sar_analysis_ready_service import register_analysis_ready_tif
|
||||
|
||||
STANDARD_MANIFEST_NAME = "gf3_standard_manifest.json"
|
||||
STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
|
||||
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
|
||||
CONVERTER_VERSION = "v1"
|
||||
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _path_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
return os.path.normpath(text) if text else ""
|
||||
|
||||
|
||||
def _db_now() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None, microsecond=0)
|
||||
|
||||
|
||||
def _path_kind(path: str) -> str:
|
||||
text = str(path or "").strip()
|
||||
if text.startswith("\\\\"):
|
||||
return "unc"
|
||||
if len(text) >= 3 and text[1:3] in {":\\", ":/"} and text[0].isalpha():
|
||||
return "windows"
|
||||
if text.startswith("/mnt/"):
|
||||
return "wsl_mount"
|
||||
if text.startswith("/"):
|
||||
return "posix"
|
||||
return "relative"
|
||||
|
||||
|
||||
def _source_asset_uid(path: str) -> str:
|
||||
normalized = os.path.normpath(str(path or "").strip())
|
||||
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
|
||||
return f"source:{digest[:32]}"
|
||||
|
||||
|
||||
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
text = default
|
||||
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text).strip("._-")
|
||||
return safe or default
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = path.with_name(f".{path.name}.tmp")
|
||||
with tmp_path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False)
|
||||
os.replace(tmp_path, path)
|
||||
|
||||
|
||||
def _json_safe(value: Any) -> Any:
|
||||
if isinstance(value, float):
|
||||
return value if math.isfinite(value) else None
|
||||
if isinstance(value, dict):
|
||||
return {key: _json_safe(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_safe(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _finite_float(value: Any) -> float | None:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
data = json.load(stream)
|
||||
return data if isinstance(data, dict) else None
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _file_fingerprint(path: Path) -> dict[str, Any] | None:
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
return None
|
||||
return {
|
||||
"path": str(path),
|
||||
"size": int(stat.st_size),
|
||||
"mtime": float(stat.st_mtime),
|
||||
"mtime_ns": int(stat.st_mtime_ns),
|
||||
}
|
||||
|
||||
|
||||
def _tree_stats(path: Path) -> dict[str, Any]:
|
||||
if path.is_file():
|
||||
info = _file_fingerprint(path)
|
||||
return {
|
||||
"size_bytes": info.get("size") if info else None,
|
||||
"mtime_epoch": info.get("mtime") if info else None,
|
||||
}
|
||||
|
||||
total = 0
|
||||
newest: float | None = None
|
||||
try:
|
||||
iterator = path.rglob("*")
|
||||
for item in iterator:
|
||||
try:
|
||||
if not item.is_file():
|
||||
continue
|
||||
stat = item.stat()
|
||||
except OSError:
|
||||
continue
|
||||
total += int(stat.st_size)
|
||||
mtime = float(stat.st_mtime)
|
||||
newest = mtime if newest is None else max(newest, mtime)
|
||||
except OSError:
|
||||
return {"size_bytes": None, "mtime_epoch": None}
|
||||
return {"size_bytes": total, "mtime_epoch": newest}
|
||||
|
||||
|
||||
async def _find_managed_root_for_path(db: AsyncSession, path: str) -> ManagedRootORM | None:
|
||||
target = os.path.normcase(os.path.normpath(str(path or "")))
|
||||
if not target:
|
||||
return None
|
||||
result = await db.execute(
|
||||
select(ManagedRootORM)
|
||||
.where(ManagedRootORM.enabled == True) # noqa: E712
|
||||
.order_by(func.length(ManagedRootORM.path).desc())
|
||||
)
|
||||
for root in result.scalars().all():
|
||||
root_path = os.path.normcase(os.path.normpath(str(root.path or "")))
|
||||
if target == root_path or target.startswith(root_path + os.sep):
|
||||
return root
|
||||
return None
|
||||
|
||||
|
||||
def _batch_name(scene_manifest: dict[str, Any]) -> str:
|
||||
raw = scene_manifest.get("batch_name") or (scene_manifest.get("metadata") or {}).get("imaging_date")
|
||||
return _safe_slug(raw, default="unknown_batch")
|
||||
|
||||
|
||||
def _standard_scene_dir(scene_manifest: dict[str, Any], storage_root: Path) -> Path:
|
||||
return storage_root / _batch_name(scene_manifest) / _safe_slug(scene_manifest.get("scene_name"))
|
||||
|
||||
|
||||
def _target_tif_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
|
||||
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
|
||||
return _standard_scene_dir(scene_manifest, storage_root) / f"{pol}_L2.tif"
|
||||
|
||||
|
||||
def _preview_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
|
||||
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
|
||||
return _standard_scene_dir(scene_manifest, storage_root) / f"preview_{pol}.png"
|
||||
|
||||
|
||||
def _quality_path(scene_manifest: dict[str, Any], asset: dict[str, Any], storage_root: Path) -> Path:
|
||||
pol = _safe_slug(asset.get("polarization"), default="UNKNOWN").upper()
|
||||
return _standard_scene_dir(scene_manifest, storage_root) / f"quality_{pol}.json"
|
||||
|
||||
|
||||
def _source_changed(asset: dict[str, Any], target_tif: Path, existing_asset: dict[str, Any] | None) -> bool:
|
||||
source_path = Path(_path_text(asset.get("path")))
|
||||
source_fp = _file_fingerprint(source_path)
|
||||
if not target_tif.is_file() or target_tif.stat().st_size <= 0:
|
||||
return True
|
||||
if not existing_asset:
|
||||
return True
|
||||
if str(existing_asset.get("source_native") or "") != str(source_path):
|
||||
return True
|
||||
if (existing_asset.get("source_fingerprint") or {}) != source_fp:
|
||||
return True
|
||||
if str(existing_asset.get("converter_version") or "") != CONVERTER_VERSION:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _existing_manifest_asset(standard_manifest: dict[str, Any] | None, polarization: str) -> dict[str, Any] | None:
|
||||
if not standard_manifest:
|
||||
return None
|
||||
target_pol = str(polarization or "").upper()
|
||||
for item in standard_manifest.get("assets") or []:
|
||||
if str(item.get("polarization") or "").upper() == target_pol:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
def _convert_native_asset_to_tif(asset: dict[str, Any], target_tif: Path) -> dict[str, Any]:
|
||||
source_path = Path(_path_text(asset.get("path")))
|
||||
if not source_path.is_file():
|
||||
raise FileNotFoundError(f"GF3 native data file does not exist: {source_path}")
|
||||
hdr_path = Path(_path_text(asset.get("hdr")))
|
||||
if not hdr_path.is_file():
|
||||
raise FileNotFoundError(f"GF3 native ENVI header does not exist: {hdr_path}")
|
||||
|
||||
target_tif.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp_tif = target_tif.with_name(f".{target_tif.name}.tmp.tif")
|
||||
if tmp_tif.exists():
|
||||
tmp_tif.unlink()
|
||||
|
||||
try:
|
||||
from osgeo import gdal
|
||||
|
||||
src_ds = gdal.Open(str(source_path), gdal.GA_ReadOnly)
|
||||
if src_ds is None:
|
||||
raise RuntimeError(f"GDAL cannot open GF3 native dataset: {source_path}")
|
||||
|
||||
creation_options = ["TILED=YES", "COMPRESS=DEFLATE", "BIGTIFF=IF_SAFER"]
|
||||
if bool(settings.SAR_ANALYSIS_OUTPUT_COG):
|
||||
creation_options.append("COPY_SRC_OVERVIEWS=YES")
|
||||
translated = gdal.Translate(
|
||||
str(tmp_tif),
|
||||
src_ds,
|
||||
format="GTiff",
|
||||
creationOptions=creation_options,
|
||||
)
|
||||
src_ds = None
|
||||
if translated is None:
|
||||
raise RuntimeError(f"GDAL Translate failed for GF3 native dataset: {source_path}")
|
||||
translated.FlushCache()
|
||||
translated = None
|
||||
except ImportError:
|
||||
import rasterio
|
||||
|
||||
with rasterio.open(source_path) as src:
|
||||
profile = src.profile.copy()
|
||||
profile.update(
|
||||
driver="GTiff",
|
||||
tiled=True,
|
||||
compress="deflate",
|
||||
BIGTIFF="IF_SAFER",
|
||||
)
|
||||
with rasterio.open(tmp_tif, "w", **profile) as dst:
|
||||
for band_idx in range(1, src.count + 1):
|
||||
for _block_index, window in src.block_windows(band_idx):
|
||||
dst.write(src.read(band_idx, window=window), band_idx, window=window)
|
||||
dst.update_tags(**src.tags())
|
||||
for band_idx in range(1, src.count + 1):
|
||||
dst.update_tags(band_idx, **src.tags(band_idx))
|
||||
|
||||
os.replace(tmp_tif, target_tif)
|
||||
|
||||
return {
|
||||
"path": str(target_tif),
|
||||
"source_native": str(source_path),
|
||||
"source_fingerprint": _file_fingerprint(source_path),
|
||||
"converter_name": CONVERTER_NAME,
|
||||
"converter_version": CONVERTER_VERSION,
|
||||
}
|
||||
|
||||
|
||||
def _raster_quality(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
except Exception as exc:
|
||||
return {"ok": False, "warning": f"rasterio unavailable: {exc}"}
|
||||
|
||||
with rasterio.open(path) as src:
|
||||
if src.height > 2048 or src.width > 2048:
|
||||
scale = min(1024 / src.width, 1024 / src.height)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
sampled = src.read(1, out_shape=(out_height, out_width), masked=True)
|
||||
else:
|
||||
sampled = src.read(1, masked=True)
|
||||
|
||||
valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)]
|
||||
bounds = src.bounds
|
||||
quality: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"driver": src.driver,
|
||||
"width": src.width,
|
||||
"height": src.height,
|
||||
"count": src.count,
|
||||
"dtype": str(src.dtypes[0]) if src.dtypes else None,
|
||||
"crs": src.crs.to_string() if src.crs else None,
|
||||
"bounds": {
|
||||
"left": bounds.left,
|
||||
"bottom": bounds.bottom,
|
||||
"right": bounds.right,
|
||||
"top": bounds.top,
|
||||
},
|
||||
"transform": list(src.transform)[:6],
|
||||
"nodata": _finite_float(src.nodata),
|
||||
"valid_sample_count": int(valid.size),
|
||||
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
|
||||
}
|
||||
if valid.size:
|
||||
quality.update(
|
||||
{
|
||||
"sample_min": float(np.nanmin(valid)),
|
||||
"sample_max": float(np.nanmax(valid)),
|
||||
"sample_mean": float(np.nanmean(valid)),
|
||||
"sample_p02": float(np.nanpercentile(valid, 2)),
|
||||
"sample_p98": float(np.nanpercentile(valid, 98)),
|
||||
}
|
||||
)
|
||||
return quality
|
||||
|
||||
|
||||
def _build_preview_png(source: Path, target: Path) -> str | None:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with rasterio.open(source) as src:
|
||||
if src.height > 1600 or src.width > 1600:
|
||||
scale = min(1600 / src.width, 1600 / src.height)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
band = src.read(1, out_shape=(out_height, out_width), masked=True)
|
||||
else:
|
||||
band = src.read(1, masked=True)
|
||||
data = band.filled(float("nan")).astype("float32")
|
||||
|
||||
valid = data[np.isfinite(data)]
|
||||
if valid.size:
|
||||
p2, p98 = np.nanpercentile(valid, [2, 98])
|
||||
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
|
||||
normalized = np.where(np.isfinite(normalized), normalized, 0)
|
||||
gray = (normalized * 255).astype("uint8")
|
||||
else:
|
||||
gray = np.zeros(data.shape, dtype="uint8")
|
||||
alpha = np.where(np.isfinite(data), 255, 0).astype("uint8")
|
||||
rgba = np.stack([gray, gray, gray, alpha], axis=-1)
|
||||
Image.fromarray(rgba, "RGBA").save(target)
|
||||
return str(target)
|
||||
|
||||
|
||||
def _build_preview_from_quicklook(source: Path | None, target: Path) -> str | None:
|
||||
if source is None or not source.is_file():
|
||||
return None
|
||||
try:
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with Image.open(source) as img:
|
||||
preview = img.copy()
|
||||
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
|
||||
preview.thumbnail((1600, 1600), resampling)
|
||||
if preview.mode in {"1", "I", "I;16", "F"}:
|
||||
preview = preview.convert("L")
|
||||
elif preview.mode not in {"L", "LA", "RGB", "RGBA"}:
|
||||
preview = preview.convert("RGB")
|
||||
preview = image_service.make_edge_dark_transparent(preview)
|
||||
preview.save(target, "PNG")
|
||||
return str(target)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _asset_quicklook_path(asset: dict[str, Any]) -> Path | None:
|
||||
text = _path_text(asset.get("quicklook"))
|
||||
if not text:
|
||||
return None
|
||||
return Path(text)
|
||||
|
||||
|
||||
def _points_look_like_lonlat(points: list[tuple[float, float]]) -> bool:
|
||||
if not points:
|
||||
return False
|
||||
for lon, lat in points:
|
||||
if not (math.isfinite(float(lon)) and math.isfinite(float(lat))):
|
||||
return False
|
||||
if not (-180.0 <= float(lon) <= 180.0 and -90.0 <= float(lat) <= 90.0):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _crs_is_geographic_lonlat(crs: Any) -> bool:
|
||||
if not crs:
|
||||
return False
|
||||
try:
|
||||
if crs.to_epsg() == 4326:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if bool(crs.is_geographic):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
wkt = str(crs.to_wkt() or "").upper()
|
||||
if "GEOGCS" in wkt and ("WGS 84" in wkt or "WORLD GEODETIC" in wkt):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
|
||||
polygon = extract_geotiff_bounds(str(path))
|
||||
if polygon and len(polygon) >= 4:
|
||||
return polygon
|
||||
try:
|
||||
import rasterio
|
||||
from rasterio.warp import transform
|
||||
|
||||
with rasterio.open(path) as src:
|
||||
if not src.crs:
|
||||
return None
|
||||
corners_xy = [
|
||||
src.transform * (0, 0),
|
||||
src.transform * (src.width, 0),
|
||||
src.transform * (src.width, src.height),
|
||||
src.transform * (0, src.height),
|
||||
]
|
||||
xs = [point[0] for point in corners_xy]
|
||||
ys = [point[1] for point in corners_xy]
|
||||
raw_points = [(float(x), float(y)) for x, y in zip(xs, ys)]
|
||||
if _crs_is_geographic_lonlat(src.crs):
|
||||
points = raw_points
|
||||
else:
|
||||
try:
|
||||
lons, lats = transform(src.crs, "EPSG:4326", xs, ys)
|
||||
points = [(float(lon), float(lat)) for lon, lat in zip(lons, lats)]
|
||||
except Exception:
|
||||
if not _points_look_like_lonlat(raw_points):
|
||||
raise
|
||||
points = raw_points
|
||||
if not _points_look_like_lonlat(points):
|
||||
return None
|
||||
points.append(points[0])
|
||||
return points
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
|
||||
if not polygon:
|
||||
return None, None
|
||||
try:
|
||||
shp = Polygon(polygon)
|
||||
if not shp.is_valid:
|
||||
shp = shp.buffer(0)
|
||||
if shp.is_valid and not shp.is_empty:
|
||||
return float(shp.centroid.x), float(shp.centroid.y)
|
||||
except Exception:
|
||||
return None, None
|
||||
return None, None
|
||||
|
||||
|
||||
def _bounds_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None, float | None, float | None]:
|
||||
if not polygon:
|
||||
return None, None, None, None
|
||||
lons = [float(point[0]) for point in polygon]
|
||||
lats = [float(point[1]) for point in polygon]
|
||||
return min(lons), min(lats), max(lons), max(lats)
|
||||
|
||||
|
||||
def _geom_from_polygon(polygon: list[tuple[float, float]] | None) -> Any | None:
|
||||
if not polygon:
|
||||
return None
|
||||
try:
|
||||
shp = Polygon(polygon)
|
||||
if not shp.is_valid:
|
||||
shp = shp.buffer(0)
|
||||
if shp.is_valid and not shp.is_empty:
|
||||
return from_shape(shp, srid=4326)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
by_pol = {str(asset.get("polarization") or "").upper(): asset for asset in assets}
|
||||
for pol in POLARIZATION_PRIORITY:
|
||||
if pol in by_pol:
|
||||
return by_pol[pol]
|
||||
return assets[0] if assets else None
|
||||
|
||||
|
||||
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
metadata = dict(scene_manifest.get("metadata") or {})
|
||||
metadata.update(
|
||||
{
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"standard_manifest": standard_manifest.get("manifest_path"),
|
||||
"standard_dir": standard_manifest.get("standard_dir"),
|
||||
"standard_assets": standard_manifest.get("assets") or [],
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
}
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
async def _upsert_source_product_asset(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
standard_manifest: dict[str, Any],
|
||||
) -> int | None:
|
||||
standard_dir_text = _path_text(standard_manifest.get("standard_dir"))
|
||||
if not standard_dir_text:
|
||||
return None
|
||||
standard_dir = Path(standard_dir_text)
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
scene_name = scene_manifest.get("scene_name") or standard_dir.name
|
||||
now = _db_now()
|
||||
root = await _find_managed_root_for_path(db, standard_dir_text)
|
||||
stats = await asyncio.to_thread(_tree_stats, standard_dir)
|
||||
asset_metadata = _json_safe(
|
||||
{
|
||||
"source": "GF3 SARscape standardized L2",
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"standard_manifest": standard_manifest.get("manifest_path"),
|
||||
"standard_dir": standard_dir_text,
|
||||
"standard_status": standard_manifest.get("status"),
|
||||
"standard_assets": standard_manifest.get("assets") or [],
|
||||
"summary": standard_manifest.get("summary") or {},
|
||||
"errors": standard_manifest.get("errors") or [],
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
}
|
||||
)
|
||||
data = {
|
||||
"asset_uid": _source_asset_uid(standard_dir_text),
|
||||
"logical_product_uid": scene_name,
|
||||
"satellite_family": "GF3",
|
||||
"satellite": "GF3",
|
||||
"source_format": SOURCE_ASSET_FORMAT,
|
||||
"product_type": metadata.get("product_type") or "SARSCAPE_L2",
|
||||
"product_level": "L2",
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": metadata.get("polarization"),
|
||||
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
|
||||
"relative_orbit": metadata.get("relative_orbit"),
|
||||
"orbit_direction": metadata.get("orbit_direction"),
|
||||
"acquisition_start_time_utc": None,
|
||||
"acquisition_stop_time_utc": None,
|
||||
"imaging_date": metadata.get("imaging_date"),
|
||||
"root_ref_id": root.id if root else None,
|
||||
"root_path": root.path if root else str(standard_dir.parent),
|
||||
"file_path": standard_dir_text,
|
||||
"archive_path": scene_manifest.get("native_dir"),
|
||||
"path_kind": _path_kind(standard_dir_text),
|
||||
"file_name": standard_dir.name,
|
||||
"file_stem": standard_dir.name,
|
||||
"file_ext": "",
|
||||
"size_bytes": stats.get("size_bytes"),
|
||||
"mtime_epoch": stats.get("mtime_epoch"),
|
||||
"checksum_status": "NOT_COMPUTED",
|
||||
"parser_name": "gf3_sarscape_standard_manifest",
|
||||
"parser_version": CONVERTER_VERSION,
|
||||
"parse_status": "OK" if standard_manifest.get("status") == "DONE" else str(standard_manifest.get("status") or "PARTIAL"),
|
||||
"parse_error": "; ".join(str(item.get("error") or item) for item in (standard_manifest.get("errors") or [])) or None,
|
||||
"parsed_at": now,
|
||||
"metadata_json": asset_metadata,
|
||||
"is_active": True,
|
||||
"missing_since": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
result = await db.execute(
|
||||
select(SourceProductAssetORM).where(
|
||||
or_(
|
||||
SourceProductAssetORM.asset_uid == data["asset_uid"],
|
||||
SourceProductAssetORM.file_path == standard_dir_text,
|
||||
)
|
||||
)
|
||||
)
|
||||
asset = result.scalars().first()
|
||||
if asset is None:
|
||||
asset = SourceProductAssetORM(**data)
|
||||
db.add(asset)
|
||||
else:
|
||||
for key, value in data.items():
|
||||
setattr(asset, key, value)
|
||||
await db.flush()
|
||||
return int(asset.id) if asset.id is not None else None
|
||||
|
||||
|
||||
async def _upsert_radar_data(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
standard_manifest: dict[str, Any],
|
||||
source_product_ref_id: int | None = None,
|
||||
) -> int | None:
|
||||
assets = standard_manifest.get("assets") or []
|
||||
default_asset = _select_default_asset(assets)
|
||||
if not default_asset:
|
||||
return None
|
||||
|
||||
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
|
||||
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
|
||||
center_lon, center_lat = _scene_center_from_polygon(polygon)
|
||||
geom = _geom_from_polygon(polygon)
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
radar_metadata = _metadata_for_radar(scene_manifest, standard_manifest)
|
||||
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
|
||||
unique_id = f"gf3_sarscape:{scene_name}"
|
||||
file_path = str(standard_manifest.get("standard_dir") or "")
|
||||
|
||||
data_to_upsert = {
|
||||
"unique_id": unique_id,
|
||||
"satellite": "GF3",
|
||||
"satellite_family": "GF3",
|
||||
"imaging_date": metadata.get("imaging_date"),
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": ",".join(
|
||||
pol
|
||||
for pol in POLARIZATION_PRIORITY
|
||||
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
|
||||
)
|
||||
or metadata.get("polarization"),
|
||||
"scene_center_lon": metadata.get("scene_center_lon") if metadata.get("scene_center_lon") is not None else center_lon,
|
||||
"scene_center_lat": metadata.get("scene_center_lat") if metadata.get("scene_center_lat") is not None else center_lat,
|
||||
"product_level": "L2",
|
||||
"product_unique_id": metadata.get("product_unique_id"),
|
||||
"source_format": SOURCE_ASSET_FORMAT,
|
||||
"source_product_ref_id": source_product_ref_id,
|
||||
"image_data_format": "GEOTIFF",
|
||||
"geocoded_flag": True,
|
||||
"metadata_json": radar_metadata,
|
||||
"file_path": file_path,
|
||||
"has_orbit_data": False,
|
||||
"orbit_file_path": None,
|
||||
"is_envi_processed": True,
|
||||
"coverage_polygon": polygon,
|
||||
"geom": geom,
|
||||
"min_lon": min_lon,
|
||||
"min_lat": min_lat,
|
||||
"max_lon": max_lon,
|
||||
"max_lat": max_lat,
|
||||
}
|
||||
|
||||
result = await db.execute(
|
||||
select(RadarDataORM).where(
|
||||
or_(
|
||||
RadarDataORM.unique_id == unique_id,
|
||||
RadarDataORM.file_path == file_path,
|
||||
)
|
||||
)
|
||||
)
|
||||
radar = result.scalars().first()
|
||||
if radar is None:
|
||||
radar = RadarDataORM(**data_to_upsert)
|
||||
db.add(radar)
|
||||
else:
|
||||
for key, value in data_to_upsert.items():
|
||||
setattr(radar, key, value)
|
||||
await db.flush()
|
||||
return int(radar.id) if radar.id is not None else None
|
||||
|
||||
|
||||
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
|
||||
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
|
||||
scene = result.scalar_one_or_none()
|
||||
if scene:
|
||||
return scene
|
||||
scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING")
|
||||
db.add(scene)
|
||||
await db.flush()
|
||||
return scene
|
||||
|
||||
|
||||
async def _register_analysis_ready(
|
||||
db: AsyncSession,
|
||||
radar_id: int,
|
||||
scene_manifest: dict[str, Any],
|
||||
standard_manifest: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
radar = await db.get(RadarDataORM, radar_id)
|
||||
if not radar:
|
||||
return None
|
||||
assets = standard_manifest.get("assets") or []
|
||||
default_asset = _select_default_asset(assets)
|
||||
if not default_asset:
|
||||
return None
|
||||
scene = await _get_or_create_scene(db, radar_id)
|
||||
return await register_analysis_ready_tif(
|
||||
db=db,
|
||||
scene=scene,
|
||||
radar=radar,
|
||||
source_tif_path=str(default_asset.get("path") or ""),
|
||||
engine="gf3_sarscape",
|
||||
profile=CONVERTER_NAME,
|
||||
backscatter_unit="unknown",
|
||||
polarization=str(default_asset.get("polarization") or "").upper() or None,
|
||||
preview_source_path=str(default_asset.get("preview") or "") or None,
|
||||
metadata={
|
||||
"source": "GF3 SARscape native _geo",
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"standard_manifest": standard_manifest.get("manifest_path"),
|
||||
"available_polarization": [asset.get("polarization") for asset in assets],
|
||||
"standard_assets": assets,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def standardize_scene_manifest(
|
||||
scene_manifest: dict[str, Any],
|
||||
*,
|
||||
storage_root: str | Path | None = None,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Convert one native scene manifest to GeoTIFF assets."""
|
||||
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
|
||||
out_dir = _standard_scene_dir(scene_manifest, root)
|
||||
manifest_path = out_dir / STANDARD_MANIFEST_NAME
|
||||
existing_manifest = _read_json(manifest_path)
|
||||
|
||||
converted = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
output_assets: list[dict[str, Any]] = []
|
||||
errors: list[dict[str, str]] = []
|
||||
|
||||
for asset in scene_manifest.get("assets") or []:
|
||||
if not asset.get("complete"):
|
||||
continue
|
||||
pol = str(asset.get("polarization") or "UNKNOWN").upper()
|
||||
target_tif = _target_tif_path(scene_manifest, asset, root)
|
||||
existing_asset = _existing_manifest_asset(existing_manifest, pol)
|
||||
|
||||
try:
|
||||
if force or _source_changed(asset, target_tif, existing_asset):
|
||||
convert_info = _convert_native_asset_to_tif(asset, target_tif)
|
||||
converted += 1
|
||||
status = "converted"
|
||||
else:
|
||||
convert_info = {
|
||||
"path": str(target_tif),
|
||||
"source_native": str(Path(_path_text(asset.get("path")))),
|
||||
"source_fingerprint": _file_fingerprint(Path(_path_text(asset.get("path")))),
|
||||
"converter_name": CONVERTER_NAME,
|
||||
"converter_version": CONVERTER_VERSION,
|
||||
}
|
||||
skipped += 1
|
||||
status = "skipped"
|
||||
|
||||
quality = _raster_quality(target_tif)
|
||||
quality_file = _quality_path(scene_manifest, asset, root)
|
||||
_write_json(quality_file, quality)
|
||||
preview_target = _preview_path(scene_manifest, asset, root)
|
||||
quicklook_path = _asset_quicklook_path(asset)
|
||||
preview = _build_preview_from_quicklook(quicklook_path, preview_target)
|
||||
preview_source = str(quicklook_path) if preview and quicklook_path else str(target_tif)
|
||||
if not preview:
|
||||
preview = _build_preview_png(target_tif, preview_target)
|
||||
output_assets.append(
|
||||
{
|
||||
"polarization": pol,
|
||||
"role": "analysis_tif",
|
||||
"path": str(target_tif),
|
||||
"source_native": convert_info["source_native"],
|
||||
"source_fingerprint": convert_info["source_fingerprint"],
|
||||
"converter_name": CONVERTER_NAME,
|
||||
"converter_version": CONVERTER_VERSION,
|
||||
"quality": str(quality_file),
|
||||
"preview": preview,
|
||||
"preview_source": preview_source,
|
||||
"status": status,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
errors.append({"polarization": pol, "source_native": str(asset.get("path") or ""), "error": str(exc)})
|
||||
|
||||
status = "DONE" if output_assets and failed == 0 else ("PARTIAL" if output_assets else "FAILED")
|
||||
standard_manifest = {
|
||||
"schema": STANDARD_MANIFEST_SCHEMA,
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"batch_name": scene_manifest.get("batch_name"),
|
||||
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"standard_dir": str(out_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"status": status,
|
||||
"converter": {"name": CONVERTER_NAME, "version": CONVERTER_VERSION},
|
||||
"assets": output_assets,
|
||||
"summary": {
|
||||
"converted": converted,
|
||||
"skipped": skipped,
|
||||
"failed": failed,
|
||||
},
|
||||
"errors": errors,
|
||||
}
|
||||
_write_json(manifest_path, standard_manifest)
|
||||
return standard_manifest
|
||||
|
||||
|
||||
async def standardize_gf3_sarscape_native_roots(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
native_dirs: list[str] | None = None,
|
||||
storage_root: str | None = None,
|
||||
force: bool = False,
|
||||
register: bool = True,
|
||||
progress_callback: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan native roots, convert complete assets, and register standard scenes."""
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
native_dirs,
|
||||
write_manifest=True,
|
||||
)
|
||||
scenes = inventory.get("scenes") or []
|
||||
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
|
||||
|
||||
converted_scenes = 0
|
||||
partial_scenes = 0
|
||||
failed_scenes = 0
|
||||
skipped_assets = 0
|
||||
converted_assets = 0
|
||||
failed_assets = 0
|
||||
registered = 0
|
||||
analysis_ready = 0
|
||||
scene_results: list[dict[str, Any]] = []
|
||||
|
||||
total = len(ready_scenes)
|
||||
for idx, scene_manifest in enumerate(ready_scenes):
|
||||
if progress_callback:
|
||||
pct = 10 + int((idx / max(total, 1)) * 80)
|
||||
progress_callback(pct, f"标准化 GF3 SARscape 原生结果 {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
|
||||
|
||||
standard_manifest = await asyncio.to_thread(
|
||||
standardize_scene_manifest,
|
||||
scene_manifest,
|
||||
storage_root=storage_root,
|
||||
force=force,
|
||||
)
|
||||
summary = standard_manifest.get("summary") or {}
|
||||
converted_assets += int(summary.get("converted") or 0)
|
||||
skipped_assets += int(summary.get("skipped") or 0)
|
||||
failed_assets += int(summary.get("failed") or 0)
|
||||
status = standard_manifest.get("status")
|
||||
if status == "DONE":
|
||||
converted_scenes += 1
|
||||
elif status == "PARTIAL":
|
||||
partial_scenes += 1
|
||||
else:
|
||||
failed_scenes += 1
|
||||
|
||||
radar_id = None
|
||||
source_asset_id = None
|
||||
analysis_manifest_path = None
|
||||
if register and status in {"DONE", "PARTIAL"}:
|
||||
source_asset_id = await _upsert_source_product_asset(db, scene_manifest, standard_manifest)
|
||||
radar_id = await _upsert_radar_data(
|
||||
db,
|
||||
scene_manifest,
|
||||
standard_manifest,
|
||||
source_product_ref_id=source_asset_id,
|
||||
)
|
||||
if radar_id:
|
||||
registered += 1
|
||||
analysis_manifest = await _register_analysis_ready(db, radar_id, scene_manifest, standard_manifest)
|
||||
if analysis_manifest:
|
||||
analysis_ready += 1
|
||||
analysis_manifest_path = analysis_manifest.get("analysis_dir")
|
||||
await db.commit()
|
||||
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"native_status": scene_manifest.get("status"),
|
||||
"standard_status": status,
|
||||
"standard_manifest": standard_manifest.get("manifest_path"),
|
||||
"source_asset_id": source_asset_id,
|
||||
"radar_id": radar_id,
|
||||
"analysis_manifest_path": analysis_manifest_path,
|
||||
"summary": summary,
|
||||
"errors": standard_manifest.get("errors") or [],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": failed_scenes == 0 and failed_assets == 0,
|
||||
"inventory": {
|
||||
key: value
|
||||
for key, value in inventory.items()
|
||||
if key != "scenes"
|
||||
},
|
||||
"scene_count": len(scenes),
|
||||
"ready_scene_count": len(ready_scenes),
|
||||
"converted_scenes": converted_scenes,
|
||||
"partial_scenes": partial_scenes,
|
||||
"failed_scenes": failed_scenes,
|
||||
"converted_assets": converted_assets,
|
||||
"skipped_assets": skipped_assets,
|
||||
"failed_assets": failed_assets,
|
||||
"registered": registered,
|
||||
"analysis_ready": analysis_ready,
|
||||
"scenes": scene_results,
|
||||
}
|
||||
@@ -879,6 +879,36 @@ def _probe_directory_status(path: str) -> Dict[str, Any]:
|
||||
return payload
|
||||
|
||||
|
||||
def _probe_file_status(path: str) -> Dict[str, Any]:
|
||||
normalized = str(path or "").strip()
|
||||
payload = {
|
||||
"path": normalized,
|
||||
"exists": False,
|
||||
"accessible": False,
|
||||
"error": None,
|
||||
}
|
||||
if not normalized:
|
||||
payload["error"] = "empty path"
|
||||
return payload
|
||||
|
||||
try:
|
||||
if os.path.isfile(normalized):
|
||||
payload["exists"] = True
|
||||
try:
|
||||
with open(normalized, "rb") as stream:
|
||||
stream.read(1)
|
||||
payload["accessible"] = True
|
||||
except Exception as exc:
|
||||
payload["error"] = str(exc)
|
||||
return payload
|
||||
|
||||
os.stat(normalized)
|
||||
payload["error"] = "path exists but is not a file"
|
||||
except Exception as exc:
|
||||
payload["error"] = str(exc)
|
||||
return payload
|
||||
|
||||
|
||||
async def _check_source_roots() -> Dict[str, Any]:
|
||||
items = []
|
||||
|
||||
@@ -908,11 +938,34 @@ async def _check_source_roots() -> Dict[str, Any]:
|
||||
status["role"] = "gf3_l1a_source"
|
||||
items.append(status)
|
||||
|
||||
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
|
||||
status = _probe_directory_status(path)
|
||||
status["role"] = "gf3_sarscape_native"
|
||||
items.append(status)
|
||||
|
||||
for path in split_env_paths(settings.GF3_STORAGE_DIRS):
|
||||
status = _probe_directory_status(path)
|
||||
status["role"] = "gf3_l2_storage"
|
||||
items.append(status)
|
||||
|
||||
wrapper_exe = str(settings.GF3_SARSCAPE_WRAPPER_EXE or "").strip()
|
||||
if wrapper_exe:
|
||||
status = _probe_file_status(wrapper_exe)
|
||||
status["role"] = "gf3_sarscape_wrapper"
|
||||
items.append(status)
|
||||
|
||||
idlrt_path = str(settings.GF3_SARSCAPE_IDLRT_PATH or "").strip()
|
||||
if idlrt_path:
|
||||
status = _probe_file_status(idlrt_path)
|
||||
status["role"] = "gf3_sarscape_idlrt"
|
||||
items.append(status)
|
||||
|
||||
dem_path = str(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or "").strip()
|
||||
if dem_path:
|
||||
status = _probe_file_status(dem_path)
|
||||
status["role"] = "gf3_sarscape_dem"
|
||||
items.append(status)
|
||||
|
||||
configured_count = len(items)
|
||||
accessible_count = sum(1 for item in items if item.get("accessible"))
|
||||
inaccessible_count = configured_count - accessible_count
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
from collections import deque
|
||||
from typing import Tuple, Optional, Dict, Any, List
|
||||
from PIL import Image
|
||||
import rasterio
|
||||
@@ -334,6 +335,55 @@ class ImageService:
|
||||
"""
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
image.save(output_path, format='WEBP', quality=quality)
|
||||
|
||||
@staticmethod
|
||||
def make_edge_dark_transparent(
|
||||
image: Image.Image,
|
||||
*,
|
||||
threshold: int = 6,
|
||||
) -> Image.Image:
|
||||
"""Make edge-connected near-black preview background transparent."""
|
||||
rgba = image.convert("RGBA")
|
||||
arr = np.array(rgba, dtype=np.uint8, copy=True)
|
||||
if arr.ndim != 3 or arr.shape[2] < 4:
|
||||
return rgba
|
||||
|
||||
alpha = arr[:, :, 3]
|
||||
dark = (alpha > 0) & (arr[:, :, :3].max(axis=2) <= int(threshold))
|
||||
if not dark.any():
|
||||
return rgba
|
||||
|
||||
h, w = dark.shape
|
||||
edge = np.zeros_like(dark, dtype=bool)
|
||||
edge[0, :] = dark[0, :]
|
||||
edge[h - 1, :] = dark[h - 1, :]
|
||||
edge[:, 0] |= dark[:, 0]
|
||||
edge[:, w - 1] |= dark[:, w - 1]
|
||||
if not edge.any():
|
||||
return rgba
|
||||
|
||||
visited = np.zeros_like(dark, dtype=bool)
|
||||
ys, xs = np.where(edge)
|
||||
queue = deque(zip(ys.tolist(), xs.tolist()))
|
||||
visited[ys, xs] = True
|
||||
|
||||
while queue:
|
||||
y, x = queue.popleft()
|
||||
if y > 0 and dark[y - 1, x] and not visited[y - 1, x]:
|
||||
visited[y - 1, x] = True
|
||||
queue.append((y - 1, x))
|
||||
if y + 1 < h and dark[y + 1, x] and not visited[y + 1, x]:
|
||||
visited[y + 1, x] = True
|
||||
queue.append((y + 1, x))
|
||||
if x > 0 and dark[y, x - 1] and not visited[y, x - 1]:
|
||||
visited[y, x - 1] = True
|
||||
queue.append((y, x - 1))
|
||||
if x + 1 < w and dark[y, x + 1] and not visited[y, x + 1]:
|
||||
visited[y, x + 1] = True
|
||||
queue.append((y, x + 1))
|
||||
|
||||
arr[:, :, 3][visited] = 0
|
||||
return Image.fromarray(arr, "RGBA")
|
||||
|
||||
@staticmethod
|
||||
def create_cached_image(
|
||||
@@ -497,12 +547,12 @@ class ImageService:
|
||||
|
||||
@staticmethod
|
||||
def _warp_preview_to_geo_bbox(
|
||||
source_rgb: np.ndarray,
|
||||
source_rgba: np.ndarray,
|
||||
inverse_h: np.ndarray,
|
||||
bbox: Tuple[float, float, float, float],
|
||||
out_size: Tuple[int, int],
|
||||
) -> np.ndarray:
|
||||
src_h, src_w = source_rgb.shape[:2]
|
||||
src_h, src_w = source_rgba.shape[:2]
|
||||
out_w, out_h = out_size
|
||||
min_lon, min_lat, max_lon, max_lat = bbox
|
||||
lon_span = max_lon - min_lon
|
||||
@@ -557,7 +607,7 @@ class ImageService:
|
||||
du = (u_valid - x0).astype(np.float32)
|
||||
dv = (v_valid - y0).astype(np.float32)
|
||||
|
||||
src_float = source_rgb.astype(np.float32, copy=False)
|
||||
src_float = source_rgba.astype(np.float32, copy=False)
|
||||
s00 = src_float[y0, x0]
|
||||
s10 = src_float[y0, x1]
|
||||
s01 = src_float[y1, x0]
|
||||
@@ -568,15 +618,14 @@ class ImageService:
|
||||
+ s01 * (1 - du)[:, None] * dv[:, None]
|
||||
+ s11 * du[:, None] * dv[:, None]
|
||||
)
|
||||
rgb = np.clip(samples, 0, 255).astype(np.uint8)
|
||||
rgba = np.clip(samples, 0, 255).astype(np.uint8)
|
||||
else:
|
||||
nearest_x = np.clip(np.round(u_valid).astype(np.int32), 0, src_w - 1)
|
||||
nearest_y = np.clip(np.round(v_valid).astype(np.int32), 0, src_h - 1)
|
||||
rgb = source_rgb[nearest_y, nearest_x]
|
||||
rgba = source_rgba[nearest_y, nearest_x]
|
||||
|
||||
flat = output.reshape(-1, 4)
|
||||
flat[valid_idx, :3] = rgb
|
||||
flat[valid_idx, 3] = 255
|
||||
flat[valid_idx] = rgba
|
||||
return output
|
||||
|
||||
@staticmethod
|
||||
@@ -608,9 +657,10 @@ class ImageService:
|
||||
return False, "invalid_bbox"
|
||||
|
||||
with Image.open(source_image_path) as image:
|
||||
source_rgb = np.asarray(image.convert("RGB"), dtype=np.uint8)
|
||||
source = ImageService.make_edge_dark_transparent(image)
|
||||
source_rgba = np.asarray(source, dtype=np.uint8)
|
||||
|
||||
src_h, src_w = source_rgb.shape[:2]
|
||||
src_h, src_w = source_rgba.shape[:2]
|
||||
if src_h < 1 or src_w < 1:
|
||||
return False, "invalid_source_image_size"
|
||||
|
||||
@@ -633,7 +683,7 @@ class ImageService:
|
||||
return False, "homography_invert_failed"
|
||||
|
||||
warped_rgba = ImageService._warp_preview_to_geo_bbox(
|
||||
source_rgb=source_rgb,
|
||||
source_rgba=source_rgba,
|
||||
inverse_h=inverse_h,
|
||||
bbox=bbox,
|
||||
out_size=out_size,
|
||||
@@ -672,7 +722,7 @@ class ImageService:
|
||||
)
|
||||
|
||||
with Image.open(source_image_path) as image:
|
||||
image = image.convert("RGB")
|
||||
image = ImageService.make_edge_dark_transparent(image)
|
||||
image.thumbnail(max_size, Image.Resampling.LANCZOS)
|
||||
ImageService.save_image_as_webp(image, cache_path, quality=82)
|
||||
return True
|
||||
|
||||
@@ -86,6 +86,9 @@ JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION"
|
||||
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
|
||||
JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
@@ -3455,7 +3458,8 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
raise ValueError("水体检测缺少输入路径 input_path")
|
||||
|
||||
output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}"
|
||||
output_dir = os.path.join(os.path.dirname(input_path), output_name)
|
||||
output_root = settings.WATER_RESULTS_DIR or os.path.join(settings.BACKEND_DIR, "water_results")
|
||||
output_dir = os.path.join(output_root, output_name)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...")
|
||||
@@ -3496,6 +3500,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
det.threshold_value = result.get("threshold_value")
|
||||
det.metadata_json = {
|
||||
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
|
||||
"value_transform": result.get("value_transform"),
|
||||
"job_id": job.job_id,
|
||||
}
|
||||
else:
|
||||
@@ -3518,6 +3523,7 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
mirror.task_id = job.task_id
|
||||
mirror.metadata_json = {
|
||||
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
|
||||
"value_transform": result.get("value_transform"),
|
||||
"legacy_detection_id": int(record_id),
|
||||
"job_id": job.job_id,
|
||||
}
|
||||
@@ -3781,6 +3787,330 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_SARSCAPE_SYNC requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
native_dirs = payload.get("native_dirs") or []
|
||||
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
|
||||
if not native_dirs:
|
||||
raise ValueError("GF3_SARSCAPE_SYNC: native_dirs is empty")
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
|
||||
|
||||
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.update_task(job.task_id, progress=progress, message=message),
|
||||
loop,
|
||||
)
|
||||
def _swallow_progress_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape] progress callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_progress_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await standardize_gf3_sarscape_native_roots(
|
||||
db,
|
||||
native_dirs=native_dirs,
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=f"GF3 SARscape 标准化失败: {exc}",
|
||||
)
|
||||
raise
|
||||
|
||||
message = (
|
||||
"GF3 SARscape 标准化完成: "
|
||||
f"发现 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
|
||||
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
|
||||
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
|
||||
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
|
||||
from .gf3_sarscape_production_service import (
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
run_gf3_sarscape_production,
|
||||
)
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
source_dirs = payload.get("source_dirs") or []
|
||||
native_dirs = payload.get("native_dirs") or []
|
||||
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
|
||||
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
|
||||
if not source_dirs:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
|
||||
if not native_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
|
||||
|
||||
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.update_task(job.task_id, progress=progress, message=message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_progress_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_progress_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
def _log_cb(level: str, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.add_log(job.task_id, level, message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_log_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_log_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
async def _production_keepalive() -> None:
|
||||
progress = 8
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
progress = min(68, progress + 1)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress,
|
||||
message="GF3 SARscape production is still running...",
|
||||
)
|
||||
|
||||
try:
|
||||
production_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
run_gf3_sarscape_production,
|
||||
source_dirs=source_dirs,
|
||||
native_root=native_root,
|
||||
wrapper_exe=payload.get("wrapper_exe"),
|
||||
dem_path=payload.get("dem_path"),
|
||||
idlrt_path=payload.get("idlrt_path"),
|
||||
polarizations=payload.get("polarizations"),
|
||||
archive_exts=payload.get("archive_exts") or [],
|
||||
max_archives_per_run=payload.get("max_archives_per_run"),
|
||||
timeout_seconds=payload.get("timeout_seconds"),
|
||||
keep_extracted=payload.get("keep_extracted"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
)
|
||||
keepalive_task = asyncio.create_task(_production_keepalive())
|
||||
try:
|
||||
production_result = await production_task
|
||||
finally:
|
||||
keepalive_task.cancel()
|
||||
try:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
standardize_result: Dict[str, Any] = {}
|
||||
if bool(payload.get("auto_standardize", True)):
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=72,
|
||||
message="GF3 SARscape production finished; standardizing native _geo outputs...",
|
||||
)
|
||||
async with AsyncSessionLocal() as db:
|
||||
standardize_result = await standardize_gf3_sarscape_native_roots(
|
||||
db,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force_standardize", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
|
||||
)
|
||||
|
||||
cleanup_result: Dict[str, Any] = {}
|
||||
production_ok = int(production_result.get("failed_count") or 0) == 0
|
||||
standardize_ok = (
|
||||
not standardize_result
|
||||
or (
|
||||
int(standardize_result.get("failed_assets") or 0) == 0
|
||||
and int(standardize_result.get("failed_scenes") or 0) == 0
|
||||
)
|
||||
)
|
||||
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=90,
|
||||
message="Cleaning GF3 SARscape intermediate files...",
|
||||
)
|
||||
cleanup_result = await asyncio.to_thread(
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
|
||||
dry_run=bool(payload.get("cleanup_dry_run", False)),
|
||||
max_scenes=payload.get("cleanup_max_scenes"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
|
||||
)
|
||||
elif bool(payload.get("clean_after_success", True)):
|
||||
_log_cb(
|
||||
"WARNING",
|
||||
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=f"GF3 SARscape production failed: {exc}",
|
||||
)
|
||||
raise
|
||||
|
||||
failed_count = int(production_result.get("failed_count") or 0)
|
||||
failed_assets = int(standardize_result.get("failed_assets") or 0)
|
||||
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
|
||||
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
|
||||
message = (
|
||||
"GF3 SARscape production chain finished: "
|
||||
f"found={int(production_result.get('found_count') or 0)}, "
|
||||
f"produced={int(production_result.get('processed_count') or 0)}, "
|
||||
f"skipped={int(production_result.get('skipped_count') or 0)}, "
|
||||
f"failed={failed_count}, "
|
||||
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
|
||||
f"registered={int(standardize_result.get('registered') or 0)}, "
|
||||
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
|
||||
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
|
||||
"""Clean GF3 SARscape intermediate files from native pool."""
|
||||
from .gf3_sarscape_production_service import cleanup_gf3_sarscape_native_pool
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_SARSCAPE_CLEAN requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
native_dirs = payload.get("native_dirs") or []
|
||||
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
|
||||
if not native_dirs:
|
||||
raise ValueError("GF3_SARSCAPE_CLEAN: native_dirs is empty")
|
||||
|
||||
await task_service.start_task(job.task_id, message="GF3 SARscape native cleanup starting...")
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.update_task(job.task_id, progress=progress, message=message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_progress_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Clean] progress callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_progress_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
def _log_cb(level: str, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.add_log(job.task_id, level, message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_log_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Clean] log callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_log_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
native_dirs=native_dirs,
|
||||
storage_root=storage_root,
|
||||
require_standardized=bool(payload.get("require_standardized", True)),
|
||||
dry_run=bool(payload.get("dry_run", False)),
|
||||
max_scenes=payload.get("max_scenes"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=f"GF3 SARscape native cleanup failed: {exc}",
|
||||
)
|
||||
raise
|
||||
|
||||
status = "FAILED" if int(result.get("error_scene_count") or 0) else "COMPLETED"
|
||||
message = (
|
||||
"GF3 SARscape native cleanup finished: "
|
||||
f"scenes={int(result.get('scene_count') or 0)}, "
|
||||
f"cleaned={int(result.get('cleaned_scene_count') or 0)}, "
|
||||
f"skipped={int(result.get('skipped_scene_count') or 0)}, "
|
||||
f"errors={int(result.get('error_scene_count') or 0)}, "
|
||||
f"bytes={int(result.get('bytes_deleted') or 0)}, "
|
||||
f"dry_run={bool(result.get('dry_run'))}"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status=status, progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_publish_dinsar_products_clean(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("PUBLISH_DINSAR_PRODUCTS requires task_id for progress tracking.")
|
||||
@@ -4671,6 +5001,9 @@ _HANDLERS = {
|
||||
JOB_TYPE_GF3_PROCESS: _handle_gf3_process,
|
||||
JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack,
|
||||
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
|
||||
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
|
||||
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
|
||||
JOB_TYPE_SBAS_INTERFEROGRAMS: _handle_sbas_interferograms,
|
||||
|
||||
@@ -262,6 +262,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_SARSCAPE_NATIVE_DIRS",
|
||||
paths=split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS),
|
||||
root_role="source_pool_gf3_sarscape_native",
|
||||
display_prefix="GF3 SARscape Native Pool",
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="SAR_ANALYSIS_READY_ROOT",
|
||||
|
||||
@@ -20,6 +20,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from ..config import settings
|
||||
from ..models import RadarDataORM, SARSceneGeoORM
|
||||
from ..utils import normalize_satellite_family
|
||||
from .image_service import image_service
|
||||
|
||||
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
_POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
||||
@@ -82,7 +83,25 @@ def scene_analysis_dir(
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
||||
json.dump(_json_safe(payload), stream, ensure_ascii=False, indent=2, default=str, allow_nan=False)
|
||||
|
||||
|
||||
def _json_safe(value: Any) -> Any:
|
||||
if isinstance(value, float):
|
||||
return value if math.isfinite(value) else None
|
||||
if isinstance(value, dict):
|
||||
return {key: _json_safe(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [_json_safe(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _finite_float(value: Any) -> float | None:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _link_or_copy(source: Path, target: Path) -> str:
|
||||
@@ -171,7 +190,7 @@ def _raster_quality(path: Path) -> dict[str, Any]:
|
||||
"top": bounds.top,
|
||||
},
|
||||
"transform": list(transform)[:6],
|
||||
"nodata": src.nodata,
|
||||
"nodata": _finite_float(src.nodata),
|
||||
"valid_sample_count": int(valid.size),
|
||||
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
|
||||
}
|
||||
@@ -231,6 +250,7 @@ def _build_preview_png(source: Path, target: Path) -> str | None:
|
||||
if valid.size:
|
||||
p2, p98 = np.nanpercentile(valid, [2, 98])
|
||||
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
|
||||
normalized = np.where(np.isfinite(normalized), normalized, 0)
|
||||
gray = (normalized * 255).astype("uint8")
|
||||
else:
|
||||
gray = np.zeros(data.shape, dtype="uint8")
|
||||
@@ -240,6 +260,31 @@ def _build_preview_png(source: Path, target: Path) -> str | None:
|
||||
return str(target)
|
||||
|
||||
|
||||
def _build_preview_from_existing(source: Path | None, target: Path) -> str | None:
|
||||
if source is None or not source.is_file():
|
||||
return None
|
||||
try:
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with Image.open(source) as img:
|
||||
preview = img.copy()
|
||||
resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS")
|
||||
preview.thumbnail((1600, 1600), resampling)
|
||||
if preview.mode in {"1", "I", "I;16", "F"}:
|
||||
preview = preview.convert("L")
|
||||
elif preview.mode not in {"L", "LA", "RGB", "RGBA"}:
|
||||
preview = preview.convert("RGB")
|
||||
preview = image_service.make_edge_dark_transparent(preview)
|
||||
preview.save(target, "PNG")
|
||||
return str(target)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
|
||||
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
|
||||
scene = result.scalar_one_or_none()
|
||||
@@ -262,6 +307,7 @@ async def register_analysis_ready_tif(
|
||||
backscatter_unit: str,
|
||||
polarization: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
preview_source_path: str | None = None,
|
||||
copy_mode: str = "link_or_copy",
|
||||
) -> dict[str, Any]:
|
||||
source = Path(os.path.normpath(str(source_tif_path or "").strip()))
|
||||
@@ -283,7 +329,10 @@ async def register_analysis_ready_tif(
|
||||
transfer = _link_or_copy(source, target_tif)
|
||||
|
||||
quality = _raster_quality(target_tif)
|
||||
preview_path = _build_preview_png(target_tif, out_dir / "preview.png")
|
||||
preview_source = Path(os.path.normpath(preview_source_path)) if preview_source_path else None
|
||||
preview_path = _build_preview_from_existing(preview_source, out_dir / "preview.png")
|
||||
if not preview_path:
|
||||
preview_path = _build_preview_png(target_tif, out_dir / "preview.png")
|
||||
manifest = {
|
||||
"scene_id": scene.id,
|
||||
"radar_data_id": scene.radar_data_id,
|
||||
@@ -296,6 +345,7 @@ async def register_analysis_ready_tif(
|
||||
"backscatter_unit": backscatter_unit,
|
||||
"polarization": polarization,
|
||||
"transfer": transfer,
|
||||
"preview_source_path": str(preview_source) if preview_path and preview_source else None,
|
||||
"metadata": metadata or {},
|
||||
"quality": quality,
|
||||
}
|
||||
@@ -309,14 +359,9 @@ async def register_analysis_ready_tif(
|
||||
scene.analysis_engine = engine
|
||||
scene.analysis_profile = profile
|
||||
scene.analysis_backscatter_unit = backscatter_unit
|
||||
nodata_value = quality.get("nodata")
|
||||
scene.analysis_nodata_value = (
|
||||
float(nodata_value)
|
||||
if nodata_value is not None
|
||||
else float(settings.SAR_ANALYSIS_NODATA_VALUE)
|
||||
)
|
||||
scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")}
|
||||
scene.analysis_quality_json = quality
|
||||
scene.analysis_nodata_value = _finite_float(quality.get("nodata")) or float(settings.SAR_ANALYSIS_NODATA_VALUE)
|
||||
scene.analysis_metadata_json = _json_safe({**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")})
|
||||
scene.analysis_quality_json = _json_safe(quality)
|
||||
scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m
|
||||
scene.status = "DONE"
|
||||
scene.error_msg = None
|
||||
|
||||
@@ -10,6 +10,7 @@ import logging
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
@@ -21,6 +22,65 @@ logger = logging.getLogger(__name__)
|
||||
# SRTM HGT helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _otsu_threshold(values: np.ndarray, bins: int = 512) -> float:
|
||||
"""Compute Otsu's threshold without depending on scikit-image."""
|
||||
finite_values = values[np.isfinite(values)]
|
||||
if finite_values.size == 0:
|
||||
raise ValueError("No finite pixels available for Otsu threshold")
|
||||
|
||||
min_value = float(np.min(finite_values))
|
||||
max_value = float(np.max(finite_values))
|
||||
if min_value == max_value:
|
||||
return min_value
|
||||
|
||||
counts, edges = np.histogram(finite_values, bins=bins, range=(min_value, max_value))
|
||||
centers = (edges[:-1] + edges[1:]) / 2.0
|
||||
total = float(counts.sum())
|
||||
if total <= 0:
|
||||
return min_value
|
||||
|
||||
weight_background = np.cumsum(counts).astype(np.float64)
|
||||
weight_foreground = total - weight_background
|
||||
cumulative_mean = np.cumsum(counts * centers)
|
||||
total_mean = cumulative_mean[-1]
|
||||
|
||||
valid = (weight_background > 0) & (weight_foreground > 0)
|
||||
between = np.zeros_like(centers, dtype=np.float64)
|
||||
mean_background = np.zeros_like(centers, dtype=np.float64)
|
||||
mean_foreground = np.zeros_like(centers, dtype=np.float64)
|
||||
mean_background[valid] = cumulative_mean[valid] / weight_background[valid]
|
||||
mean_foreground[valid] = (total_mean - cumulative_mean[valid]) / weight_foreground[valid]
|
||||
between[valid] = (
|
||||
weight_background[valid]
|
||||
* weight_foreground[valid]
|
||||
* (mean_background[valid] - mean_foreground[valid]) ** 2
|
||||
)
|
||||
return float(centers[int(np.argmax(between))])
|
||||
|
||||
|
||||
def _disk_structure(radius: int) -> np.ndarray:
|
||||
y, x = np.ogrid[-radius: radius + 1, -radius: radius + 1]
|
||||
return (x * x + y * y) <= radius * radius
|
||||
|
||||
|
||||
def _prepare_detection_image(img: np.ndarray, valid: np.ndarray) -> tuple[np.ndarray, np.ndarray, str]:
|
||||
"""Normalize SAR values for water thresholding and keep the original mask."""
|
||||
working = img.astype(np.float32, copy=True)
|
||||
working[~valid] = np.nan
|
||||
valid_values = working[valid]
|
||||
if valid_values.size == 0:
|
||||
return working, valid, "raw"
|
||||
|
||||
min_value = float(np.nanmin(valid_values))
|
||||
p99_value = float(np.nanpercentile(valid_values, 99))
|
||||
max_value = float(np.nanmax(valid_values))
|
||||
if min_value >= 0.0 and (p99_value > 1.0 or max_value > 5.0):
|
||||
positive_valid = valid & (working > 0)
|
||||
converted = np.full_like(working, np.nan, dtype=np.float32)
|
||||
converted[positive_valid] = 10.0 * np.log10(np.maximum(working[positive_valid], 1e-12))
|
||||
return converted, positive_valid, "linear_to_db"
|
||||
return working, valid, "raw"
|
||||
|
||||
def _read_hgt(filepath: str) -> np.ndarray:
|
||||
"""Read a single SRTM .hgt file. Auto-detect SRTM1 (3601) vs SRTM3 (1201)."""
|
||||
file_size = os.path.getsize(filepath)
|
||||
@@ -96,6 +156,70 @@ def _load_srtm3_dem(
|
||||
return mosaic, dem_bounds
|
||||
|
||||
|
||||
def _candidate_dem_paths(dem_path: str) -> list[str]:
|
||||
text = str(dem_path or "").strip()
|
||||
if not text:
|
||||
return []
|
||||
path = Path(text)
|
||||
candidates: list[Path] = []
|
||||
for suffix in (".vrt", ".wgs84.vrt", ".tif", ".tiff", ".img", ".wgs84"):
|
||||
candidate = Path(text + suffix)
|
||||
if candidate.exists() and candidate not in candidates:
|
||||
candidates.append(candidate)
|
||||
if path.is_file() and path not in candidates:
|
||||
candidates.append(path)
|
||||
if path.is_dir():
|
||||
for pattern in ("*.vrt", "*.tif", "*.tiff", "*.img"):
|
||||
for candidate in path.glob(pattern):
|
||||
if candidate not in candidates:
|
||||
candidates.append(candidate)
|
||||
return [str(candidate) for candidate in candidates]
|
||||
|
||||
|
||||
def _load_raster_dem(
|
||||
bounds: Tuple[float, float, float, float],
|
||||
dem_path: str,
|
||||
out_shape: tuple[int, int],
|
||||
) -> Optional[np.ndarray]:
|
||||
"""Read a DEM raster subset and resample it to the SAR image grid size."""
|
||||
import rasterio
|
||||
from rasterio.enums import Resampling
|
||||
from rasterio.windows import from_bounds
|
||||
|
||||
height, width = out_shape
|
||||
for candidate in _candidate_dem_paths(dem_path):
|
||||
try:
|
||||
with rasterio.open(candidate) as src:
|
||||
if src.crs and not src.crs.is_geographic:
|
||||
continue
|
||||
dem_bounds = src.bounds
|
||||
min_lon, min_lat, max_lon, max_lat = bounds
|
||||
if (
|
||||
max_lon <= dem_bounds.left
|
||||
or min_lon >= dem_bounds.right
|
||||
or max_lat <= dem_bounds.bottom
|
||||
or min_lat >= dem_bounds.top
|
||||
):
|
||||
continue
|
||||
window = from_bounds(min_lon, min_lat, max_lon, max_lat, transform=src.transform)
|
||||
data = src.read(
|
||||
1,
|
||||
window=window,
|
||||
out_shape=(height, width),
|
||||
boundless=True,
|
||||
fill_value=np.nan,
|
||||
resampling=Resampling.bilinear,
|
||||
).astype(np.float32)
|
||||
nodata = src.nodata
|
||||
if nodata is not None and np.isfinite(nodata):
|
||||
data[data == np.float32(nodata)] = np.nan
|
||||
logger.info("[WaterDetect] DEM raster loaded: %s", candidate)
|
||||
return data
|
||||
except Exception as exc:
|
||||
logger.warning("[WaterDetect] DEM raster candidate skipped: %s (%s)", candidate, exc)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core detection
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -119,14 +243,12 @@ def run_water_detection(
|
||||
Returns dict with keys: ok, output_path, water_area_km2, water_pixel_count, otsu_threshold_db
|
||||
"""
|
||||
import rasterio
|
||||
from scipy import ndimage
|
||||
from scipy.ndimage import median_filter, gaussian_filter, label, zoom
|
||||
from skimage.filters import threshold_otsu
|
||||
from skimage.morphology import disk, binary_dilation, binary_erosion
|
||||
from skimage.measure import regionprops
|
||||
|
||||
from ..config import settings
|
||||
|
||||
dem_dir = settings.SRTM_DEM_DIR
|
||||
dem_path = settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH or settings.SRTM_DEM_DIR
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
logger.info("[WaterDetect] Reading input: %s", geo_tiff_path)
|
||||
@@ -136,6 +258,7 @@ def run_water_detection(
|
||||
img = src.read(1).astype(np.float32)
|
||||
transform = src.transform
|
||||
crs = src.crs
|
||||
nodata = src.nodata
|
||||
height, width = img.shape
|
||||
pixel_size_x = transform.a # degrees per pixel (x)
|
||||
pixel_size_y = transform.e # degrees per pixel (y, negative)
|
||||
@@ -155,17 +278,23 @@ def run_water_detection(
|
||||
|
||||
# Step 5: Valid mask
|
||||
valid = np.isfinite(img) & (img != 0)
|
||||
if nodata is not None and np.isfinite(nodata):
|
||||
valid &= img != np.float32(nodata)
|
||||
|
||||
if np.count_nonzero(valid) < 100:
|
||||
return {"ok": False, "error": "Too few valid pixels in input image"}
|
||||
|
||||
detection_img, valid, value_transform = _prepare_detection_image(img, valid)
|
||||
logger.info("[WaterDetect] Value transform: %s", value_transform)
|
||||
|
||||
# Step 6: Otsu threshold on valid pixels
|
||||
valid_pixels = img[valid]
|
||||
thresh = threshold_otsu(valid_pixels)
|
||||
valid_pixels = detection_img[valid]
|
||||
thresh = _otsu_threshold(valid_pixels)
|
||||
logger.info("[WaterDetect] Otsu threshold: %.4f", thresh)
|
||||
|
||||
# Step 7-8: Median + Gaussian filtering
|
||||
filtered = median_filter(img, size=3)
|
||||
filtered_input = np.where(valid, detection_img, np.nanmedian(valid_pixels)).astype(np.float32)
|
||||
filtered = median_filter(filtered_input, size=3)
|
||||
filtered = gaussian_filter(filtered, sigma=1.0)
|
||||
|
||||
# Step 9: Initial water mask
|
||||
@@ -173,8 +302,11 @@ def run_water_detection(
|
||||
|
||||
# Step 3-4: Load and resample DEM (if available)
|
||||
dem_applied = False
|
||||
if dem_dir and os.path.isdir(dem_dir):
|
||||
dem, dem_bounds = _load_srtm3_dem(bounds, dem_dir)
|
||||
if dem_path and (os.path.isdir(dem_path) or os.path.isfile(dem_path)):
|
||||
dem_resampled = _load_raster_dem(bounds, dem_path, (height, width))
|
||||
dem, dem_bounds = (None, None)
|
||||
if dem_resampled is None and os.path.isdir(dem_path):
|
||||
dem, dem_bounds = _load_srtm3_dem(bounds, dem_path)
|
||||
if dem is not None and dem_bounds is not None:
|
||||
# Resample DEM to image resolution
|
||||
zoom_y = height / dem.shape[0]
|
||||
@@ -183,6 +315,7 @@ def run_water_detection(
|
||||
# Clip to match image shape exactly
|
||||
dem_resampled = dem_resampled[:height, :width]
|
||||
|
||||
if dem_resampled is not None:
|
||||
# Step 10: DEM height constraint (0m <= DEM <= 1000m)
|
||||
dem_valid = np.isfinite(dem_resampled)
|
||||
height_mask = dem_valid & (dem_resampled >= 0) & (dem_resampled <= 1000)
|
||||
@@ -204,14 +337,14 @@ def run_water_detection(
|
||||
else:
|
||||
logger.warning("[WaterDetect] DEM not available, skipping DEM constraints")
|
||||
else:
|
||||
logger.warning("[WaterDetect] SRTM_DEM_DIR not configured, skipping DEM constraints")
|
||||
logger.warning("[WaterDetect] DEM path not configured, skipping DEM constraints")
|
||||
|
||||
# Step 12: Morphological processing — disk(5) dilate→erode→dilate→erode
|
||||
selem = disk(5)
|
||||
water = binary_dilation(water, selem)
|
||||
water = binary_erosion(water, selem)
|
||||
water = binary_dilation(water, selem)
|
||||
water = binary_erosion(water, selem)
|
||||
selem = _disk_structure(5)
|
||||
water = ndimage.binary_dilation(water, structure=selem)
|
||||
water = ndimage.binary_erosion(water, structure=selem)
|
||||
water = ndimage.binary_dilation(water, structure=selem)
|
||||
water = ndimage.binary_erosion(water, structure=selem)
|
||||
|
||||
# Step 13: Connected component filtering
|
||||
labeled, num_features = label(water)
|
||||
@@ -220,18 +353,22 @@ def run_water_detection(
|
||||
pixel_area_m2 = abs(pixel_size_x) * 111320 * abs(pixel_size_y) * 111320
|
||||
min_pixels_by_area = max(1, int(3000 / max(pixel_area_m2, 1)))
|
||||
|
||||
props = regionprops(labeled)
|
||||
areas = [p.area for p in props]
|
||||
if areas:
|
||||
areas = ndimage.sum(
|
||||
np.ones_like(labeled, dtype=np.uint8),
|
||||
labeled,
|
||||
index=np.arange(1, num_features + 1),
|
||||
)
|
||||
areas = np.asarray(areas, dtype=np.float64)
|
||||
if areas.size:
|
||||
median_area = float(np.median(areas))
|
||||
min_area = max(min_pixels_by_area, int(median_area))
|
||||
else:
|
||||
min_area = min_pixels_by_area
|
||||
|
||||
# Remove small components
|
||||
for prop in props:
|
||||
if prop.area < min_area:
|
||||
water[labeled == prop.label] = False
|
||||
small_labels = np.where(areas < min_area)[0] + 1
|
||||
if small_labels.size:
|
||||
water[np.isin(labeled, small_labels)] = False
|
||||
logger.info("[WaterDetect] Connected component filter: min_area=%d pixels, kept %d/%d components",
|
||||
min_area, np.count_nonzero(np.unique(labeled[water])), num_features)
|
||||
|
||||
@@ -265,4 +402,5 @@ def run_water_detection(
|
||||
"water_area_km2": round(water_area, 4),
|
||||
"water_pixel_count": water_pixel_count,
|
||||
"otsu_threshold_db": round(float(thresh), 4),
|
||||
"value_transform": value_transform,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user