Refactor local InSAR asset and production workflows

This commit is contained in:
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
File diff suppressed because it is too large Load Diff
+231 -13
View File
@@ -16,6 +16,8 @@ import json
import asyncio
import hashlib
import re
import tarfile
import zipfile
from typing import List, Dict, Optional, Tuple, Any
from datetime import datetime
from shapely.geometry import Point, Polygon, shape, mapping
@@ -240,6 +242,8 @@ def _chunked(items: List[Any], size: int):
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
_RADAR_ARCHIVE_SUFFIXES = (".zip", ".tar", ".tgz", ".tar.gz")
_RADAR_ARCHIVE_PREVIEW_MAX_BYTES = 256 * 1024 * 1024
def _sanitize_cache_name(name: str) -> str:
@@ -265,6 +269,208 @@ def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_versio
return f"RGID_{safe_version}_{digest}_{base}.webp"
def _has_radar_archive_suffix(path: str) -> bool:
return str(path or "").lower().endswith(_RADAR_ARCHIVE_SUFFIXES)
def _archive_product_stem(path: str) -> str:
base = os.path.basename(str(path or ""))
lower_base = base.lower()
for suffix in (".tar.gz", ".tgz", ".tar", ".zip"):
if lower_base.endswith(suffix):
return base[: -len(suffix)]
return os.path.splitext(base)[0]
def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> Optional[int]:
product_stem = _archive_product_stem(archive_path)
if not product_stem:
return None
member = str(member_name or "").replace("\\", "/").strip("/")
member_lower = member.lower()
product_lower = product_stem.lower()
expected_names = [
f"{product_lower}/{product_lower}.browse.jpg",
f"{product_lower}/{product_lower}.browse.jpeg",
f"{product_lower}/{product_lower}.browse.png",
f"{product_lower}/{product_lower}.quicklook.jpg",
f"{product_lower}/{product_lower}.quicklook.png",
f"{product_lower}/{product_lower}.quick-look.png",
f"{product_lower}/{product_lower}.thumb.jpg",
f"{product_lower}/{product_lower}.thumb.jpeg",
f"{product_lower}/{product_lower}.thumb.png",
f"{product_lower}/preview/quick-look.png",
f"{product_lower}.browse.jpg",
f"{product_lower}.browse.jpeg",
f"{product_lower}.browse.png",
f"{product_lower}.thumb.jpg",
f"{product_lower}.thumb.jpeg",
f"{product_lower}.thumb.png",
]
try:
return expected_names.index(member_lower)
except ValueError:
return None
def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]:
lower_name = str(member_name or "").replace("\\", "/").lower()
base_name = os.path.basename(lower_name)
if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
return None
if size_bytes and size_bytes > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
return None
has_keyword = any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS)
if base_name.endswith((".tif", ".tiff")) and not has_keyword:
return None
keyword_score = 0 if has_keyword else 2
if base_name == "quick-look.png":
keyword_score = -4
elif base_name == "quicklook.png":
keyword_score = -3
elif base_name.startswith("quick-look.") or base_name.startswith("quicklook."):
keyword_score = min(keyword_score, -2)
if "/preview/" in lower_name:
keyword_score -= 1
if "/icons/" in lower_name:
keyword_score += 2
if base_name.endswith((".jpg", ".jpeg", ".png", ".webp")):
ext_score = 0
elif base_name.endswith(".bmp"):
ext_score = 1
else:
ext_score = 2
depth = lower_name.count("/")
size_score = -int(size_bytes or 0)
return (keyword_score, depth, ext_score, size_score, member_name)
def _radar_archive_preview_cache_path(archive_path: str, member_name: str) -> str:
digest = hashlib.sha1(
f"{archive_path}|{member_name}".encode("utf-8", errors="ignore")
).hexdigest()[:16]
raw_base = os.path.basename(str(member_name or "preview"))
base = _sanitize_cache_name(raw_base)
_, ext = os.path.splitext(raw_base)
ext = _RADAR_CACHE_NAME_RE.sub("", ext.lower())[:12]
if ext and not base.lower().endswith(ext):
base = f"{base[:max(1, 48 - len(ext))]}{ext}"
return os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources", f"APS_{digest}_{base}")
def _write_archive_preview_cache(target_path: str, source_obj: Any, archive_mtime: float) -> Optional[str]:
try:
os.makedirs(os.path.dirname(target_path), exist_ok=True)
if os.path.exists(target_path) and _safe_mtime(target_path) >= archive_mtime:
return target_path
tmp_path = f"{target_path}.tmp"
with open(tmp_path, "wb") as target:
while True:
chunk = source_obj.read(1024 * 1024)
if not chunk:
break
target.write(chunk)
os.replace(tmp_path, target_path)
if archive_mtime:
os.utime(target_path, (archive_mtime, archive_mtime))
return target_path
except Exception:
try:
if "tmp_path" in locals() and os.path.exists(tmp_path):
os.remove(tmp_path)
except OSError:
pass
return None
def _find_radar_archive_preview_source(archive_path: str) -> Optional[str]:
if not archive_path or not os.path.isfile(archive_path) or not _has_radar_archive_suffix(archive_path):
return None
archive_mtime = _safe_mtime(archive_path)
lower_path = archive_path.lower()
try:
if lower_path.endswith(".zip"):
with zipfile.ZipFile(archive_path) as archive:
candidates = []
info_by_name = {}
for info in archive.infolist():
if info.is_dir():
continue
score = _radar_archive_preview_score(info.filename, int(info.file_size or 0))
if score is None:
continue
candidates.append(score)
info_by_name[info.filename] = info
if not candidates:
return None
candidates.sort()
member_name = candidates[0][-1]
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
with archive.open(info_by_name[member_name], "r") as source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
with tarfile.open(archive_path, "r:*") as archive:
candidates = []
member_by_name = {}
best_rank: Optional[int] = None
best_member: Optional[tarfile.TarInfo] = None
for member in archive:
if not member.isfile():
continue
expected_rank = _radar_archive_expected_preview_rank(archive_path, member.name)
if expected_rank is not None:
if best_rank is None or expected_rank < best_rank:
best_rank = expected_rank
best_member = member
if expected_rank == 0:
source_obj = archive.extractfile(member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
continue
score = _radar_archive_preview_score(member.name, int(member.size or 0))
if score is None:
if best_member is not None and int(member.size or 0) > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
break
continue
candidates.append(score)
member_by_name[member.name] = member
if os.path.basename(str(member.name or "").lower()) in {"quick-look.png", "quicklook.png"}:
source_obj = archive.extractfile(member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
if best_member is not None:
source_obj = archive.extractfile(best_member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, best_member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
if not candidates:
return None
candidates.sort()
member_name = candidates[0][-1]
source_obj = archive.extractfile(member_by_name[member_name])
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
except Exception:
return None
return None
def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]:
"""Extract coverage polygon from a GeoTIFF file using GDAL.
@@ -380,6 +586,8 @@ class DataService:
@staticmethod
def find_radar_preview_source(scene_dir: str) -> Optional[str]:
if scene_dir and os.path.isfile(scene_dir) and _has_radar_archive_suffix(scene_dir):
return _find_radar_archive_preview_source(scene_dir)
if not scene_dir or not os.path.isdir(scene_dir):
return None
@@ -520,21 +728,26 @@ class DataService:
if orbit_dir and orbit_files_map:
update_progress("正在同步精轨到本地引擎池...", 8)
try:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
sync_result = await asyncio.to_thread(
sync_orbit_pools,
orbit_dir,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_POOL_LANDSAR,
bool(settings.ISCE2_ENABLED),
)
envi_new = len(sync_result.get("envi", {}).get("copied", []))
envi_updated = len(sync_result.get("envi", {}).get("updated", []))
isce2_new = len(sync_result.get("isce2", {}).get("converted", []))
isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", []))
print(
f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated},"
f"ISCE2 转换 {isce2_new}、重转 {isce2_updated}"
)
if settings.ISCE2_ENABLED:
print(
f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},"
f"ISCE2 XML 转换 {isce2_new}、重转 {isce2_updated}"
)
else:
print(f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},ISCE2 已停用")
invalid_orbit_stems = {
item.get("name")
for item in sync_result.get("invalid_sources", [])
@@ -550,7 +763,11 @@ class DataService:
sync_errors = (
sync_result.get("source", {}).get("errors", [])
+ [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])]
+ [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
+ (
[item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
if settings.ISCE2_ENABLED
else []
)
)
if task_id and sync_errors:
await task_service.add_log(
@@ -560,15 +777,16 @@ class DataService:
"精轨同步存在异常: "
f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, "
f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, "
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))} 项"
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', [])) if settings.ISCE2_ENABLED else 0} 项"
),
)
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
await task_service.add_log(
task_id,
"WARN",
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
)
if settings.ISCE2_ENABLED:
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
await task_service.add_log(
task_id,
"WARN",
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
)
if task_id and invalid_orbit_stems:
await task_service.add_log(
task_id,
@@ -17,10 +17,13 @@ from ..utils import parse_gf3_l2_dirname
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes"
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
SKIP_DIR_NAMES = {
".git",
FLAT_SCENE_CATALOG_DIR_NAME,
".gf3_extract",
".gf3_runtime",
".sarmap",
"__pycache__",
"temp",
@@ -87,6 +90,16 @@ def _polarization_from_geo_name(name: str) -> str | None:
return None
def _scene_name_from_geo_name(name: str) -> str:
text = str(name or "").strip()
match = re.match(r"^(?P<scene>.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE)
if match:
return match.group("scene")
if text.lower().endswith("_geo"):
return text[:-4]
return Path(text).stem
def _is_geo_native_data_file(path: Path) -> bool:
return path.is_file() and path.name.lower().endswith("_geo")
@@ -128,6 +141,62 @@ def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict
return metadata
def _native_asset_from_base(base: Path) -> dict[str, Any]:
hdr = Path(str(base) + ".hdr")
sml = Path(str(base) + ".sml")
aux_xml = Path(str(base) + ".aux.xml")
ovr = Path(str(base) + ".ovr")
kml = Path(str(base) + ".kml")
quicklook = base.with_name(base.name + "_ql.tif")
ql_kml = base.with_name(base.name + "_ql.kml")
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
return {
"polarization": polarization,
"role": "geo_native",
"path": str(base),
"hdr": str(hdr) if hdr.exists() else None,
"sml": str(sml) if sml.exists() else None,
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
"ovr": str(ovr) if ovr.exists() else None,
"quicklook": str(quicklook) if quicklook.exists() else None,
"kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None),
"complete": bool(complete),
"source": _safe_stat(base),
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
"sml_info": _safe_stat(sml) if sml.exists() else None,
}
def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]:
return {
"polarization": str(asset.get("polarization") or "").upper(),
"path": asset.get("path"),
"complete": bool(asset.get("complete")),
"source": asset.get("source"),
"hdr": asset.get("hdr_info"),
"sml": asset.get("sml_info"),
}
def _native_fingerprint(assets: list[dict[str, Any]]) -> str:
payload = [
_asset_fingerprint_part(asset)
for asset in sorted(
assets,
key=lambda item: (
str(item.get("polarization") or ""),
str(item.get("path") or ""),
),
)
]
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
import hashlib
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
assets: list[dict[str, Any]] = []
try:
@@ -139,43 +208,25 @@ def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
if not _is_geo_native_data_file(path):
continue
base = path
hdr = Path(str(base) + ".hdr")
sml = Path(str(base) + ".sml")
aux_xml = Path(str(base) + ".aux.xml")
ovr = Path(str(base) + ".ovr")
kml = Path(str(base) + ".kml")
quicklook = base.with_name(base.name + "_ql.tif")
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
asset: dict[str, Any] = {
"polarization": polarization,
"role": "geo_native",
"path": str(base),
"hdr": str(hdr) if hdr.exists() else None,
"sml": str(sml) if sml.exists() else None,
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
"ovr": str(ovr) if ovr.exists() else None,
"quicklook": str(quicklook) if quicklook.exists() else None,
"kml": str(kml) if kml.exists() else None,
"complete": bool(complete),
"source": _safe_stat(base),
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
"sml_info": _safe_stat(sml) if sml.exists() else None,
}
assets.append(asset)
assets.append(_native_asset_from_base(path))
return assets
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
assets = _collect_native_assets(scene_dir)
def _build_scene_manifest(
root: Path,
*,
scene_name: str,
native_dir: Path,
manifest_dir: Path,
assets: list[dict[str, Any]],
source_dir: Path,
storage_layout: str,
) -> dict[str, Any] | None:
if not assets:
return None
scene_name = scene_dir.name
batch_name = _scene_batch_name(root, scene_dir, scene_name)
batch_name = _scene_batch_name(root, source_dir, scene_name)
complete_assets = [asset for asset in assets if asset.get("complete")]
complete_pols = [
pol
@@ -200,25 +251,30 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
logs = []
for name in ("gf3_sarscape_cli.log",):
log_path = scene_dir / name
log_path = source_dir / name
if log_path.is_file():
logs.append(str(log_path))
try:
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
except OSError:
pass
metadata = _parse_scene_metadata(scene_name, assets)
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
native_fingerprint = _native_fingerprint(assets)
metadata["native_fingerprint"] = native_fingerprint
manifest_path = manifest_dir / NATIVE_MANIFEST_NAME
return {
"schema": NATIVE_MANIFEST_SCHEMA,
"generated_at": _utc_now(),
"scene_name": scene_name,
"batch_name": batch_name,
"native_root": str(root),
"native_dir": str(scene_dir),
"native_dir": str(native_dir),
"source_dir": str(source_dir),
"manifest_path": str(manifest_path),
"source_archive": None,
"storage_layout": storage_layout,
"native_fingerprint": native_fingerprint,
"status": status,
"polarizations": complete_pols,
"metadata": metadata,
@@ -227,6 +283,49 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
}
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
assets = _collect_native_assets(scene_dir)
return _build_scene_manifest(
root,
scene_name=scene_dir.name,
native_dir=scene_dir,
manifest_dir=scene_dir,
assets=assets,
source_dir=scene_dir,
storage_layout="scene_dir",
)
def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]:
try:
entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower())
except OSError:
return []
grouped_assets: dict[str, list[dict[str, Any]]] = {}
for path in entries:
if not _is_geo_native_data_file(path):
continue
scene_name = _scene_name_from_geo_name(path.name)
grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path))
manifests: list[dict[str, Any]] = []
for scene_name in sorted(grouped_assets):
scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name
manifest = _build_scene_manifest(
root,
scene_name=scene_name,
native_dir=scene_catalog_dir,
manifest_dir=scene_catalog_dir,
assets=grouped_assets[scene_name],
source_dir=source_dir,
storage_layout="flat_files",
)
if manifest:
manifests.append(manifest)
return manifests
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
roots: list[Path] = []
missing: list[str] = []
@@ -251,6 +350,7 @@ def scan_gf3_sarscape_native_roots(
native_dirs: list[str] | tuple[str, ...] | None,
*,
write_manifest: bool = True,
scene_dirs_only: bool = False,
) -> dict[str, Any]:
"""Scan configured native roots and return discovered scene manifests."""
roots, missing_roots = _normalize_roots(native_dirs)
@@ -258,12 +358,56 @@ def scan_gf3_sarscape_native_roots(
seen_scene_dirs: set[str] = set()
write_errors: list[dict[str, str]] = []
def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None:
seen_scene_dirs.add(scene_key)
if write_manifest:
try:
_write_json(Path(manifest["manifest_path"]), manifest)
except OSError as exc:
write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)})
scenes.append(manifest)
for root in roots:
if scene_dirs_only:
try:
candidate_dirs = [root] + [
child
for child in sorted(root.iterdir(), key=lambda item: item.name.lower())
if child.is_dir()
and child.name not in SKIP_DIR_NAMES
and not child.name.startswith(".SARscape")
and not child.name.startswith(".gf3_")
]
except OSError:
candidate_dirs = [root]
for scene_dir in candidate_dirs:
scene_key = str(scene_dir).lower()
if scene_key in seen_scene_dirs:
continue
if not scene_dir.name.upper().startswith("GF3_"):
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
if flat_manifests:
for manifest in flat_manifests:
flat_key = str(manifest.get("native_dir") or "").lower()
if flat_key and flat_key not in seen_scene_dirs:
append_manifest(manifest, flat_key, scene_dir)
continue
manifest = _collect_scene_manifest(root, scene_dir)
if not manifest:
continue
append_manifest(manifest, scene_key, scene_dir)
continue
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
and not name.startswith(".gf3_")
]
scene_dir = Path(current_dir)
scene_key = str(scene_dir).lower()
@@ -271,17 +415,20 @@ def scan_gf3_sarscape_native_roots(
dir_names[:] = []
continue
if not scene_dir.name.upper().startswith("GF3_"):
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
if flat_manifests:
for manifest in flat_manifests:
flat_key = str(manifest.get("native_dir") or "").lower()
if flat_key and flat_key not in seen_scene_dirs:
append_manifest(manifest, flat_key, scene_dir)
continue
manifest = _collect_scene_manifest(root, scene_dir)
if not manifest:
continue
seen_scene_dirs.add(scene_key)
if write_manifest:
try:
_write_json(Path(manifest["manifest_path"]), manifest)
except OSError as exc:
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
scenes.append(manifest)
append_manifest(manifest, scene_key, scene_dir)
dir_names[:] = []
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
+737 -9
View File
@@ -15,12 +15,13 @@ from shapely.geometry import Polygon
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..config import settings, split_env_paths
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
from .data_service import extract_geotiff_bounds
from .data_service import DataService, extract_geotiff_bounds
from .gf3_native_inventory_service import (
NATIVE_MANIFEST_NAME,
POLARIZATION_PRIORITY,
SKIP_DIR_NAMES,
scan_gf3_sarscape_native_roots,
)
from .image_service import image_service
@@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
CONVERTER_VERSION = "v1"
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK"
SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2"
def _utc_now() -> str:
@@ -72,6 +76,112 @@ def _path_kind(path: str) -> str:
return "relative"
def _path_key(path: str | Path) -> str:
return os.path.normcase(os.path.normpath(str(path))).lower()
def _has_native_geo_assets(path: Path) -> bool:
try:
return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir())
except OSError:
return False
def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]:
"""Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning.
Operators usually configure a durable GF3 root rather than a date folder.
Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer
direct ``*_geo`` children and fall back to recursive discovery.
"""
roots: list[str] = []
seen: set[str] = set()
def add(path: str | Path) -> None:
text = os.path.normpath(str(path))
key = _path_key(text)
if key and key not in seen:
seen.add(key)
roots.append(text)
for raw in native_dirs or []:
text = str(raw or "").strip()
if not text:
continue
if any(ch in text for ch in "*?"):
glob_root = Path(os.path.normpath(text))
try:
matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower())
except OSError:
matches = []
if matches:
for match in matches:
if match.is_dir():
add(match)
continue
add(glob_root)
continue
root = Path(os.path.normpath(text))
try:
is_dir = root.is_dir()
except OSError:
is_dir = False
if not is_dir:
add(root)
continue
if root.name.lower().endswith("_geo"):
add(root)
try:
immediate_geo_roots = sorted(
(
child
for child in root.iterdir()
if child.is_dir() and child.name.lower().endswith("_geo")
),
key=lambda item: item.name.lower(),
)
except OSError:
immediate_geo_roots = []
if immediate_geo_roots:
for child in immediate_geo_roots:
add(child)
continue
if _has_native_geo_assets(root):
add(root)
continue
found_recursive = 0
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
and not name.startswith(".gf3_")
]
selected: list[str] = []
remaining: list[str] = []
for name in dir_names:
if name.lower().endswith("_geo"):
selected.append(name)
else:
remaining.append(name)
for name in selected:
add(Path(current_dir) / name)
found_recursive += len(selected)
dir_names[:] = remaining
if found_recursive == 0:
add(root)
return roots
def _source_asset_uid(path: str) -> str:
normalized = os.path.normpath(str(path or "").strip())
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
@@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool:
return False
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None:
polygon = extract_geotiff_bounds(str(path))
if polygon and len(polygon) >= 4:
return polygon
@@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
return None
def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None:
if not asset:
return None
for key in ("path", "quicklook"):
text = _path_text(asset.get(key))
if not text:
continue
path = Path(text)
if not path.is_file():
continue
polygon = _polygon_from_raster(path)
if polygon and len(polygon) >= 4:
return polygon
return None
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
if not polygon:
return None, None
@@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None
return assets[0] if assets else None
def _is_quicklook_raster_path(path: Any) -> bool:
text = _path_text(path).lower()
return text.endswith("_ql.tif") or text.endswith("_ql.tiff")
def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
candidates = [
asset
for asset in assets
if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path"))
]
complete = [asset for asset in candidates if asset.get("complete") is not False]
return _select_default_asset(complete or candidates)
def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]:
output_assets: list[dict[str, Any]] = []
for asset in scene_manifest.get("assets") or []:
native_path = _path_text(asset.get("path"))
if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True:
continue
output_assets.append(
{
"polarization": str(asset.get("polarization") or "UNKNOWN").upper(),
"role": "native_geo",
"path": native_path,
"hdr": _path_text(asset.get("hdr")) or None,
"sml": _path_text(asset.get("sml")) or None,
"quicklook": _path_text(asset.get("quicklook")) or None,
"complete": bool(asset.get("complete")),
"status": "registered_native_geo",
}
)
return output_assets
def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]:
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
out_dir = _standard_scene_dir(scene_manifest, root)
manifest_path = out_dir / "gf3_native_preview_manifest.json"
assets = _quicklook_assets_from_scene_manifest(scene_manifest)
manifest = {
"schema": "gf3_sarscape_native_preview.v1",
"generated_at": _utc_now(),
"scene_name": scene_manifest.get("scene_name"),
"batch_name": scene_manifest.get("batch_name"),
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
"native_dir": scene_manifest.get("native_dir"),
"standard_dir": str(out_dir),
"manifest_path": str(manifest_path),
"status": "NATIVE_READY" if assets else "FAILED",
"assets": assets,
"summary": {
"native_assets": len(assets),
"quicklook_assets": 0,
"full_raster_materialized": True,
"webp_source": "native_geo",
},
"errors": [] if assets else [{"error": "no native _geo assets found"}],
}
_write_json(manifest_path, manifest)
return manifest
def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool:
"""Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``."""
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except Exception:
return False
try:
target = Path(target_path)
target.parent.mkdir(parents=True, exist_ok=True)
limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600)
with rasterio.open(source_path) as src:
scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
band = src.read(
1,
out_shape=(out_height, out_width),
masked=True,
resampling=Resampling.bilinear,
)
data = band.filled(0)
if data.dtype != np.uint8:
valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)]
if valid.size:
p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98])
scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1)
data = (scaled * 255).astype("uint8")
else:
data = np.zeros(data.shape, dtype="uint8")
else:
data = data.astype("uint8", copy=False)
mask = getattr(band, "mask", None)
if mask is None or np.ndim(mask) == 0:
alpha = np.full(data.shape, 255, dtype="uint8")
else:
alpha = np.where(mask, 0, 255).astype("uint8")
rgba = np.stack([data, data, data, alpha], axis=-1)
image = Image.fromarray(rgba, "RGBA")
image = image_service.make_edge_dark_transparent(image)
image_service.save_image_as_webp(image, target_path, quality=82)
return target.exists() and target.stat().st_size > 0
except Exception:
return False
def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str:
candidates: list[Any] = [metadata.get("default_native_path")]
for key in ("native_assets", "quicklook_assets"):
for asset in metadata.get(key) or []:
if isinstance(asset, dict):
candidates.append(asset.get("path"))
for candidate in candidates:
text = _path_text(candidate)
if text and not _is_quicklook_raster_path(text):
return text
return ""
def _path_is_under_any(path: str, roots: list[str]) -> bool:
target = _path_key(path)
if not target:
return False
for root in roots:
root_key = _path_key(root)
if root_key and (target == root_key or target.startswith(root_key + os.sep)):
return True
return False
def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]:
roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if not roots:
roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
return [os.path.normpath(root) for root in roots if str(root or "").strip()]
def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str:
return str(
scene_manifest.get("native_fingerprint")
or (scene_manifest.get("metadata") or {}).get("native_fingerprint")
or ""
).strip()
def _record_has_bounds(record: RadarDataORM) -> bool:
return (
record.coverage_polygon is not None
and record.min_lon is not None
and record.min_lat is not None
and record.max_lon is not None
and record.max_lat is not None
)
async def _find_existing_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> RadarDataORM | None:
native_dir = str(scene_manifest.get("native_dir") or "")
scene_name = scene_manifest.get("scene_name") or Path(native_dir).name
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
return result.scalars().first()
async def _can_skip_unchanged_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> tuple[bool, int | None, str]:
native_fingerprint = _scene_native_fingerprint(scene_manifest)
if not native_fingerprint:
return False, None, "missing_native_fingerprint"
record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest)
if record is None:
return False, None, "missing_record"
metadata = record.metadata_json or {}
if str(metadata.get("native_fingerprint") or "") != native_fingerprint:
return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed"
if str(metadata.get("registration_mode") or "") != "native_preview":
return False, int(record.id) if record.id is not None else None, "registration_mode_changed"
if not _record_has_bounds(record):
return False, int(record.id) if record.id is not None else None, "missing_bounds"
return True, int(record.id) if record.id is not None else None, "unchanged"
async def generate_gf3_quicklook_webp_cache(
db: AsyncSession,
*,
force: bool = False,
max_records: int | None = None,
native_dirs: list[str] | tuple[str, ...] | None = None,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters."""
stmt = (
select(RadarDataORM)
.where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT)
.order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc())
)
result = await db.execute(stmt)
configured_roots = _configured_gf3_native_roots(native_dirs)
records = [
record
for record in result.scalars().all()
if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots)
]
if max_records and max_records > 0:
records = records[: int(max_records)]
total = len(records)
generated = 0
skipped = 0
failed = 0
record_results: list[dict[str, Any]] = []
for idx, record in enumerate(records):
metadata = record.metadata_json or {}
source_path = _native_preview_source_from_metadata(metadata)
cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path)
if progress_callback:
pct = 5 + int((idx / max(total, 1)) * 90)
progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}")
if (
not force
and (record.preview_cache_status or "NONE") == "READY"
and record.preview_cache_path
and os.path.exists(record.preview_cache_path)
):
skipped += 1
record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path})
continue
if not source_path:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "default_native_path_missing"
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
continue
ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path)
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
if ok and os.path.exists(cache_path):
generated += 1
record.preview_cache_status = "READY"
record.preview_cache_path = cache_path
record.preview_cache_error = None
record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path})
else:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "native_webp_build_failed"
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
return {
"ok": failed == 0,
"mode": "gf3_native_webp",
"total": total,
"generated": generated,
"skipped": skipped,
"failed": failed,
"records": record_results,
}
async def _upsert_quicklook_source_product_asset(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
) -> int | None:
native_dir_text = _path_text(scene_manifest.get("native_dir"))
if not native_dir_text:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name
now = _db_now()
root = await _find_managed_root_for_path(db, native_dir_text)
asset_metadata = _json_safe(
{
"source": "GF3 SARscape native _geo",
"native_dir": native_dir_text,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": quicklook_manifest.get("assets") or [],
"full_raster_materialized": True,
"analysis_engine": "gf3_sarscape",
}
)
data = {
"asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"),
"logical_product_uid": scene_name,
"satellite_family": "GF3",
"satellite": "GF3",
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"product_type": metadata.get("product_type") or "SARSCAPE_NATIVE",
"product_level": "L2_NATIVE",
"imaging_mode": metadata.get("imaging_mode"),
"polarization": metadata.get("polarization"),
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"orbit_direction": metadata.get("orbit_direction"),
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"imaging_date": imaging_date,
"root_ref_id": root.id if root else None,
"root_path": root.path if root else str(Path(native_dir_text).parent),
"file_path": native_dir_text,
"archive_path": scene_manifest.get("source_archive"),
"path_kind": _path_kind(native_dir_text),
"file_name": Path(native_dir_text).name,
"file_stem": Path(native_dir_text).name,
"file_ext": "",
"size_bytes": None,
"mtime_epoch": None,
"checksum_status": "NOT_COMPUTED",
"parser_name": "gf3_sarscape_native_preview_manifest",
"parser_version": CONVERTER_VERSION,
"parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED",
"parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None,
"parsed_at": now,
"metadata_json": asset_metadata,
"is_active": True,
"missing_since": None,
"updated_at": now,
}
result = await db.execute(
select(SourceProductAssetORM).where(
or_(
SourceProductAssetORM.asset_uid == data["asset_uid"],
SourceProductAssetORM.file_path == native_dir_text,
)
)
)
asset = result.scalars().first()
if asset is None:
asset = SourceProductAssetORM(**data)
db.add(asset)
else:
for key, value in data.items():
setattr(asset, key, value)
await db.flush()
return int(asset.id) if asset.id is not None else None
async def _upsert_quicklook_radar_data(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
source_product_ref_id: int | None = None,
) -> int | None:
assets = quicklook_manifest.get("assets") or []
default_asset = _select_default_native_asset(assets)
if not default_asset:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
native_dir = str(scene_manifest.get("native_dir") or "")
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
default_native_path = str(default_asset.get("path") or "")
default_quicklook_path = str(default_asset.get("quicklook") or "")
polygon = _polygon_from_native_asset(default_asset)
min_lon = min_lat = max_lon = max_lat = None
geom = None
if polygon:
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
else:
center_lon = metadata.get("scene_center_lon")
center_lat = metadata.get("scene_center_lat")
preview_cache_path = None
preview_cache_kind = "deferred"
preview_cache_error = "native_webp_not_generated"
radar_metadata = _json_safe(
{
**metadata,
"native_dir": native_dir,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": assets,
"quicklook_assets": [],
"default_native_path": default_native_path,
"default_quicklook_path": default_quicklook_path,
"coverage_source": "native_geo_or_quicklook" if polygon else "scene_name",
"preview_cache_kind": preview_cache_kind,
"analysis_engine": "gf3_sarscape",
"registration_mode": "native_preview",
"full_raster_materialized": True,
}
)
data_to_upsert = {
"unique_id": unique_id,
"satellite": "GF3",
"satellite_family": "GF3",
"imaging_date": imaging_date,
"imaging_mode": metadata.get("imaging_mode"),
"polarization": ",".join(
pol
for pol in POLARIZATION_PRIORITY
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
)
or metadata.get("polarization"),
"scene_center_lon": center_lon,
"scene_center_lat": center_lat,
"acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None,
"product_level": "L2_NATIVE",
"product_unique_id": metadata.get("product_unique_id") or scene_name,
"source_product_token": scene_name,
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"source_product_ref_id": source_product_ref_id,
"image_data_format": "ENVI_NATIVE",
"geocoded_flag": True,
"metadata_json": radar_metadata,
"file_path": native_dir,
"has_orbit_data": False,
"orbit_file_path": None,
"is_envi_processed": True,
"coverage_polygon": polygon,
"geom": geom,
"min_lon": min_lon,
"min_lat": min_lat,
"max_lon": max_lon,
"max_lat": max_lat,
"preview_cache_status": "NONE",
"preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION,
"preview_cache_path": preview_cache_path,
"preview_cache_updated_at": _db_now(),
"preview_cache_error": preview_cache_error,
}
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
radar = result.scalars().first()
if (
radar is not None
and (radar.preview_cache_status or "").upper() == "READY"
and radar.preview_cache_path
and os.path.exists(radar.preview_cache_path)
and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path
):
data_to_upsert["preview_cache_status"] = radar.preview_cache_status
data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION
data_to_upsert["preview_cache_path"] = radar.preview_cache_path
data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at
data_to_upsert["preview_cache_error"] = radar.preview_cache_error
if radar is None:
radar = RadarDataORM(**data_to_upsert)
db.add(radar)
else:
for key, value in data_to_upsert.items():
setattr(radar, key, value)
await db.flush()
return int(radar.id) if radar.id is not None else None
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
metadata = dict(scene_manifest.get("metadata") or {})
metadata.update(
@@ -635,7 +1260,7 @@ async def _upsert_radar_data(
if not default_asset:
return None
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path"))))
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
@@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots(
storage_root: str | None = None,
force: bool = False,
register: bool = True,
quicklook_only: bool = False,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Scan native roots, convert complete assets, and register standard scenes."""
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
native_dirs,
write_manifest=True,
)
requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if quicklook_only:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
else:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
scenes = inventory.get("scenes") or []
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
if quicklook_only:
registered = 0
quicklook_assets = 0
failed_scenes = 0
skipped_unchanged = 0
scene_results: list[dict[str, Any]] = []
total = len(ready_scenes)
for idx, scene_manifest in enumerate(ready_scenes):
if progress_callback:
pct = 10 + int((idx / max(total, 1)) * 80)
progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
quicklook_manifest = await asyncio.to_thread(
_quicklook_manifest_for_scene,
scene_manifest,
storage_root,
)
assets = quicklook_manifest.get("assets") or []
quicklook_assets += len(assets)
source_asset_id = None
radar_id = None
skipped_reason = None
if register and assets:
can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar(
db,
scene_manifest=scene_manifest,
)
if can_skip:
skipped_unchanged += 1
radar_id = existing_radar_id
skipped_reason = skip_reason
if progress_callback:
pct = 10 + int(((idx + 1) / max(total, 1)) * 80)
progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
else:
skipped_reason = skip_reason
source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest)
radar_id = await _upsert_quicklook_radar_data(
db,
scene_manifest,
quicklook_manifest,
source_product_ref_id=source_asset_id,
)
if radar_id:
registered += 1
await db.commit()
if not assets:
failed_scenes += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"native_status": scene_manifest.get("status"),
"standard_status": quicklook_manifest.get("status"),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"source_asset_id": source_asset_id,
"radar_id": radar_id,
"skipped_unchanged": bool(skipped_reason == "unchanged"),
"skip_reason": skipped_reason,
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"summary": quicklook_manifest.get("summary") or {},
"errors": quicklook_manifest.get("errors") or [],
}
)
return {
"ok": failed_scenes == 0,
"mode": "native_preview",
"inventory": {
key: value
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": 0,
"partial_scenes": 0,
"failed_scenes": failed_scenes,
"converted_assets": 0,
"skipped_assets": skipped_unchanged,
"skipped_unchanged": skipped_unchanged,
"failed_assets": failed_scenes,
"quicklook_assets": quicklook_assets,
"native_assets": quicklook_assets,
"registered": registered,
"analysis_ready": 0,
"scenes": scene_results,
}
converted_scenes = 0
partial_scenes = 0
failed_scenes = 0
@@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots(
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": converted_scenes,
+130 -161
View File
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
from sqlalchemy import select
from .. import database
from ..config import settings
from ..config import settings, split_env_paths
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job
from .data_service import data_service
@@ -88,6 +88,7 @@ JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
JOB_TYPE_GF3_QUICKLOOK_WEBP = "GF3_QUICKLOOK_WEBP"
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
@@ -97,6 +98,7 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY"
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM"
JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS"
@@ -253,9 +255,12 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
families=payload.get("families") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
build_previews=bool(payload.get("build_previews", True)),
task_id=job.task_id,
)
preview = result.get("preview_cache") or {}
await task_service.update_task(
job.task_id,
status="COMPLETED",
@@ -265,7 +270,42 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
f"sources={result.get('source_assets', 0)}, "
f"orbits={result.get('orbit_assets', 0)}, "
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
f"missing={((result.get('binding') or {}).get('missing_count', 0))}, "
f"previews_ready={preview.get('ready', 0)}, "
f"previews_skipped={preview.get('skipped_ready', 0)}, "
f"previews_failed={preview.get('failed', 0)}, "
f"previews_missing={preview.get('missing_source', 0)}"
),
db=db,
)
async def _handle_archive_integrity_audit(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("AUDIT_SOURCE_ARCHIVE_INTEGRITY requires task_id for progress tracking.")
payload = job.payload or {}
await task_service.start_task(job.task_id, message="Source archive integrity audit started")
async with AsyncSessionLocal() as db:
result = await asset_inventory_service.audit_source_archive_integrity(
db,
families=payload.get("families") or None,
source_formats=payload.get("source_formats") or None,
asset_ids=payload.get("asset_ids") or None,
force=bool(payload.get("force", False)),
limit=payload.get("limit"),
task_id=job.task_id,
)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"Source archive integrity audit completed: "
f"checked={result.get('checked', 0)}, "
f"skipped={result.get('skipped', 0)}, "
f"ok={result.get('ok', 0)}, "
f"failed={result.get('failed', 0)}, "
f"unsupported={result.get('unsupported', 0)}"
),
db=db,
)
@@ -345,7 +385,7 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses"))
include_orbit_files = bool(payload.get("include_orbit_files"))
export_zip = bool(payload.get("export_zip"))
package_mode = str(payload.get("package_mode") or ("task_zip" if export_zip else "task_folder")).strip().lower()
package_mode = str(payload.get("package_mode") or ("source_bundle" if export_zip else "task_folder")).strip().lower()
skip_existing = payload.get("skip_existing") is not False
max_items = _normalize_positive_int(payload.get("max_items"))
@@ -445,17 +485,27 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
skip_existing=skip_existing,
max_items=max_items,
)
else:
return
if package_mode in {"task_folder", "task"}:
await run_dinsar_copy_items(
job.task_id,
items,
dest_dir,
include_orbit_files=include_orbit_files,
export_zip=(package_mode == "task_zip" or export_zip),
export_zip=False,
skip_existing=skip_existing,
max_items=max_items,
)
return
return
if package_mode == "task_zip":
raise ValueError(
"D-InSAR Task ZIP export is not a production-preparation format. "
"Use package_mode=task_folder for Task_Pool preparation or source_bundle for archive distribution."
)
raise ValueError(
"Unsupported D-InSAR package_mode. "
"Use task_folder for production preparation or source_bundle for archive distribution."
)
raise ValueError(f"Unknown COPY_DATA file_type: {file_type}")
@@ -884,7 +934,9 @@ async def _handle_ai_diagnosis(job: SystemJobORM) -> None:
from ..ai_service import analyze_map_with_vlm
diagnosis_markdown = await analyze_map_with_vlm(
images_base64=[img_base64],
prompt=full_prompt
prompt=full_prompt,
model_name=model_name,
raise_on_error=True,
)
# 8. 解析风险等级和置信度(简单正则匹配)
@@ -3646,7 +3698,7 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
@@ -3725,7 +3777,7 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
if not job.task_id:
@@ -3792,7 +3844,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
@@ -3901,7 +3953,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
"""Scan SARscape native GF3 _geo outputs and register requested assets."""
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
if not job.task_id:
@@ -3915,7 +3967,15 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
if not storage_root:
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
quicklook_only = bool(payload.get("quicklook_only", False))
await task_service.start_task(
job.task_id,
message=(
"按 GF3 日期/场景命名规则登记本机 SARscape _geo 原生结果..."
if quicklook_only
else "递归扫描 GF3 SARscape 原生 _geo 结果池..."
),
)
loop = asyncio.get_running_loop()
@@ -3943,6 +4003,7 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
storage_root=storage_root,
force=bool(payload.get("force", False)),
register=bool(payload.get("register", True)),
quicklook_only=quicklook_only,
progress_callback=_progress_cb,
)
except Exception as exc:
@@ -3954,44 +4015,44 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
)
raise
message = (
"GF3 SARscape 标准化完成: "
f"发现 {int(result.get('scene_count') or 0)} 景, "
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)} 景"
)
if quicklook_only:
message = (
"GF3 _geo 原生结果登记完成: "
f"规则匹配 {int(result.get('scene_count') or 0)} 景, "
f"可登记 {int(result.get('ready_scene_count') or 0)} 景, "
f"原生资产 {int(result.get('native_assets') or result.get('quicklook_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)} 景, "
f"跳过未变化 {int(result.get('skipped_unchanged') or 0)} 景, "
f"失败 {int(result.get('failed_scenes') or 0)} 景"
)
else:
message = (
"GF3 SARscape 标准化完成: "
f"发现 {int(result.get('scene_count') or 0)} 景, "
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)} 景"
)
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
from .gf3_sarscape_production_service import (
cleanup_gf3_sarscape_native_pool,
run_gf3_sarscape_production,
)
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
async def _handle_gf3_quicklook_webp(job: SystemJobORM) -> None:
"""Generate local WebP previews from registered GF3 SARscape native _geo records."""
from .gf3_standardize_service import generate_gf3_quicklook_webp_cache
if not job.task_id:
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
raise ValueError("GF3_QUICKLOOK_WEBP requires task_id for progress tracking.")
payload = job.payload or {}
source_dirs = payload.get("source_dirs") or []
native_dirs = payload.get("native_dirs") or []
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
if not source_dirs:
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
if not native_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
if not storage_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
force = bool(payload.get("force", False))
max_records = _normalize_positive_int(payload.get("max_records"))
native_dirs = payload.get("native_dirs") or split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
await task_service.start_task(job.task_id, message="开始从 GF3 _geo 原生数据生成 WebP 缓存...")
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
loop = asyncio.get_running_loop()
def _progress_cb(progress: int, message: str) -> None:
@@ -4005,143 +4066,49 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
logger.warning("[GF3 native WebP] progress callback failed: %s", exc)
future.add_done_callback(_swallow_progress_error)
except RuntimeError:
return
def _log_cb(level: str, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.add_log(job.task_id, level, message),
loop,
)
def _swallow_log_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
future.add_done_callback(_swallow_log_error)
except RuntimeError:
return
async def _production_keepalive() -> None:
progress = 8
while True:
await asyncio.sleep(60)
progress = min(68, progress + 1)
await task_service.update_task(
job.task_id,
progress=progress,
message="GF3 SARscape production is still running...",
)
try:
production_task = asyncio.create_task(
asyncio.to_thread(
run_gf3_sarscape_production,
source_dirs=source_dirs,
native_root=native_root,
wrapper_exe=payload.get("wrapper_exe"),
dem_path=payload.get("dem_path"),
idlrt_path=payload.get("idlrt_path"),
polarizations=payload.get("polarizations"),
archive_exts=payload.get("archive_exts") or [],
max_archives_per_run=payload.get("max_archives_per_run"),
selected_dates=payload.get("selected_dates") or [],
task_id=job.task_id,
local_staging_root=payload.get("local_staging_root") or settings.GF3_TASK_POOL_ROOT,
timeout_seconds=payload.get("timeout_seconds"),
keep_extracted=payload.get("keep_extracted"),
log_callback=_log_cb,
async with AsyncSessionLocal() as db:
result = await generate_gf3_quicklook_webp_cache(
db,
force=force,
max_records=max_records,
native_dirs=native_dirs,
progress_callback=_progress_cb,
)
)
keepalive_task = asyncio.create_task(_production_keepalive())
try:
production_result = await production_task
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
standardize_result: Dict[str, Any] = {}
if bool(payload.get("auto_standardize", True)):
await task_service.update_task(
job.task_id,
progress=72,
message="GF3 SARscape production finished; standardizing native _geo outputs...",
)
async with AsyncSessionLocal() as db:
standardize_result = await standardize_gf3_sarscape_native_roots(
db,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
force=bool(payload.get("force_standardize", False)),
register=bool(payload.get("register", True)),
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
)
cleanup_result: Dict[str, Any] = {}
production_ok = int(production_result.get("failed_count") or 0) == 0
standardize_ok = (
not standardize_result
or (
int(standardize_result.get("failed_assets") or 0) == 0
and int(standardize_result.get("failed_scenes") or 0) == 0
)
)
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
await task_service.update_task(
job.task_id,
progress=90,
message="Cleaning GF3 SARscape intermediate files...",
)
cleanup_result = await asyncio.to_thread(
cleanup_gf3_sarscape_native_pool,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
dry_run=bool(payload.get("cleanup_dry_run", False)),
max_scenes=payload.get("cleanup_max_scenes"),
log_callback=_log_cb,
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
)
elif bool(payload.get("clean_after_success", True)):
_log_cb(
"WARNING",
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
)
except Exception as exc:
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=f"GF3 SARscape production failed: {exc}",
message=f"GF3 _geo WebP 生成失败: {exc}",
)
raise
failed_count = int(production_result.get("failed_count") or 0)
failed_assets = int(standardize_result.get("failed_assets") or 0)
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
message = (
"GF3 SARscape production chain finished: "
f"found={int(production_result.get('found_count') or 0)}, "
f"produced={int(production_result.get('processed_count') or 0)}, "
f"skipped={int(production_result.get('skipped_count') or 0)}, "
f"failed={failed_count}, "
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
f"registered={int(standardize_result.get('registered') or 0)}, "
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
"GF3 _geo WebP 生成完成: "
f"总数 {int(result.get('total') or 0)} 景, "
f"生成 {int(result.get('generated') or 0)} 景, "
f"跳过 {int(result.get('skipped') or 0)} 景, "
f"失败 {int(result.get('failed') or 0)} 景"
)
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
"""Reject GF3 production jobs on this management machine."""
message = (
"GF3 SARscape production is disabled on this management machine. "
"Run production on the SARscape host and register local _geo native results here."
)
if job.task_id:
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=message)
raise ValueError(message)
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
@@ -5263,6 +5230,7 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None:
_HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit,
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
@@ -5301,6 +5269,7 @@ _HANDLERS = {
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
JOB_TYPE_GF3_QUICKLOOK_WEBP: _handle_gf3_quicklook_webp,
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
+71 -6
View File
@@ -24,6 +24,8 @@ from ..dinsar_engines.landsar_engine import (
_norm_path,
_path_search_dirs,
_summarize_landsar_failure,
derive_landsar_dem_bbox_from_xml_paths,
prepare_landsar_dem_crop,
)
@@ -633,6 +635,16 @@ class LandsarSbasService:
"errors": errors,
}
def _derive_task_dem_bbox(self, task: dict[str, Any], input_dir: str) -> tuple[float, float, float, float] | None:
xml_paths = [
str(scene.get("xml") or "")
for scene in (task.get("scenes") or [])
if str(scene.get("xml") or "").strip()
]
if not xml_paths and input_dir and os.path.isdir(input_dir):
xml_paths = [str(path) for path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower())]
return derive_landsar_dem_bbox_from_xml_paths(xml_paths, fallback_paths=xml_paths)
def materialize_stack(
self,
*,
@@ -949,7 +961,7 @@ class LandsarSbasService:
params_payload = self.normalize_params(params or {})
normalized_dem = _norm_path(dem_path or self.default_dem_path)
if not normalized_dem or not os.path.isfile(normalized_dem):
raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or '<empty>'}")
raise ValueError(f"LandSAR SBAS DEM source file is missing: {normalized_dem or '<empty>'}")
from .sbas_insar_production_service import sbas_insar_production_service
@@ -1035,6 +1047,8 @@ class LandsarSbasService:
"input_task_root": import_dest_root,
"publish_root": str(publish_root),
"dem_path": normalized_dem,
"dem_source_path": normalized_dem,
"dem_role": "global_prepared_source",
"params": params_payload,
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
"import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
@@ -1541,9 +1555,9 @@ class LandsarSbasService:
if not bool(settings.LANDSAR_SBAS_ENABLED):
raise ValueError("LandSAR SBAS is disabled.")
params = self.normalize_params(extra or {})
dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
if not dem_path or not os.path.isfile(dem_path):
raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or '<empty>'}")
dem_source_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
if not dem_source_path or not os.path.isfile(dem_source_path):
raise ValueError(f"LandSAR SBAS DEM source file is missing: {dem_source_path or '<empty>'}")
validation = self.validate_root_dir(
root_dir,
min_scenes=min_scenes,
@@ -1585,7 +1599,9 @@ class LandsarSbasService:
"work_root_strategy": "short_landsar_execution_path",
"native_root": str(native_root),
"publish_root": str(publish_root),
"dem_path": dem_path,
"dem_path": dem_source_path,
"dem_source_path": dem_source_path,
"dem_role": "global_prepared_source",
"params": params,
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
"min_scenes": validation.get("min_scenes"),
@@ -1716,10 +1732,56 @@ class LandsarSbasService:
native_output_dir = native_root / task_alias / "Output_Data"
native_output_dir.mkdir(parents=True, exist_ok=True)
project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias
dem_source_path = _norm_path(str(manifest.get("dem_path") or ""))
effective_dem_path = dem_source_path
dem_crop_info: dict[str, Any] = {}
try:
dem_bbox = self._derive_task_dem_bbox(task, input_dir)
if not dem_bbox:
raise ValueError("cannot derive DEM crop bbox from LandSAR SBAS Input_Data XML corner coordinates")
dem_crop_info = prepare_landsar_dem_crop(
dem_source_path,
str(native_root / task_alias / "dem_crop"),
dem_bbox,
label=task_alias,
)
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
self._emit(
progress_callback,
"INFO",
f"[{index}/{len(tasks)}] LandSAR SBAS DEM crop ready: {effective_dem_path}",
)
except Exception as exc:
failed_count += 1
error = f"LandSAR SBAS DEM crop failed: {exc}"
self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] {task_name} failed: {error}")
task_results.append(
{
"task_name": task_name,
"task_alias": task_alias,
"input_data_dir": input_dir,
"native_output_dir": str(native_output_dir),
"param_file": "",
"process_name": SBAS_PROCESS_NAME,
"command": "",
"returncode": -2,
"success": False,
"timed_out": False,
"failure_kind": "dem_crop_failed",
"error": error,
"stdout_tail": "",
"native_logs": {},
"publish": {},
"dem_source_path": dem_source_path,
"dem_path": effective_dem_path,
"dem_crop": dem_crop_info,
}
)
continue
param_file = _generate_sbas_param_file(
str(native_output_dir / f"{SBAS_PROID}.txt"),
slc_folder=input_dir,
dem_path=str(manifest.get("dem_path") or ""),
dem_path=effective_dem_path,
output_dir=str(native_output_dir),
project_name=project_name,
params=dict(manifest.get("params") or {}),
@@ -1784,6 +1846,9 @@ class LandsarSbasService:
"stdout_tail": _collect_tail(stdout_text, 4000),
"native_logs": log_publish_result,
"publish": publish_result,
"dem_source_path": dem_source_path,
"dem_path": effective_dem_path,
"dem_crop": dem_crop_info,
}
)
+40 -14
View File
@@ -1,7 +1,7 @@
"""轨道文件管理与格式转换服务。
目录约定:
MONITOR_ORBIT_DIR/ 源精轨目录(可为 UNC),平铺 .txt
MONITOR_ORBIT_DIR/ 本机源精轨目录,平铺 .txt
ORBIT_POOL_ENVI/ ENVI 本地精轨池
LT1A/ LT-1A 精轨 .txt
LT1B/ LT-1B 精轨 .txt
@@ -320,16 +320,17 @@ def summarize_source_orbit_gaps(
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
isce2_enabled = bool(str(isce2_pool or "").strip())
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
source_stems = set(source_files)
envi_stems = set(envi_files)
isce2_stems = set(isce2_files)
source_without_isce2 = sorted(source_stems - isce2_stems)
source_without_isce2 = sorted(source_stems - isce2_stems) if isce2_enabled else []
source_without_envi = sorted(source_stems - envi_stems)
envi_without_source = sorted(envi_stems - source_stems)
isce2_without_source = sorted(isce2_stems - source_stems)
isce2_without_source = sorted(isce2_stems - source_stems) if isce2_enabled else []
sample_limit = 20
def _make_samples(
@@ -349,7 +350,7 @@ def summarize_source_orbit_gaps(
)
return samples
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) if isce2_enabled else []
bad_source_samples = [
item for item in suspect_bad_samples
if item.get("has_corruption_signal")
@@ -358,6 +359,7 @@ def summarize_source_orbit_gaps(
return {
"sample_limit": sample_limit,
"quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
"isce2_enabled": isce2_enabled,
"source_without_isce2_count": len(source_without_isce2),
"source_without_envi_count": len(source_without_envi),
"envi_without_source_count": len(envi_without_source),
@@ -809,6 +811,7 @@ def sync_orbit_pools(
envi_pool: str = "",
isce2_pool: str = "",
landsar_pool: str = "",
validate_without_isce2: bool = True,
) -> Dict:
"""从 source_dir 扫描平铺 .txt 精轨,同步到各引擎本地池。
@@ -863,11 +866,16 @@ def sync_orbit_pools(
xml_needs_refresh = bool(isce2_pool) and (
not os.path.exists(xml_path) or _needs_generated_refresh(fpath, xml_path)
)
requires_validation = xml_needs_refresh or (not isce2_pool and envi_needs_refresh)
requires_validation = xml_needs_refresh or (
bool(validate_without_isce2) and not isce2_pool and envi_needs_refresh
)
staged_dir = ""
staged_xml = ""
try:
health = _inspect_orbit_txt_health(fpath)
if _has_orbit_corruption_signal(health):
raise RuntimeError("Source TXT contains NUL bytes or cannot be read as a healthy LT-1 orbit text file.")
if requires_validation:
staged_dir, staged_xml = _stage_isce2_xml(
fpath,
@@ -924,8 +932,15 @@ def repair_orbit_pools(
isce2_pool: str = "",
landsar_pool: str = "",
) -> Dict[str, Any]:
isce2_enabled = bool(str(isce2_pool or "").strip())
before = check_orbit_consistency(envi_pool, isce2_pool)
sync_result = sync_orbit_pools(source_dir, envi_pool, isce2_pool, landsar_pool)
sync_result = sync_orbit_pools(
source_dir,
envi_pool,
isce2_pool,
landsar_pool,
validate_without_isce2=isce2_enabled,
)
repaired_from_envi: List[str] = []
repair_errors: List[Dict[str, str]] = []
@@ -934,7 +949,7 @@ def repair_orbit_pools(
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
if isce2_pool:
if isce2_enabled:
os.makedirs(isce2_pool, exist_ok=True)
for stem in sorted(set(envi_files) - set(isce2_files)):
txt_path = envi_files[stem]["path"]
@@ -952,6 +967,7 @@ def repair_orbit_pools(
"before": before,
"after": after,
"sync_result": sync_result,
"isce2_enabled": isce2_enabled,
"repaired_from_envi": repaired_from_envi,
"repair_error_count": len(repair_errors),
"repair_errors": repair_errors,
@@ -968,17 +984,19 @@ def quarantine_bad_orbits(
isce2_pool: str = "",
quarantine_root: str = "",
) -> Dict[str, Any]:
isce2_enabled = bool(str(isce2_pool or "").strip())
source_inventory = get_source_orbit_inventory(source_dir, recursive=True)
pool_inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
quarantine_dir = _default_quarantine_root(source_dir, quarantine_root)
candidate_stems = sorted(set(source_files) - set(isce2_files))
candidate_stems = sorted(set(source_files) - set(isce2_files)) if isce2_enabled else []
result: Dict[str, Any] = {
"isce2_enabled": isce2_enabled,
"quarantine_root": quarantine_dir,
"candidate_count": len(candidate_stems),
"validated_count": 0,
@@ -1083,14 +1101,18 @@ def check_orbit_consistency(
"healthy": bool,
}
"""
isce2_enabled = bool(str(isce2_pool or "").strip())
inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
envi_files: Dict[str, Dict[str, str]] = inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"]
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] if isce2_enabled else {}
by_satellite: Dict[str, int] = inventory["envi"]["by_satellite"]
# 对比
envi_stems = set(envi_files)
isce2_stems = set(isce2_files)
if not isce2_enabled:
envi_stems = set()
isce2_stems = set()
mismatches = []
for stem in envi_stems - isce2_stems:
mismatches.append(
@@ -1112,6 +1134,9 @@ def check_orbit_consistency(
)
mismatches.sort(key=lambda item: item["name"])
errors = list(inventory["envi"]["errors"])
if isce2_enabled:
errors.extend(inventory["isce2"]["errors"])
return {
"envi": {
@@ -1121,11 +1146,12 @@ def check_orbit_consistency(
},
"isce2": {
"path": isce2_pool,
"enabled": isce2_enabled,
"total": len(isce2_files),
},
"error_count": len(inventory["envi"]["errors"]) + len(inventory["isce2"]["errors"]),
"errors": list(inventory["envi"]["errors"]) + list(inventory["isce2"]["errors"]),
"error_count": len(errors),
"errors": errors,
"mismatch_count": len(mismatches),
"mismatches": mismatches,
"healthy": len(mismatches) == 0,
"healthy": len(mismatches) == 0 and len(errors) == 0,
}
@@ -52,6 +52,15 @@ def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str:
)
def _supported_dinsar_family_pair_expr(left_alias: str, right_alias: str) -> str:
left_family = _satellite_family_expr(left_alias)
right_family = _satellite_family_expr(right_alias)
return (
f"(({left_family} = 'LT1' AND {right_family} = 'LT1') "
f"OR ({left_family} = 'S1' AND {right_family} = 'S1'))"
)
def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
return (
f"(NULLIF({left_alias}.look_direction, '') IS NULL "
@@ -60,6 +69,60 @@ def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
)
def _relative_orbit_value_expr(alias: str) -> str:
return f"NULLIF(upper(trim(COALESCE({alias}.relative_orbit, ''))), '')"
def _same_relative_orbit_expr(left_alias: str, right_alias: str) -> str:
left_rel = _relative_orbit_value_expr(left_alias)
right_rel = _relative_orbit_value_expr(right_alias)
return f"({left_rel} IS NOT NULL AND {right_rel} IS NOT NULL AND {left_rel} = {right_rel})"
def _dinsar_quality_tier_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"CASE "
f"WHEN {same_rel} AND {center_distance_expr} <= 5000 THEN 'A' "
f"WHEN {center_distance_expr} <= 5000 THEN 'B' "
f"WHEN {center_distance_expr} <= 12000 THEN 'C' "
"ELSE 'REJECT' END"
)
def _dinsar_readiness_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
tier = _dinsar_quality_tier_expr(left_alias, right_alias, center_distance_expr)
return (
"CASE "
f"WHEN {tier} IN ('A', 'B') THEN 'RECOMMENDED' "
f"WHEN {tier} = 'C' THEN 'CANDIDATE' "
"ELSE 'NOT_RECOMMENDED' END"
)
def _dinsar_quality_score_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"GREATEST(0.0, LEAST(1.0, "
f"0.45 * COALESCE((ST_Area(ST_Intersection({left_alias}.geom, {right_alias}.geom)::geography) / "
f"NULLIF(GREATEST(ST_Area({left_alias}.geom::geography), ST_Area({right_alias}.geom::geography)), 0)), 0) + "
f"0.30 * CASE WHEN {same_rel} THEN 1 ELSE 0 END + "
f"0.25 * GREATEST(0.0, 1.0 - ({center_distance_expr} / 12000.0))"
"))"
)
def _dinsar_reasons_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"COALESCE(jsonb_path_query_array(jsonb_build_array("
f"CASE WHEN NOT ({same_rel}) THEN 'relative_orbit_missing_or_mismatch' END, "
f"CASE WHEN {center_distance_expr} > 5000 THEN 'center_distance_over_5km' END, "
f"CASE WHEN {center_distance_expr} > 12000 THEN 'center_distance_over_12km' END"
"), '$[*] ? (@ != null)')::json, '[]'::json)"
)
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
left_uid = _scene_uid_expr(left_alias)
right_uid = _scene_uid_expr(right_alias)
@@ -82,6 +145,7 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
f"AND COALESCE({left_alias}.insar_source_ready, false) "
f"AND COALESCE({right_alias}.insar_source_ready, false) "
f"AND { _supported_dinsar_family_pair_expr(left_alias, right_alias) } "
f"AND { _same_look_direction_expr(left_alias, right_alias) } "
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
)
@@ -106,7 +170,15 @@ def _full_rebuild_insert_sql() -> str:
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
pair_aoi_overlap_ratio,
orbit_direction,
same_relative_orbit,
master_relative_orbit,
slave_relative_orbit,
dinsar_quality_tier,
dinsar_quality_score,
dinsar_readiness,
dinsar_reasons_json,
same_satellite,
same_satellite_family,
same_look_direction,
@@ -144,7 +216,15 @@ def _full_rebuild_insert_sql() -> str:
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
NULL::double precision AS pair_aoi_overlap_ratio,
m.orbit_direction,
{ _same_relative_orbit_expr('m', 's') } AS same_relative_orbit,
{_relative_orbit_value_expr('m')} AS master_relative_orbit,
{_relative_orbit_value_expr('s')} AS slave_relative_orbit,
{ _dinsar_quality_tier_expr('m', 's', center_distance) } AS dinsar_quality_tier,
{ _dinsar_quality_score_expr('m', 's', center_distance) } AS dinsar_quality_score,
{ _dinsar_readiness_expr('m', 's', center_distance) } AS dinsar_readiness,
{ _dinsar_reasons_expr('m', 's', center_distance) } AS dinsar_reasons_json,
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
{ _same_satellite_family_expr('m', 's') } AS same_satellite_family,
{ _same_look_direction_expr('m', 's') } AS same_look_direction,
@@ -201,7 +281,15 @@ def _incremental_insert_sql() -> str:
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
pair_aoi_overlap_ratio,
orbit_direction,
same_relative_orbit,
master_relative_orbit,
slave_relative_orbit,
dinsar_quality_tier,
dinsar_quality_score,
dinsar_readiness,
dinsar_reasons_json,
same_satellite,
same_satellite_family,
same_look_direction,
@@ -244,7 +332,15 @@ def _incremental_insert_sql() -> str:
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
NULL::double precision AS pair_aoi_overlap_ratio,
d.orbit_direction,
{ _same_relative_orbit_expr('d', 'o') } AS same_relative_orbit,
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('d')} ELSE {_relative_orbit_value_expr('o')} END AS master_relative_orbit,
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('o')} ELSE {_relative_orbit_value_expr('d')} END AS slave_relative_orbit,
{ _dinsar_quality_tier_expr('d', 'o', center_distance) } AS dinsar_quality_tier,
{ _dinsar_quality_score_expr('d', 'o', center_distance) } AS dinsar_quality_score,
{ _dinsar_readiness_expr('d', 'o', center_distance) } AS dinsar_readiness,
{ _dinsar_reasons_expr('d', 'o', center_distance) } AS dinsar_reasons_json,
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
{ _same_satellite_family_expr('d', 'o') } AS same_satellite_family,
{ _same_look_direction_expr('d', 'o') } AS same_look_direction,
@@ -18,7 +18,7 @@ from ..models import (
PAIRING_CACHE_SCOPE_GLOBAL = "global"
DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1"
DEFAULT_PAIRING_METRIC_VERSION = "2026.06.dinsar.family.v1"
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
@@ -235,15 +235,6 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="directory_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_ARCHIVE_SOURCE_DIRS",
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
root_role="source_pool_gf3_archive",
display_prefix="GF3 Archive Pool",
scan_mode="archive_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_SOURCE_DIRS",
+603 -46
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import and_, case, cast, func, or_
from sqlalchemy import and_, case, cast, func, or_, text
from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
@@ -24,6 +24,8 @@ from shapely.geometry import Polygon
from shapely.ops import unary_union
from ..models import (
DinsarProductionRunItemORM,
DinsarProductionRunORM,
HazardPoint,
HazardPointORM,
PairingNetworkEdgeORM,
@@ -43,7 +45,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
PAIRING_POLICY_VERSION = "2026.06.dinsar-production.v1"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000
logger = logging.getLogger(__name__)
@@ -145,10 +147,10 @@ class SpatialService:
require_orbit_data=require_orbit_data,
)
if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
if len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
raise RuntimeError(
f"全部配对命中 {len(candidate_pool)} 条候选边,超过系统一次性返回上限 "
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略,或收紧 AOI、日期范围、重叠率。"
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请收紧 AOI、日期范围、重叠率或中心距离阈值。"
)
if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD:
@@ -156,15 +158,21 @@ class SpatialService:
f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}(当前: {len(candidate_pool)}),建议收紧参数或缩小 AOI。"
)
selected_candidates, strategy_warnings = self._apply_strategy(
candidate_pool,
effective_params,
aoi_wkt=aoi_wkt,
)
selected_candidates, strategy_warnings = self._apply_dinsar_production_strategy(candidate_pool, effective_params)
warnings.extend(strategy_warnings)
if not selected_candidates:
warnings.extend(
await self._build_empty_pairing_diagnostics(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
)
for candidate in selected_candidates:
candidate.setdefault("selection_strategy", effective_params.strategy)
candidate["selection_strategy"] = "dinsar_production"
self._ensure_candidate_identity(candidate)
network_run_id = await self._persist_network_run(
db,
@@ -176,6 +184,7 @@ class SpatialService:
selected_candidates=selected_candidates,
)
await self._attach_dinsar_production_summaries(db, selected_candidates)
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
metadata = {
"fallback_used": False,
@@ -227,13 +236,20 @@ class SpatialService:
PairingMetricCacheORM.status == "READY",
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.master_imaging_date < PairingMetricCacheORM.slave_imaging_date,
PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold,
PairingMetricCacheORM.same_look_direction.is_(True),
PairingMetricCacheORM.dinsar_readiness.in_(["RECOMMENDED", "CANDIDATE"]),
)
)
if params.limit_footprint_center_distance:
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
stmt = stmt.where(
or_(
and_(master_family_expr == "LT1", slave_family_expr == "LT1"),
and_(master_family_expr == "S1", slave_family_expr == "S1"),
)
)
if require_orbit_data:
stmt = stmt.where(
@@ -241,8 +257,7 @@ class SpatialService:
slave_alias.has_orbit_data.is_(True),
)
if not params.cross_satellite_pairing:
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
if params.require_same_imaging_mode:
stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True))
@@ -259,20 +274,19 @@ class SpatialService:
)
if params.allowed_satellites:
allowed_satellites = [
str(item).strip().upper()
for item in params.allowed_satellites
if str(item).strip()
]
allowed_satellites = []
for item in params.allowed_satellites:
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
allowed_satellites.append("LT1")
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
allowed_satellites.append("S1")
allowed_satellites = list(dict.fromkeys(allowed_satellites))
if not allowed_satellites:
return []
stmt = stmt.where(
or_(
func.upper(master_alias.satellite).in_(allowed_satellites),
func.upper(master_alias.satellite_family).in_(allowed_satellites),
),
or_(
func.upper(slave_alias.satellite).in_(allowed_satellites),
func.upper(slave_alias.satellite_family).in_(allowed_satellites),
),
master_family_expr.in_(allowed_satellites),
slave_family_expr.in_(allowed_satellites),
)
if params.master_date_from:
@@ -286,23 +300,23 @@ class SpatialService:
if aoi_wkt:
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
aoi_geog = cast(aoi_geom, Geography)
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
pair_overlap_geom = ST_Intersection(master_alias.geom, slave_alias.geom)
pair_aoi_geom = ST_Intersection(pair_overlap_geom, aoi_geom)
pair_aoi_overlap_expr = (ST_Area(cast(pair_aoi_geom, Geography)) / aoi_area).label("pair_aoi_overlap_ratio")
stmt = stmt.add_columns(pair_aoi_overlap_expr)
stmt = stmt.where(
ST_Intersects(master_alias.geom, aoi_geom),
ST_Intersects(slave_alias.geom, aoi_geom),
ST_Intersects(pair_overlap_geom, aoi_geom),
)
if params.aoi_overlap_threshold is not None:
aoi_geog = cast(aoi_geom, Geography)
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
master_inter_geog = cast(ST_Intersection(master_alias.geom, aoi_geom), Geography)
slave_inter_geog = cast(ST_Intersection(slave_alias.geom, aoi_geom), Geography)
stmt = stmt.where(
ST_Area(master_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
ST_Area(slave_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
)
stmt = stmt.where(pair_aoi_overlap_expr >= params.aoi_overlap_threshold)
stmt = stmt.order_by(
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
PairingMetricCacheORM.dinsar_quality_tier.asc(),
func.coalesce(PairingMetricCacheORM.dinsar_quality_score, 0).desc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
center_distance_expr.asc(),
PairingMetricCacheORM.pair_uid.asc(),
@@ -310,7 +324,9 @@ class SpatialService:
result = await db.execute(stmt)
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
for row in result.all():
metric_row, master_row, slave_row = row[0], row[1], row[2]
pair_aoi_overlap_ratio = row[3] if len(row) > 3 else None
center_distance = float(
metric_row.scene_center_distance_meters
if metric_row.scene_center_distance_meters is not None
@@ -328,16 +344,54 @@ class SpatialService:
"dist": center_distance,
"scene_center_distance_meters": center_distance,
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0),
"dinsar_quality_tier": metric_row.dinsar_quality_tier,
"dinsar_quality_score": (
float(metric_row.dinsar_quality_score)
if metric_row.dinsar_quality_score is not None
else None
),
"dinsar_readiness": metric_row.dinsar_readiness,
"dinsar_reasons": [
str(item)
for item in (metric_row.dinsar_reasons_json or [])
if isinstance(item, str) and item.strip()
],
"same_relative_orbit": bool(metric_row.same_relative_orbit),
"master_relative_orbit": metric_row.master_relative_orbit,
"slave_relative_orbit": metric_row.slave_relative_orbit,
"pair_aoi_overlap_ratio": (
float(pair_aoi_overlap_ratio)
if pair_aoi_overlap_ratio is not None
else None
),
}
)
return candidate_pool
def _ensure_candidate_identity(self, candidate: dict) -> Tuple[str, str]:
master = candidate["master"]
slave = candidate["slave"]
task_alias = str(candidate.get("task_alias") or "").strip() or build_task_alias(
master.imaging_date,
slave.imaging_date,
)
pair_key = str(candidate.get("pair_key") or "").strip() or build_pair_key(
master.file_path,
slave.file_path,
master.imaging_date,
slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
)
candidate["task_alias"] = task_alias
candidate["pair_key"] = pair_key
return task_alias, pair_key
def _build_radar_pairs(self, selected_candidates: List[dict]) -> List[RadarPair]:
result_pairs: List[RadarPair] = []
for candidate in selected_candidates:
master = candidate["master"]
slave = candidate["slave"]
task_alias = build_task_alias(master.imaging_date, slave.imaging_date)
task_alias, pair_key = self._ensure_candidate_identity(candidate)
selection_score = candidate.get("selection_score")
result_pairs.append(
RadarPair(
@@ -345,13 +399,7 @@ class SpatialService:
slave=slave,
task_name=task_alias,
task_alias=task_alias,
pair_key=build_pair_key(
master.file_path,
slave.file_path,
master.imaging_date,
slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
),
pair_key=pair_key,
pair_uid=candidate.get("pair_uid"),
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
network_run_id=candidate.get("network_run_id"),
@@ -367,10 +415,292 @@ class SpatialService:
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0
),
scene_overlap_ratio=float(candidate.get("overlap_ratio") or 0.0),
pair_aoi_overlap_ratio=(
float(candidate["pair_aoi_overlap_ratio"])
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
dinsar_quality_tier=candidate.get("dinsar_quality_tier"),
dinsar_quality_score=(
float(candidate["dinsar_quality_score"])
if candidate.get("dinsar_quality_score") is not None
else None
),
dinsar_readiness=candidate.get("dinsar_readiness"),
dinsar_reasons=candidate.get("dinsar_reasons") or [],
same_relative_orbit=bool(candidate.get("same_relative_orbit")),
master_relative_orbit=candidate.get("master_relative_orbit"),
slave_relative_orbit=candidate.get("slave_relative_orbit"),
production_summary=candidate.get("production_summary"),
)
)
return result_pairs
async def _attach_dinsar_production_summaries(
self,
db: AsyncSession,
selected_candidates: List[dict],
) -> None:
if not selected_candidates:
return
pair_uids: set[str] = set()
pair_keys: set[str] = set()
aliases: set[str] = set()
for candidate in selected_candidates:
task_alias, pair_key = self._ensure_candidate_identity(candidate)
pair_uid = str(candidate.get("pair_uid") or "").strip()
if pair_uid:
pair_uids.add(pair_uid)
if pair_key:
pair_keys.add(pair_key)
if task_alias:
aliases.add(task_alias)
run_conditions = []
if pair_uids:
run_conditions.append(DinsarProductionRunItemORM.pair_uid.in_(pair_uids))
if pair_keys:
run_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys))
if aliases:
run_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases))
run_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases))
product_conditions = []
if pair_uids:
product_conditions.append(ResultProductORM.pair_uid.in_(pair_uids))
if pair_keys:
product_conditions.append(ResultProductORM.pair_key.in_(pair_keys))
if aliases:
product_conditions.append(ResultProductORM.task_alias.in_(aliases))
product_conditions.append(ResultProductORM.task_name.in_(aliases))
run_rows = []
if run_conditions:
result = await db.execute(
select(DinsarProductionRunItemORM, DinsarProductionRunORM)
.join(DinsarProductionRunORM, DinsarProductionRunItemORM.run_id == DinsarProductionRunORM.run_id)
.where(or_(*run_conditions))
.order_by(
DinsarProductionRunItemORM.updated_at.desc().nullslast(),
DinsarProductionRunItemORM.id.desc(),
)
)
run_rows = result.all()
products = []
if product_conditions:
result = await db.execute(
select(ResultProductORM)
.where(ResultProductORM.catalog_name == "dinsar")
.where(or_(*product_conditions))
.order_by(
ResultProductORM.published_at.desc().nullslast(),
ResultProductORM.id.desc(),
)
)
products = result.scalars().all()
run_by_uid: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
run_by_key: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
run_by_alias: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
for item, run in run_rows:
pair_uid = str(item.pair_uid or "").strip()
pair_key = str(item.pair_key or "").strip()
if pair_uid:
run_by_uid[pair_uid].append((item, run))
if pair_key:
run_by_key[pair_key].append((item, run))
for alias in {str(item.task_alias or "").strip(), str(item.task_name or "").strip()}:
if alias:
run_by_alias[alias].append((item, run))
products_by_uid: Dict[str, List[ResultProductORM]] = defaultdict(list)
products_by_key: Dict[str, List[ResultProductORM]] = defaultdict(list)
products_by_alias: Dict[str, List[ResultProductORM]] = defaultdict(list)
for product in products:
pair_uid = str(product.pair_uid or "").strip()
pair_key = str(product.pair_key or "").strip()
if pair_uid:
products_by_uid[pair_uid].append(product)
if pair_key:
products_by_key[pair_key].append(product)
for alias in {str(product.task_alias or "").strip(), str(product.task_name or "").strip()}:
if alias:
products_by_alias[alias].append(product)
for candidate in selected_candidates:
task_alias, pair_key = self._ensure_candidate_identity(candidate)
pair_uid = str(candidate.get("pair_uid") or "").strip()
exact_runs = self._dedupe_by_object_id(
[*run_by_uid.get(pair_uid, []), *run_by_key.get(pair_key, [])],
key=lambda row: getattr(row[0], "id", None),
)
alias_runs = self._dedupe_by_object_id(
run_by_alias.get(task_alias, []),
key=lambda row: getattr(row[0], "id", None),
)
exact_products = self._dedupe_by_object_id(
[*products_by_uid.get(pair_uid, []), *products_by_key.get(pair_key, [])],
key=lambda product: getattr(product, "id", None),
)
alias_products = self._dedupe_by_object_id(
products_by_alias.get(task_alias, []),
key=lambda product: getattr(product, "id", None),
)
matched_runs = exact_runs or alias_runs
matched_products = exact_products or alias_products
candidate["production_summary"] = self._summarize_dinsar_production(
matched_runs,
matched_products,
match_level="identity" if (exact_runs or exact_products) else ("task_alias" if (alias_runs or alias_products) else "none"),
)
def _summarize_dinsar_production(
self,
run_rows: List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]],
products: List[ResultProductORM],
*,
match_level: str = "none",
) -> Dict[str, Any]:
latest_run_row = max(
run_rows,
key=lambda row: self._datetime_sort_key(
row[0].updated_at,
row[0].ended_at,
row[0].started_at,
row[0].created_at,
),
default=None,
)
latest_product = max(
products,
key=lambda product: self._datetime_sort_key(
product.published_at,
product.produced_at,
product.updated_at,
product.registered_at,
),
default=None,
)
ready_products = [product for product in products if self._is_ready_result_product(product)]
completed_statuses = {"COMPLETED", "READY", "SUCCESS", "PUBLISHED"}
failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
completed_run_count = sum(
1
for item, run in run_rows
if str(item.status or "").strip().upper() in completed_statuses
or str(run.status or "").strip().upper() in completed_statuses
)
failed_run_count = sum(
1
for item, run in run_rows
if str(item.status or "").strip().upper() in failed_statuses
or str(run.status or "").strip().upper() in failed_statuses
)
latest_item = latest_run_row[0] if latest_run_row else None
latest_run = latest_run_row[1] if latest_run_row else None
if ready_products and latest_product is not None:
latest_status = str(latest_product.status or "").strip().upper()
elif latest_item is not None or latest_run is not None:
latest_status = str(
(latest_item.status if latest_item is not None else None)
or (latest_run.status if latest_run is not None else None)
or ""
).strip().upper()
elif latest_product is not None:
latest_status = str(latest_product.status or "").strip().upper()
else:
latest_status = ""
if ready_products:
status = "READY"
elif completed_run_count > 0:
status = "COMPLETED"
elif latest_status:
status = latest_status
else:
status = "MISSING"
engine_codes = sorted(
{
str(value or "").strip().lower()
for value in [
*(product.engine_code for product in products),
*(run.engine_code for _, run in run_rows),
]
if str(value or "").strip()
}
)
return {
"has_record": bool(run_rows or products),
"is_produced": bool(ready_products or completed_run_count > 0),
"status": status,
"match_level": match_level,
"run_item_count": len(run_rows),
"completed_run_count": completed_run_count,
"failed_run_count": failed_run_count,
"product_count": len(products),
"ready_product_count": len(ready_products),
"engine_codes": engine_codes,
"latest_engine_code": (
str(latest_product.engine_code or "").strip().lower()
if latest_product is not None and latest_product.engine_code
else (
str(latest_run.engine_code or "").strip().lower()
if latest_run is not None and latest_run.engine_code
else None
)
),
"latest_run_id": latest_run.run_id if latest_run is not None else None,
"latest_run_status": latest_run.status if latest_run is not None else None,
"latest_item_status": latest_item.status if latest_item is not None else None,
"latest_output_dir": latest_item.latest_output_dir if latest_item is not None else None,
"latest_product_id": latest_product.id if latest_product is not None else None,
"latest_product_identifier": latest_product.product_id if latest_product is not None else None,
"latest_product_status": latest_product.status if latest_product is not None else None,
"latest_product_health": latest_product.health_status if latest_product is not None else None,
"latest_product_published_at": latest_product.published_at if latest_product is not None else None,
"updated_at": (
latest_item.updated_at
if latest_item is not None
else (
latest_product.updated_at
if latest_product is not None
else None
)
),
}
def _is_ready_result_product(self, product: ResultProductORM) -> bool:
status = str(product.status or "").strip().upper()
health = str(product.health_status or "").strip().upper()
return status in {"READY", "COMPLETED", "SUCCESS"} and health not in {"ERROR", "FAILED"}
def _datetime_sort_key(self, *values: Any) -> float:
for value in values:
if value is None:
continue
try:
return float(value.timestamp())
except Exception:
continue
return 0.0
def _dedupe_by_object_id(self, items: List[Any], *, key) -> List[Any]:
seen: set[Any] = set()
output: List[Any] = []
for item in items:
item_key = key(item)
if item_key is None:
item_key = id(item)
if item_key in seen:
continue
seen.add(item_key)
output.append(item)
return output
async def _persist_network_run(
self,
db: AsyncSession,
@@ -451,6 +781,8 @@ class SpatialService:
"master_scene_uid": candidate.get("master_scene_uid"),
"slave_scene_uid": candidate.get("slave_scene_uid"),
"pair_uid": candidate.get("pair_uid"),
"pair_key": candidate.get("pair_key"),
"task_alias": candidate.get("task_alias"),
"time_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_center_distance_meters": float(
@@ -460,6 +792,11 @@ class SpatialService:
),
"legacy_spatial_baseline_field": "scene_center_distance_meters",
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
"pair_aoi_overlap_ratio": (
float(candidate["pair_aoi_overlap_ratio"])
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
}
def _stable_sha1(self, value: Any) -> str:
@@ -814,6 +1151,226 @@ class SpatialService:
"scenes": scene_payloads,
}
def _apply_dinsar_production_strategy(
self,
candidate_pool: List[dict],
params: PairingRequest,
) -> Tuple[List[dict], List[str]]:
if not candidate_pool:
return [], []
warnings: List[str] = []
rejected_count = sum(
1
for candidate in candidate_pool
if str(candidate.get("dinsar_readiness") or "").upper() == "NOT_RECOMMENDED"
)
if rejected_count:
warnings.append(f"{rejected_count}条候选因D-InSAR生产前置条件不足被过滤。")
selected = []
for candidate in candidate_pool:
readiness = str(candidate.get("dinsar_readiness") or "CANDIDATE").upper()
if readiness == "NOT_RECOMMENDED":
continue
tier = str(candidate.get("dinsar_quality_tier") or "C").upper()
quality_score = candidate.get("dinsar_quality_score")
selected.append(
{
**candidate,
"selection_reason": f"dinsar_{readiness.lower()}_{tier.lower()}",
"selection_score": (
float(quality_score)
if quality_score is not None
else self._score_pair_candidate(candidate, params)
),
}
)
selected.sort(
key=lambda item: (
{"A": 0, "B": 1, "C": 2}.get(str(item.get("dinsar_quality_tier") or "C").upper(), 9),
-float(item.get("selection_score") or 0),
int(item.get("days") or 0),
float(item.get("dist") or 0),
str(getattr(item.get("master"), "imaging_date", "") or ""),
str(getattr(item.get("slave"), "imaging_date", "") or ""),
)
)
return selected, warnings
async def _build_empty_pairing_diagnostics(
self,
db: AsyncSession,
params: PairingRequest,
*,
aoi_wkt: Optional[str],
require_orbit_data: bool,
) -> List[str]:
allowed_families: List[str] = []
for item in params.allowed_satellites or []:
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
allowed_families.append("LT1")
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
allowed_families.append("S1")
allowed_families = list(dict.fromkeys(allowed_families))
sql = text(
"""
WITH base AS (
SELECT
pmc.*,
m.satellite AS master_satellite_actual,
s.satellite AS slave_satellite_actual,
m.has_orbit_data AS master_has_orbit,
s.has_orbit_data AS slave_has_orbit,
COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters) AS center_m
FROM pairing_metric_cache pmc
JOIN radar_data m ON m.id = pmc.master_scene_ref_id
JOIN radar_data s ON s.id = pmc.slave_scene_ref_id
WHERE pmc.metric_version = :metric_version
AND pmc.status = 'READY'
AND pmc.master_imaging_date < pmc.slave_imaging_date
AND pmc.same_look_direction IS TRUE
AND pmc.same_satellite_family IS TRUE
AND pmc.dinsar_readiness IN ('RECOMMENDED', 'CANDIDATE')
AND (:require_orbit_data IS FALSE OR (m.has_orbit_data IS TRUE AND s.has_orbit_data IS TRUE))
AND (:require_same_imaging_mode IS FALSE OR pmc.same_imaging_mode IS TRUE)
AND (:require_same_polarization IS FALSE OR pmc.same_polarization IS TRUE)
AND (
:allowed_families_is_empty IS TRUE
OR pmc.master_satellite_family = ANY(:allowed_families)
OR pmc.master_satellite = ANY(:allowed_families)
)
AND (CAST(:master_date_from AS text) IS NULL OR pmc.master_imaging_date >= CAST(:master_date_from AS text))
AND (CAST(:master_date_to AS text) IS NULL OR pmc.master_imaging_date <= CAST(:master_date_to AS text))
AND (CAST(:slave_date_from AS text) IS NULL OR pmc.slave_imaging_date >= CAST(:slave_date_from AS text))
AND (CAST(:slave_date_to AS text) IS NULL OR pmc.slave_imaging_date <= CAST(:slave_date_to AS text))
AND (
CAST(:aoi_wkt AS text) IS NULL
OR ST_Intersects(
ST_Intersection(m.geom, s.geom),
ST_GeomFromText(CAST(:aoi_wkt AS text), 4326)
)
)
),
time_ok AS (
SELECT * FROM base
WHERE time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
),
overlap_ok AS (
SELECT * FROM time_ok
WHERE scene_overlap_ratio >= :overlap_threshold
),
center_ok AS (
SELECT * FROM overlap_ok
WHERE center_m <= :center_distance_max
)
SELECT
(SELECT count(*) FROM base) AS base_count,
(SELECT count(*) FROM time_ok) AS time_ok_count,
(SELECT count(*) FROM overlap_ok) AS overlap_ok_count,
(SELECT count(*) FROM center_ok) AS center_ok_count,
(
SELECT json_build_object(
'master_date', master_imaging_date,
'slave_date', slave_imaging_date,
'master_satellite', master_satellite_actual,
'slave_satellite', slave_satellite_actual,
'time_baseline_days', time_baseline_days,
'center_meters', center_m,
'overlap_ratio', scene_overlap_ratio,
'quality_tier', dinsar_quality_tier,
'readiness', dinsar_readiness
)
FROM base
ORDER BY
CASE
WHEN time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
THEN 0 ELSE 1
END,
CASE WHEN scene_overlap_ratio >= :overlap_threshold THEN 0 ELSE 1 END,
abs(time_baseline_days - :time_baseline_max),
center_m ASC NULLS LAST
LIMIT 1
) AS nearest_candidate;
"""
)
result = await db.execute(
sql,
{
"metric_version": pairing_state_service.metric_version,
"require_orbit_data": require_orbit_data,
"require_same_imaging_mode": bool(params.require_same_imaging_mode),
"require_same_polarization": bool(params.require_same_polarization),
"allowed_families": allowed_families or ["__NONE__"],
"allowed_families_is_empty": not allowed_families,
"master_date_from": params.master_date_from or None,
"master_date_to": params.master_date_to or None,
"slave_date_from": params.slave_date_from or None,
"slave_date_to": params.slave_date_to or None,
"aoi_wkt": aoi_wkt,
"time_baseline_min": int(params.time_baseline_min),
"time_baseline_max": int(params.time_baseline_max),
"overlap_threshold": float(params.overlap_threshold),
"center_distance_max": float(params.spatial_baseline_max_meters),
},
)
row = result.mappings().first()
if not row:
return ["未找到满足条件的 D-InSAR 配对;诊断查询未返回统计结果。"]
base_count = int(row.get("base_count") or 0)
time_ok_count = int(row.get("time_ok_count") or 0)
overlap_ok_count = int(row.get("overlap_ok_count") or 0)
center_ok_count = int(row.get("center_ok_count") or 0)
if base_count <= 0:
family_text = "、".join(allowed_families) if allowed_families else "LT-1/Sentinel-1"
return [
f"未找到 {family_text} 的可生产基础候选边。请检查数据体系、主从日期范围、AOI、精轨绑定和配对缓存状态。"
]
messages = [
(
"当前筛选下基础候选 {base} 条;时间基线 {min_days}-{max_days} 天后剩 {time_ok} 条;"
"重叠率 >= {overlap:.2f} 后剩 {overlap_ok} 条;footprint 中心距离 <= {center:.0f} 米后剩 {center_ok} 条。"
).format(
base=base_count,
min_days=int(params.time_baseline_min),
max_days=int(params.time_baseline_max),
time_ok=time_ok_count,
overlap=float(params.overlap_threshold),
overlap_ok=overlap_ok_count,
center=float(params.spatial_baseline_max_meters),
center_ok=center_ok_count,
)
]
nearest = row.get("nearest_candidate")
if isinstance(nearest, str):
try:
nearest = json.loads(nearest)
except Exception:
nearest = None
if isinstance(nearest, dict):
messages.append(
(
"最接近的一对是 {master_satellite} {master_date} -> {slave_satellite} {slave_date},"
"时间基线 {days} 天,中心距离 {center:.1f} 米,重叠率 {overlap:.3f},质量 {tier}/{readiness}。"
).format(
master_satellite=nearest.get("master_satellite") or "?",
master_date=nearest.get("master_date") or "?",
slave_satellite=nearest.get("slave_satellite") or "?",
slave_date=nearest.get("slave_date") or "?",
days=int(nearest.get("time_baseline_days") or 0),
center=float(nearest.get("center_meters") or 0.0),
overlap=float(nearest.get("overlap_ratio") or 0.0),
tier=nearest.get("quality_tier") or "?",
readiness=nearest.get("readiness") or "?",
)
)
return messages
def _apply_strategy(
self,
candidate_pool: List[dict],
+3
View File
@@ -259,12 +259,15 @@ class TaskService:
task = result.scalar_one_or_none()
if task:
previous_message = task.message
if status:
task.status = status
if progress is not None:
task.progress = progress
if message:
task.message = message
if message != previous_message:
await self._add_log(task_id, "INFO", message, db=db)
# 如果任务结束,更新结束时间
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
+1 -1
View File
@@ -39,7 +39,7 @@ def get_unpack_config() -> Dict[str, Any]:
"source_dirs": source_dirs,
"insar_storage_dirs": insar_storage_dirs,
"min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")),
"delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")),
"delete_archive": False,
"tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"),
"archive_exts": archive_exts,
"scan_workers": module.parse_int(