Refactor local InSAR asset and production workflows
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,8 @@ import json
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import tarfile
|
||||
import zipfile
|
||||
from typing import List, Dict, Optional, Tuple, Any
|
||||
from datetime import datetime
|
||||
from shapely.geometry import Point, Polygon, shape, mapping
|
||||
@@ -240,6 +242,8 @@ def _chunked(items: List[Any], size: int):
|
||||
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
_RADAR_ARCHIVE_SUFFIXES = (".zip", ".tar", ".tgz", ".tar.gz")
|
||||
_RADAR_ARCHIVE_PREVIEW_MAX_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def _sanitize_cache_name(name: str) -> str:
|
||||
@@ -265,6 +269,208 @@ def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_versio
|
||||
return f"RGID_{safe_version}_{digest}_{base}.webp"
|
||||
|
||||
|
||||
def _has_radar_archive_suffix(path: str) -> bool:
|
||||
return str(path or "").lower().endswith(_RADAR_ARCHIVE_SUFFIXES)
|
||||
|
||||
|
||||
def _archive_product_stem(path: str) -> str:
|
||||
base = os.path.basename(str(path or ""))
|
||||
lower_base = base.lower()
|
||||
for suffix in (".tar.gz", ".tgz", ".tar", ".zip"):
|
||||
if lower_base.endswith(suffix):
|
||||
return base[: -len(suffix)]
|
||||
return os.path.splitext(base)[0]
|
||||
|
||||
|
||||
def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> Optional[int]:
|
||||
product_stem = _archive_product_stem(archive_path)
|
||||
if not product_stem:
|
||||
return None
|
||||
|
||||
member = str(member_name or "").replace("\\", "/").strip("/")
|
||||
member_lower = member.lower()
|
||||
product_lower = product_stem.lower()
|
||||
expected_names = [
|
||||
f"{product_lower}/{product_lower}.browse.jpg",
|
||||
f"{product_lower}/{product_lower}.browse.jpeg",
|
||||
f"{product_lower}/{product_lower}.browse.png",
|
||||
f"{product_lower}/{product_lower}.quicklook.jpg",
|
||||
f"{product_lower}/{product_lower}.quicklook.png",
|
||||
f"{product_lower}/{product_lower}.quick-look.png",
|
||||
f"{product_lower}/{product_lower}.thumb.jpg",
|
||||
f"{product_lower}/{product_lower}.thumb.jpeg",
|
||||
f"{product_lower}/{product_lower}.thumb.png",
|
||||
f"{product_lower}/preview/quick-look.png",
|
||||
f"{product_lower}.browse.jpg",
|
||||
f"{product_lower}.browse.jpeg",
|
||||
f"{product_lower}.browse.png",
|
||||
f"{product_lower}.thumb.jpg",
|
||||
f"{product_lower}.thumb.jpeg",
|
||||
f"{product_lower}.thumb.png",
|
||||
]
|
||||
try:
|
||||
return expected_names.index(member_lower)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]:
|
||||
lower_name = str(member_name or "").replace("\\", "/").lower()
|
||||
base_name = os.path.basename(lower_name)
|
||||
if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
|
||||
return None
|
||||
if size_bytes and size_bytes > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
|
||||
return None
|
||||
|
||||
has_keyword = any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS)
|
||||
if base_name.endswith((".tif", ".tiff")) and not has_keyword:
|
||||
return None
|
||||
|
||||
keyword_score = 0 if has_keyword else 2
|
||||
if base_name == "quick-look.png":
|
||||
keyword_score = -4
|
||||
elif base_name == "quicklook.png":
|
||||
keyword_score = -3
|
||||
elif base_name.startswith("quick-look.") or base_name.startswith("quicklook."):
|
||||
keyword_score = min(keyword_score, -2)
|
||||
if "/preview/" in lower_name:
|
||||
keyword_score -= 1
|
||||
if "/icons/" in lower_name:
|
||||
keyword_score += 2
|
||||
if base_name.endswith((".jpg", ".jpeg", ".png", ".webp")):
|
||||
ext_score = 0
|
||||
elif base_name.endswith(".bmp"):
|
||||
ext_score = 1
|
||||
else:
|
||||
ext_score = 2
|
||||
depth = lower_name.count("/")
|
||||
size_score = -int(size_bytes or 0)
|
||||
return (keyword_score, depth, ext_score, size_score, member_name)
|
||||
|
||||
|
||||
def _radar_archive_preview_cache_path(archive_path: str, member_name: str) -> str:
|
||||
digest = hashlib.sha1(
|
||||
f"{archive_path}|{member_name}".encode("utf-8", errors="ignore")
|
||||
).hexdigest()[:16]
|
||||
raw_base = os.path.basename(str(member_name or "preview"))
|
||||
base = _sanitize_cache_name(raw_base)
|
||||
_, ext = os.path.splitext(raw_base)
|
||||
ext = _RADAR_CACHE_NAME_RE.sub("", ext.lower())[:12]
|
||||
if ext and not base.lower().endswith(ext):
|
||||
base = f"{base[:max(1, 48 - len(ext))]}{ext}"
|
||||
return os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources", f"APS_{digest}_{base}")
|
||||
|
||||
|
||||
def _write_archive_preview_cache(target_path: str, source_obj: Any, archive_mtime: float) -> Optional[str]:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(target_path), exist_ok=True)
|
||||
if os.path.exists(target_path) and _safe_mtime(target_path) >= archive_mtime:
|
||||
return target_path
|
||||
tmp_path = f"{target_path}.tmp"
|
||||
with open(tmp_path, "wb") as target:
|
||||
while True:
|
||||
chunk = source_obj.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
target.write(chunk)
|
||||
os.replace(tmp_path, target_path)
|
||||
if archive_mtime:
|
||||
os.utime(target_path, (archive_mtime, archive_mtime))
|
||||
return target_path
|
||||
except Exception:
|
||||
try:
|
||||
if "tmp_path" in locals() and os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _find_radar_archive_preview_source(archive_path: str) -> Optional[str]:
|
||||
if not archive_path or not os.path.isfile(archive_path) or not _has_radar_archive_suffix(archive_path):
|
||||
return None
|
||||
|
||||
archive_mtime = _safe_mtime(archive_path)
|
||||
lower_path = archive_path.lower()
|
||||
try:
|
||||
if lower_path.endswith(".zip"):
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
candidates = []
|
||||
info_by_name = {}
|
||||
for info in archive.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
score = _radar_archive_preview_score(info.filename, int(info.file_size or 0))
|
||||
if score is None:
|
||||
continue
|
||||
candidates.append(score)
|
||||
info_by_name[info.filename] = info
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort()
|
||||
member_name = candidates[0][-1]
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
|
||||
with archive.open(info_by_name[member_name], "r") as source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
|
||||
with tarfile.open(archive_path, "r:*") as archive:
|
||||
candidates = []
|
||||
member_by_name = {}
|
||||
best_rank: Optional[int] = None
|
||||
best_member: Optional[tarfile.TarInfo] = None
|
||||
for member in archive:
|
||||
if not member.isfile():
|
||||
continue
|
||||
expected_rank = _radar_archive_expected_preview_rank(archive_path, member.name)
|
||||
if expected_rank is not None:
|
||||
if best_rank is None or expected_rank < best_rank:
|
||||
best_rank = expected_rank
|
||||
best_member = member
|
||||
if expected_rank == 0:
|
||||
source_obj = archive.extractfile(member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
continue
|
||||
|
||||
score = _radar_archive_preview_score(member.name, int(member.size or 0))
|
||||
if score is None:
|
||||
if best_member is not None and int(member.size or 0) > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
|
||||
break
|
||||
continue
|
||||
candidates.append(score)
|
||||
member_by_name[member.name] = member
|
||||
if os.path.basename(str(member.name or "").lower()) in {"quick-look.png", "quicklook.png"}:
|
||||
source_obj = archive.extractfile(member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
if best_member is not None:
|
||||
source_obj = archive.extractfile(best_member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, best_member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort()
|
||||
member_name = candidates[0][-1]
|
||||
source_obj = archive.extractfile(member_by_name[member_name])
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]:
|
||||
"""Extract coverage polygon from a GeoTIFF file using GDAL.
|
||||
|
||||
@@ -380,6 +586,8 @@ class DataService:
|
||||
|
||||
@staticmethod
|
||||
def find_radar_preview_source(scene_dir: str) -> Optional[str]:
|
||||
if scene_dir and os.path.isfile(scene_dir) and _has_radar_archive_suffix(scene_dir):
|
||||
return _find_radar_archive_preview_source(scene_dir)
|
||||
if not scene_dir or not os.path.isdir(scene_dir):
|
||||
return None
|
||||
|
||||
@@ -520,21 +728,26 @@ class DataService:
|
||||
if orbit_dir and orbit_files_map:
|
||||
update_progress("正在同步精轨到本地引擎池...", 8)
|
||||
try:
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
sync_result = await asyncio.to_thread(
|
||||
sync_orbit_pools,
|
||||
orbit_dir,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
settings.ORBIT_POOL_LANDSAR,
|
||||
bool(settings.ISCE2_ENABLED),
|
||||
)
|
||||
envi_new = len(sync_result.get("envi", {}).get("copied", []))
|
||||
envi_updated = len(sync_result.get("envi", {}).get("updated", []))
|
||||
isce2_new = len(sync_result.get("isce2", {}).get("converted", []))
|
||||
isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", []))
|
||||
print(
|
||||
f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated},"
|
||||
f"ISCE2 转换 {isce2_new}、重转 {isce2_updated}"
|
||||
)
|
||||
if settings.ISCE2_ENABLED:
|
||||
print(
|
||||
f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},"
|
||||
f"ISCE2 XML 转换 {isce2_new}、重转 {isce2_updated}"
|
||||
)
|
||||
else:
|
||||
print(f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},ISCE2 已停用")
|
||||
invalid_orbit_stems = {
|
||||
item.get("name")
|
||||
for item in sync_result.get("invalid_sources", [])
|
||||
@@ -550,7 +763,11 @@ class DataService:
|
||||
sync_errors = (
|
||||
sync_result.get("source", {}).get("errors", [])
|
||||
+ [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])]
|
||||
+ [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
|
||||
+ (
|
||||
[item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
|
||||
if settings.ISCE2_ENABLED
|
||||
else []
|
||||
)
|
||||
)
|
||||
if task_id and sync_errors:
|
||||
await task_service.add_log(
|
||||
@@ -560,15 +777,16 @@ class DataService:
|
||||
"精轨同步存在异常: "
|
||||
f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, "
|
||||
f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, "
|
||||
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))} 项"
|
||||
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', [])) if settings.ISCE2_ENABLED else 0} 项"
|
||||
),
|
||||
)
|
||||
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
"WARN",
|
||||
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
|
||||
)
|
||||
if settings.ISCE2_ENABLED:
|
||||
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
"WARN",
|
||||
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
|
||||
)
|
||||
if task_id and invalid_orbit_stems:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
|
||||
@@ -17,10 +17,13 @@ from ..utils import parse_gf3_l2_dirname
|
||||
|
||||
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
|
||||
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
|
||||
FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes"
|
||||
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
||||
SKIP_DIR_NAMES = {
|
||||
".git",
|
||||
FLAT_SCENE_CATALOG_DIR_NAME,
|
||||
".gf3_extract",
|
||||
".gf3_runtime",
|
||||
".sarmap",
|
||||
"__pycache__",
|
||||
"temp",
|
||||
@@ -87,6 +90,16 @@ def _polarization_from_geo_name(name: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _scene_name_from_geo_name(name: str) -> str:
|
||||
text = str(name or "").strip()
|
||||
match = re.match(r"^(?P<scene>.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE)
|
||||
if match:
|
||||
return match.group("scene")
|
||||
if text.lower().endswith("_geo"):
|
||||
return text[:-4]
|
||||
return Path(text).stem
|
||||
|
||||
|
||||
def _is_geo_native_data_file(path: Path) -> bool:
|
||||
return path.is_file() and path.name.lower().endswith("_geo")
|
||||
|
||||
@@ -128,6 +141,62 @@ def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict
|
||||
return metadata
|
||||
|
||||
|
||||
def _native_asset_from_base(base: Path) -> dict[str, Any]:
|
||||
hdr = Path(str(base) + ".hdr")
|
||||
sml = Path(str(base) + ".sml")
|
||||
aux_xml = Path(str(base) + ".aux.xml")
|
||||
ovr = Path(str(base) + ".ovr")
|
||||
kml = Path(str(base) + ".kml")
|
||||
quicklook = base.with_name(base.name + "_ql.tif")
|
||||
ql_kml = base.with_name(base.name + "_ql.kml")
|
||||
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
||||
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
||||
|
||||
return {
|
||||
"polarization": polarization,
|
||||
"role": "geo_native",
|
||||
"path": str(base),
|
||||
"hdr": str(hdr) if hdr.exists() else None,
|
||||
"sml": str(sml) if sml.exists() else None,
|
||||
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
||||
"ovr": str(ovr) if ovr.exists() else None,
|
||||
"quicklook": str(quicklook) if quicklook.exists() else None,
|
||||
"kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None),
|
||||
"complete": bool(complete),
|
||||
"source": _safe_stat(base),
|
||||
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
||||
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
||||
}
|
||||
|
||||
|
||||
def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"polarization": str(asset.get("polarization") or "").upper(),
|
||||
"path": asset.get("path"),
|
||||
"complete": bool(asset.get("complete")),
|
||||
"source": asset.get("source"),
|
||||
"hdr": asset.get("hdr_info"),
|
||||
"sml": asset.get("sml_info"),
|
||||
}
|
||||
|
||||
|
||||
def _native_fingerprint(assets: list[dict[str, Any]]) -> str:
|
||||
payload = [
|
||||
_asset_fingerprint_part(asset)
|
||||
for asset in sorted(
|
||||
assets,
|
||||
key=lambda item: (
|
||||
str(item.get("polarization") or ""),
|
||||
str(item.get("path") or ""),
|
||||
),
|
||||
)
|
||||
]
|
||||
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()
|
||||
|
||||
|
||||
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
||||
assets: list[dict[str, Any]] = []
|
||||
try:
|
||||
@@ -139,43 +208,25 @@ def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
||||
if not _is_geo_native_data_file(path):
|
||||
continue
|
||||
|
||||
base = path
|
||||
hdr = Path(str(base) + ".hdr")
|
||||
sml = Path(str(base) + ".sml")
|
||||
aux_xml = Path(str(base) + ".aux.xml")
|
||||
ovr = Path(str(base) + ".ovr")
|
||||
kml = Path(str(base) + ".kml")
|
||||
quicklook = base.with_name(base.name + "_ql.tif")
|
||||
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
||||
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
||||
|
||||
asset: dict[str, Any] = {
|
||||
"polarization": polarization,
|
||||
"role": "geo_native",
|
||||
"path": str(base),
|
||||
"hdr": str(hdr) if hdr.exists() else None,
|
||||
"sml": str(sml) if sml.exists() else None,
|
||||
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
||||
"ovr": str(ovr) if ovr.exists() else None,
|
||||
"quicklook": str(quicklook) if quicklook.exists() else None,
|
||||
"kml": str(kml) if kml.exists() else None,
|
||||
"complete": bool(complete),
|
||||
"source": _safe_stat(base),
|
||||
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
||||
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
||||
}
|
||||
assets.append(asset)
|
||||
assets.append(_native_asset_from_base(path))
|
||||
|
||||
return assets
|
||||
|
||||
|
||||
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
||||
assets = _collect_native_assets(scene_dir)
|
||||
def _build_scene_manifest(
|
||||
root: Path,
|
||||
*,
|
||||
scene_name: str,
|
||||
native_dir: Path,
|
||||
manifest_dir: Path,
|
||||
assets: list[dict[str, Any]],
|
||||
source_dir: Path,
|
||||
storage_layout: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if not assets:
|
||||
return None
|
||||
|
||||
scene_name = scene_dir.name
|
||||
batch_name = _scene_batch_name(root, scene_dir, scene_name)
|
||||
batch_name = _scene_batch_name(root, source_dir, scene_name)
|
||||
complete_assets = [asset for asset in assets if asset.get("complete")]
|
||||
complete_pols = [
|
||||
pol
|
||||
@@ -200,25 +251,30 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
|
||||
|
||||
logs = []
|
||||
for name in ("gf3_sarscape_cli.log",):
|
||||
log_path = scene_dir / name
|
||||
log_path = source_dir / name
|
||||
if log_path.is_file():
|
||||
logs.append(str(log_path))
|
||||
try:
|
||||
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
||||
logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
metadata = _parse_scene_metadata(scene_name, assets)
|
||||
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
|
||||
native_fingerprint = _native_fingerprint(assets)
|
||||
metadata["native_fingerprint"] = native_fingerprint
|
||||
manifest_path = manifest_dir / NATIVE_MANIFEST_NAME
|
||||
return {
|
||||
"schema": NATIVE_MANIFEST_SCHEMA,
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_name,
|
||||
"batch_name": batch_name,
|
||||
"native_root": str(root),
|
||||
"native_dir": str(scene_dir),
|
||||
"native_dir": str(native_dir),
|
||||
"source_dir": str(source_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"source_archive": None,
|
||||
"storage_layout": storage_layout,
|
||||
"native_fingerprint": native_fingerprint,
|
||||
"status": status,
|
||||
"polarizations": complete_pols,
|
||||
"metadata": metadata,
|
||||
@@ -227,6 +283,49 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
|
||||
}
|
||||
|
||||
|
||||
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
||||
assets = _collect_native_assets(scene_dir)
|
||||
return _build_scene_manifest(
|
||||
root,
|
||||
scene_name=scene_dir.name,
|
||||
native_dir=scene_dir,
|
||||
manifest_dir=scene_dir,
|
||||
assets=assets,
|
||||
source_dir=scene_dir,
|
||||
storage_layout="scene_dir",
|
||||
)
|
||||
|
||||
|
||||
def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]:
|
||||
try:
|
||||
entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower())
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
grouped_assets: dict[str, list[dict[str, Any]]] = {}
|
||||
for path in entries:
|
||||
if not _is_geo_native_data_file(path):
|
||||
continue
|
||||
scene_name = _scene_name_from_geo_name(path.name)
|
||||
grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path))
|
||||
|
||||
manifests: list[dict[str, Any]] = []
|
||||
for scene_name in sorted(grouped_assets):
|
||||
scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name
|
||||
manifest = _build_scene_manifest(
|
||||
root,
|
||||
scene_name=scene_name,
|
||||
native_dir=scene_catalog_dir,
|
||||
manifest_dir=scene_catalog_dir,
|
||||
assets=grouped_assets[scene_name],
|
||||
source_dir=source_dir,
|
||||
storage_layout="flat_files",
|
||||
)
|
||||
if manifest:
|
||||
manifests.append(manifest)
|
||||
return manifests
|
||||
|
||||
|
||||
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
@@ -251,6 +350,7 @@ def scan_gf3_sarscape_native_roots(
|
||||
native_dirs: list[str] | tuple[str, ...] | None,
|
||||
*,
|
||||
write_manifest: bool = True,
|
||||
scene_dirs_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan configured native roots and return discovered scene manifests."""
|
||||
roots, missing_roots = _normalize_roots(native_dirs)
|
||||
@@ -258,12 +358,56 @@ def scan_gf3_sarscape_native_roots(
|
||||
seen_scene_dirs: set[str] = set()
|
||||
write_errors: list[dict[str, str]] = []
|
||||
|
||||
def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None:
|
||||
seen_scene_dirs.add(scene_key)
|
||||
if write_manifest:
|
||||
try:
|
||||
_write_json(Path(manifest["manifest_path"]), manifest)
|
||||
except OSError as exc:
|
||||
write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)})
|
||||
scenes.append(manifest)
|
||||
|
||||
for root in roots:
|
||||
if scene_dirs_only:
|
||||
try:
|
||||
candidate_dirs = [root] + [
|
||||
child
|
||||
for child in sorted(root.iterdir(), key=lambda item: item.name.lower())
|
||||
if child.is_dir()
|
||||
and child.name not in SKIP_DIR_NAMES
|
||||
and not child.name.startswith(".SARscape")
|
||||
and not child.name.startswith(".gf3_")
|
||||
]
|
||||
except OSError:
|
||||
candidate_dirs = [root]
|
||||
|
||||
for scene_dir in candidate_dirs:
|
||||
scene_key = str(scene_dir).lower()
|
||||
if scene_key in seen_scene_dirs:
|
||||
continue
|
||||
|
||||
if not scene_dir.name.upper().startswith("GF3_"):
|
||||
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
||||
if flat_manifests:
|
||||
for manifest in flat_manifests:
|
||||
flat_key = str(manifest.get("native_dir") or "").lower()
|
||||
if flat_key and flat_key not in seen_scene_dirs:
|
||||
append_manifest(manifest, flat_key, scene_dir)
|
||||
continue
|
||||
|
||||
manifest = _collect_scene_manifest(root, scene_dir)
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
append_manifest(manifest, scene_key, scene_dir)
|
||||
continue
|
||||
|
||||
for current_dir, dir_names, _file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
||||
and not name.startswith(".gf3_")
|
||||
]
|
||||
scene_dir = Path(current_dir)
|
||||
scene_key = str(scene_dir).lower()
|
||||
@@ -271,17 +415,20 @@ def scan_gf3_sarscape_native_roots(
|
||||
dir_names[:] = []
|
||||
continue
|
||||
|
||||
if not scene_dir.name.upper().startswith("GF3_"):
|
||||
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
||||
if flat_manifests:
|
||||
for manifest in flat_manifests:
|
||||
flat_key = str(manifest.get("native_dir") or "").lower()
|
||||
if flat_key and flat_key not in seen_scene_dirs:
|
||||
append_manifest(manifest, flat_key, scene_dir)
|
||||
continue
|
||||
|
||||
manifest = _collect_scene_manifest(root, scene_dir)
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
seen_scene_dirs.add(scene_key)
|
||||
if write_manifest:
|
||||
try:
|
||||
_write_json(Path(manifest["manifest_path"]), manifest)
|
||||
except OSError as exc:
|
||||
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
|
||||
scenes.append(manifest)
|
||||
append_manifest(manifest, scene_key, scene_dir)
|
||||
dir_names[:] = []
|
||||
|
||||
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
|
||||
|
||||
@@ -15,12 +15,13 @@ from shapely.geometry import Polygon
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import settings
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
|
||||
from .data_service import extract_geotiff_bounds
|
||||
from .data_service import DataService, extract_geotiff_bounds
|
||||
from .gf3_native_inventory_service import (
|
||||
NATIVE_MANIFEST_NAME,
|
||||
POLARIZATION_PRIORITY,
|
||||
SKIP_DIR_NAMES,
|
||||
scan_gf3_sarscape_native_roots,
|
||||
)
|
||||
from .image_service import image_service
|
||||
@@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
|
||||
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
|
||||
CONVERTER_VERSION = "v1"
|
||||
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
|
||||
SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK"
|
||||
SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
|
||||
QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2"
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
@@ -72,6 +76,112 @@ def _path_kind(path: str) -> str:
|
||||
return "relative"
|
||||
|
||||
|
||||
def _path_key(path: str | Path) -> str:
|
||||
return os.path.normcase(os.path.normpath(str(path))).lower()
|
||||
|
||||
|
||||
def _has_native_geo_assets(path: Path) -> bool:
|
||||
try:
|
||||
return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]:
|
||||
"""Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning.
|
||||
|
||||
Operators usually configure a durable GF3 root rather than a date folder.
|
||||
Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer
|
||||
direct ``*_geo`` children and fall back to recursive discovery.
|
||||
"""
|
||||
roots: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(path: str | Path) -> None:
|
||||
text = os.path.normpath(str(path))
|
||||
key = _path_key(text)
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
roots.append(text)
|
||||
|
||||
for raw in native_dirs or []:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
if any(ch in text for ch in "*?"):
|
||||
glob_root = Path(os.path.normpath(text))
|
||||
try:
|
||||
matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower())
|
||||
except OSError:
|
||||
matches = []
|
||||
if matches:
|
||||
for match in matches:
|
||||
if match.is_dir():
|
||||
add(match)
|
||||
continue
|
||||
add(glob_root)
|
||||
continue
|
||||
|
||||
root = Path(os.path.normpath(text))
|
||||
try:
|
||||
is_dir = root.is_dir()
|
||||
except OSError:
|
||||
is_dir = False
|
||||
if not is_dir:
|
||||
add(root)
|
||||
continue
|
||||
|
||||
if root.name.lower().endswith("_geo"):
|
||||
add(root)
|
||||
|
||||
try:
|
||||
immediate_geo_roots = sorted(
|
||||
(
|
||||
child
|
||||
for child in root.iterdir()
|
||||
if child.is_dir() and child.name.lower().endswith("_geo")
|
||||
),
|
||||
key=lambda item: item.name.lower(),
|
||||
)
|
||||
except OSError:
|
||||
immediate_geo_roots = []
|
||||
|
||||
if immediate_geo_roots:
|
||||
for child in immediate_geo_roots:
|
||||
add(child)
|
||||
continue
|
||||
|
||||
if _has_native_geo_assets(root):
|
||||
add(root)
|
||||
continue
|
||||
|
||||
found_recursive = 0
|
||||
for current_dir, dir_names, _file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
||||
and not name.startswith(".gf3_")
|
||||
]
|
||||
selected: list[str] = []
|
||||
remaining: list[str] = []
|
||||
for name in dir_names:
|
||||
if name.lower().endswith("_geo"):
|
||||
selected.append(name)
|
||||
else:
|
||||
remaining.append(name)
|
||||
for name in selected:
|
||||
add(Path(current_dir) / name)
|
||||
found_recursive += len(selected)
|
||||
dir_names[:] = remaining
|
||||
|
||||
if found_recursive == 0:
|
||||
add(root)
|
||||
|
||||
return roots
|
||||
|
||||
|
||||
def _source_asset_uid(path: str) -> str:
|
||||
normalized = os.path.normpath(str(path or "").strip())
|
||||
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
|
||||
@@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
|
||||
def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None:
|
||||
polygon = extract_geotiff_bounds(str(path))
|
||||
if polygon and len(polygon) >= 4:
|
||||
return polygon
|
||||
@@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None:
|
||||
if not asset:
|
||||
return None
|
||||
for key in ("path", "quicklook"):
|
||||
text = _path_text(asset.get(key))
|
||||
if not text:
|
||||
continue
|
||||
path = Path(text)
|
||||
if not path.is_file():
|
||||
continue
|
||||
polygon = _polygon_from_raster(path)
|
||||
if polygon and len(polygon) >= 4:
|
||||
return polygon
|
||||
return None
|
||||
|
||||
|
||||
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
|
||||
if not polygon:
|
||||
return None, None
|
||||
@@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None
|
||||
return assets[0] if assets else None
|
||||
|
||||
|
||||
def _is_quicklook_raster_path(path: Any) -> bool:
|
||||
text = _path_text(path).lower()
|
||||
return text.endswith("_ql.tif") or text.endswith("_ql.tiff")
|
||||
|
||||
|
||||
def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
candidates = [
|
||||
asset
|
||||
for asset in assets
|
||||
if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path"))
|
||||
]
|
||||
complete = [asset for asset in candidates if asset.get("complete") is not False]
|
||||
return _select_default_asset(complete or candidates)
|
||||
|
||||
|
||||
def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
output_assets: list[dict[str, Any]] = []
|
||||
for asset in scene_manifest.get("assets") or []:
|
||||
native_path = _path_text(asset.get("path"))
|
||||
if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True:
|
||||
continue
|
||||
output_assets.append(
|
||||
{
|
||||
"polarization": str(asset.get("polarization") or "UNKNOWN").upper(),
|
||||
"role": "native_geo",
|
||||
"path": native_path,
|
||||
"hdr": _path_text(asset.get("hdr")) or None,
|
||||
"sml": _path_text(asset.get("sml")) or None,
|
||||
"quicklook": _path_text(asset.get("quicklook")) or None,
|
||||
"complete": bool(asset.get("complete")),
|
||||
"status": "registered_native_geo",
|
||||
}
|
||||
)
|
||||
return output_assets
|
||||
|
||||
|
||||
def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]:
|
||||
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
|
||||
out_dir = _standard_scene_dir(scene_manifest, root)
|
||||
manifest_path = out_dir / "gf3_native_preview_manifest.json"
|
||||
assets = _quicklook_assets_from_scene_manifest(scene_manifest)
|
||||
manifest = {
|
||||
"schema": "gf3_sarscape_native_preview.v1",
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"batch_name": scene_manifest.get("batch_name"),
|
||||
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"standard_dir": str(out_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"status": "NATIVE_READY" if assets else "FAILED",
|
||||
"assets": assets,
|
||||
"summary": {
|
||||
"native_assets": len(assets),
|
||||
"quicklook_assets": 0,
|
||||
"full_raster_materialized": True,
|
||||
"webp_source": "native_geo",
|
||||
},
|
||||
"errors": [] if assets else [{"error": "no native _geo assets found"}],
|
||||
}
|
||||
_write_json(manifest_path, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool:
|
||||
"""Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``."""
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
from rasterio.enums import Resampling
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
try:
|
||||
target = Path(target_path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600)
|
||||
with rasterio.open(source_path) as src:
|
||||
scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
band = src.read(
|
||||
1,
|
||||
out_shape=(out_height, out_width),
|
||||
masked=True,
|
||||
resampling=Resampling.bilinear,
|
||||
)
|
||||
data = band.filled(0)
|
||||
if data.dtype != np.uint8:
|
||||
valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)]
|
||||
if valid.size:
|
||||
p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98])
|
||||
scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1)
|
||||
data = (scaled * 255).astype("uint8")
|
||||
else:
|
||||
data = np.zeros(data.shape, dtype="uint8")
|
||||
else:
|
||||
data = data.astype("uint8", copy=False)
|
||||
|
||||
mask = getattr(band, "mask", None)
|
||||
if mask is None or np.ndim(mask) == 0:
|
||||
alpha = np.full(data.shape, 255, dtype="uint8")
|
||||
else:
|
||||
alpha = np.where(mask, 0, 255).astype("uint8")
|
||||
rgba = np.stack([data, data, data, alpha], axis=-1)
|
||||
|
||||
image = Image.fromarray(rgba, "RGBA")
|
||||
image = image_service.make_edge_dark_transparent(image)
|
||||
image_service.save_image_as_webp(image, target_path, quality=82)
|
||||
return target.exists() and target.stat().st_size > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str:
|
||||
candidates: list[Any] = [metadata.get("default_native_path")]
|
||||
for key in ("native_assets", "quicklook_assets"):
|
||||
for asset in metadata.get(key) or []:
|
||||
if isinstance(asset, dict):
|
||||
candidates.append(asset.get("path"))
|
||||
for candidate in candidates:
|
||||
text = _path_text(candidate)
|
||||
if text and not _is_quicklook_raster_path(text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _path_is_under_any(path: str, roots: list[str]) -> bool:
|
||||
target = _path_key(path)
|
||||
if not target:
|
||||
return False
|
||||
for root in roots:
|
||||
root_key = _path_key(root)
|
||||
if root_key and (target == root_key or target.startswith(root_key + os.sep)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]:
|
||||
roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
|
||||
if not roots:
|
||||
roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
return [os.path.normpath(root) for root in roots if str(root or "").strip()]
|
||||
|
||||
|
||||
def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str:
|
||||
return str(
|
||||
scene_manifest.get("native_fingerprint")
|
||||
or (scene_manifest.get("metadata") or {}).get("native_fingerprint")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def _record_has_bounds(record: RadarDataORM) -> bool:
|
||||
return (
|
||||
record.coverage_polygon is not None
|
||||
and record.min_lon is not None
|
||||
and record.min_lat is not None
|
||||
and record.max_lon is not None
|
||||
and record.max_lat is not None
|
||||
)
|
||||
|
||||
|
||||
async def _find_existing_quicklook_radar(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
scene_manifest: dict[str, Any],
|
||||
) -> RadarDataORM | None:
|
||||
native_dir = str(scene_manifest.get("native_dir") or "")
|
||||
scene_name = scene_manifest.get("scene_name") or Path(native_dir).name
|
||||
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
|
||||
result = await db.execute(
|
||||
select(RadarDataORM).where(
|
||||
or_(
|
||||
RadarDataORM.unique_id == unique_id,
|
||||
RadarDataORM.file_path == native_dir,
|
||||
)
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def _can_skip_unchanged_quicklook_radar(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
scene_manifest: dict[str, Any],
|
||||
) -> tuple[bool, int | None, str]:
|
||||
native_fingerprint = _scene_native_fingerprint(scene_manifest)
|
||||
if not native_fingerprint:
|
||||
return False, None, "missing_native_fingerprint"
|
||||
|
||||
record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest)
|
||||
if record is None:
|
||||
return False, None, "missing_record"
|
||||
|
||||
metadata = record.metadata_json or {}
|
||||
if str(metadata.get("native_fingerprint") or "") != native_fingerprint:
|
||||
return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed"
|
||||
if str(metadata.get("registration_mode") or "") != "native_preview":
|
||||
return False, int(record.id) if record.id is not None else None, "registration_mode_changed"
|
||||
if not _record_has_bounds(record):
|
||||
return False, int(record.id) if record.id is not None else None, "missing_bounds"
|
||||
|
||||
return True, int(record.id) if record.id is not None else None, "unchanged"
|
||||
|
||||
|
||||
async def generate_gf3_quicklook_webp_cache(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
force: bool = False,
|
||||
max_records: int | None = None,
|
||||
native_dirs: list[str] | tuple[str, ...] | None = None,
|
||||
progress_callback: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters."""
|
||||
stmt = (
|
||||
select(RadarDataORM)
|
||||
.where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT)
|
||||
.order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc())
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
configured_roots = _configured_gf3_native_roots(native_dirs)
|
||||
records = [
|
||||
record
|
||||
for record in result.scalars().all()
|
||||
if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots)
|
||||
]
|
||||
if max_records and max_records > 0:
|
||||
records = records[: int(max_records)]
|
||||
total = len(records)
|
||||
generated = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
record_results: list[dict[str, Any]] = []
|
||||
|
||||
for idx, record in enumerate(records):
|
||||
metadata = record.metadata_json or {}
|
||||
source_path = _native_preview_source_from_metadata(metadata)
|
||||
cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path)
|
||||
if progress_callback:
|
||||
pct = 5 + int((idx / max(total, 1)) * 90)
|
||||
progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}")
|
||||
|
||||
if (
|
||||
not force
|
||||
and (record.preview_cache_status or "NONE") == "READY"
|
||||
and record.preview_cache_path
|
||||
and os.path.exists(record.preview_cache_path)
|
||||
):
|
||||
skipped += 1
|
||||
record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path})
|
||||
continue
|
||||
|
||||
if not source_path:
|
||||
failed += 1
|
||||
record.preview_cache_status = "FAILED"
|
||||
record.preview_cache_path = None
|
||||
record.preview_cache_error = "default_native_path_missing"
|
||||
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
record.preview_cache_updated_at = _db_now()
|
||||
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path)
|
||||
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
record.preview_cache_updated_at = _db_now()
|
||||
if ok and os.path.exists(cache_path):
|
||||
generated += 1
|
||||
record.preview_cache_status = "READY"
|
||||
record.preview_cache_path = cache_path
|
||||
record.preview_cache_error = None
|
||||
record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path})
|
||||
else:
|
||||
failed += 1
|
||||
record.preview_cache_status = "FAILED"
|
||||
record.preview_cache_path = None
|
||||
record.preview_cache_error = "native_webp_build_failed"
|
||||
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"ok": failed == 0,
|
||||
"mode": "gf3_native_webp",
|
||||
"total": total,
|
||||
"generated": generated,
|
||||
"skipped": skipped,
|
||||
"failed": failed,
|
||||
"records": record_results,
|
||||
}
|
||||
|
||||
|
||||
async def _upsert_quicklook_source_product_asset(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
quicklook_manifest: dict[str, Any],
|
||||
) -> int | None:
|
||||
native_dir_text = _path_text(scene_manifest.get("native_dir"))
|
||||
if not native_dir_text:
|
||||
return None
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
|
||||
acquisition_start = _date_to_naive_utc(imaging_date)
|
||||
scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name
|
||||
now = _db_now()
|
||||
root = await _find_managed_root_for_path(db, native_dir_text)
|
||||
asset_metadata = _json_safe(
|
||||
{
|
||||
"source": "GF3 SARscape native _geo",
|
||||
"native_dir": native_dir_text,
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"native_assets": quicklook_manifest.get("assets") or [],
|
||||
"full_raster_materialized": True,
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
}
|
||||
)
|
||||
data = {
|
||||
"asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"),
|
||||
"logical_product_uid": scene_name,
|
||||
"satellite_family": "GF3",
|
||||
"satellite": "GF3",
|
||||
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
|
||||
"product_type": metadata.get("product_type") or "SARSCAPE_NATIVE",
|
||||
"product_level": "L2_NATIVE",
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": metadata.get("polarization"),
|
||||
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
|
||||
"relative_orbit": metadata.get("relative_orbit"),
|
||||
"orbit_direction": metadata.get("orbit_direction"),
|
||||
"acquisition_start_time_utc": acquisition_start,
|
||||
"acquisition_stop_time_utc": None,
|
||||
"imaging_date": imaging_date,
|
||||
"root_ref_id": root.id if root else None,
|
||||
"root_path": root.path if root else str(Path(native_dir_text).parent),
|
||||
"file_path": native_dir_text,
|
||||
"archive_path": scene_manifest.get("source_archive"),
|
||||
"path_kind": _path_kind(native_dir_text),
|
||||
"file_name": Path(native_dir_text).name,
|
||||
"file_stem": Path(native_dir_text).name,
|
||||
"file_ext": "",
|
||||
"size_bytes": None,
|
||||
"mtime_epoch": None,
|
||||
"checksum_status": "NOT_COMPUTED",
|
||||
"parser_name": "gf3_sarscape_native_preview_manifest",
|
||||
"parser_version": CONVERTER_VERSION,
|
||||
"parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED",
|
||||
"parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None,
|
||||
"parsed_at": now,
|
||||
"metadata_json": asset_metadata,
|
||||
"is_active": True,
|
||||
"missing_since": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
result = await db.execute(
|
||||
select(SourceProductAssetORM).where(
|
||||
or_(
|
||||
SourceProductAssetORM.asset_uid == data["asset_uid"],
|
||||
SourceProductAssetORM.file_path == native_dir_text,
|
||||
)
|
||||
)
|
||||
)
|
||||
asset = result.scalars().first()
|
||||
if asset is None:
|
||||
asset = SourceProductAssetORM(**data)
|
||||
db.add(asset)
|
||||
else:
|
||||
for key, value in data.items():
|
||||
setattr(asset, key, value)
|
||||
await db.flush()
|
||||
return int(asset.id) if asset.id is not None else None
|
||||
|
||||
|
||||
async def _upsert_quicklook_radar_data(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
quicklook_manifest: dict[str, Any],
|
||||
source_product_ref_id: int | None = None,
|
||||
) -> int | None:
|
||||
assets = quicklook_manifest.get("assets") or []
|
||||
default_asset = _select_default_native_asset(assets)
|
||||
if not default_asset:
|
||||
return None
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
|
||||
acquisition_start = _date_to_naive_utc(imaging_date)
|
||||
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
|
||||
native_dir = str(scene_manifest.get("native_dir") or "")
|
||||
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
|
||||
default_native_path = str(default_asset.get("path") or "")
|
||||
default_quicklook_path = str(default_asset.get("quicklook") or "")
|
||||
polygon = _polygon_from_native_asset(default_asset)
|
||||
min_lon = min_lat = max_lon = max_lat = None
|
||||
geom = None
|
||||
if polygon:
|
||||
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
|
||||
center_lon, center_lat = _scene_center_from_polygon(polygon)
|
||||
geom = _geom_from_polygon(polygon)
|
||||
else:
|
||||
center_lon = metadata.get("scene_center_lon")
|
||||
center_lat = metadata.get("scene_center_lat")
|
||||
preview_cache_path = None
|
||||
preview_cache_kind = "deferred"
|
||||
preview_cache_error = "native_webp_not_generated"
|
||||
radar_metadata = _json_safe(
|
||||
{
|
||||
**metadata,
|
||||
"native_dir": native_dir,
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"native_assets": assets,
|
||||
"quicklook_assets": [],
|
||||
"default_native_path": default_native_path,
|
||||
"default_quicklook_path": default_quicklook_path,
|
||||
"coverage_source": "native_geo_or_quicklook" if polygon else "scene_name",
|
||||
"preview_cache_kind": preview_cache_kind,
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
"registration_mode": "native_preview",
|
||||
"full_raster_materialized": True,
|
||||
}
|
||||
)
|
||||
data_to_upsert = {
|
||||
"unique_id": unique_id,
|
||||
"satellite": "GF3",
|
||||
"satellite_family": "GF3",
|
||||
"imaging_date": imaging_date,
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": ",".join(
|
||||
pol
|
||||
for pol in POLARIZATION_PRIORITY
|
||||
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
|
||||
)
|
||||
or metadata.get("polarization"),
|
||||
"scene_center_lon": center_lon,
|
||||
"scene_center_lat": center_lat,
|
||||
"acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None,
|
||||
"product_level": "L2_NATIVE",
|
||||
"product_unique_id": metadata.get("product_unique_id") or scene_name,
|
||||
"source_product_token": scene_name,
|
||||
"acquisition_start_time_utc": acquisition_start,
|
||||
"acquisition_stop_time_utc": None,
|
||||
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
|
||||
"relative_orbit": metadata.get("relative_orbit"),
|
||||
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
|
||||
"source_product_ref_id": source_product_ref_id,
|
||||
"image_data_format": "ENVI_NATIVE",
|
||||
"geocoded_flag": True,
|
||||
"metadata_json": radar_metadata,
|
||||
"file_path": native_dir,
|
||||
"has_orbit_data": False,
|
||||
"orbit_file_path": None,
|
||||
"is_envi_processed": True,
|
||||
"coverage_polygon": polygon,
|
||||
"geom": geom,
|
||||
"min_lon": min_lon,
|
||||
"min_lat": min_lat,
|
||||
"max_lon": max_lon,
|
||||
"max_lat": max_lat,
|
||||
"preview_cache_status": "NONE",
|
||||
"preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION,
|
||||
"preview_cache_path": preview_cache_path,
|
||||
"preview_cache_updated_at": _db_now(),
|
||||
"preview_cache_error": preview_cache_error,
|
||||
}
|
||||
result = await db.execute(
|
||||
select(RadarDataORM).where(
|
||||
or_(
|
||||
RadarDataORM.unique_id == unique_id,
|
||||
RadarDataORM.file_path == native_dir,
|
||||
)
|
||||
)
|
||||
)
|
||||
radar = result.scalars().first()
|
||||
if (
|
||||
radar is not None
|
||||
and (radar.preview_cache_status or "").upper() == "READY"
|
||||
and radar.preview_cache_path
|
||||
and os.path.exists(radar.preview_cache_path)
|
||||
and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path
|
||||
):
|
||||
data_to_upsert["preview_cache_status"] = radar.preview_cache_status
|
||||
data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
data_to_upsert["preview_cache_path"] = radar.preview_cache_path
|
||||
data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at
|
||||
data_to_upsert["preview_cache_error"] = radar.preview_cache_error
|
||||
if radar is None:
|
||||
radar = RadarDataORM(**data_to_upsert)
|
||||
db.add(radar)
|
||||
else:
|
||||
for key, value in data_to_upsert.items():
|
||||
setattr(radar, key, value)
|
||||
await db.flush()
|
||||
return int(radar.id) if radar.id is not None else None
|
||||
|
||||
|
||||
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
metadata = dict(scene_manifest.get("metadata") or {})
|
||||
metadata.update(
|
||||
@@ -635,7 +1260,7 @@ async def _upsert_radar_data(
|
||||
if not default_asset:
|
||||
return None
|
||||
|
||||
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
|
||||
polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path"))))
|
||||
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
|
||||
center_lon, center_lat = _scene_center_from_polygon(polygon)
|
||||
geom = _geom_from_polygon(polygon)
|
||||
@@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots(
|
||||
storage_root: str | None = None,
|
||||
force: bool = False,
|
||||
register: bool = True,
|
||||
quicklook_only: bool = False,
|
||||
progress_callback: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan native roots, convert complete assets, and register standard scenes."""
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
native_dirs,
|
||||
write_manifest=True,
|
||||
)
|
||||
requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
|
||||
if quicklook_only:
|
||||
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
scan_roots,
|
||||
write_manifest=True,
|
||||
)
|
||||
else:
|
||||
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
scan_roots,
|
||||
write_manifest=True,
|
||||
)
|
||||
scenes = inventory.get("scenes") or []
|
||||
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
|
||||
|
||||
if quicklook_only:
|
||||
registered = 0
|
||||
quicklook_assets = 0
|
||||
failed_scenes = 0
|
||||
skipped_unchanged = 0
|
||||
scene_results: list[dict[str, Any]] = []
|
||||
total = len(ready_scenes)
|
||||
for idx, scene_manifest in enumerate(ready_scenes):
|
||||
if progress_callback:
|
||||
pct = 10 + int((idx / max(total, 1)) * 80)
|
||||
progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
|
||||
|
||||
quicklook_manifest = await asyncio.to_thread(
|
||||
_quicklook_manifest_for_scene,
|
||||
scene_manifest,
|
||||
storage_root,
|
||||
)
|
||||
assets = quicklook_manifest.get("assets") or []
|
||||
quicklook_assets += len(assets)
|
||||
source_asset_id = None
|
||||
radar_id = None
|
||||
skipped_reason = None
|
||||
if register and assets:
|
||||
can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar(
|
||||
db,
|
||||
scene_manifest=scene_manifest,
|
||||
)
|
||||
if can_skip:
|
||||
skipped_unchanged += 1
|
||||
radar_id = existing_radar_id
|
||||
skipped_reason = skip_reason
|
||||
if progress_callback:
|
||||
pct = 10 + int(((idx + 1) / max(total, 1)) * 80)
|
||||
progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
|
||||
else:
|
||||
skipped_reason = skip_reason
|
||||
source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest)
|
||||
radar_id = await _upsert_quicklook_radar_data(
|
||||
db,
|
||||
scene_manifest,
|
||||
quicklook_manifest,
|
||||
source_product_ref_id=source_asset_id,
|
||||
)
|
||||
if radar_id:
|
||||
registered += 1
|
||||
await db.commit()
|
||||
if not assets:
|
||||
failed_scenes += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"native_status": scene_manifest.get("status"),
|
||||
"standard_status": quicklook_manifest.get("status"),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"source_asset_id": source_asset_id,
|
||||
"radar_id": radar_id,
|
||||
"skipped_unchanged": bool(skipped_reason == "unchanged"),
|
||||
"skip_reason": skipped_reason,
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"summary": quicklook_manifest.get("summary") or {},
|
||||
"errors": quicklook_manifest.get("errors") or [],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": failed_scenes == 0,
|
||||
"mode": "native_preview",
|
||||
"inventory": {
|
||||
key: value
|
||||
for key, value in inventory.items()
|
||||
if key != "scenes"
|
||||
},
|
||||
"requested_native_dirs": requested_native_dirs,
|
||||
"scan_roots": scan_roots,
|
||||
"scene_count": len(scenes),
|
||||
"ready_scene_count": len(ready_scenes),
|
||||
"converted_scenes": 0,
|
||||
"partial_scenes": 0,
|
||||
"failed_scenes": failed_scenes,
|
||||
"converted_assets": 0,
|
||||
"skipped_assets": skipped_unchanged,
|
||||
"skipped_unchanged": skipped_unchanged,
|
||||
"failed_assets": failed_scenes,
|
||||
"quicklook_assets": quicklook_assets,
|
||||
"native_assets": quicklook_assets,
|
||||
"registered": registered,
|
||||
"analysis_ready": 0,
|
||||
"scenes": scene_results,
|
||||
}
|
||||
|
||||
converted_scenes = 0
|
||||
partial_scenes = 0
|
||||
failed_scenes = 0
|
||||
@@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots(
|
||||
for key, value in inventory.items()
|
||||
if key != "scenes"
|
||||
},
|
||||
"requested_native_dirs": requested_native_dirs,
|
||||
"scan_roots": scan_roots,
|
||||
"scene_count": len(scenes),
|
||||
"ready_scene_count": len(ready_scenes),
|
||||
"converted_scenes": converted_scenes,
|
||||
|
||||
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
from sqlalchemy import select
|
||||
|
||||
from .. import database
|
||||
from ..config import settings
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..scheduler import scan_data_job
|
||||
from .data_service import data_service
|
||||
@@ -88,6 +88,7 @@ JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
|
||||
JOB_TYPE_GF3_QUICKLOOK_WEBP = "GF3_QUICKLOOK_WEBP"
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
@@ -97,6 +98,7 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY"
|
||||
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
|
||||
JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM"
|
||||
JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS"
|
||||
@@ -253,9 +255,12 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
families=payload.get("families") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
build_previews=bool(payload.get("build_previews", True)),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
preview = result.get("preview_cache") or {}
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
@@ -265,7 +270,42 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
f"sources={result.get('source_assets', 0)}, "
|
||||
f"orbits={result.get('orbit_assets', 0)}, "
|
||||
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}, "
|
||||
f"previews_ready={preview.get('ready', 0)}, "
|
||||
f"previews_skipped={preview.get('skipped_ready', 0)}, "
|
||||
f"previews_failed={preview.get('failed', 0)}, "
|
||||
f"previews_missing={preview.get('missing_source', 0)}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_archive_integrity_audit(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("AUDIT_SOURCE_ARCHIVE_INTEGRITY requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
await task_service.start_task(job.task_id, message="Source archive integrity audit started")
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await asset_inventory_service.audit_source_archive_integrity(
|
||||
db,
|
||||
families=payload.get("families") or None,
|
||||
source_formats=payload.get("source_formats") or None,
|
||||
asset_ids=payload.get("asset_ids") or None,
|
||||
force=bool(payload.get("force", False)),
|
||||
limit=payload.get("limit"),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
"Source archive integrity audit completed: "
|
||||
f"checked={result.get('checked', 0)}, "
|
||||
f"skipped={result.get('skipped', 0)}, "
|
||||
f"ok={result.get('ok', 0)}, "
|
||||
f"failed={result.get('failed', 0)}, "
|
||||
f"unsupported={result.get('unsupported', 0)}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
@@ -345,7 +385,7 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
|
||||
copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses"))
|
||||
include_orbit_files = bool(payload.get("include_orbit_files"))
|
||||
export_zip = bool(payload.get("export_zip"))
|
||||
package_mode = str(payload.get("package_mode") or ("task_zip" if export_zip else "task_folder")).strip().lower()
|
||||
package_mode = str(payload.get("package_mode") or ("source_bundle" if export_zip else "task_folder")).strip().lower()
|
||||
skip_existing = payload.get("skip_existing") is not False
|
||||
max_items = _normalize_positive_int(payload.get("max_items"))
|
||||
|
||||
@@ -445,17 +485,27 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
|
||||
skip_existing=skip_existing,
|
||||
max_items=max_items,
|
||||
)
|
||||
else:
|
||||
return
|
||||
if package_mode in {"task_folder", "task"}:
|
||||
await run_dinsar_copy_items(
|
||||
job.task_id,
|
||||
items,
|
||||
dest_dir,
|
||||
include_orbit_files=include_orbit_files,
|
||||
export_zip=(package_mode == "task_zip" or export_zip),
|
||||
export_zip=False,
|
||||
skip_existing=skip_existing,
|
||||
max_items=max_items,
|
||||
)
|
||||
return
|
||||
return
|
||||
if package_mode == "task_zip":
|
||||
raise ValueError(
|
||||
"D-InSAR Task ZIP export is not a production-preparation format. "
|
||||
"Use package_mode=task_folder for Task_Pool preparation or source_bundle for archive distribution."
|
||||
)
|
||||
raise ValueError(
|
||||
"Unsupported D-InSAR package_mode. "
|
||||
"Use task_folder for production preparation or source_bundle for archive distribution."
|
||||
)
|
||||
|
||||
raise ValueError(f"Unknown COPY_DATA file_type: {file_type}")
|
||||
|
||||
@@ -884,7 +934,9 @@ async def _handle_ai_diagnosis(job: SystemJobORM) -> None:
|
||||
from ..ai_service import analyze_map_with_vlm
|
||||
diagnosis_markdown = await analyze_map_with_vlm(
|
||||
images_base64=[img_base64],
|
||||
prompt=full_prompt
|
||||
prompt=full_prompt,
|
||||
model_name=model_name,
|
||||
raise_on_error=True,
|
||||
)
|
||||
|
||||
# 8. 解析风险等级和置信度(简单正则匹配)
|
||||
@@ -3646,7 +3698,7 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 Python/GDAL preprocessing is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
payload = job.payload or {}
|
||||
@@ -3725,7 +3777,7 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 archive unpack is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
if not job.task_id:
|
||||
@@ -3792,7 +3844,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 Python/GDAL preprocessing is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
payload = job.payload or {}
|
||||
@@ -3901,7 +3953,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
|
||||
"""Scan SARscape native GF3 _geo outputs and register requested assets."""
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
|
||||
if not job.task_id:
|
||||
@@ -3915,7 +3967,15 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
|
||||
|
||||
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
|
||||
quicklook_only = bool(payload.get("quicklook_only", False))
|
||||
await task_service.start_task(
|
||||
job.task_id,
|
||||
message=(
|
||||
"按 GF3 日期/场景命名规则登记本机 SARscape _geo 原生结果..."
|
||||
if quicklook_only
|
||||
else "递归扫描 GF3 SARscape 原生 _geo 结果池..."
|
||||
),
|
||||
)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -3943,6 +4003,7 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
quicklook_only=quicklook_only,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -3954,44 +4015,44 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
)
|
||||
raise
|
||||
|
||||
message = (
|
||||
"GF3 SARscape 标准化完成: "
|
||||
f"发现 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
|
||||
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
|
||||
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
|
||||
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景"
|
||||
)
|
||||
if quicklook_only:
|
||||
message = (
|
||||
"GF3 _geo 原生结果登记完成: "
|
||||
f"规则匹配 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可登记 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"原生资产 {int(result.get('native_assets') or result.get('quicklook_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景, "
|
||||
f"跳过未变化 {int(result.get('skipped_unchanged') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景"
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"GF3 SARscape 标准化完成: "
|
||||
f"发现 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
|
||||
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
|
||||
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
|
||||
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
|
||||
from .gf3_sarscape_production_service import (
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
run_gf3_sarscape_production,
|
||||
)
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
async def _handle_gf3_quicklook_webp(job: SystemJobORM) -> None:
|
||||
"""Generate local WebP previews from registered GF3 SARscape native _geo records."""
|
||||
from .gf3_standardize_service import generate_gf3_quicklook_webp_cache
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
|
||||
raise ValueError("GF3_QUICKLOOK_WEBP requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
source_dirs = payload.get("source_dirs") or []
|
||||
native_dirs = payload.get("native_dirs") or []
|
||||
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
|
||||
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
|
||||
if not source_dirs:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
|
||||
if not native_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
|
||||
force = bool(payload.get("force", False))
|
||||
max_records = _normalize_positive_int(payload.get("max_records"))
|
||||
native_dirs = payload.get("native_dirs") or split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
await task_service.start_task(job.task_id, message="开始从 GF3 _geo 原生数据生成 WebP 缓存...")
|
||||
|
||||
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
@@ -4005,143 +4066,49 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
|
||||
logger.warning("[GF3 native WebP] progress callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_progress_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
def _log_cb(level: str, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.add_log(job.task_id, level, message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_log_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_log_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
async def _production_keepalive() -> None:
|
||||
progress = 8
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
progress = min(68, progress + 1)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress,
|
||||
message="GF3 SARscape production is still running...",
|
||||
)
|
||||
|
||||
try:
|
||||
production_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
run_gf3_sarscape_production,
|
||||
source_dirs=source_dirs,
|
||||
native_root=native_root,
|
||||
wrapper_exe=payload.get("wrapper_exe"),
|
||||
dem_path=payload.get("dem_path"),
|
||||
idlrt_path=payload.get("idlrt_path"),
|
||||
polarizations=payload.get("polarizations"),
|
||||
archive_exts=payload.get("archive_exts") or [],
|
||||
max_archives_per_run=payload.get("max_archives_per_run"),
|
||||
selected_dates=payload.get("selected_dates") or [],
|
||||
task_id=job.task_id,
|
||||
local_staging_root=payload.get("local_staging_root") or settings.GF3_TASK_POOL_ROOT,
|
||||
timeout_seconds=payload.get("timeout_seconds"),
|
||||
keep_extracted=payload.get("keep_extracted"),
|
||||
log_callback=_log_cb,
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await generate_gf3_quicklook_webp_cache(
|
||||
db,
|
||||
force=force,
|
||||
max_records=max_records,
|
||||
native_dirs=native_dirs,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
)
|
||||
keepalive_task = asyncio.create_task(_production_keepalive())
|
||||
try:
|
||||
production_result = await production_task
|
||||
finally:
|
||||
keepalive_task.cancel()
|
||||
try:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
standardize_result: Dict[str, Any] = {}
|
||||
if bool(payload.get("auto_standardize", True)):
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=72,
|
||||
message="GF3 SARscape production finished; standardizing native _geo outputs...",
|
||||
)
|
||||
async with AsyncSessionLocal() as db:
|
||||
standardize_result = await standardize_gf3_sarscape_native_roots(
|
||||
db,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force_standardize", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
|
||||
)
|
||||
|
||||
cleanup_result: Dict[str, Any] = {}
|
||||
production_ok = int(production_result.get("failed_count") or 0) == 0
|
||||
standardize_ok = (
|
||||
not standardize_result
|
||||
or (
|
||||
int(standardize_result.get("failed_assets") or 0) == 0
|
||||
and int(standardize_result.get("failed_scenes") or 0) == 0
|
||||
)
|
||||
)
|
||||
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=90,
|
||||
message="Cleaning GF3 SARscape intermediate files...",
|
||||
)
|
||||
cleanup_result = await asyncio.to_thread(
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
|
||||
dry_run=bool(payload.get("cleanup_dry_run", False)),
|
||||
max_scenes=payload.get("cleanup_max_scenes"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
|
||||
)
|
||||
elif bool(payload.get("clean_after_success", True)):
|
||||
_log_cb(
|
||||
"WARNING",
|
||||
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=f"GF3 SARscape production failed: {exc}",
|
||||
message=f"GF3 _geo WebP 生成失败: {exc}",
|
||||
)
|
||||
raise
|
||||
|
||||
failed_count = int(production_result.get("failed_count") or 0)
|
||||
failed_assets = int(standardize_result.get("failed_assets") or 0)
|
||||
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
|
||||
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
|
||||
message = (
|
||||
"GF3 SARscape production chain finished: "
|
||||
f"found={int(production_result.get('found_count') or 0)}, "
|
||||
f"produced={int(production_result.get('processed_count') or 0)}, "
|
||||
f"skipped={int(production_result.get('skipped_count') or 0)}, "
|
||||
f"failed={failed_count}, "
|
||||
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
|
||||
f"registered={int(standardize_result.get('registered') or 0)}, "
|
||||
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
|
||||
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
|
||||
"GF3 _geo WebP 生成完成: "
|
||||
f"总数 {int(result.get('total') or 0)} 景, "
|
||||
f"生成 {int(result.get('generated') or 0)} 景, "
|
||||
f"跳过 {int(result.get('skipped') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed') or 0)} 景"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
"""Reject GF3 production jobs on this management machine."""
|
||||
message = (
|
||||
"GF3 SARscape production is disabled on this management machine. "
|
||||
"Run production on the SARscape host and register local _geo native results here."
|
||||
)
|
||||
if job.task_id:
|
||||
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=message)
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
|
||||
@@ -5263,6 +5230,7 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None:
|
||||
_HANDLERS = {
|
||||
JOB_TYPE_SCAN_DATA: _handle_scan_data,
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
|
||||
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit,
|
||||
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
|
||||
@@ -5301,6 +5269,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
|
||||
JOB_TYPE_GF3_QUICKLOOK_WEBP: _handle_gf3_quicklook_webp,
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
|
||||
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
|
||||
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
|
||||
|
||||
@@ -24,6 +24,8 @@ from ..dinsar_engines.landsar_engine import (
|
||||
_norm_path,
|
||||
_path_search_dirs,
|
||||
_summarize_landsar_failure,
|
||||
derive_landsar_dem_bbox_from_xml_paths,
|
||||
prepare_landsar_dem_crop,
|
||||
)
|
||||
|
||||
|
||||
@@ -633,6 +635,16 @@ class LandsarSbasService:
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def _derive_task_dem_bbox(self, task: dict[str, Any], input_dir: str) -> tuple[float, float, float, float] | None:
|
||||
xml_paths = [
|
||||
str(scene.get("xml") or "")
|
||||
for scene in (task.get("scenes") or [])
|
||||
if str(scene.get("xml") or "").strip()
|
||||
]
|
||||
if not xml_paths and input_dir and os.path.isdir(input_dir):
|
||||
xml_paths = [str(path) for path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower())]
|
||||
return derive_landsar_dem_bbox_from_xml_paths(xml_paths, fallback_paths=xml_paths)
|
||||
|
||||
def materialize_stack(
|
||||
self,
|
||||
*,
|
||||
@@ -949,7 +961,7 @@ class LandsarSbasService:
|
||||
params_payload = self.normalize_params(params or {})
|
||||
normalized_dem = _norm_path(dem_path or self.default_dem_path)
|
||||
if not normalized_dem or not os.path.isfile(normalized_dem):
|
||||
raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or '<empty>'}")
|
||||
raise ValueError(f"LandSAR SBAS DEM source file is missing: {normalized_dem or '<empty>'}")
|
||||
|
||||
from .sbas_insar_production_service import sbas_insar_production_service
|
||||
|
||||
@@ -1035,6 +1047,8 @@ class LandsarSbasService:
|
||||
"input_task_root": import_dest_root,
|
||||
"publish_root": str(publish_root),
|
||||
"dem_path": normalized_dem,
|
||||
"dem_source_path": normalized_dem,
|
||||
"dem_role": "global_prepared_source",
|
||||
"params": params_payload,
|
||||
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
"import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
@@ -1541,9 +1555,9 @@ class LandsarSbasService:
|
||||
if not bool(settings.LANDSAR_SBAS_ENABLED):
|
||||
raise ValueError("LandSAR SBAS is disabled.")
|
||||
params = self.normalize_params(extra or {})
|
||||
dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
|
||||
if not dem_path or not os.path.isfile(dem_path):
|
||||
raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or '<empty>'}")
|
||||
dem_source_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
|
||||
if not dem_source_path or not os.path.isfile(dem_source_path):
|
||||
raise ValueError(f"LandSAR SBAS DEM source file is missing: {dem_source_path or '<empty>'}")
|
||||
validation = self.validate_root_dir(
|
||||
root_dir,
|
||||
min_scenes=min_scenes,
|
||||
@@ -1585,7 +1599,9 @@ class LandsarSbasService:
|
||||
"work_root_strategy": "short_landsar_execution_path",
|
||||
"native_root": str(native_root),
|
||||
"publish_root": str(publish_root),
|
||||
"dem_path": dem_path,
|
||||
"dem_path": dem_source_path,
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_role": "global_prepared_source",
|
||||
"params": params,
|
||||
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
"min_scenes": validation.get("min_scenes"),
|
||||
@@ -1716,10 +1732,56 @@ class LandsarSbasService:
|
||||
native_output_dir = native_root / task_alias / "Output_Data"
|
||||
native_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias
|
||||
dem_source_path = _norm_path(str(manifest.get("dem_path") or ""))
|
||||
effective_dem_path = dem_source_path
|
||||
dem_crop_info: dict[str, Any] = {}
|
||||
try:
|
||||
dem_bbox = self._derive_task_dem_bbox(task, input_dir)
|
||||
if not dem_bbox:
|
||||
raise ValueError("cannot derive DEM crop bbox from LandSAR SBAS Input_Data XML corner coordinates")
|
||||
dem_crop_info = prepare_landsar_dem_crop(
|
||||
dem_source_path,
|
||||
str(native_root / task_alias / "dem_crop"),
|
||||
dem_bbox,
|
||||
label=task_alias,
|
||||
)
|
||||
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
|
||||
self._emit(
|
||||
progress_callback,
|
||||
"INFO",
|
||||
f"[{index}/{len(tasks)}] LandSAR SBAS DEM crop ready: {effective_dem_path}",
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_count += 1
|
||||
error = f"LandSAR SBAS DEM crop failed: {exc}"
|
||||
self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] {task_name} failed: {error}")
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"input_data_dir": input_dir,
|
||||
"native_output_dir": str(native_output_dir),
|
||||
"param_file": "",
|
||||
"process_name": SBAS_PROCESS_NAME,
|
||||
"command": "",
|
||||
"returncode": -2,
|
||||
"success": False,
|
||||
"timed_out": False,
|
||||
"failure_kind": "dem_crop_failed",
|
||||
"error": error,
|
||||
"stdout_tail": "",
|
||||
"native_logs": {},
|
||||
"publish": {},
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_path": effective_dem_path,
|
||||
"dem_crop": dem_crop_info,
|
||||
}
|
||||
)
|
||||
continue
|
||||
param_file = _generate_sbas_param_file(
|
||||
str(native_output_dir / f"{SBAS_PROID}.txt"),
|
||||
slc_folder=input_dir,
|
||||
dem_path=str(manifest.get("dem_path") or ""),
|
||||
dem_path=effective_dem_path,
|
||||
output_dir=str(native_output_dir),
|
||||
project_name=project_name,
|
||||
params=dict(manifest.get("params") or {}),
|
||||
@@ -1784,6 +1846,9 @@ class LandsarSbasService:
|
||||
"stdout_tail": _collect_tail(stdout_text, 4000),
|
||||
"native_logs": log_publish_result,
|
||||
"publish": publish_result,
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_path": effective_dem_path,
|
||||
"dem_crop": dem_crop_info,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""轨道文件管理与格式转换服务。
|
||||
|
||||
目录约定:
|
||||
MONITOR_ORBIT_DIR/ 源精轨目录(可为 UNC),平铺 .txt
|
||||
MONITOR_ORBIT_DIR/ 本机源精轨目录,平铺 .txt
|
||||
ORBIT_POOL_ENVI/ ENVI 本地精轨池
|
||||
LT1A/ LT-1A 精轨 .txt
|
||||
LT1B/ LT-1B 精轨 .txt
|
||||
@@ -320,16 +320,17 @@ def summarize_source_orbit_gaps(
|
||||
|
||||
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
|
||||
source_stems = set(source_files)
|
||||
envi_stems = set(envi_files)
|
||||
isce2_stems = set(isce2_files)
|
||||
|
||||
source_without_isce2 = sorted(source_stems - isce2_stems)
|
||||
source_without_isce2 = sorted(source_stems - isce2_stems) if isce2_enabled else []
|
||||
source_without_envi = sorted(source_stems - envi_stems)
|
||||
envi_without_source = sorted(envi_stems - source_stems)
|
||||
isce2_without_source = sorted(isce2_stems - source_stems)
|
||||
isce2_without_source = sorted(isce2_stems - source_stems) if isce2_enabled else []
|
||||
sample_limit = 20
|
||||
|
||||
def _make_samples(
|
||||
@@ -349,7 +350,7 @@ def summarize_source_orbit_gaps(
|
||||
)
|
||||
return samples
|
||||
|
||||
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
|
||||
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) if isce2_enabled else []
|
||||
bad_source_samples = [
|
||||
item for item in suspect_bad_samples
|
||||
if item.get("has_corruption_signal")
|
||||
@@ -358,6 +359,7 @@ def summarize_source_orbit_gaps(
|
||||
return {
|
||||
"sample_limit": sample_limit,
|
||||
"quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"source_without_isce2_count": len(source_without_isce2),
|
||||
"source_without_envi_count": len(source_without_envi),
|
||||
"envi_without_source_count": len(envi_without_source),
|
||||
@@ -809,6 +811,7 @@ def sync_orbit_pools(
|
||||
envi_pool: str = "",
|
||||
isce2_pool: str = "",
|
||||
landsar_pool: str = "",
|
||||
validate_without_isce2: bool = True,
|
||||
) -> Dict:
|
||||
"""从 source_dir 扫描平铺 .txt 精轨,同步到各引擎本地池。
|
||||
|
||||
@@ -863,11 +866,16 @@ def sync_orbit_pools(
|
||||
xml_needs_refresh = bool(isce2_pool) and (
|
||||
not os.path.exists(xml_path) or _needs_generated_refresh(fpath, xml_path)
|
||||
)
|
||||
requires_validation = xml_needs_refresh or (not isce2_pool and envi_needs_refresh)
|
||||
requires_validation = xml_needs_refresh or (
|
||||
bool(validate_without_isce2) and not isce2_pool and envi_needs_refresh
|
||||
)
|
||||
|
||||
staged_dir = ""
|
||||
staged_xml = ""
|
||||
try:
|
||||
health = _inspect_orbit_txt_health(fpath)
|
||||
if _has_orbit_corruption_signal(health):
|
||||
raise RuntimeError("Source TXT contains NUL bytes or cannot be read as a healthy LT-1 orbit text file.")
|
||||
if requires_validation:
|
||||
staged_dir, staged_xml = _stage_isce2_xml(
|
||||
fpath,
|
||||
@@ -924,8 +932,15 @@ def repair_orbit_pools(
|
||||
isce2_pool: str = "",
|
||||
landsar_pool: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
before = check_orbit_consistency(envi_pool, isce2_pool)
|
||||
sync_result = sync_orbit_pools(source_dir, envi_pool, isce2_pool, landsar_pool)
|
||||
sync_result = sync_orbit_pools(
|
||||
source_dir,
|
||||
envi_pool,
|
||||
isce2_pool,
|
||||
landsar_pool,
|
||||
validate_without_isce2=isce2_enabled,
|
||||
)
|
||||
|
||||
repaired_from_envi: List[str] = []
|
||||
repair_errors: List[Dict[str, str]] = []
|
||||
@@ -934,7 +949,7 @@ def repair_orbit_pools(
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
|
||||
if isce2_pool:
|
||||
if isce2_enabled:
|
||||
os.makedirs(isce2_pool, exist_ok=True)
|
||||
for stem in sorted(set(envi_files) - set(isce2_files)):
|
||||
txt_path = envi_files[stem]["path"]
|
||||
@@ -952,6 +967,7 @@ def repair_orbit_pools(
|
||||
"before": before,
|
||||
"after": after,
|
||||
"sync_result": sync_result,
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"repaired_from_envi": repaired_from_envi,
|
||||
"repair_error_count": len(repair_errors),
|
||||
"repair_errors": repair_errors,
|
||||
@@ -968,17 +984,19 @@ def quarantine_bad_orbits(
|
||||
isce2_pool: str = "",
|
||||
quarantine_root: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
source_inventory = get_source_orbit_inventory(source_dir, recursive=True)
|
||||
pool_inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
|
||||
|
||||
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
|
||||
quarantine_dir = _default_quarantine_root(source_dir, quarantine_root)
|
||||
candidate_stems = sorted(set(source_files) - set(isce2_files))
|
||||
candidate_stems = sorted(set(source_files) - set(isce2_files)) if isce2_enabled else []
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"quarantine_root": quarantine_dir,
|
||||
"candidate_count": len(candidate_stems),
|
||||
"validated_count": 0,
|
||||
@@ -1083,14 +1101,18 @@ def check_orbit_consistency(
|
||||
"healthy": bool,
|
||||
}
|
||||
"""
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
|
||||
envi_files: Dict[str, Dict[str, str]] = inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
by_satellite: Dict[str, int] = inventory["envi"]["by_satellite"]
|
||||
|
||||
# 对比
|
||||
envi_stems = set(envi_files)
|
||||
isce2_stems = set(isce2_files)
|
||||
if not isce2_enabled:
|
||||
envi_stems = set()
|
||||
isce2_stems = set()
|
||||
mismatches = []
|
||||
for stem in envi_stems - isce2_stems:
|
||||
mismatches.append(
|
||||
@@ -1112,6 +1134,9 @@ def check_orbit_consistency(
|
||||
)
|
||||
|
||||
mismatches.sort(key=lambda item: item["name"])
|
||||
errors = list(inventory["envi"]["errors"])
|
||||
if isce2_enabled:
|
||||
errors.extend(inventory["isce2"]["errors"])
|
||||
|
||||
return {
|
||||
"envi": {
|
||||
@@ -1121,11 +1146,12 @@ def check_orbit_consistency(
|
||||
},
|
||||
"isce2": {
|
||||
"path": isce2_pool,
|
||||
"enabled": isce2_enabled,
|
||||
"total": len(isce2_files),
|
||||
},
|
||||
"error_count": len(inventory["envi"]["errors"]) + len(inventory["isce2"]["errors"]),
|
||||
"errors": list(inventory["envi"]["errors"]) + list(inventory["isce2"]["errors"]),
|
||||
"error_count": len(errors),
|
||||
"errors": errors,
|
||||
"mismatch_count": len(mismatches),
|
||||
"mismatches": mismatches,
|
||||
"healthy": len(mismatches) == 0,
|
||||
"healthy": len(mismatches) == 0 and len(errors) == 0,
|
||||
}
|
||||
|
||||
@@ -52,6 +52,15 @@ def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _supported_dinsar_family_pair_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_family = _satellite_family_expr(left_alias)
|
||||
right_family = _satellite_family_expr(right_alias)
|
||||
return (
|
||||
f"(({left_family} = 'LT1' AND {right_family} = 'LT1') "
|
||||
f"OR ({left_family} = 'S1' AND {right_family} = 'S1'))"
|
||||
)
|
||||
|
||||
|
||||
def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
|
||||
return (
|
||||
f"(NULLIF({left_alias}.look_direction, '') IS NULL "
|
||||
@@ -60,6 +69,60 @@ def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _relative_orbit_value_expr(alias: str) -> str:
|
||||
return f"NULLIF(upper(trim(COALESCE({alias}.relative_orbit, ''))), '')"
|
||||
|
||||
|
||||
def _same_relative_orbit_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_rel = _relative_orbit_value_expr(left_alias)
|
||||
right_rel = _relative_orbit_value_expr(right_alias)
|
||||
return f"({left_rel} IS NOT NULL AND {right_rel} IS NOT NULL AND {left_rel} = {right_rel})"
|
||||
|
||||
|
||||
def _dinsar_quality_tier_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"CASE "
|
||||
f"WHEN {same_rel} AND {center_distance_expr} <= 5000 THEN 'A' "
|
||||
f"WHEN {center_distance_expr} <= 5000 THEN 'B' "
|
||||
f"WHEN {center_distance_expr} <= 12000 THEN 'C' "
|
||||
"ELSE 'REJECT' END"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_readiness_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
tier = _dinsar_quality_tier_expr(left_alias, right_alias, center_distance_expr)
|
||||
return (
|
||||
"CASE "
|
||||
f"WHEN {tier} IN ('A', 'B') THEN 'RECOMMENDED' "
|
||||
f"WHEN {tier} = 'C' THEN 'CANDIDATE' "
|
||||
"ELSE 'NOT_RECOMMENDED' END"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_quality_score_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"GREATEST(0.0, LEAST(1.0, "
|
||||
f"0.45 * COALESCE((ST_Area(ST_Intersection({left_alias}.geom, {right_alias}.geom)::geography) / "
|
||||
f"NULLIF(GREATEST(ST_Area({left_alias}.geom::geography), ST_Area({right_alias}.geom::geography)), 0)), 0) + "
|
||||
f"0.30 * CASE WHEN {same_rel} THEN 1 ELSE 0 END + "
|
||||
f"0.25 * GREATEST(0.0, 1.0 - ({center_distance_expr} / 12000.0))"
|
||||
"))"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_reasons_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"COALESCE(jsonb_path_query_array(jsonb_build_array("
|
||||
f"CASE WHEN NOT ({same_rel}) THEN 'relative_orbit_missing_or_mismatch' END, "
|
||||
f"CASE WHEN {center_distance_expr} > 5000 THEN 'center_distance_over_5km' END, "
|
||||
f"CASE WHEN {center_distance_expr} > 12000 THEN 'center_distance_over_12km' END"
|
||||
"), '$[*] ? (@ != null)')::json, '[]'::json)"
|
||||
)
|
||||
|
||||
|
||||
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_uid = _scene_uid_expr(left_alias)
|
||||
right_uid = _scene_uid_expr(right_alias)
|
||||
@@ -82,6 +145,7 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
|
||||
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
|
||||
f"AND COALESCE({left_alias}.insar_source_ready, false) "
|
||||
f"AND COALESCE({right_alias}.insar_source_ready, false) "
|
||||
f"AND { _supported_dinsar_family_pair_expr(left_alias, right_alias) } "
|
||||
f"AND { _same_look_direction_expr(left_alias, right_alias) } "
|
||||
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
|
||||
)
|
||||
@@ -106,7 +170,15 @@ def _full_rebuild_insert_sql() -> str:
|
||||
spatial_baseline_meters,
|
||||
scene_center_distance_meters,
|
||||
scene_overlap_ratio,
|
||||
pair_aoi_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_relative_orbit,
|
||||
master_relative_orbit,
|
||||
slave_relative_orbit,
|
||||
dinsar_quality_tier,
|
||||
dinsar_quality_score,
|
||||
dinsar_readiness,
|
||||
dinsar_reasons_json,
|
||||
same_satellite,
|
||||
same_satellite_family,
|
||||
same_look_direction,
|
||||
@@ -144,7 +216,15 @@ def _full_rebuild_insert_sql() -> str:
|
||||
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
NULL::double precision AS pair_aoi_overlap_ratio,
|
||||
m.orbit_direction,
|
||||
{ _same_relative_orbit_expr('m', 's') } AS same_relative_orbit,
|
||||
{_relative_orbit_value_expr('m')} AS master_relative_orbit,
|
||||
{_relative_orbit_value_expr('s')} AS slave_relative_orbit,
|
||||
{ _dinsar_quality_tier_expr('m', 's', center_distance) } AS dinsar_quality_tier,
|
||||
{ _dinsar_quality_score_expr('m', 's', center_distance) } AS dinsar_quality_score,
|
||||
{ _dinsar_readiness_expr('m', 's', center_distance) } AS dinsar_readiness,
|
||||
{ _dinsar_reasons_expr('m', 's', center_distance) } AS dinsar_reasons_json,
|
||||
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
|
||||
{ _same_satellite_family_expr('m', 's') } AS same_satellite_family,
|
||||
{ _same_look_direction_expr('m', 's') } AS same_look_direction,
|
||||
@@ -201,7 +281,15 @@ def _incremental_insert_sql() -> str:
|
||||
spatial_baseline_meters,
|
||||
scene_center_distance_meters,
|
||||
scene_overlap_ratio,
|
||||
pair_aoi_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_relative_orbit,
|
||||
master_relative_orbit,
|
||||
slave_relative_orbit,
|
||||
dinsar_quality_tier,
|
||||
dinsar_quality_score,
|
||||
dinsar_readiness,
|
||||
dinsar_reasons_json,
|
||||
same_satellite,
|
||||
same_satellite_family,
|
||||
same_look_direction,
|
||||
@@ -244,7 +332,15 @@ def _incremental_insert_sql() -> str:
|
||||
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
NULL::double precision AS pair_aoi_overlap_ratio,
|
||||
d.orbit_direction,
|
||||
{ _same_relative_orbit_expr('d', 'o') } AS same_relative_orbit,
|
||||
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('d')} ELSE {_relative_orbit_value_expr('o')} END AS master_relative_orbit,
|
||||
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('o')} ELSE {_relative_orbit_value_expr('d')} END AS slave_relative_orbit,
|
||||
{ _dinsar_quality_tier_expr('d', 'o', center_distance) } AS dinsar_quality_tier,
|
||||
{ _dinsar_quality_score_expr('d', 'o', center_distance) } AS dinsar_quality_score,
|
||||
{ _dinsar_readiness_expr('d', 'o', center_distance) } AS dinsar_readiness,
|
||||
{ _dinsar_reasons_expr('d', 'o', center_distance) } AS dinsar_reasons_json,
|
||||
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
|
||||
{ _same_satellite_family_expr('d', 'o') } AS same_satellite_family,
|
||||
{ _same_look_direction_expr('d', 'o') } AS same_look_direction,
|
||||
|
||||
@@ -18,7 +18,7 @@ from ..models import (
|
||||
|
||||
|
||||
PAIRING_CACHE_SCOPE_GLOBAL = "global"
|
||||
DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1"
|
||||
DEFAULT_PAIRING_METRIC_VERSION = "2026.06.dinsar.family.v1"
|
||||
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
|
||||
|
||||
|
||||
|
||||
@@ -235,15 +235,6 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="directory_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_ARCHIVE_SOURCE_DIRS",
|
||||
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
root_role="source_pool_gf3_archive",
|
||||
display_prefix="GF3 Archive Pool",
|
||||
scan_mode="archive_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_SOURCE_DIRS",
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy import and_, case, cast, func, or_
|
||||
from sqlalchemy import and_, case, cast, func, or_, text
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from geoalchemy2 import Geography
|
||||
@@ -24,6 +24,8 @@ from shapely.geometry import Polygon
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from ..models import (
|
||||
DinsarProductionRunItemORM,
|
||||
DinsarProductionRunORM,
|
||||
HazardPoint,
|
||||
HazardPointORM,
|
||||
PairingNetworkEdgeORM,
|
||||
@@ -43,7 +45,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
|
||||
from .pairing_state_service import pairing_state_service
|
||||
|
||||
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
|
||||
PAIRING_POLICY_VERSION = "2026.06.dinsar-production.v1"
|
||||
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
|
||||
PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -145,10 +147,10 @@ class SpatialService:
|
||||
require_orbit_data=require_orbit_data,
|
||||
)
|
||||
|
||||
if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
|
||||
if len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
|
||||
raise RuntimeError(
|
||||
f"全部配对命中 {len(candidate_pool)} 条候选边,超过系统一次性返回上限 "
|
||||
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略,或收紧 AOI、日期范围、重叠率。"
|
||||
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请收紧 AOI、日期范围、重叠率或中心距离阈值。"
|
||||
)
|
||||
|
||||
if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD:
|
||||
@@ -156,15 +158,21 @@ class SpatialService:
|
||||
f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}(当前: {len(candidate_pool)}),建议收紧参数或缩小 AOI。"
|
||||
)
|
||||
|
||||
selected_candidates, strategy_warnings = self._apply_strategy(
|
||||
candidate_pool,
|
||||
effective_params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
)
|
||||
selected_candidates, strategy_warnings = self._apply_dinsar_production_strategy(candidate_pool, effective_params)
|
||||
warnings.extend(strategy_warnings)
|
||||
if not selected_candidates:
|
||||
warnings.extend(
|
||||
await self._build_empty_pairing_diagnostics(
|
||||
db,
|
||||
effective_params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
require_orbit_data=require_orbit_data,
|
||||
)
|
||||
)
|
||||
|
||||
for candidate in selected_candidates:
|
||||
candidate.setdefault("selection_strategy", effective_params.strategy)
|
||||
candidate["selection_strategy"] = "dinsar_production"
|
||||
self._ensure_candidate_identity(candidate)
|
||||
|
||||
network_run_id = await self._persist_network_run(
|
||||
db,
|
||||
@@ -176,6 +184,7 @@ class SpatialService:
|
||||
selected_candidates=selected_candidates,
|
||||
)
|
||||
|
||||
await self._attach_dinsar_production_summaries(db, selected_candidates)
|
||||
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
|
||||
metadata = {
|
||||
"fallback_used": False,
|
||||
@@ -227,13 +236,20 @@ class SpatialService:
|
||||
PairingMetricCacheORM.status == "READY",
|
||||
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
|
||||
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
|
||||
PairingMetricCacheORM.master_imaging_date < PairingMetricCacheORM.slave_imaging_date,
|
||||
PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold,
|
||||
PairingMetricCacheORM.same_look_direction.is_(True),
|
||||
PairingMetricCacheORM.dinsar_readiness.in_(["RECOMMENDED", "CANDIDATE"]),
|
||||
)
|
||||
)
|
||||
|
||||
if params.limit_footprint_center_distance:
|
||||
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
|
||||
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
and_(master_family_expr == "LT1", slave_family_expr == "LT1"),
|
||||
and_(master_family_expr == "S1", slave_family_expr == "S1"),
|
||||
)
|
||||
)
|
||||
|
||||
if require_orbit_data:
|
||||
stmt = stmt.where(
|
||||
@@ -241,8 +257,7 @@ class SpatialService:
|
||||
slave_alias.has_orbit_data.is_(True),
|
||||
)
|
||||
|
||||
if not params.cross_satellite_pairing:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
|
||||
|
||||
if params.require_same_imaging_mode:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True))
|
||||
@@ -259,20 +274,19 @@ class SpatialService:
|
||||
)
|
||||
|
||||
if params.allowed_satellites:
|
||||
allowed_satellites = [
|
||||
str(item).strip().upper()
|
||||
for item in params.allowed_satellites
|
||||
if str(item).strip()
|
||||
]
|
||||
allowed_satellites = []
|
||||
for item in params.allowed_satellites:
|
||||
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
allowed_satellites.append("LT1")
|
||||
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
allowed_satellites.append("S1")
|
||||
allowed_satellites = list(dict.fromkeys(allowed_satellites))
|
||||
if not allowed_satellites:
|
||||
return []
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
func.upper(master_alias.satellite).in_(allowed_satellites),
|
||||
func.upper(master_alias.satellite_family).in_(allowed_satellites),
|
||||
),
|
||||
or_(
|
||||
func.upper(slave_alias.satellite).in_(allowed_satellites),
|
||||
func.upper(slave_alias.satellite_family).in_(allowed_satellites),
|
||||
),
|
||||
master_family_expr.in_(allowed_satellites),
|
||||
slave_family_expr.in_(allowed_satellites),
|
||||
)
|
||||
|
||||
if params.master_date_from:
|
||||
@@ -286,23 +300,23 @@ class SpatialService:
|
||||
|
||||
if aoi_wkt:
|
||||
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
|
||||
aoi_geog = cast(aoi_geom, Geography)
|
||||
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
|
||||
pair_overlap_geom = ST_Intersection(master_alias.geom, slave_alias.geom)
|
||||
pair_aoi_geom = ST_Intersection(pair_overlap_geom, aoi_geom)
|
||||
pair_aoi_overlap_expr = (ST_Area(cast(pair_aoi_geom, Geography)) / aoi_area).label("pair_aoi_overlap_ratio")
|
||||
stmt = stmt.add_columns(pair_aoi_overlap_expr)
|
||||
stmt = stmt.where(
|
||||
ST_Intersects(master_alias.geom, aoi_geom),
|
||||
ST_Intersects(slave_alias.geom, aoi_geom),
|
||||
ST_Intersects(pair_overlap_geom, aoi_geom),
|
||||
)
|
||||
if params.aoi_overlap_threshold is not None:
|
||||
aoi_geog = cast(aoi_geom, Geography)
|
||||
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
|
||||
master_inter_geog = cast(ST_Intersection(master_alias.geom, aoi_geom), Geography)
|
||||
slave_inter_geog = cast(ST_Intersection(slave_alias.geom, aoi_geom), Geography)
|
||||
stmt = stmt.where(
|
||||
ST_Area(master_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
|
||||
ST_Area(slave_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
|
||||
)
|
||||
stmt = stmt.where(pair_aoi_overlap_expr >= params.aoi_overlap_threshold)
|
||||
|
||||
stmt = stmt.order_by(
|
||||
PairingMetricCacheORM.master_imaging_date.asc(),
|
||||
PairingMetricCacheORM.slave_imaging_date.asc(),
|
||||
PairingMetricCacheORM.dinsar_quality_tier.asc(),
|
||||
func.coalesce(PairingMetricCacheORM.dinsar_quality_score, 0).desc(),
|
||||
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
|
||||
center_distance_expr.asc(),
|
||||
PairingMetricCacheORM.pair_uid.asc(),
|
||||
@@ -310,7 +324,9 @@ class SpatialService:
|
||||
|
||||
result = await db.execute(stmt)
|
||||
candidate_pool: List[dict] = []
|
||||
for metric_row, master_row, slave_row in result.all():
|
||||
for row in result.all():
|
||||
metric_row, master_row, slave_row = row[0], row[1], row[2]
|
||||
pair_aoi_overlap_ratio = row[3] if len(row) > 3 else None
|
||||
center_distance = float(
|
||||
metric_row.scene_center_distance_meters
|
||||
if metric_row.scene_center_distance_meters is not None
|
||||
@@ -328,16 +344,54 @@ class SpatialService:
|
||||
"dist": center_distance,
|
||||
"scene_center_distance_meters": center_distance,
|
||||
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0),
|
||||
"dinsar_quality_tier": metric_row.dinsar_quality_tier,
|
||||
"dinsar_quality_score": (
|
||||
float(metric_row.dinsar_quality_score)
|
||||
if metric_row.dinsar_quality_score is not None
|
||||
else None
|
||||
),
|
||||
"dinsar_readiness": metric_row.dinsar_readiness,
|
||||
"dinsar_reasons": [
|
||||
str(item)
|
||||
for item in (metric_row.dinsar_reasons_json or [])
|
||||
if isinstance(item, str) and item.strip()
|
||||
],
|
||||
"same_relative_orbit": bool(metric_row.same_relative_orbit),
|
||||
"master_relative_orbit": metric_row.master_relative_orbit,
|
||||
"slave_relative_orbit": metric_row.slave_relative_orbit,
|
||||
"pair_aoi_overlap_ratio": (
|
||||
float(pair_aoi_overlap_ratio)
|
||||
if pair_aoi_overlap_ratio is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
)
|
||||
return candidate_pool
|
||||
|
||||
def _ensure_candidate_identity(self, candidate: dict) -> Tuple[str, str]:
|
||||
master = candidate["master"]
|
||||
slave = candidate["slave"]
|
||||
task_alias = str(candidate.get("task_alias") or "").strip() or build_task_alias(
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
)
|
||||
pair_key = str(candidate.get("pair_key") or "").strip() or build_pair_key(
|
||||
master.file_path,
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
)
|
||||
candidate["task_alias"] = task_alias
|
||||
candidate["pair_key"] = pair_key
|
||||
return task_alias, pair_key
|
||||
|
||||
def _build_radar_pairs(self, selected_candidates: List[dict]) -> List[RadarPair]:
|
||||
result_pairs: List[RadarPair] = []
|
||||
for candidate in selected_candidates:
|
||||
master = candidate["master"]
|
||||
slave = candidate["slave"]
|
||||
task_alias = build_task_alias(master.imaging_date, slave.imaging_date)
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
selection_score = candidate.get("selection_score")
|
||||
result_pairs.append(
|
||||
RadarPair(
|
||||
@@ -345,13 +399,7 @@ class SpatialService:
|
||||
slave=slave,
|
||||
task_name=task_alias,
|
||||
task_alias=task_alias,
|
||||
pair_key=build_pair_key(
|
||||
master.file_path,
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
),
|
||||
pair_key=pair_key,
|
||||
pair_uid=candidate.get("pair_uid"),
|
||||
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
|
||||
network_run_id=candidate.get("network_run_id"),
|
||||
@@ -367,10 +415,292 @@ class SpatialService:
|
||||
if candidate.get("scene_center_distance_meters") is not None
|
||||
else candidate.get("dist") or 0
|
||||
),
|
||||
scene_overlap_ratio=float(candidate.get("overlap_ratio") or 0.0),
|
||||
pair_aoi_overlap_ratio=(
|
||||
float(candidate["pair_aoi_overlap_ratio"])
|
||||
if candidate.get("pair_aoi_overlap_ratio") is not None
|
||||
else None
|
||||
),
|
||||
dinsar_quality_tier=candidate.get("dinsar_quality_tier"),
|
||||
dinsar_quality_score=(
|
||||
float(candidate["dinsar_quality_score"])
|
||||
if candidate.get("dinsar_quality_score") is not None
|
||||
else None
|
||||
),
|
||||
dinsar_readiness=candidate.get("dinsar_readiness"),
|
||||
dinsar_reasons=candidate.get("dinsar_reasons") or [],
|
||||
same_relative_orbit=bool(candidate.get("same_relative_orbit")),
|
||||
master_relative_orbit=candidate.get("master_relative_orbit"),
|
||||
slave_relative_orbit=candidate.get("slave_relative_orbit"),
|
||||
production_summary=candidate.get("production_summary"),
|
||||
)
|
||||
)
|
||||
return result_pairs
|
||||
|
||||
async def _attach_dinsar_production_summaries(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
selected_candidates: List[dict],
|
||||
) -> None:
|
||||
if not selected_candidates:
|
||||
return
|
||||
|
||||
pair_uids: set[str] = set()
|
||||
pair_keys: set[str] = set()
|
||||
aliases: set[str] = set()
|
||||
for candidate in selected_candidates:
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
pair_uid = str(candidate.get("pair_uid") or "").strip()
|
||||
if pair_uid:
|
||||
pair_uids.add(pair_uid)
|
||||
if pair_key:
|
||||
pair_keys.add(pair_key)
|
||||
if task_alias:
|
||||
aliases.add(task_alias)
|
||||
|
||||
run_conditions = []
|
||||
if pair_uids:
|
||||
run_conditions.append(DinsarProductionRunItemORM.pair_uid.in_(pair_uids))
|
||||
if pair_keys:
|
||||
run_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys))
|
||||
if aliases:
|
||||
run_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases))
|
||||
run_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases))
|
||||
|
||||
product_conditions = []
|
||||
if pair_uids:
|
||||
product_conditions.append(ResultProductORM.pair_uid.in_(pair_uids))
|
||||
if pair_keys:
|
||||
product_conditions.append(ResultProductORM.pair_key.in_(pair_keys))
|
||||
if aliases:
|
||||
product_conditions.append(ResultProductORM.task_alias.in_(aliases))
|
||||
product_conditions.append(ResultProductORM.task_name.in_(aliases))
|
||||
|
||||
run_rows = []
|
||||
if run_conditions:
|
||||
result = await db.execute(
|
||||
select(DinsarProductionRunItemORM, DinsarProductionRunORM)
|
||||
.join(DinsarProductionRunORM, DinsarProductionRunItemORM.run_id == DinsarProductionRunORM.run_id)
|
||||
.where(or_(*run_conditions))
|
||||
.order_by(
|
||||
DinsarProductionRunItemORM.updated_at.desc().nullslast(),
|
||||
DinsarProductionRunItemORM.id.desc(),
|
||||
)
|
||||
)
|
||||
run_rows = result.all()
|
||||
|
||||
products = []
|
||||
if product_conditions:
|
||||
result = await db.execute(
|
||||
select(ResultProductORM)
|
||||
.where(ResultProductORM.catalog_name == "dinsar")
|
||||
.where(or_(*product_conditions))
|
||||
.order_by(
|
||||
ResultProductORM.published_at.desc().nullslast(),
|
||||
ResultProductORM.id.desc(),
|
||||
)
|
||||
)
|
||||
products = result.scalars().all()
|
||||
|
||||
run_by_uid: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
run_by_key: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
run_by_alias: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
for item, run in run_rows:
|
||||
pair_uid = str(item.pair_uid or "").strip()
|
||||
pair_key = str(item.pair_key or "").strip()
|
||||
if pair_uid:
|
||||
run_by_uid[pair_uid].append((item, run))
|
||||
if pair_key:
|
||||
run_by_key[pair_key].append((item, run))
|
||||
for alias in {str(item.task_alias or "").strip(), str(item.task_name or "").strip()}:
|
||||
if alias:
|
||||
run_by_alias[alias].append((item, run))
|
||||
|
||||
products_by_uid: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
products_by_key: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
products_by_alias: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
for product in products:
|
||||
pair_uid = str(product.pair_uid or "").strip()
|
||||
pair_key = str(product.pair_key or "").strip()
|
||||
if pair_uid:
|
||||
products_by_uid[pair_uid].append(product)
|
||||
if pair_key:
|
||||
products_by_key[pair_key].append(product)
|
||||
for alias in {str(product.task_alias or "").strip(), str(product.task_name or "").strip()}:
|
||||
if alias:
|
||||
products_by_alias[alias].append(product)
|
||||
|
||||
for candidate in selected_candidates:
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
pair_uid = str(candidate.get("pair_uid") or "").strip()
|
||||
exact_runs = self._dedupe_by_object_id(
|
||||
[*run_by_uid.get(pair_uid, []), *run_by_key.get(pair_key, [])],
|
||||
key=lambda row: getattr(row[0], "id", None),
|
||||
)
|
||||
alias_runs = self._dedupe_by_object_id(
|
||||
run_by_alias.get(task_alias, []),
|
||||
key=lambda row: getattr(row[0], "id", None),
|
||||
)
|
||||
exact_products = self._dedupe_by_object_id(
|
||||
[*products_by_uid.get(pair_uid, []), *products_by_key.get(pair_key, [])],
|
||||
key=lambda product: getattr(product, "id", None),
|
||||
)
|
||||
alias_products = self._dedupe_by_object_id(
|
||||
products_by_alias.get(task_alias, []),
|
||||
key=lambda product: getattr(product, "id", None),
|
||||
)
|
||||
matched_runs = exact_runs or alias_runs
|
||||
matched_products = exact_products or alias_products
|
||||
candidate["production_summary"] = self._summarize_dinsar_production(
|
||||
matched_runs,
|
||||
matched_products,
|
||||
match_level="identity" if (exact_runs or exact_products) else ("task_alias" if (alias_runs or alias_products) else "none"),
|
||||
)
|
||||
|
||||
def _summarize_dinsar_production(
|
||||
self,
|
||||
run_rows: List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]],
|
||||
products: List[ResultProductORM],
|
||||
*,
|
||||
match_level: str = "none",
|
||||
) -> Dict[str, Any]:
|
||||
latest_run_row = max(
|
||||
run_rows,
|
||||
key=lambda row: self._datetime_sort_key(
|
||||
row[0].updated_at,
|
||||
row[0].ended_at,
|
||||
row[0].started_at,
|
||||
row[0].created_at,
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
latest_product = max(
|
||||
products,
|
||||
key=lambda product: self._datetime_sort_key(
|
||||
product.published_at,
|
||||
product.produced_at,
|
||||
product.updated_at,
|
||||
product.registered_at,
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
ready_products = [product for product in products if self._is_ready_result_product(product)]
|
||||
completed_statuses = {"COMPLETED", "READY", "SUCCESS", "PUBLISHED"}
|
||||
failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
|
||||
completed_run_count = sum(
|
||||
1
|
||||
for item, run in run_rows
|
||||
if str(item.status or "").strip().upper() in completed_statuses
|
||||
or str(run.status or "").strip().upper() in completed_statuses
|
||||
)
|
||||
failed_run_count = sum(
|
||||
1
|
||||
for item, run in run_rows
|
||||
if str(item.status or "").strip().upper() in failed_statuses
|
||||
or str(run.status or "").strip().upper() in failed_statuses
|
||||
)
|
||||
|
||||
latest_item = latest_run_row[0] if latest_run_row else None
|
||||
latest_run = latest_run_row[1] if latest_run_row else None
|
||||
if ready_products and latest_product is not None:
|
||||
latest_status = str(latest_product.status or "").strip().upper()
|
||||
elif latest_item is not None or latest_run is not None:
|
||||
latest_status = str(
|
||||
(latest_item.status if latest_item is not None else None)
|
||||
or (latest_run.status if latest_run is not None else None)
|
||||
or ""
|
||||
).strip().upper()
|
||||
elif latest_product is not None:
|
||||
latest_status = str(latest_product.status or "").strip().upper()
|
||||
else:
|
||||
latest_status = ""
|
||||
if ready_products:
|
||||
status = "READY"
|
||||
elif completed_run_count > 0:
|
||||
status = "COMPLETED"
|
||||
elif latest_status:
|
||||
status = latest_status
|
||||
else:
|
||||
status = "MISSING"
|
||||
|
||||
engine_codes = sorted(
|
||||
{
|
||||
str(value or "").strip().lower()
|
||||
for value in [
|
||||
*(product.engine_code for product in products),
|
||||
*(run.engine_code for _, run in run_rows),
|
||||
]
|
||||
if str(value or "").strip()
|
||||
}
|
||||
)
|
||||
return {
|
||||
"has_record": bool(run_rows or products),
|
||||
"is_produced": bool(ready_products or completed_run_count > 0),
|
||||
"status": status,
|
||||
"match_level": match_level,
|
||||
"run_item_count": len(run_rows),
|
||||
"completed_run_count": completed_run_count,
|
||||
"failed_run_count": failed_run_count,
|
||||
"product_count": len(products),
|
||||
"ready_product_count": len(ready_products),
|
||||
"engine_codes": engine_codes,
|
||||
"latest_engine_code": (
|
||||
str(latest_product.engine_code or "").strip().lower()
|
||||
if latest_product is not None and latest_product.engine_code
|
||||
else (
|
||||
str(latest_run.engine_code or "").strip().lower()
|
||||
if latest_run is not None and latest_run.engine_code
|
||||
else None
|
||||
)
|
||||
),
|
||||
"latest_run_id": latest_run.run_id if latest_run is not None else None,
|
||||
"latest_run_status": latest_run.status if latest_run is not None else None,
|
||||
"latest_item_status": latest_item.status if latest_item is not None else None,
|
||||
"latest_output_dir": latest_item.latest_output_dir if latest_item is not None else None,
|
||||
"latest_product_id": latest_product.id if latest_product is not None else None,
|
||||
"latest_product_identifier": latest_product.product_id if latest_product is not None else None,
|
||||
"latest_product_status": latest_product.status if latest_product is not None else None,
|
||||
"latest_product_health": latest_product.health_status if latest_product is not None else None,
|
||||
"latest_product_published_at": latest_product.published_at if latest_product is not None else None,
|
||||
"updated_at": (
|
||||
latest_item.updated_at
|
||||
if latest_item is not None
|
||||
else (
|
||||
latest_product.updated_at
|
||||
if latest_product is not None
|
||||
else None
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
def _is_ready_result_product(self, product: ResultProductORM) -> bool:
|
||||
status = str(product.status or "").strip().upper()
|
||||
health = str(product.health_status or "").strip().upper()
|
||||
return status in {"READY", "COMPLETED", "SUCCESS"} and health not in {"ERROR", "FAILED"}
|
||||
|
||||
def _datetime_sort_key(self, *values: Any) -> float:
|
||||
for value in values:
|
||||
if value is None:
|
||||
continue
|
||||
try:
|
||||
return float(value.timestamp())
|
||||
except Exception:
|
||||
continue
|
||||
return 0.0
|
||||
|
||||
def _dedupe_by_object_id(self, items: List[Any], *, key) -> List[Any]:
|
||||
seen: set[Any] = set()
|
||||
output: List[Any] = []
|
||||
for item in items:
|
||||
item_key = key(item)
|
||||
if item_key is None:
|
||||
item_key = id(item)
|
||||
if item_key in seen:
|
||||
continue
|
||||
seen.add(item_key)
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
async def _persist_network_run(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
@@ -451,6 +781,8 @@ class SpatialService:
|
||||
"master_scene_uid": candidate.get("master_scene_uid"),
|
||||
"slave_scene_uid": candidate.get("slave_scene_uid"),
|
||||
"pair_uid": candidate.get("pair_uid"),
|
||||
"pair_key": candidate.get("pair_key"),
|
||||
"task_alias": candidate.get("task_alias"),
|
||||
"time_baseline_days": int(candidate.get("days") or 0),
|
||||
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
|
||||
"scene_center_distance_meters": float(
|
||||
@@ -460,6 +792,11 @@ class SpatialService:
|
||||
),
|
||||
"legacy_spatial_baseline_field": "scene_center_distance_meters",
|
||||
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
|
||||
"pair_aoi_overlap_ratio": (
|
||||
float(candidate["pair_aoi_overlap_ratio"])
|
||||
if candidate.get("pair_aoi_overlap_ratio") is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
def _stable_sha1(self, value: Any) -> str:
|
||||
@@ -814,6 +1151,226 @@ class SpatialService:
|
||||
"scenes": scene_payloads,
|
||||
}
|
||||
|
||||
def _apply_dinsar_production_strategy(
|
||||
self,
|
||||
candidate_pool: List[dict],
|
||||
params: PairingRequest,
|
||||
) -> Tuple[List[dict], List[str]]:
|
||||
if not candidate_pool:
|
||||
return [], []
|
||||
|
||||
warnings: List[str] = []
|
||||
rejected_count = sum(
|
||||
1
|
||||
for candidate in candidate_pool
|
||||
if str(candidate.get("dinsar_readiness") or "").upper() == "NOT_RECOMMENDED"
|
||||
)
|
||||
if rejected_count:
|
||||
warnings.append(f"{rejected_count}条候选因D-InSAR生产前置条件不足被过滤。")
|
||||
|
||||
selected = []
|
||||
for candidate in candidate_pool:
|
||||
readiness = str(candidate.get("dinsar_readiness") or "CANDIDATE").upper()
|
||||
if readiness == "NOT_RECOMMENDED":
|
||||
continue
|
||||
tier = str(candidate.get("dinsar_quality_tier") or "C").upper()
|
||||
quality_score = candidate.get("dinsar_quality_score")
|
||||
selected.append(
|
||||
{
|
||||
**candidate,
|
||||
"selection_reason": f"dinsar_{readiness.lower()}_{tier.lower()}",
|
||||
"selection_score": (
|
||||
float(quality_score)
|
||||
if quality_score is not None
|
||||
else self._score_pair_candidate(candidate, params)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
selected.sort(
|
||||
key=lambda item: (
|
||||
{"A": 0, "B": 1, "C": 2}.get(str(item.get("dinsar_quality_tier") or "C").upper(), 9),
|
||||
-float(item.get("selection_score") or 0),
|
||||
int(item.get("days") or 0),
|
||||
float(item.get("dist") or 0),
|
||||
str(getattr(item.get("master"), "imaging_date", "") or ""),
|
||||
str(getattr(item.get("slave"), "imaging_date", "") or ""),
|
||||
)
|
||||
)
|
||||
return selected, warnings
|
||||
|
||||
async def _build_empty_pairing_diagnostics(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
params: PairingRequest,
|
||||
*,
|
||||
aoi_wkt: Optional[str],
|
||||
require_orbit_data: bool,
|
||||
) -> List[str]:
|
||||
allowed_families: List[str] = []
|
||||
for item in params.allowed_satellites or []:
|
||||
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
allowed_families.append("LT1")
|
||||
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
allowed_families.append("S1")
|
||||
allowed_families = list(dict.fromkeys(allowed_families))
|
||||
|
||||
sql = text(
|
||||
"""
|
||||
WITH base AS (
|
||||
SELECT
|
||||
pmc.*,
|
||||
m.satellite AS master_satellite_actual,
|
||||
s.satellite AS slave_satellite_actual,
|
||||
m.has_orbit_data AS master_has_orbit,
|
||||
s.has_orbit_data AS slave_has_orbit,
|
||||
COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters) AS center_m
|
||||
FROM pairing_metric_cache pmc
|
||||
JOIN radar_data m ON m.id = pmc.master_scene_ref_id
|
||||
JOIN radar_data s ON s.id = pmc.slave_scene_ref_id
|
||||
WHERE pmc.metric_version = :metric_version
|
||||
AND pmc.status = 'READY'
|
||||
AND pmc.master_imaging_date < pmc.slave_imaging_date
|
||||
AND pmc.same_look_direction IS TRUE
|
||||
AND pmc.same_satellite_family IS TRUE
|
||||
AND pmc.dinsar_readiness IN ('RECOMMENDED', 'CANDIDATE')
|
||||
AND (:require_orbit_data IS FALSE OR (m.has_orbit_data IS TRUE AND s.has_orbit_data IS TRUE))
|
||||
AND (:require_same_imaging_mode IS FALSE OR pmc.same_imaging_mode IS TRUE)
|
||||
AND (:require_same_polarization IS FALSE OR pmc.same_polarization IS TRUE)
|
||||
AND (
|
||||
:allowed_families_is_empty IS TRUE
|
||||
OR pmc.master_satellite_family = ANY(:allowed_families)
|
||||
OR pmc.master_satellite = ANY(:allowed_families)
|
||||
)
|
||||
AND (CAST(:master_date_from AS text) IS NULL OR pmc.master_imaging_date >= CAST(:master_date_from AS text))
|
||||
AND (CAST(:master_date_to AS text) IS NULL OR pmc.master_imaging_date <= CAST(:master_date_to AS text))
|
||||
AND (CAST(:slave_date_from AS text) IS NULL OR pmc.slave_imaging_date >= CAST(:slave_date_from AS text))
|
||||
AND (CAST(:slave_date_to AS text) IS NULL OR pmc.slave_imaging_date <= CAST(:slave_date_to AS text))
|
||||
AND (
|
||||
CAST(:aoi_wkt AS text) IS NULL
|
||||
OR ST_Intersects(
|
||||
ST_Intersection(m.geom, s.geom),
|
||||
ST_GeomFromText(CAST(:aoi_wkt AS text), 4326)
|
||||
)
|
||||
)
|
||||
),
|
||||
time_ok AS (
|
||||
SELECT * FROM base
|
||||
WHERE time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
|
||||
),
|
||||
overlap_ok AS (
|
||||
SELECT * FROM time_ok
|
||||
WHERE scene_overlap_ratio >= :overlap_threshold
|
||||
),
|
||||
center_ok AS (
|
||||
SELECT * FROM overlap_ok
|
||||
WHERE center_m <= :center_distance_max
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM base) AS base_count,
|
||||
(SELECT count(*) FROM time_ok) AS time_ok_count,
|
||||
(SELECT count(*) FROM overlap_ok) AS overlap_ok_count,
|
||||
(SELECT count(*) FROM center_ok) AS center_ok_count,
|
||||
(
|
||||
SELECT json_build_object(
|
||||
'master_date', master_imaging_date,
|
||||
'slave_date', slave_imaging_date,
|
||||
'master_satellite', master_satellite_actual,
|
||||
'slave_satellite', slave_satellite_actual,
|
||||
'time_baseline_days', time_baseline_days,
|
||||
'center_meters', center_m,
|
||||
'overlap_ratio', scene_overlap_ratio,
|
||||
'quality_tier', dinsar_quality_tier,
|
||||
'readiness', dinsar_readiness
|
||||
)
|
||||
FROM base
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
|
||||
THEN 0 ELSE 1
|
||||
END,
|
||||
CASE WHEN scene_overlap_ratio >= :overlap_threshold THEN 0 ELSE 1 END,
|
||||
abs(time_baseline_days - :time_baseline_max),
|
||||
center_m ASC NULLS LAST
|
||||
LIMIT 1
|
||||
) AS nearest_candidate;
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{
|
||||
"metric_version": pairing_state_service.metric_version,
|
||||
"require_orbit_data": require_orbit_data,
|
||||
"require_same_imaging_mode": bool(params.require_same_imaging_mode),
|
||||
"require_same_polarization": bool(params.require_same_polarization),
|
||||
"allowed_families": allowed_families or ["__NONE__"],
|
||||
"allowed_families_is_empty": not allowed_families,
|
||||
"master_date_from": params.master_date_from or None,
|
||||
"master_date_to": params.master_date_to or None,
|
||||
"slave_date_from": params.slave_date_from or None,
|
||||
"slave_date_to": params.slave_date_to or None,
|
||||
"aoi_wkt": aoi_wkt,
|
||||
"time_baseline_min": int(params.time_baseline_min),
|
||||
"time_baseline_max": int(params.time_baseline_max),
|
||||
"overlap_threshold": float(params.overlap_threshold),
|
||||
"center_distance_max": float(params.spatial_baseline_max_meters),
|
||||
},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ["未找到满足条件的 D-InSAR 配对;诊断查询未返回统计结果。"]
|
||||
|
||||
base_count = int(row.get("base_count") or 0)
|
||||
time_ok_count = int(row.get("time_ok_count") or 0)
|
||||
overlap_ok_count = int(row.get("overlap_ok_count") or 0)
|
||||
center_ok_count = int(row.get("center_ok_count") or 0)
|
||||
if base_count <= 0:
|
||||
family_text = "、".join(allowed_families) if allowed_families else "LT-1/Sentinel-1"
|
||||
return [
|
||||
f"未找到 {family_text} 的可生产基础候选边。请检查数据体系、主从日期范围、AOI、精轨绑定和配对缓存状态。"
|
||||
]
|
||||
|
||||
messages = [
|
||||
(
|
||||
"当前筛选下基础候选 {base} 条;时间基线 {min_days}-{max_days} 天后剩 {time_ok} 条;"
|
||||
"重叠率 >= {overlap:.2f} 后剩 {overlap_ok} 条;footprint 中心距离 <= {center:.0f} 米后剩 {center_ok} 条。"
|
||||
).format(
|
||||
base=base_count,
|
||||
min_days=int(params.time_baseline_min),
|
||||
max_days=int(params.time_baseline_max),
|
||||
time_ok=time_ok_count,
|
||||
overlap=float(params.overlap_threshold),
|
||||
overlap_ok=overlap_ok_count,
|
||||
center=float(params.spatial_baseline_max_meters),
|
||||
center_ok=center_ok_count,
|
||||
)
|
||||
]
|
||||
nearest = row.get("nearest_candidate")
|
||||
if isinstance(nearest, str):
|
||||
try:
|
||||
nearest = json.loads(nearest)
|
||||
except Exception:
|
||||
nearest = None
|
||||
if isinstance(nearest, dict):
|
||||
messages.append(
|
||||
(
|
||||
"最接近的一对是 {master_satellite} {master_date} -> {slave_satellite} {slave_date},"
|
||||
"时间基线 {days} 天,中心距离 {center:.1f} 米,重叠率 {overlap:.3f},质量 {tier}/{readiness}。"
|
||||
).format(
|
||||
master_satellite=nearest.get("master_satellite") or "?",
|
||||
master_date=nearest.get("master_date") or "?",
|
||||
slave_satellite=nearest.get("slave_satellite") or "?",
|
||||
slave_date=nearest.get("slave_date") or "?",
|
||||
days=int(nearest.get("time_baseline_days") or 0),
|
||||
center=float(nearest.get("center_meters") or 0.0),
|
||||
overlap=float(nearest.get("overlap_ratio") or 0.0),
|
||||
tier=nearest.get("quality_tier") or "?",
|
||||
readiness=nearest.get("readiness") or "?",
|
||||
)
|
||||
)
|
||||
return messages
|
||||
|
||||
def _apply_strategy(
|
||||
self,
|
||||
candidate_pool: List[dict],
|
||||
|
||||
@@ -259,12 +259,15 @@ class TaskService:
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if task:
|
||||
previous_message = task.message
|
||||
if status:
|
||||
task.status = status
|
||||
if progress is not None:
|
||||
task.progress = progress
|
||||
if message:
|
||||
task.message = message
|
||||
if message != previous_message:
|
||||
await self._add_log(task_id, "INFO", message, db=db)
|
||||
|
||||
# 如果任务结束,更新结束时间
|
||||
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
|
||||
|
||||
@@ -39,7 +39,7 @@ def get_unpack_config() -> Dict[str, Any]:
|
||||
"source_dirs": source_dirs,
|
||||
"insar_storage_dirs": insar_storage_dirs,
|
||||
"min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")),
|
||||
"delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")),
|
||||
"delete_archive": False,
|
||||
"tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"),
|
||||
"archive_exts": archive_exts,
|
||||
"scan_workers": module.parse_int(
|
||||
|
||||
Reference in New Issue
Block a user