feat: add SBAS AOI discovery and result management
This commit is contained in:
+27
-1
@@ -19,6 +19,7 @@ from .services.pairing_state_service import pairing_state_service
|
||||
from .services.psinsar_catalog_service import psinsar_catalog_service
|
||||
from .services.result_catalog_service import result_catalog_service
|
||||
from .services.root_registry_service import root_registry_service
|
||||
from .services.sbas_insar_catalog_service import sbas_insar_catalog_service
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
@@ -83,6 +84,18 @@ async def lifespan(app: FastAPI):
|
||||
"queued": False,
|
||||
"error": str(exc),
|
||||
}
|
||||
try:
|
||||
sbas_catalog_bootstrap = await sbas_insar_catalog_service.bootstrap_catalog_on_startup_clean()
|
||||
except Exception as exc:
|
||||
sbas_catalog_bootstrap = {
|
||||
"storage_root": settings.GAMMA_SBAS_WORK_ROOT,
|
||||
"manifest_count": 0,
|
||||
"db_count": 0,
|
||||
"needs_rebuild": False,
|
||||
"rebuilt": False,
|
||||
"queued": False,
|
||||
"error": str(exc),
|
||||
}
|
||||
try:
|
||||
pairing_bootstrap = await pairing_state_service.bootstrap_pairing_cache_state()
|
||||
except Exception as exc:
|
||||
@@ -178,6 +191,17 @@ async def lifespan(app: FastAPI):
|
||||
)
|
||||
if ps_catalog_bootstrap.get("error"):
|
||||
print(f">>> [Timeseries Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}")
|
||||
print(
|
||||
">>> [Gamma SBAS Catalog] root={0} runs={1} db={2} rebuild={3} rebuilt={4}".format(
|
||||
sbas_catalog_bootstrap.get("storage_root") or "?",
|
||||
sbas_catalog_bootstrap.get("manifest_count", 0),
|
||||
sbas_catalog_bootstrap.get("db_count", 0),
|
||||
"YES" if sbas_catalog_bootstrap.get("needs_rebuild") else "NO",
|
||||
"YES" if sbas_catalog_bootstrap.get("rebuilt") else "NO",
|
||||
)
|
||||
)
|
||||
if sbas_catalog_bootstrap.get("error"):
|
||||
print(f">>> [Gamma SBAS Catalog] Startup bootstrap failed: {sbas_catalog_bootstrap['error']}")
|
||||
print(
|
||||
">>> [Pairing] status={0} scenes={1} pairs={2} dirty={3} metric={4} rebuild={5}".format(
|
||||
pairing_bootstrap.get("status") or "?",
|
||||
@@ -214,17 +238,19 @@ async def lifespan(app: FastAPI):
|
||||
health.get("timeseries_result_catalog", {})
|
||||
or health.get("psinsar_result_catalog", {})
|
||||
).get("ok")
|
||||
sbas_catalog_ok = health.get("sbas_insar_result_catalog", {}).get("ok")
|
||||
pairing_ok = health.get("pairing_system", {}).get("ok")
|
||||
idl_ok = health.get("idl", {}).get("ok")
|
||||
product_packages_ok = health.get("product_packages", {}).get("ok")
|
||||
wsl_runtime_ok = health.get("wsl_runtime", {}).get("ok")
|
||||
print(
|
||||
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} Timeseries-Catalog:{4} Packages:{5} WSL:{6} Pairing:{7} IDL:{8}".format(
|
||||
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} Timeseries-Catalog:{4} SBAS-Catalog:{5} Packages:{6} WSL:{7} Pairing:{8} IDL:{9}".format(
|
||||
"OK" if db_ok else "FAIL",
|
||||
"OK" if schema_ok else "FAIL",
|
||||
"OK" if worker_ok else "FAIL",
|
||||
"OK" if dinsar_catalog_ok else "FAIL",
|
||||
"OK" if psinsar_catalog_ok else "FAIL",
|
||||
"OK" if sbas_catalog_ok else "FAIL",
|
||||
"OK" if product_packages_ok else "FAIL",
|
||||
"OK" if wsl_runtime_ok else "FAIL",
|
||||
"OK" if pairing_ok else "FAIL",
|
||||
|
||||
@@ -20,6 +20,7 @@ from . import (
|
||||
orbit,
|
||||
pairing,
|
||||
ps_products,
|
||||
sbas_insar_products,
|
||||
radar,
|
||||
root_registry,
|
||||
sbas_insar_production,
|
||||
@@ -55,6 +56,7 @@ def include_all_routers(router: APIRouter) -> None:
|
||||
router.include_router(dinsar_products.router)
|
||||
router.include_router(dinsar_production.router)
|
||||
router.include_router(sbas_insar_production.router)
|
||||
router.include_router(sbas_insar_products.router)
|
||||
router.include_router(timeseries_production.router)
|
||||
router.include_router(ps_products.router)
|
||||
router.include_router(ai.router)
|
||||
|
||||
@@ -6,7 +6,7 @@ import subprocess
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.sbas_insar_production_service import sbas_insar_production_service
|
||||
@@ -16,6 +16,19 @@ from ..services.task_service import task_service
|
||||
router = APIRouter(prefix="/sbas-insar-production", tags=["sbas-insar-production"])
|
||||
|
||||
|
||||
class SbasAoiBbox(BaseModel):
|
||||
min_lon: float = Field(ge=-180, le=180)
|
||||
min_lat: float = Field(ge=-90, le=90)
|
||||
max_lon: float = Field(ge=-180, le=180)
|
||||
max_lat: float = Field(ge=-90, le=90)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_order(self):
|
||||
if self.min_lon >= self.max_lon or self.min_lat >= self.max_lat:
|
||||
raise ValueError("aoi_bbox min values must be smaller than max values")
|
||||
return self
|
||||
|
||||
|
||||
class SbasStackDiscoverRequest(BaseModel):
|
||||
source_roots: list[str] | None = None
|
||||
orbit_roots: list[str] | None = None
|
||||
@@ -26,6 +39,11 @@ class SbasStackDiscoverRequest(BaseModel):
|
||||
platform: str | None = Field(default=None, max_length=16)
|
||||
relative_orbit: str | None = Field(default=None, max_length=32)
|
||||
orbit_direction: str | None = Field(default=None, max_length=32)
|
||||
admin_region: str | None = Field(default=None, max_length=120)
|
||||
discovery_mode: str = Field(default="strict", pattern="^(strict|aoi)$")
|
||||
aoi_bbox: SbasAoiBbox | None = None
|
||||
min_aoi_coverage_ratio: float = Field(default=0.01, ge=0, le=1)
|
||||
min_common_overlap_ratio: float = Field(default=0.0, ge=0, le=1)
|
||||
|
||||
@field_validator("source_roots", "orbit_roots", mode="before")
|
||||
@classmethod
|
||||
@@ -39,7 +57,7 @@ class SbasStackDiscoverRequest(BaseModel):
|
||||
cleaned = [str(item or "").strip() for item in items if str(item or "").strip()]
|
||||
return cleaned or None
|
||||
|
||||
@field_validator("platform", "relative_orbit", "orbit_direction", mode="before")
|
||||
@field_validator("platform", "relative_orbit", "orbit_direction", "admin_region", mode="before")
|
||||
@classmethod
|
||||
def _normalize_optional_text(cls, value):
|
||||
if value is None:
|
||||
@@ -47,6 +65,12 @@ class SbasStackDiscoverRequest(BaseModel):
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
@field_validator("discovery_mode", mode="before")
|
||||
@classmethod
|
||||
def _normalize_discovery_mode(cls, value):
|
||||
text = str(value or "strict").strip().lower()
|
||||
return text or "strict"
|
||||
|
||||
|
||||
class SbasMonitorPoint(BaseModel):
|
||||
point_id: str | None = Field(default=None, max_length=64)
|
||||
@@ -58,7 +82,7 @@ class SbasMonitorPoint(BaseModel):
|
||||
class SbasRunSubmitRequest(SbasStackDiscoverRequest):
|
||||
run_label: str | None = Field(default=None, max_length=120)
|
||||
dry_run: bool = True
|
||||
monitor_point_strategy: str = Field(default="auto_low_sigma_high_rate", max_length=64)
|
||||
monitor_point_strategy: str = Field(default="auto_representative_points", max_length=64)
|
||||
monitor_points: list[SbasMonitorPoint] | None = None
|
||||
|
||||
|
||||
@@ -160,6 +184,11 @@ async def discover_sbas_insar_stacks(request: SbasStackDiscoverRequest):
|
||||
platform=request.platform,
|
||||
relative_orbit=request.relative_orbit,
|
||||
orbit_direction=request.orbit_direction,
|
||||
admin_region=request.admin_region,
|
||||
discovery_mode=request.discovery_mode,
|
||||
aoi_bbox=request.aoi_bbox.model_dump() if request.aoi_bbox else None,
|
||||
min_aoi_coverage_ratio=request.min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=request.min_common_overlap_ratio,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
@@ -175,6 +204,11 @@ async def audit_sbas_insar_stack(stack_id: str, request: SbasStackDiscoverReques
|
||||
orbit_roots=request.orbit_roots,
|
||||
min_scenes=request.min_scenes,
|
||||
require_orbits=request.require_orbits,
|
||||
discovery_mode=request.discovery_mode,
|
||||
admin_region=request.admin_region,
|
||||
aoi_bbox=request.aoi_bbox.model_dump() if request.aoi_bbox else None,
|
||||
min_aoi_coverage_ratio=request.min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=request.min_common_overlap_ratio,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
@@ -193,6 +227,11 @@ async def submit_sbas_insar_run(stack_id: str, request: SbasRunSubmitRequest):
|
||||
orbit_roots=request.orbit_roots,
|
||||
min_scenes=request.min_scenes,
|
||||
require_orbits=request.require_orbits,
|
||||
discovery_mode=request.discovery_mode,
|
||||
admin_region=request.admin_region,
|
||||
aoi_bbox=request.aoi_bbox.model_dump() if request.aoi_bbox else None,
|
||||
min_aoi_coverage_ratio=request.min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=request.min_common_overlap_ratio,
|
||||
monitor_points=[
|
||||
point.model_dump(exclude_none=True)
|
||||
for point in (request.monitor_points or [])
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import AuthUserORM
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.sbas_insar_catalog_service import (
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG,
|
||||
TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG,
|
||||
sbas_insar_catalog_service,
|
||||
)
|
||||
from ..services.task_service import task_service
|
||||
from .dependencies import _add_operation_audit_log, _get_current_user, _require_admin
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SbasInsarCatalogRebuildRequest(BaseModel):
|
||||
full_rebuild: bool = True
|
||||
|
||||
|
||||
@router.get("/sbas-insar-products/catalog-status")
|
||||
async def get_sbas_insar_catalog_status(
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await sbas_insar_catalog_service.get_catalog_status(db)
|
||||
|
||||
|
||||
@router.post("/sbas-insar-products/rebuild", status_code=202)
|
||||
async def queue_sbas_insar_catalog_rebuild(
|
||||
request: SbasInsarCatalogRebuildRequest,
|
||||
http_request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
task_id = await task_service.create_task(
|
||||
TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG,
|
||||
"SBAS-InSAR result catalog rebuild",
|
||||
params={"full_rebuild": request.full_rebuild},
|
||||
db=db,
|
||||
)
|
||||
await job_queue_service.create_job(
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG,
|
||||
payload={"full_rebuild": request.full_rebuild},
|
||||
task_id=task_id,
|
||||
db=db,
|
||||
)
|
||||
await _add_operation_audit_log(
|
||||
db,
|
||||
request=http_request,
|
||||
action="sbas_insar_catalog_rebuild_queued",
|
||||
resource="sbas-insar-products/rebuild",
|
||||
detail={"task_id": task_id, "full_rebuild": request.full_rebuild},
|
||||
)
|
||||
await db.commit()
|
||||
return {
|
||||
"message": "SBAS-InSAR result catalog rebuild has been queued.",
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/sbas-insar-products")
|
||||
async def list_sbas_insar_products(
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
query: str | None = None,
|
||||
admin_region: str | None = None,
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await sbas_insar_catalog_service.list_products(
|
||||
db,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
status=status,
|
||||
query=query,
|
||||
admin_region=admin_region,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/sbas-insar-products/{product_db_id}")
|
||||
async def get_sbas_insar_product_detail(
|
||||
product_db_id: int,
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
detail = await sbas_insar_catalog_service.get_product_detail(db, product_db_id=product_db_id)
|
||||
if detail is None:
|
||||
raise HTTPException(status_code=404, detail="SBAS-InSAR product not found")
|
||||
return detail
|
||||
|
||||
|
||||
@router.get("/sbas-insar-products/{product_db_id}/preview")
|
||||
async def get_sbas_insar_product_preview(
|
||||
product_db_id: int,
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
detail = await sbas_insar_catalog_service.get_product_detail(db, product_db_id=product_db_id)
|
||||
if detail is None:
|
||||
raise HTTPException(status_code=404, detail="SBAS-InSAR product not found")
|
||||
preview_path = str(detail.get("preview_path") or "").strip()
|
||||
if not preview_path or not os.path.isfile(preview_path):
|
||||
raise HTTPException(status_code=404, detail="Preview not found")
|
||||
return FileResponse(preview_path, media_type="image/png")
|
||||
|
||||
|
||||
@router.get("/sbas-insar-products/{product_db_id}/assets/{asset_id}")
|
||||
async def get_sbas_insar_product_asset(
|
||||
product_db_id: int,
|
||||
asset_id: int,
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
asset = await sbas_insar_catalog_service.get_asset(db, product_db_id=product_db_id, asset_id=asset_id)
|
||||
if asset is None:
|
||||
raise HTTPException(status_code=404, detail="SBAS-InSAR product asset not found")
|
||||
if not asset.absolute_path or not os.path.isfile(asset.absolute_path):
|
||||
raise HTTPException(status_code=404, detail="Asset file not found")
|
||||
return FileResponse(
|
||||
asset.absolute_path,
|
||||
media_type=asset.media_type or "application/octet-stream",
|
||||
filename=asset.asset_name or os.path.basename(asset.absolute_path),
|
||||
)
|
||||
@@ -0,0 +1,392 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from shapely.geometry import Point, shape
|
||||
from shapely.ops import unary_union
|
||||
|
||||
try:
|
||||
from shapely.validation import make_valid as _make_valid_geometry
|
||||
except Exception: # pragma: no cover - depends on the installed Shapely version.
|
||||
_make_valid_geometry = None
|
||||
|
||||
|
||||
_LEVEL_RANK = {
|
||||
"country": 0,
|
||||
"province": 1,
|
||||
"city": 2,
|
||||
"district": 3,
|
||||
"county": 3,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RegionGeometryRecord:
|
||||
tree_id: str
|
||||
name: str
|
||||
level: str | None
|
||||
adcode: str | None
|
||||
geometry: Any
|
||||
area: float
|
||||
|
||||
|
||||
_REGION_GEOMETRY_CACHE: list[_RegionGeometryRecord] | None = None
|
||||
_REGION_BY_ID_LOOKUP_CACHE: dict[str, dict[str, Any]] | None = None
|
||||
_REGION_LOAD_ERROR: str | None = None
|
||||
|
||||
|
||||
def _backend_geojson_dir() -> Path:
|
||||
return Path(__file__).resolve().parents[2] / "geojson"
|
||||
|
||||
|
||||
def _normalize_region_index_node(raw: dict[str, Any]) -> dict[str, Any] | None:
|
||||
tree_id = str(raw.get("treeID") or raw.get("tree_id") or raw.get("treeId") or "").strip()
|
||||
if not tree_id:
|
||||
return None
|
||||
parent_raw = raw.get("parent")
|
||||
parent_tree_id = str(parent_raw).strip() if parent_raw is not None else None
|
||||
if parent_tree_id == "":
|
||||
parent_tree_id = None
|
||||
depth = len(tree_id.split("-"))
|
||||
level = {1: "country", 2: "province", 3: "city", 4: "district"}.get(depth, "unknown")
|
||||
return {
|
||||
"tree_id": tree_id,
|
||||
"parent_tree_id": parent_tree_id,
|
||||
"name": str(raw.get("name") or tree_id).strip(),
|
||||
"level": level,
|
||||
}
|
||||
|
||||
|
||||
def _load_region_index_from_files() -> dict[str, dict[str, Any]]:
|
||||
geojson_dir = _backend_geojson_dir()
|
||||
candidates = [geojson_dir / "层级映射.json", *sorted(geojson_dir.glob("*.json"))]
|
||||
for path in candidates:
|
||||
if not path.is_file() or path.name == "treeid_fill_report.json":
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(payload, list):
|
||||
continue
|
||||
nodes: dict[str, dict[str, Any]] = {}
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
node = _normalize_region_index_node(item)
|
||||
if node:
|
||||
nodes[node["tree_id"]] = node
|
||||
if nodes:
|
||||
return nodes
|
||||
return {}
|
||||
|
||||
|
||||
def _load_region_geometry_from_files() -> dict[str, list[dict[str, Any]]]:
|
||||
geojson_dir = _backend_geojson_dir()
|
||||
candidates = [
|
||||
geojson_dir / "全国行政区.geojson",
|
||||
geojson_dir / "中华人民共和国.geojson",
|
||||
*sorted(geojson_dir.glob("*.geojson"), key=lambda item: item.stat().st_size if item.exists() else 0, reverse=True),
|
||||
]
|
||||
for path in candidates:
|
||||
if not path.is_file():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
features = payload.get("features") if isinstance(payload, dict) else payload
|
||||
if not isinstance(features, list):
|
||||
continue
|
||||
feature_index: dict[str, list[dict[str, Any]]] = {}
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or feature.get("type") != "Feature":
|
||||
continue
|
||||
props = feature.get("properties") or {}
|
||||
if not isinstance(props, dict):
|
||||
continue
|
||||
tree_id = str(props.get("treeID") or props.get("tree_id") or props.get("treeId") or "").strip()
|
||||
if tree_id:
|
||||
feature_index.setdefault(tree_id, []).append(feature)
|
||||
if feature_index:
|
||||
return feature_index
|
||||
return {}
|
||||
|
||||
|
||||
def _repair_geometry(geometry):
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
if getattr(geometry, "is_valid", True):
|
||||
return geometry
|
||||
if _make_valid_geometry is not None:
|
||||
try:
|
||||
fixed = _make_valid_geometry(geometry)
|
||||
if fixed is not None and not fixed.is_empty:
|
||||
return fixed
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
fixed = geometry.buffer(0)
|
||||
if fixed is not None and not fixed.is_empty:
|
||||
return fixed
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _merge_region_geometries(geometries: list[Any]):
|
||||
fixed_geometries = []
|
||||
for geometry in geometries:
|
||||
fixed = _repair_geometry(geometry)
|
||||
if fixed is not None and not fixed.is_empty:
|
||||
fixed_geometries.append(fixed)
|
||||
if not fixed_geometries:
|
||||
return None
|
||||
if len(fixed_geometries) == 1:
|
||||
return fixed_geometries[0]
|
||||
try:
|
||||
merged = unary_union(fixed_geometries)
|
||||
return _repair_geometry(merged)
|
||||
except Exception:
|
||||
repaired_buffers = []
|
||||
for geometry in fixed_geometries:
|
||||
try:
|
||||
buffered = geometry.buffer(0)
|
||||
except Exception:
|
||||
continue
|
||||
if buffered is not None and not buffered.is_empty:
|
||||
repaired_buffers.append(buffered)
|
||||
if not repaired_buffers:
|
||||
return None
|
||||
try:
|
||||
merged = unary_union(repaired_buffers)
|
||||
return _repair_geometry(merged)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_region_path(tree_id: str, region_by_id: dict[str, dict[str, Any]]) -> tuple[list[str], list[str]]:
|
||||
names: list[str] = []
|
||||
tree_ids: list[str] = []
|
||||
current = tree_id
|
||||
guard = 0
|
||||
while current and guard < 12:
|
||||
node = region_by_id.get(current) or {}
|
||||
name = str(node.get("name") or current).strip()
|
||||
if name:
|
||||
names.append(name)
|
||||
tree_ids.append(current)
|
||||
current = str(node.get("parent_tree_id") or "").strip()
|
||||
guard += 1
|
||||
names.reverse()
|
||||
tree_ids.reverse()
|
||||
if names and names[0] in {"中国", "中华人民共和国"}:
|
||||
names = names[1:]
|
||||
if tree_ids and tree_ids[0] == "1":
|
||||
tree_ids = tree_ids[1:]
|
||||
return names, tree_ids
|
||||
|
||||
|
||||
def _load_region_records() -> tuple[list[_RegionGeometryRecord], dict[str, dict[str, Any]], str | None]:
|
||||
global _REGION_BY_ID_LOOKUP_CACHE, _REGION_GEOMETRY_CACHE, _REGION_LOAD_ERROR
|
||||
if _REGION_GEOMETRY_CACHE is not None:
|
||||
return _REGION_GEOMETRY_CACHE, _REGION_BY_ID_LOOKUP_CACHE or {}, _REGION_LOAD_ERROR
|
||||
try:
|
||||
from ..routers import dependencies as deps
|
||||
|
||||
deps._load_region_index()
|
||||
deps._load_region_geometry_index()
|
||||
region_by_id = deps._REGION_BY_ID_CACHE or {}
|
||||
geometry_by_id = deps._REGION_GEOMETRY_BY_ID_CACHE or {}
|
||||
except Exception:
|
||||
region_by_id = _load_region_index_from_files()
|
||||
geometry_by_id = _load_region_geometry_from_files()
|
||||
if not region_by_id or not geometry_by_id:
|
||||
_REGION_LOAD_ERROR = "AOI region index or geometry file is unavailable."
|
||||
return [], {}, _REGION_LOAD_ERROR
|
||||
try:
|
||||
records: list[_RegionGeometryRecord] = []
|
||||
for tree_id, features in geometry_by_id.items():
|
||||
geometries = []
|
||||
merged_props: dict[str, Any] = {}
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or not feature.get("geometry"):
|
||||
continue
|
||||
try:
|
||||
geom = shape(feature["geometry"])
|
||||
except Exception:
|
||||
continue
|
||||
if geom.is_empty:
|
||||
continue
|
||||
geometries.append(geom)
|
||||
props = feature.get("properties") or {}
|
||||
if isinstance(props, dict):
|
||||
merged_props.update({key: value for key, value in props.items() if value not in (None, "")})
|
||||
if not geometries:
|
||||
continue
|
||||
geometry = _merge_region_geometries(geometries)
|
||||
if geometry is None or geometry.is_empty:
|
||||
continue
|
||||
node = region_by_id.get(tree_id) or {}
|
||||
records.append(
|
||||
_RegionGeometryRecord(
|
||||
tree_id=str(tree_id),
|
||||
name=str(merged_props.get("name") or node.get("name") or tree_id).strip(),
|
||||
level=str(merged_props.get("level") or node.get("level") or "").strip() or None,
|
||||
adcode=str(merged_props.get("adcode") or "").strip() or None,
|
||||
geometry=geometry,
|
||||
area=float(getattr(geometry, "area", 0.0) or 0.0),
|
||||
)
|
||||
)
|
||||
records.sort(
|
||||
key=lambda item: (
|
||||
-_LEVEL_RANK.get(str(item.level or "").lower(), len(item.tree_id.split("-"))),
|
||||
item.area,
|
||||
item.tree_id,
|
||||
)
|
||||
)
|
||||
_REGION_GEOMETRY_CACHE = records
|
||||
_REGION_BY_ID_LOOKUP_CACHE = region_by_id
|
||||
_REGION_LOAD_ERROR = None
|
||||
return records, region_by_id, None
|
||||
except Exception as exc:
|
||||
_REGION_LOAD_ERROR = str(exc)
|
||||
return [], {}, _REGION_LOAD_ERROR
|
||||
|
||||
|
||||
def lookup_admin_region_for_point(lon: Any, lat: Any) -> dict[str, Any] | None:
|
||||
try:
|
||||
lon_value = float(lon)
|
||||
lat_value = float(lat)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not (-180 <= lon_value <= 180 and -90 <= lat_value <= 90):
|
||||
return None
|
||||
|
||||
records, region_by_id, error = _load_region_records()
|
||||
if error:
|
||||
return {
|
||||
"match_status": "unavailable",
|
||||
"message": error,
|
||||
"center": {"lon": lon_value, "lat": lat_value},
|
||||
}
|
||||
point = Point(lon_value, lat_value)
|
||||
best: _RegionGeometryRecord | None = None
|
||||
for record in records:
|
||||
try:
|
||||
if record.geometry.covers(point):
|
||||
best = record
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if best is None:
|
||||
return {
|
||||
"match_status": "not_matched",
|
||||
"center": {"lon": lon_value, "lat": lat_value},
|
||||
}
|
||||
|
||||
path_names, path_tree_ids = _build_region_path(best.tree_id, region_by_id)
|
||||
display_name = " / ".join(path_names or [best.name])
|
||||
return {
|
||||
"match_status": "matched",
|
||||
"tree_id": best.tree_id,
|
||||
"name": best.name,
|
||||
"level": best.level,
|
||||
"adcode": best.adcode,
|
||||
"path_names": path_names,
|
||||
"path_tree_ids": path_tree_ids,
|
||||
"display_name": display_name,
|
||||
"center": {"lon": lon_value, "lat": lat_value},
|
||||
"source": "aoi_region_geometry",
|
||||
}
|
||||
|
||||
|
||||
def lookup_admin_region_geometry(query: str | None) -> dict[str, Any] | None:
|
||||
text = str(query or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
records, region_by_id, error = _load_region_records()
|
||||
if error:
|
||||
return {
|
||||
"match_status": "unavailable",
|
||||
"message": error,
|
||||
"query": text,
|
||||
}
|
||||
|
||||
query_lower = text.lower()
|
||||
matches: list[tuple[tuple[int, int, float, str], _RegionGeometryRecord, dict[str, Any]]] = []
|
||||
for record in records:
|
||||
path_names, path_tree_ids = _build_region_path(record.tree_id, region_by_id)
|
||||
display_name = " / ".join(path_names or [record.name])
|
||||
name_lower = str(record.name or "").lower()
|
||||
display_lower = display_name.lower()
|
||||
adcode_lower = str(record.adcode or "").lower()
|
||||
tree_id_lower = str(record.tree_id or "").lower()
|
||||
path_lowers = [str(item or "").lower() for item in path_names]
|
||||
|
||||
score: int | None = None
|
||||
if query_lower in {name_lower, display_lower, adcode_lower, tree_id_lower}:
|
||||
score = 0
|
||||
elif query_lower in path_lowers:
|
||||
score = 1
|
||||
elif query_lower and query_lower in name_lower:
|
||||
score = 2
|
||||
elif query_lower and query_lower in display_lower:
|
||||
score = 3
|
||||
elif query_lower and query_lower in " ".join(path_lowers + [adcode_lower, tree_id_lower]):
|
||||
score = 4
|
||||
if score is None:
|
||||
continue
|
||||
|
||||
level_rank = _LEVEL_RANK.get(str(record.level or "").lower(), len(record.tree_id.split("-")))
|
||||
summary = {
|
||||
"match_status": "matched",
|
||||
"query": text,
|
||||
"tree_id": record.tree_id,
|
||||
"name": record.name,
|
||||
"level": record.level,
|
||||
"adcode": record.adcode,
|
||||
"path_names": path_names,
|
||||
"path_tree_ids": path_tree_ids,
|
||||
"display_name": display_name,
|
||||
"bbox": {
|
||||
"min_lon": float(record.geometry.bounds[0]),
|
||||
"min_lat": float(record.geometry.bounds[1]),
|
||||
"max_lon": float(record.geometry.bounds[2]),
|
||||
"max_lat": float(record.geometry.bounds[3]),
|
||||
},
|
||||
"source": "aoi_region_geometry",
|
||||
}
|
||||
matches.append(((score, level_rank, record.area, record.tree_id), record, summary))
|
||||
|
||||
if not matches:
|
||||
return {
|
||||
"match_status": "not_matched",
|
||||
"query": text,
|
||||
}
|
||||
|
||||
matches.sort(key=lambda item: item[0])
|
||||
_, record, summary = matches[0]
|
||||
return {
|
||||
**summary,
|
||||
"geometry": record.geometry,
|
||||
}
|
||||
|
||||
|
||||
def admin_region_matches(region: dict[str, Any] | None, query: str | None) -> bool:
|
||||
text = str(query or "").strip().lower()
|
||||
if not text:
|
||||
return True
|
||||
if not isinstance(region, dict):
|
||||
return False
|
||||
values: list[str] = []
|
||||
for key in ("display_name", "name", "tree_id", "adcode", "level"):
|
||||
value = region.get(key)
|
||||
if value:
|
||||
values.append(str(value))
|
||||
for item in region.get("path_names") or []:
|
||||
values.append(str(item))
|
||||
return text in " ".join(values).lower()
|
||||
@@ -298,6 +298,14 @@ async def _check_timeseries_result_catalog() -> Dict[str, Any]:
|
||||
)
|
||||
|
||||
|
||||
async def _check_sbas_insar_result_catalog() -> Dict[str, Any]:
|
||||
return await _check_catalog(
|
||||
catalog_name="sbas_insar",
|
||||
storage_root=os.path.join(settings.GAMMA_SBAS_WORK_ROOT, "runs"),
|
||||
enabled=bool(settings.GAMMA_SBAS_ENABLED),
|
||||
)
|
||||
|
||||
|
||||
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
|
||||
return await _check_timeseries_result_catalog()
|
||||
|
||||
@@ -472,6 +480,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
payload.get("timeseries_result_catalog", {}) or payload.get("psinsar_result_catalog", {}) or {}
|
||||
)
|
||||
psinsar_result_catalog = timeseries_result_catalog
|
||||
sbas_insar_result_catalog = payload.get("sbas_insar_result_catalog", {}) or {}
|
||||
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
sar_analysis_ready = payload.get("sar_analysis_ready", {}) or {}
|
||||
@@ -486,6 +495,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sanitized_dinsar_catalog = _sanitize_catalog_status(dinsar_result_catalog)
|
||||
sanitized_timeseries_catalog = _sanitize_catalog_status(timeseries_result_catalog)
|
||||
sanitized_psinsar_catalog = sanitized_timeseries_catalog
|
||||
sanitized_sbas_insar_catalog = _sanitize_catalog_status(sbas_insar_result_catalog)
|
||||
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
|
||||
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
|
||||
sanitized_sar_analysis_ready = _sanitize_sar_analysis_ready_status(sar_analysis_ready)
|
||||
@@ -511,10 +521,12 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"dinsar_result_catalog": sanitized_dinsar_catalog,
|
||||
"timeseries_result_catalog": sanitized_timeseries_catalog,
|
||||
"psinsar_result_catalog": sanitized_psinsar_catalog,
|
||||
"sbas_insar_result_catalog": sanitized_sbas_insar_catalog,
|
||||
"catalogs": {
|
||||
"dinsar": sanitized_dinsar_catalog,
|
||||
"timeseries": sanitized_timeseries_catalog,
|
||||
"psinsar": sanitized_psinsar_catalog,
|
||||
"sbas_insar": sanitized_sbas_insar_catalog,
|
||||
},
|
||||
"dinsar_bridge": sanitized_dinsar_bridge,
|
||||
"source_roots": sanitized_source_roots,
|
||||
@@ -1340,6 +1352,7 @@ async def get_health_status(
|
||||
result_catalog_status = await _check_result_catalog()
|
||||
timeseries_result_catalog_status = await _check_timeseries_result_catalog()
|
||||
psinsar_result_catalog_status = timeseries_result_catalog_status
|
||||
sbas_insar_result_catalog_status = await _check_sbas_insar_result_catalog()
|
||||
dinsar_bridge_status = await _check_dinsar_bridge()
|
||||
source_roots_status = await _check_source_roots()
|
||||
sar_analysis_ready_status = await _check_sar_analysis_ready()
|
||||
@@ -1366,6 +1379,7 @@ async def get_health_status(
|
||||
wsl_runtime_status.get("ok"),
|
||||
pairing_system_status.get("ok"),
|
||||
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
|
||||
(not settings.GAMMA_SBAS_ENABLED) or sbas_insar_result_catalog_status.get("ok"),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -1378,10 +1392,12 @@ async def get_health_status(
|
||||
"dinsar_result_catalog": result_catalog_status,
|
||||
"timeseries_result_catalog": timeseries_result_catalog_status,
|
||||
"psinsar_result_catalog": psinsar_result_catalog_status,
|
||||
"sbas_insar_result_catalog": sbas_insar_result_catalog_status,
|
||||
"catalogs": {
|
||||
"dinsar": result_catalog_status,
|
||||
"timeseries": timeseries_result_catalog_status,
|
||||
"psinsar": psinsar_result_catalog_status,
|
||||
"sbas_insar": sbas_insar_result_catalog_status,
|
||||
},
|
||||
"dinsar_bridge": dinsar_bridge_status,
|
||||
"source_roots": source_roots_status,
|
||||
|
||||
@@ -37,6 +37,7 @@ from .engine_lock_service import engine_lock_service
|
||||
from .envi_service import build_envi_runner_command, get_envi_runner_cwd, get_envi_runner_env
|
||||
from .psinsar_catalog_service import psinsar_catalog_service
|
||||
from .result_catalog_service import result_catalog_service
|
||||
from .sbas_insar_catalog_service import sbas_insar_catalog_service
|
||||
from .task_service import task_service
|
||||
from .timeseries_service import (
|
||||
JOB_TYPE_TIMESERIES_MATERIALIZE,
|
||||
@@ -90,6 +91,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
|
||||
JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM"
|
||||
@@ -4188,6 +4190,30 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_rebuild_sbas_insar_catalog(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("REBUILD_SBAS_INSAR_CATALOG requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
full_rebuild = bool(payload.get("full_rebuild", True))
|
||||
|
||||
await task_service.start_task(job.task_id, message="Rebuilding SBAS-InSAR result catalog...")
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await sbas_insar_catalog_service.rebuild_catalog(
|
||||
db,
|
||||
full_rebuild=full_rebuild,
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
f"SBAS-InSAR result catalog rebuilt: runs={result.get('run_count', 0)}, "
|
||||
f"registered={result.get('registered', 0)}, failed={result.get('failed', 0)}, "
|
||||
f"issues={result.get('issue_count', 0)}"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _handle_sbas_coregistration(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("SBAS_COREGISTRATION requires task_id for progress tracking.")
|
||||
@@ -4623,6 +4649,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH: _handle_timeseries_export_publish,
|
||||
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT: _handle_timeseries_register_product,
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG: _handle_rebuild_sbas_insar_catalog,
|
||||
JOB_TYPE_COPY_DATA: _handle_copy_data,
|
||||
JOB_TYPE_UNPACK: _handle_unpack_archives,
|
||||
JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1,
|
||||
|
||||
@@ -0,0 +1,912 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import Polygon
|
||||
from sqlalchemy import String, cast, delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import settings
|
||||
from ..models import ResultAssetORM, ResultCatalogStateORM, ResultIssueORM, ResultProductORM
|
||||
from .admin_region_lookup_service import lookup_admin_region_for_point
|
||||
from .sbas_insar_production_service import sbas_insar_production_service
|
||||
|
||||
|
||||
SBAS_INSAR_CATALOG_NAME = "sbas_insar"
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
|
||||
TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
|
||||
|
||||
_READY_STATUSES = {"PRODUCTS_READY", "MONITOR_POINTS_READY", "WORKFLOW_COMPLETED"}
|
||||
_REQUIRED_ASSET_ROLES = {"primary_geotiff", "quality_geotiff"}
|
||||
|
||||
_CORE_ASSETS = (
|
||||
("run_manifest", "Run manifest", "run_manifest.json", True, False),
|
||||
("stack_manifest", "Stack manifest", "stack_manifest.json", True, False),
|
||||
("workflow_summary", "Workflow summary", "workflow_summary.json", False, False),
|
||||
("product_summary", "Product summary", "product_summary.json", False, False),
|
||||
("quality_summary", "Quality summary", "quality_summary.json", False, False),
|
||||
("monitor_points_summary", "Monitor points summary", "monitor_points_summary.json", False, False),
|
||||
(
|
||||
"point_vector_summary",
|
||||
"LOS point-vector summary",
|
||||
"publish/vectors/los_rate_points_summary.json",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"point_vector_geojson_gz",
|
||||
"LOS point-vector GeoJSON.gz",
|
||||
"publish/vectors/los_rate_points.geojson.gz",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"primary_geocoded_preview",
|
||||
"LOS velocity preview, toward radar positive",
|
||||
"publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png",
|
||||
False,
|
||||
True,
|
||||
),
|
||||
(
|
||||
"quality_geocoded_preview",
|
||||
"LOS velocity sigma preview",
|
||||
"publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"primary_geotiff",
|
||||
"LOS velocity GeoTIFF, toward radar positive",
|
||||
"publish/geotiff/los_rate_toward_m_per_year.tif",
|
||||
True,
|
||||
True,
|
||||
),
|
||||
(
|
||||
"alternate_geotiff",
|
||||
"LOS velocity GeoTIFF, away from radar positive",
|
||||
"publish/geotiff/los_rate_away_m_per_year.tif",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"quality_geotiff",
|
||||
"LOS velocity sigma GeoTIFF",
|
||||
"publish/geotiff/los_sigma_m_per_year.tif",
|
||||
True,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"primary_rgb_geotiff",
|
||||
"LOS velocity RGB GeoTIFF",
|
||||
"publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"quality_rgb_geotiff",
|
||||
"LOS velocity sigma RGB GeoTIFF",
|
||||
"publish/geotiff/los_sigma_m_per_year.cc.geo_rgb.tif",
|
||||
False,
|
||||
False,
|
||||
),
|
||||
("gamma_phase_rate", "Gamma phase-rate GeoTIFF", "publish/geotiff/ts_rate_rad_per_year.tif", False, False),
|
||||
("gamma_sigma_rate", "Gamma sigma-rate GeoTIFF", "publish/geotiff/sigma_rate_rad_per_year.tif", False, False),
|
||||
("height_correction", "Height correction GeoTIFF", "publish/geotiff/hgt_correction_m.tif", False, False),
|
||||
)
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.utcnow()
|
||||
|
||||
|
||||
def _normalize_path(path: str | os.PathLike[str]) -> str:
|
||||
return os.path.normpath(os.path.abspath(os.fspath(path)))
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _safe_read_json(path: Path) -> dict[str, Any]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
return _read_json(path)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _safe_float(value: Any) -> Optional[float]:
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if parsed != parsed:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> Optional[int]:
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> Optional[datetime]:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
try:
|
||||
return datetime.fromisoformat(text).replace(tzinfo=None)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _stable_digest(*parts: Any, length: int = 20) -> str:
|
||||
payload = "||".join(str(part or "") for part in parts)
|
||||
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
|
||||
|
||||
|
||||
def _asset_format(path: str) -> Optional[str]:
|
||||
lowered = path.lower()
|
||||
if lowered.endswith(".geojson.gz"):
|
||||
return "geojson.gz"
|
||||
ext = Path(path).suffix.lower()
|
||||
return {
|
||||
".bmp": "bmp",
|
||||
".csv": "csv",
|
||||
".geo": "gamma_binary",
|
||||
".gz": "gzip",
|
||||
".json": "json",
|
||||
".log": "log",
|
||||
".png": "png",
|
||||
".sh": "shell",
|
||||
".tif": "geotiff",
|
||||
".tiff": "geotiff",
|
||||
".txt": "text",
|
||||
}.get(ext)
|
||||
|
||||
|
||||
def _media_type(path: str) -> Optional[str]:
|
||||
lowered = path.lower()
|
||||
if lowered.endswith(".geojson.gz"):
|
||||
return "application/gzip"
|
||||
ext = Path(path).suffix.lower()
|
||||
explicit = {
|
||||
".bmp": "image/bmp",
|
||||
".csv": "text/csv",
|
||||
".gz": "application/gzip",
|
||||
".json": "application/json",
|
||||
".log": "text/plain",
|
||||
".png": "image/png",
|
||||
".sh": "text/x-shellscript",
|
||||
".tif": "image/tiff",
|
||||
".tiff": "image/tiff",
|
||||
".txt": "text/plain",
|
||||
}
|
||||
return explicit.get(ext) or mimetypes.guess_type(path)[0]
|
||||
|
||||
|
||||
def _bbox_polygon(
|
||||
min_lon: Optional[float],
|
||||
min_lat: Optional[float],
|
||||
max_lon: Optional[float],
|
||||
max_lat: Optional[float],
|
||||
):
|
||||
if None in (min_lon, min_lat, max_lon, max_lat):
|
||||
return None
|
||||
if min_lon == max_lon or min_lat == max_lat:
|
||||
return None
|
||||
return Polygon(
|
||||
[
|
||||
(min_lon, min_lat),
|
||||
(max_lon, min_lat),
|
||||
(max_lon, max_lat),
|
||||
(min_lon, max_lat),
|
||||
(min_lon, min_lat),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _stack_dates_from_manifest(stack_manifest: dict[str, Any], manifest: dict[str, Any], stack: dict[str, Any]) -> list[str]:
|
||||
values: list[str] = []
|
||||
for source in (
|
||||
stack_manifest.get("dates"),
|
||||
stack.get("dates"),
|
||||
manifest.get("dates"),
|
||||
[scene.get("date") for scene in stack_manifest.get("scenes") or [] if isinstance(scene, dict)],
|
||||
[scene.get("date") for scene in manifest.get("scenes") or [] if isinstance(scene, dict)],
|
||||
):
|
||||
if not isinstance(source, list):
|
||||
continue
|
||||
for item in source:
|
||||
text = str(item or "").strip()
|
||||
if text:
|
||||
values.append(text)
|
||||
return sorted(dict.fromkeys(values))
|
||||
|
||||
|
||||
class SbasInsarCatalogService:
|
||||
def get_run_root(self) -> str:
|
||||
root = Path(settings.GAMMA_SBAS_WORK_ROOT or Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production")
|
||||
run_root = root / "runs"
|
||||
run_root.mkdir(parents=True, exist_ok=True)
|
||||
return _normalize_path(run_root)
|
||||
|
||||
def _iter_run_manifest_paths(self, run_root: Optional[str] = None) -> list[str]:
|
||||
root = Path(run_root or self.get_run_root())
|
||||
if not root.is_dir():
|
||||
return []
|
||||
return [
|
||||
_normalize_path(path)
|
||||
for path in sorted(root.glob("*/run_manifest.json"))
|
||||
if self._is_publish_ready(path.parent, _safe_read_json(path))
|
||||
]
|
||||
|
||||
def _is_publish_ready(self, run_dir: Path, manifest: dict[str, Any]) -> bool:
|
||||
status = str(manifest.get("status") or "").strip().upper()
|
||||
required_outputs_ready = all(
|
||||
(run_dir / relative_path).is_file()
|
||||
for role, _name, relative_path, is_required, _is_primary in _CORE_ASSETS
|
||||
if role in _REQUIRED_ASSET_ROLES and is_required
|
||||
)
|
||||
return status in _READY_STATUSES or required_outputs_ready
|
||||
|
||||
def _tree_fingerprint(self, manifest_paths: list[str]) -> str:
|
||||
records: list[dict[str, Any]] = []
|
||||
for raw_path in manifest_paths:
|
||||
manifest_path = Path(raw_path)
|
||||
run_dir = manifest_path.parent
|
||||
tracked_paths = [
|
||||
manifest_path,
|
||||
run_dir / "product_summary.json",
|
||||
run_dir / "quality_summary.json",
|
||||
run_dir / "monitor_points_summary.json",
|
||||
]
|
||||
tracked_paths.extend(run_dir / relative_path for _role, _name, relative_path, _required, _primary in _CORE_ASSETS)
|
||||
for path in tracked_paths:
|
||||
if not path.exists():
|
||||
continue
|
||||
stat = path.stat()
|
||||
records.append(
|
||||
{
|
||||
"path": str(path.relative_to(run_dir)).replace("\\", "/"),
|
||||
"run": run_dir.name,
|
||||
"size": stat.st_size,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
}
|
||||
)
|
||||
encoded = json.dumps(records, sort_keys=True, ensure_ascii=True)
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
async def _get_or_create_catalog_state(self, db: AsyncSession, *, storage_root: str) -> ResultCatalogStateORM:
|
||||
result = await db.execute(
|
||||
select(ResultCatalogStateORM).where(ResultCatalogStateORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
if state is None:
|
||||
state = ResultCatalogStateORM(
|
||||
catalog_name=SBAS_INSAR_CATALOG_NAME,
|
||||
product_family="timeseries",
|
||||
storage_root=storage_root,
|
||||
status="READY",
|
||||
needs_rebuild=False,
|
||||
)
|
||||
db.add(state)
|
||||
await db.flush()
|
||||
elif state.storage_root != storage_root:
|
||||
state.storage_root = storage_root
|
||||
if state.product_family != "timeseries":
|
||||
state.product_family = "timeseries"
|
||||
return state
|
||||
|
||||
def _asset_row(
|
||||
self,
|
||||
run_dir: Path,
|
||||
*,
|
||||
role: str,
|
||||
name: str,
|
||||
relative_path: str,
|
||||
is_required: bool,
|
||||
is_primary: bool,
|
||||
) -> ResultAssetORM:
|
||||
absolute_path = run_dir / relative_path
|
||||
exists = absolute_path.is_file()
|
||||
return ResultAssetORM(
|
||||
asset_role=role[:32],
|
||||
asset_name=name,
|
||||
relative_path=relative_path.replace("\\", "/"),
|
||||
absolute_path=_normalize_path(absolute_path),
|
||||
format=_asset_format(relative_path),
|
||||
media_type=_media_type(relative_path),
|
||||
is_required=is_required,
|
||||
is_primary=is_primary,
|
||||
exists_flag=exists,
|
||||
file_size=absolute_path.stat().st_size if exists else None,
|
||||
srid=4326 if (
|
||||
(relative_path.lower().endswith((".tif", ".tiff")) and "/geotiff/" in relative_path)
|
||||
or relative_path.lower().endswith(".geojson.gz")
|
||||
) else None,
|
||||
)
|
||||
|
||||
def _monitor_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]:
|
||||
monitor_dir = run_dir / "publish" / "monitor_points"
|
||||
if not monitor_dir.is_dir():
|
||||
return []
|
||||
rows: list[ResultAssetORM] = []
|
||||
for path in sorted(monitor_dir.iterdir()):
|
||||
if not path.is_file():
|
||||
continue
|
||||
suffix = path.suffix.lower()
|
||||
if suffix not in {".png", ".csv", ".json"}:
|
||||
continue
|
||||
role = {
|
||||
".png": "monitor_point_curve",
|
||||
".csv": "monitor_point_csv",
|
||||
".json": "monitor_point_metadata",
|
||||
}[suffix]
|
||||
relative_path = str(path.relative_to(run_dir)).replace("\\", "/")
|
||||
rows.append(
|
||||
self._asset_row(
|
||||
run_dir,
|
||||
role=role,
|
||||
name=path.name,
|
||||
relative_path=relative_path,
|
||||
is_required=False,
|
||||
is_primary=False,
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
def _build_product(self, manifest_path: str) -> ResultProductORM:
|
||||
manifest_file = Path(manifest_path)
|
||||
run_dir = manifest_file.parent
|
||||
manifest = _read_json(manifest_file)
|
||||
if not self._is_publish_ready(run_dir, manifest):
|
||||
raise ValueError(f"run is not publish-ready: {manifest.get('status') or 'UNKNOWN'}")
|
||||
|
||||
detail = sbas_insar_production_service.get_run_detail(run_dir.name)
|
||||
coverage = detail.get("geographic_coverage") or {}
|
||||
stack_manifest = _safe_read_json(run_dir / "stack_manifest.json")
|
||||
product_summary = _safe_read_json(run_dir / "product_summary.json")
|
||||
quality_summary = _safe_read_json(run_dir / "quality_summary.json")
|
||||
monitor_summary = _safe_read_json(run_dir / "monitor_points_summary.json")
|
||||
point_vector_summary = _safe_read_json(run_dir / "publish" / "vectors" / "los_rate_points_summary.json")
|
||||
workflow_summary = _safe_read_json(run_dir / "workflow_summary.json")
|
||||
|
||||
bbox = coverage.get("bbox") or {}
|
||||
min_lon = _safe_float(bbox.get("min_lon"))
|
||||
min_lat = _safe_float(bbox.get("min_lat"))
|
||||
max_lon = _safe_float(bbox.get("max_lon"))
|
||||
max_lat = _safe_float(bbox.get("max_lat"))
|
||||
poly = _bbox_polygon(min_lon, min_lat, max_lon, max_lat)
|
||||
|
||||
run_id = str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name
|
||||
stack = stack_manifest.get("stack") or manifest.get("stack") or {}
|
||||
stack_id = str(manifest.get("stack_id") or stack_manifest.get("stack_id") or stack.get("stack_id") or "").strip()
|
||||
stack_dates = _stack_dates_from_manifest(stack_manifest, manifest, stack)
|
||||
reference_date = str(
|
||||
manifest.get("reference_date")
|
||||
or stack.get("reference_date")
|
||||
or (manifest.get("coregistration") or {}).get("reference_date")
|
||||
or ""
|
||||
).strip() or None
|
||||
display_name = stack_id or f"Gamma SBAS {run_id}"
|
||||
product_id = str(manifest.get("product_id") or "").strip() or f"gamma_sbas_{run_id}"
|
||||
if len(product_id) > 64:
|
||||
product_id = f"gamma_sbas_{_stable_digest(product_id, run_dir, length=32)}"
|
||||
|
||||
assets: list[ResultAssetORM] = [
|
||||
self._asset_row(
|
||||
run_dir,
|
||||
role=role,
|
||||
name=name,
|
||||
relative_path=relative_path,
|
||||
is_required=is_required,
|
||||
is_primary=is_primary,
|
||||
)
|
||||
for role, name, relative_path, is_required, is_primary in _CORE_ASSETS
|
||||
]
|
||||
assets.extend(self._monitor_asset_rows(run_dir))
|
||||
preview_asset = next((asset for asset in assets if asset.asset_role == "primary_geocoded_preview" and asset.exists_flag), None)
|
||||
primary_asset = next((asset for asset in assets if asset.asset_role == "primary_geotiff" and asset.exists_flag), None)
|
||||
missing_required = [asset for asset in assets if asset.is_required and not asset.exists_flag]
|
||||
|
||||
produced_at = (
|
||||
_parse_datetime(monitor_summary.get("generated_at"))
|
||||
or _parse_datetime(product_summary.get("generated_at"))
|
||||
or _parse_datetime(workflow_summary.get("generated_at"))
|
||||
or _parse_datetime(manifest.get("updated_at"))
|
||||
or _parse_datetime(manifest.get("created_at"))
|
||||
)
|
||||
center = coverage.get("center") or {}
|
||||
admin_region = coverage.get("admin_region") or lookup_admin_region_for_point(center.get("lon"), center.get("lat"))
|
||||
scene_count = (
|
||||
_safe_int(manifest.get("scene_count"))
|
||||
or len(stack_manifest.get("scenes") or [])
|
||||
or len(stack_dates)
|
||||
)
|
||||
|
||||
summary_json = {
|
||||
"schema": "insar.gamma-sbas-result-catalog-summary/v1",
|
||||
"run_id": run_id,
|
||||
"stack_id": stack_id or None,
|
||||
"stack": stack,
|
||||
"reference_date": reference_date,
|
||||
"stack_dates": stack_dates,
|
||||
"stack_size": len(stack_dates),
|
||||
"date_start": stack_dates[0] if stack_dates else None,
|
||||
"date_end": stack_dates[-1] if stack_dates else None,
|
||||
"scene_count": scene_count,
|
||||
"pair_count": _safe_int(manifest.get("pair_count")),
|
||||
"status": manifest.get("status"),
|
||||
"next_stage": manifest.get("next_stage"),
|
||||
"los_sign_convention": (
|
||||
product_summary.get("los_sign_convention")
|
||||
or "toward radar positive; away from radar negative"
|
||||
),
|
||||
"default_los_product": product_summary.get("default_los_product") or "los_rate_toward_m_per_year",
|
||||
"center": center or None,
|
||||
"admin_region": admin_region,
|
||||
"geographic_coverage": coverage,
|
||||
"quality": quality_summary,
|
||||
"monitor_points": monitor_summary,
|
||||
"point_vector": point_vector_summary,
|
||||
"workflow": {
|
||||
"status": ((manifest.get("workflow") or {}).get("status")),
|
||||
"summary": ((manifest.get("workflow") or {}).get("summary")) or workflow_summary,
|
||||
},
|
||||
"source_run_dir": str(run_dir),
|
||||
}
|
||||
|
||||
product = ResultProductORM(
|
||||
product_id=product_id,
|
||||
catalog_name=SBAS_INSAR_CATALOG_NAME,
|
||||
product_family="timeseries",
|
||||
product_type="sbas_insar",
|
||||
display_name=display_name,
|
||||
task_name="Gamma SBAS-InSAR",
|
||||
task_alias=run_id,
|
||||
stack_key=stack_id or run_id,
|
||||
run_key=run_id,
|
||||
profile_code=str(stack.get("relative_orbit") or manifest.get("relative_orbit") or "").strip() or None,
|
||||
engine_code="gamma",
|
||||
engine_version=str((manifest.get("engine") or {}).get("version") or "").strip() or None,
|
||||
package_schema=str(manifest.get("schema") or "").strip() or "insar.gamma-sbas-run/v1",
|
||||
package_layout="gamma_sbas_expert_workflow_run",
|
||||
processor_code="gamma_ipta_sbas",
|
||||
runtime_id=settings.GAMMA_SBAS_RUNTIME_ID,
|
||||
status="READY" if not missing_required else "INCOMPLETE",
|
||||
health_status="OK" if not missing_required else "WARN",
|
||||
publish_dir=_normalize_path(run_dir / "publish"),
|
||||
manifest_path=_normalize_path(manifest_file),
|
||||
source_primary_path=primary_asset.absolute_path if primary_asset else None,
|
||||
native_output_dir=_normalize_path(run_dir),
|
||||
preview_path=preview_asset.absolute_path if preview_asset else None,
|
||||
primary_asset_path=primary_asset.absolute_path if primary_asset else None,
|
||||
summary_json=summary_json,
|
||||
tags_json={
|
||||
"sensor": stack.get("satellite") or manifest.get("platform"),
|
||||
"orbit_direction": stack.get("orbit_direction") or manifest.get("direction"),
|
||||
"product": "Gamma SBAS",
|
||||
"admin_region": (admin_region or {}).get("display_name") if isinstance(admin_region, dict) else None,
|
||||
},
|
||||
min_lon=min_lon,
|
||||
min_lat=min_lat,
|
||||
max_lon=max_lon,
|
||||
max_lat=max_lat,
|
||||
geom=from_shape(poly, srid=4326) if poly is not None else None,
|
||||
coverage_polygon=(coverage.get("geojson") or coverage.get("scene_footprints_geojson")),
|
||||
produced_at=produced_at,
|
||||
published_at=produced_at,
|
||||
)
|
||||
for asset in assets:
|
||||
product.assets.append(asset)
|
||||
if asset.is_required and not asset.exists_flag:
|
||||
product.issues.append(
|
||||
ResultIssueORM(
|
||||
asset=asset,
|
||||
issue_code="MISSING_REQUIRED_ASSET",
|
||||
severity="ERROR",
|
||||
status="OPEN",
|
||||
scope="file",
|
||||
message=f"Required SBAS asset is missing: {asset.relative_path}",
|
||||
)
|
||||
)
|
||||
if not preview_asset:
|
||||
product.issues.append(
|
||||
ResultIssueORM(
|
||||
issue_code="MISSING_PREVIEW",
|
||||
severity="WARN",
|
||||
status="OPEN",
|
||||
scope="product",
|
||||
message="Primary geocoded preview PNG is missing.",
|
||||
)
|
||||
)
|
||||
if poly is None:
|
||||
product.issues.append(
|
||||
ResultIssueORM(
|
||||
issue_code="MISSING_COVERAGE",
|
||||
severity="WARN",
|
||||
status="OPEN",
|
||||
scope="product",
|
||||
message="No valid EPSG:4326 geographic coverage bbox was found.",
|
||||
)
|
||||
)
|
||||
return product
|
||||
|
||||
async def rebuild_catalog(self, db: AsyncSession, *, full_rebuild: bool = True) -> dict[str, Any]:
|
||||
run_root = self.get_run_root()
|
||||
manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root)
|
||||
fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths)
|
||||
state = await self._get_or_create_catalog_state(db, storage_root=run_root)
|
||||
state.status = "REBUILDING"
|
||||
state.needs_rebuild = False
|
||||
state.last_message = "SBAS catalog rebuild in progress"
|
||||
await db.commit()
|
||||
|
||||
if full_rebuild:
|
||||
await db.execute(delete(ResultProductORM).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME))
|
||||
await db.commit()
|
||||
|
||||
registered = 0
|
||||
failed = 0
|
||||
issue_count = 0
|
||||
details: list[dict[str, Any]] = []
|
||||
for manifest_path in manifest_paths:
|
||||
try:
|
||||
product = await asyncio.to_thread(self._build_product, manifest_path)
|
||||
product_issue_count = len(product.issues)
|
||||
product_id = product.product_id
|
||||
product_status = product.status
|
||||
db.add(product)
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
registered += 1
|
||||
issue_count += product_issue_count
|
||||
details.append(
|
||||
{
|
||||
"manifest_path": manifest_path,
|
||||
"product_id": product_id,
|
||||
"status": product_status,
|
||||
"issues": product_issue_count,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
failed += 1
|
||||
issue_count += 1
|
||||
details.append({"manifest_path": manifest_path, "status": "error", "message": str(exc)})
|
||||
|
||||
await db.commit()
|
||||
db_count_result = await db.execute(
|
||||
select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
)
|
||||
db_count = int(db_count_result.scalar_one() or 0)
|
||||
state = await self._get_or_create_catalog_state(db, storage_root=run_root)
|
||||
state.manifest_count = len(manifest_paths)
|
||||
state.manifest_fingerprint = fingerprint
|
||||
state.db_count = db_count
|
||||
state.issue_count = issue_count
|
||||
state.needs_rebuild = False
|
||||
state.status = "READY" if failed == 0 else "WARN"
|
||||
now = _utcnow()
|
||||
state.last_full_rebuild_at = now
|
||||
state.last_incremental_scan_at = now
|
||||
state.last_message = (
|
||||
f"SBAS catalog rebuild finished: runs={len(manifest_paths)}, "
|
||||
f"registered={registered}, failed={failed}, issues={issue_count}"
|
||||
)
|
||||
await db.commit()
|
||||
return {
|
||||
"catalog_name": SBAS_INSAR_CATALOG_NAME,
|
||||
"storage_root": run_root,
|
||||
"run_count": len(manifest_paths),
|
||||
"manifest_count": len(manifest_paths),
|
||||
"manifest_fingerprint": fingerprint,
|
||||
"registered": registered,
|
||||
"failed": failed,
|
||||
"issue_count": issue_count,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
async def list_products(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
status: Optional[str] = None,
|
||||
query: Optional[str] = None,
|
||||
admin_region: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
safe_limit = max(1, min(int(limit or 100), 500))
|
||||
safe_offset = max(0, int(offset or 0))
|
||||
stmt = select(ResultProductORM).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
count_stmt = select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
if status:
|
||||
stmt = stmt.where(ResultProductORM.status == status)
|
||||
count_stmt = count_stmt.where(ResultProductORM.status == status)
|
||||
if query:
|
||||
like_value = f"%{query.strip()}%"
|
||||
predicate = or_(
|
||||
ResultProductORM.display_name.ilike(like_value),
|
||||
ResultProductORM.product_id.ilike(like_value),
|
||||
ResultProductORM.run_key.ilike(like_value),
|
||||
ResultProductORM.stack_key.ilike(like_value),
|
||||
)
|
||||
stmt = stmt.where(predicate)
|
||||
count_stmt = count_stmt.where(predicate)
|
||||
if admin_region:
|
||||
like_value = f"%{admin_region.strip()}%"
|
||||
predicate = or_(
|
||||
cast(ResultProductORM.summary_json, String).ilike(like_value),
|
||||
cast(ResultProductORM.tags_json, String).ilike(like_value),
|
||||
)
|
||||
stmt = stmt.where(predicate)
|
||||
count_stmt = count_stmt.where(predicate)
|
||||
total_result = await db.execute(count_stmt)
|
||||
total = int(total_result.scalar_one() or 0)
|
||||
result = await db.execute(
|
||||
stmt.order_by(ResultProductORM.published_at.desc().nullslast(), ResultProductORM.id.desc())
|
||||
.offset(safe_offset)
|
||||
.limit(safe_limit)
|
||||
)
|
||||
items: list[dict[str, Any]] = []
|
||||
for product in result.scalars().all():
|
||||
summary = product.summary_json or {}
|
||||
items.append(
|
||||
{
|
||||
"id": product.id,
|
||||
"product_id": product.product_id,
|
||||
"display_name": product.display_name,
|
||||
"run_key": product.run_key,
|
||||
"stack_key": product.stack_key,
|
||||
"engine_code": product.engine_code,
|
||||
"processor_code": product.processor_code,
|
||||
"runtime_id": product.runtime_id,
|
||||
"status": product.status,
|
||||
"health_status": product.health_status,
|
||||
"preview_path": product.preview_path,
|
||||
"primary_asset_path": product.primary_asset_path,
|
||||
"reference_date": summary.get("reference_date"),
|
||||
"date_start": summary.get("date_start"),
|
||||
"date_end": summary.get("date_end"),
|
||||
"stack_dates": summary.get("stack_dates") or [],
|
||||
"stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []),
|
||||
"scene_count": summary.get("scene_count"),
|
||||
"pair_count": summary.get("pair_count"),
|
||||
"los_sign_convention": summary.get("los_sign_convention"),
|
||||
"center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")),
|
||||
"admin_region": summary.get("admin_region") or ((summary.get("geographic_coverage") or {}).get("admin_region")),
|
||||
"min_lon": product.min_lon,
|
||||
"min_lat": product.min_lat,
|
||||
"max_lon": product.max_lon,
|
||||
"max_lat": product.max_lat,
|
||||
"published_at": product.published_at,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"items": items,
|
||||
"total": total,
|
||||
"limit": safe_limit,
|
||||
"offset": safe_offset,
|
||||
"has_more": safe_offset + len(items) < total,
|
||||
}
|
||||
|
||||
async def get_product_detail(self, db: AsyncSession, *, product_db_id: int) -> Optional[dict[str, Any]]:
|
||||
result = await db.execute(select(ResultProductORM).where(ResultProductORM.id == product_db_id))
|
||||
product = result.scalar_one_or_none()
|
||||
if product is None or product.catalog_name != SBAS_INSAR_CATALOG_NAME:
|
||||
return None
|
||||
assets_result = await db.execute(
|
||||
select(ResultAssetORM)
|
||||
.where(ResultAssetORM.product_ref_id == product.id)
|
||||
.order_by(ResultAssetORM.is_primary.desc(), ResultAssetORM.asset_role.asc(), ResultAssetORM.id.asc())
|
||||
)
|
||||
issues_result = await db.execute(
|
||||
select(ResultIssueORM)
|
||||
.where(ResultIssueORM.product_ref_id == product.id)
|
||||
.order_by(ResultIssueORM.severity.asc(), ResultIssueORM.id.asc())
|
||||
)
|
||||
summary = product.summary_json or {}
|
||||
return {
|
||||
"id": product.id,
|
||||
"product_id": product.product_id,
|
||||
"catalog_name": product.catalog_name,
|
||||
"product_type": product.product_type,
|
||||
"display_name": product.display_name,
|
||||
"run_key": product.run_key,
|
||||
"run_id": summary.get("run_id") or product.run_key,
|
||||
"stack_key": product.stack_key,
|
||||
"profile_code": product.profile_code,
|
||||
"engine_code": product.engine_code,
|
||||
"engine_version": product.engine_version,
|
||||
"package_schema": product.package_schema,
|
||||
"package_layout": product.package_layout,
|
||||
"processor_code": product.processor_code,
|
||||
"runtime_id": product.runtime_id,
|
||||
"status": product.status,
|
||||
"health_status": product.health_status,
|
||||
"publish_dir": product.publish_dir,
|
||||
"manifest_path": product.manifest_path,
|
||||
"source_primary_path": product.source_primary_path,
|
||||
"native_output_dir": product.native_output_dir,
|
||||
"preview_path": product.preview_path,
|
||||
"primary_asset_path": product.primary_asset_path,
|
||||
"reference_date": summary.get("reference_date"),
|
||||
"date_start": summary.get("date_start"),
|
||||
"date_end": summary.get("date_end"),
|
||||
"stack_dates": summary.get("stack_dates") or [],
|
||||
"stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []),
|
||||
"scene_count": summary.get("scene_count"),
|
||||
"pair_count": summary.get("pair_count"),
|
||||
"los_sign_convention": summary.get("los_sign_convention"),
|
||||
"default_los_product": summary.get("default_los_product"),
|
||||
"quality": summary.get("quality"),
|
||||
"monitor_points": summary.get("monitor_points"),
|
||||
"point_vector": summary.get("point_vector"),
|
||||
"workflow": summary.get("workflow"),
|
||||
"geographic_coverage": summary.get("geographic_coverage"),
|
||||
"center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")),
|
||||
"admin_region": summary.get("admin_region") or ((summary.get("geographic_coverage") or {}).get("admin_region")),
|
||||
"coverage_polygon": product.coverage_polygon,
|
||||
"min_lon": product.min_lon,
|
||||
"min_lat": product.min_lat,
|
||||
"max_lon": product.max_lon,
|
||||
"max_lat": product.max_lat,
|
||||
"produced_at": product.produced_at,
|
||||
"published_at": product.published_at,
|
||||
"registered_at": product.registered_at,
|
||||
"updated_at": product.updated_at,
|
||||
"assets": [
|
||||
{
|
||||
"id": asset.id,
|
||||
"asset_role": asset.asset_role,
|
||||
"asset_name": asset.asset_name,
|
||||
"relative_path": asset.relative_path,
|
||||
"absolute_path": asset.absolute_path,
|
||||
"format": asset.format,
|
||||
"media_type": asset.media_type,
|
||||
"is_required": asset.is_required,
|
||||
"is_primary": asset.is_primary,
|
||||
"exists_flag": asset.exists_flag,
|
||||
"file_size": asset.file_size,
|
||||
"srid": asset.srid,
|
||||
}
|
||||
for asset in assets_result.scalars().all()
|
||||
],
|
||||
"issues": [
|
||||
{
|
||||
"id": issue.id,
|
||||
"issue_code": issue.issue_code,
|
||||
"severity": issue.severity,
|
||||
"status": issue.status,
|
||||
"scope": issue.scope,
|
||||
"message": issue.message,
|
||||
"detected_at": issue.detected_at,
|
||||
}
|
||||
for issue in issues_result.scalars().all()
|
||||
],
|
||||
}
|
||||
|
||||
async def get_asset(self, db: AsyncSession, *, product_db_id: int, asset_id: int) -> Optional[ResultAssetORM]:
|
||||
result = await db.execute(
|
||||
select(ResultAssetORM)
|
||||
.join(ResultProductORM, ResultProductORM.id == ResultAssetORM.product_ref_id)
|
||||
.where(
|
||||
ResultProductORM.id == product_db_id,
|
||||
ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME,
|
||||
ResultAssetORM.id == asset_id,
|
||||
)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
async def get_catalog_status(self, db: AsyncSession) -> dict[str, Any]:
|
||||
run_root = self.get_run_root()
|
||||
manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root)
|
||||
fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths)
|
||||
state = await self._get_or_create_catalog_state(db, storage_root=run_root)
|
||||
db_count_result = await db.execute(
|
||||
select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
)
|
||||
db_count = int(db_count_result.scalar_one() or 0)
|
||||
needs_rebuild = (
|
||||
state.manifest_count != len(manifest_paths)
|
||||
or state.db_count != db_count
|
||||
or state.manifest_fingerprint != fingerprint
|
||||
)
|
||||
state.manifest_count = len(manifest_paths)
|
||||
state.db_count = db_count
|
||||
state.needs_rebuild = needs_rebuild
|
||||
state.last_incremental_scan_at = _utcnow()
|
||||
state.status = "WARN" if needs_rebuild else "READY"
|
||||
state.last_message = (
|
||||
f"SBAS catalog rebuild required: runs={len(manifest_paths)}, db={db_count}"
|
||||
if needs_rebuild
|
||||
else "SBAS catalog is in sync"
|
||||
)
|
||||
await db.commit()
|
||||
return {
|
||||
"catalog_name": state.catalog_name,
|
||||
"product_family": state.product_family,
|
||||
"storage_root": state.storage_root,
|
||||
"status": state.status,
|
||||
"needs_rebuild": state.needs_rebuild,
|
||||
"run_count": len(manifest_paths),
|
||||
"manifest_count": state.manifest_count,
|
||||
"manifest_fingerprint": state.manifest_fingerprint,
|
||||
"current_manifest_fingerprint": fingerprint,
|
||||
"db_count": db_count,
|
||||
"issue_count": state.issue_count,
|
||||
"last_message": state.last_message,
|
||||
"last_boot_check_at": state.last_boot_check_at,
|
||||
"last_full_rebuild_at": state.last_full_rebuild_at,
|
||||
"last_incremental_scan_at": state.last_incremental_scan_at,
|
||||
}
|
||||
|
||||
async def bootstrap_catalog_on_startup_clean(self) -> dict[str, Any]:
|
||||
from ..database import AsyncSessionLocal
|
||||
|
||||
if AsyncSessionLocal is None:
|
||||
raise RuntimeError("Database session factory is not initialized.")
|
||||
async with AsyncSessionLocal() as db:
|
||||
run_root = self.get_run_root()
|
||||
manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root)
|
||||
fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths)
|
||||
state = await self._get_or_create_catalog_state(db, storage_root=run_root)
|
||||
db_count_result = await db.execute(
|
||||
select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)
|
||||
)
|
||||
db_count = int(db_count_result.scalar_one() or 0)
|
||||
needs_rebuild = (
|
||||
state.manifest_count != len(manifest_paths)
|
||||
or state.db_count != db_count
|
||||
or state.manifest_fingerprint != fingerprint
|
||||
)
|
||||
state.last_boot_check_at = _utcnow()
|
||||
await db.commit()
|
||||
|
||||
rebuilt = False
|
||||
result: dict[str, Any] = {}
|
||||
if needs_rebuild and settings.RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP:
|
||||
result = await self.rebuild_catalog(db, full_rebuild=True)
|
||||
rebuilt = True
|
||||
db_count = int(result.get("registered") or db_count)
|
||||
else:
|
||||
state.manifest_count = len(manifest_paths)
|
||||
state.db_count = db_count
|
||||
state.manifest_fingerprint = fingerprint if not needs_rebuild else state.manifest_fingerprint
|
||||
state.needs_rebuild = needs_rebuild
|
||||
state.status = "WARN" if needs_rebuild else "READY"
|
||||
state.last_message = "SBAS boot check complete"
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"storage_root": run_root,
|
||||
"manifest_count": len(manifest_paths),
|
||||
"current_manifest_fingerprint": fingerprint,
|
||||
"indexed_manifest_fingerprint": state.manifest_fingerprint,
|
||||
"db_count": db_count,
|
||||
"needs_rebuild": needs_rebuild and not rebuilt,
|
||||
"rebuilt": rebuilt,
|
||||
"queued": False,
|
||||
"registered": result.get("registered"),
|
||||
"failed": result.get("failed"),
|
||||
}
|
||||
|
||||
|
||||
sbas_insar_catalog_service = SbasInsarCatalogService()
|
||||
@@ -13,7 +13,14 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from shapely.geometry import box as shapely_box
|
||||
|
||||
from ..config import settings
|
||||
from .admin_region_lookup_service import (
|
||||
admin_region_matches,
|
||||
lookup_admin_region_for_point,
|
||||
lookup_admin_region_geometry,
|
||||
)
|
||||
|
||||
|
||||
PRODUCT_DEFINITIONS = (
|
||||
@@ -745,7 +752,7 @@ class SbasInsarProductionService:
|
||||
"default_strategy": "gamma_geocode_back_data2geotiff_los_sign_conversion",
|
||||
"geocoded_preview_source": "EPSG:4326 GeoTIFF",
|
||||
},
|
||||
"monitor_point_modes": ["auto_low_sigma_high_rate", "manual_lonlat"],
|
||||
"monitor_point_modes": ["auto_representative_points", "auto_low_sigma_high_rate", "manual_lonlat"],
|
||||
"default_los_convention": {
|
||||
"key": "los_rate_toward_mm_per_year",
|
||||
"description": "toward radar positive; away from radar negative",
|
||||
@@ -778,10 +785,20 @@ class SbasInsarProductionService:
|
||||
platform: str | None = None,
|
||||
relative_orbit: str | None = None,
|
||||
orbit_direction: str | None = None,
|
||||
admin_region: str | None = None,
|
||||
discovery_mode: str = "strict",
|
||||
aoi_bbox: dict[str, Any] | None = None,
|
||||
min_aoi_coverage_ratio: float = 0.01,
|
||||
min_common_overlap_ratio: float = 0.0,
|
||||
force_refresh: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
source_paths = self._resolve_source_roots(source_roots)
|
||||
orbit_paths = self._resolve_orbit_roots(orbit_roots)
|
||||
normalized_mode = self._normalize_discovery_mode(discovery_mode)
|
||||
min_aoi_coverage_ratio = max(0.0, min(1.0, float(min_aoi_coverage_ratio or 0.0)))
|
||||
min_common_overlap_ratio = max(0.0, min(1.0, float(min_common_overlap_ratio or 0.0)))
|
||||
discovery_aoi = self._build_discovery_aoi(admin_region=admin_region, aoi_bbox=aoi_bbox)
|
||||
effective_mode = "aoi" if normalized_mode == "aoi" and discovery_aoi.get("geometry") is not None else "strict"
|
||||
cache_key = self._discovery_cache_key(
|
||||
source_paths=source_paths,
|
||||
orbit_paths=orbit_paths,
|
||||
@@ -792,6 +809,11 @@ class SbasInsarProductionService:
|
||||
platform=platform,
|
||||
relative_orbit=relative_orbit,
|
||||
orbit_direction=orbit_direction,
|
||||
admin_region=admin_region,
|
||||
discovery_mode=effective_mode,
|
||||
aoi_bbox=aoi_bbox,
|
||||
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=min_common_overlap_ratio,
|
||||
)
|
||||
if not force_refresh:
|
||||
cached = self._read_discovery_cache(cache_key)
|
||||
@@ -804,6 +826,7 @@ class SbasInsarProductionService:
|
||||
platform_filter = str(platform or "").strip().upper()
|
||||
rel_filter = str(relative_orbit or "").strip()
|
||||
direction_filter = str(orbit_direction or "").strip().upper()
|
||||
aoi_geometry = discovery_aoi.get("geometry") if effective_mode == "aoi" else None
|
||||
|
||||
for root in source_paths:
|
||||
try:
|
||||
@@ -819,18 +842,54 @@ class SbasInsarProductionService:
|
||||
continue
|
||||
if direction_filter and str(scene.get("orbit_direction") or "").upper() != direction_filter:
|
||||
continue
|
||||
if aoi_geometry is not None:
|
||||
scene = self._scene_with_aoi_metrics(scene, aoi_geometry)
|
||||
if not scene.get("aoi_intersects"):
|
||||
continue
|
||||
if float(scene.get("aoi_overlap_ratio") or 0.0) < min_aoi_coverage_ratio:
|
||||
continue
|
||||
scenes.append(scene)
|
||||
except Exception as exc:
|
||||
errors.append({"source_root": str(root), "error": str(exc)})
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
grouped_initial: dict[str, list[dict[str, Any]]] = {}
|
||||
for scene in scenes:
|
||||
grouped.setdefault(self._stack_group_key(scene), []).append(scene)
|
||||
group_key = self._aoi_stack_group_key(scene) if effective_mode == "aoi" else self._stack_group_key(scene)
|
||||
grouped_initial.setdefault(group_key, []).append(scene)
|
||||
|
||||
if effective_mode == "aoi":
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
for observation_key, group_scenes in grouped_initial.items():
|
||||
for cluster in self._cluster_aoi_scenes(group_scenes):
|
||||
cluster_key = self._aoi_cluster_key(observation_key, cluster)
|
||||
clustered_scenes = [
|
||||
{
|
||||
**scene,
|
||||
"aoi_cluster_key": cluster_key,
|
||||
"aoi_cluster_source": "footprint_common_overlap",
|
||||
}
|
||||
for scene in cluster
|
||||
]
|
||||
grouped[cluster_key] = clustered_scenes
|
||||
else:
|
||||
grouped = grouped_initial
|
||||
|
||||
candidates = [
|
||||
self._build_stack_candidate(group_scenes, min_scenes=min_scenes, require_orbits=require_orbits)
|
||||
self._build_stack_candidate(
|
||||
group_scenes,
|
||||
min_scenes=min_scenes,
|
||||
require_orbits=require_orbits,
|
||||
discovery_mode=effective_mode,
|
||||
aoi_summary=discovery_aoi.get("summary"),
|
||||
min_common_overlap_ratio=min_common_overlap_ratio,
|
||||
)
|
||||
for group_scenes in grouped.values()
|
||||
]
|
||||
if admin_region and effective_mode != "aoi":
|
||||
candidates = [
|
||||
candidate for candidate in candidates
|
||||
if admin_region_matches(candidate.get("admin_region"), admin_region)
|
||||
]
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
int(item.get("status") != "READY"),
|
||||
@@ -852,6 +911,11 @@ class SbasInsarProductionService:
|
||||
"orbit_roots": [str(path) for path in orbit_paths],
|
||||
"min_scenes": min_scenes,
|
||||
"require_orbits": require_orbits,
|
||||
"discovery_mode": effective_mode,
|
||||
"requested_discovery_mode": normalized_mode,
|
||||
"aoi": discovery_aoi.get("summary"),
|
||||
"min_aoi_coverage_ratio": min_aoi_coverage_ratio,
|
||||
"min_common_overlap_ratio": min_common_overlap_ratio,
|
||||
"scene_count": len(scenes),
|
||||
"candidate_count": len(candidates),
|
||||
"errors": errors[:50],
|
||||
@@ -874,6 +938,11 @@ class SbasInsarProductionService:
|
||||
orbit_roots: list[str] | None = None,
|
||||
min_scenes: int = 3,
|
||||
require_orbits: bool = True,
|
||||
discovery_mode: str = "strict",
|
||||
admin_region: str | None = None,
|
||||
aoi_bbox: dict[str, Any] | None = None,
|
||||
min_aoi_coverage_ratio: float = 0.01,
|
||||
min_common_overlap_ratio: float = 0.0,
|
||||
) -> dict[str, Any]:
|
||||
discovery = self.discover_stacks(
|
||||
source_roots=source_roots,
|
||||
@@ -882,6 +951,11 @@ class SbasInsarProductionService:
|
||||
require_orbits=require_orbits,
|
||||
include_scenes=True,
|
||||
limit=0,
|
||||
discovery_mode=discovery_mode,
|
||||
admin_region=admin_region,
|
||||
aoi_bbox=aoi_bbox,
|
||||
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=min_common_overlap_ratio,
|
||||
)
|
||||
candidate = next(
|
||||
(item for item in discovery.get("items", []) if item.get("stack_id") == stack_id),
|
||||
@@ -927,6 +1001,9 @@ class SbasInsarProductionService:
|
||||
"status": "READY_FOR_GAMMA_BASELINE_AUDIT" if not blockers else "BLOCKED",
|
||||
"require_orbits": require_orbits,
|
||||
"min_scenes": min_scenes,
|
||||
"discovery_mode": candidate.get("discovery_mode") or discovery.get("discovery_mode") or "strict",
|
||||
"aoi": candidate.get("aoi") or discovery.get("aoi"),
|
||||
"common_overlap_ratio": candidate.get("common_overlap_ratio"),
|
||||
"stack": {
|
||||
key: candidate.get(key)
|
||||
for key in [
|
||||
@@ -941,6 +1018,7 @@ class SbasInsarProductionService:
|
||||
"reference_date",
|
||||
]
|
||||
},
|
||||
"geographic_coverage": self._build_stack_geographic_coverage({"scenes": usable_scenes}),
|
||||
"scenes": usable_scenes,
|
||||
"excluded_scenes": [
|
||||
scene for scene in candidate.get("scenes", [])
|
||||
@@ -983,7 +1061,12 @@ class SbasInsarProductionService:
|
||||
min_scenes: int = 3,
|
||||
require_orbits: bool = True,
|
||||
monitor_points: list[dict[str, Any]] | None = None,
|
||||
monitor_point_strategy: str = "auto_low_sigma_high_rate",
|
||||
monitor_point_strategy: str = "auto_representative_points",
|
||||
discovery_mode: str = "strict",
|
||||
admin_region: str | None = None,
|
||||
aoi_bbox: dict[str, Any] | None = None,
|
||||
min_aoi_coverage_ratio: float = 0.01,
|
||||
min_common_overlap_ratio: float = 0.0,
|
||||
dry_run: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
audit = self.audit_stack(
|
||||
@@ -992,6 +1075,11 @@ class SbasInsarProductionService:
|
||||
orbit_roots=orbit_roots,
|
||||
min_scenes=min_scenes,
|
||||
require_orbits=require_orbits,
|
||||
discovery_mode=discovery_mode,
|
||||
admin_region=admin_region,
|
||||
aoi_bbox=aoi_bbox,
|
||||
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
|
||||
min_common_overlap_ratio=min_common_overlap_ratio,
|
||||
)
|
||||
manifest = audit["manifest"]
|
||||
if manifest.get("status") != "READY_FOR_GAMMA_BASELINE_AUDIT":
|
||||
@@ -1023,6 +1111,9 @@ class SbasInsarProductionService:
|
||||
"status": "WORKFLOW_READY",
|
||||
"created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"stack_id": stack_id,
|
||||
"discovery_mode": manifest.get("discovery_mode"),
|
||||
"aoi": manifest.get("aoi"),
|
||||
"common_overlap_ratio": manifest.get("common_overlap_ratio"),
|
||||
"stack_manifest_path": audit["manifest_path"],
|
||||
"pair_network_path": audit["pair_network_path"],
|
||||
"workflow_manifest_path": str(run_dir / "manifest.json"),
|
||||
@@ -1106,6 +1197,7 @@ class SbasInsarProductionService:
|
||||
}
|
||||
workflow_state = self._read_optional_json(run_dir / "state" / "step_status.json")
|
||||
monitor_points = self._read_optional_json(run_dir / "monitor_points.json")
|
||||
geographic_coverage = self._build_run_geographic_coverage(run_dir, manifest)
|
||||
return {
|
||||
"run": self._build_run_card(run_dir, manifest),
|
||||
"manifest": manifest,
|
||||
@@ -1113,6 +1205,7 @@ class SbasInsarProductionService:
|
||||
"workflow_manifest": workflow_manifest,
|
||||
"workflow_state": workflow_state,
|
||||
"monitor_points": monitor_points,
|
||||
"geographic_coverage": geographic_coverage,
|
||||
"artifacts": self._build_run_artifacts(run_dir),
|
||||
}
|
||||
|
||||
@@ -2483,6 +2576,9 @@ class SbasInsarProductionService:
|
||||
},
|
||||
"outputs": {
|
||||
"export_dir": str(export_dir),
|
||||
"vector_dir": str(run_dir / "publish" / "vectors"),
|
||||
"point_vector_geojson_gz": str(run_dir / "publish" / "vectors" / "los_rate_points.geojson.gz"),
|
||||
"point_vector_summary": str(run_dir / "publish" / "vectors" / "los_rate_points_summary.json"),
|
||||
"product_summary": str(run_dir / "product_summary.json"),
|
||||
"quality_summary": str(run_dir / "quality_summary.json"),
|
||||
},
|
||||
@@ -3614,6 +3710,11 @@ class SbasInsarProductionService:
|
||||
platform: str | None,
|
||||
relative_orbit: str | None,
|
||||
orbit_direction: str | None,
|
||||
admin_region: str | None,
|
||||
discovery_mode: str,
|
||||
aoi_bbox: dict[str, Any] | None,
|
||||
min_aoi_coverage_ratio: float,
|
||||
min_common_overlap_ratio: float,
|
||||
) -> str:
|
||||
payload = {
|
||||
"source_paths": [os.path.normcase(str(path.resolve())) for path in source_paths],
|
||||
@@ -3633,6 +3734,12 @@ class SbasInsarProductionService:
|
||||
"platform": str(platform or "").strip().upper(),
|
||||
"relative_orbit": str(relative_orbit or "").strip(),
|
||||
"orbit_direction": str(orbit_direction or "").strip().upper(),
|
||||
"admin_region": str(admin_region or "").strip(),
|
||||
"discovery_mode": str(discovery_mode or "strict").strip().lower(),
|
||||
"aoi_bbox": SbasInsarProductionService._normalize_bbox(aoi_bbox),
|
||||
"min_aoi_coverage_ratio": float(min_aoi_coverage_ratio),
|
||||
"min_common_overlap_ratio": float(min_common_overlap_ratio),
|
||||
"response_shape": "aoi_discovery_v1",
|
||||
}
|
||||
return hashlib.sha1(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
@@ -4433,6 +4540,249 @@ class SbasInsarProductionService:
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_bbox(value: Any) -> dict[str, float] | None:
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
try:
|
||||
min_lon = float(value["min_lon"])
|
||||
min_lat = float(value["min_lat"])
|
||||
max_lon = float(value["max_lon"])
|
||||
max_lat = float(value["max_lat"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
if min_lon >= max_lon or min_lat >= max_lat:
|
||||
return None
|
||||
return {
|
||||
"min_lon": min_lon,
|
||||
"min_lat": min_lat,
|
||||
"max_lon": max_lon,
|
||||
"max_lat": max_lat,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _bbox_to_geojson_feature(cls, bbox: dict[str, Any] | None, *, properties: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
||||
normalized = cls._normalize_bbox(bbox)
|
||||
if not normalized:
|
||||
return None
|
||||
min_lon = normalized["min_lon"]
|
||||
min_lat = normalized["min_lat"]
|
||||
max_lon = normalized["max_lon"]
|
||||
max_lat = normalized["max_lat"]
|
||||
return {
|
||||
"type": "Feature",
|
||||
"properties": properties or {},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[min_lon, min_lat],
|
||||
[max_lon, min_lat],
|
||||
[max_lon, max_lat],
|
||||
[min_lon, max_lat],
|
||||
[min_lon, min_lat],
|
||||
]],
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _point_to_geojson_feature(point: dict[str, Any] | None, *, properties: dict[str, Any] | None = None) -> dict[str, Any] | None:
|
||||
if not isinstance(point, dict):
|
||||
return None
|
||||
try:
|
||||
lon = float(point["lon"])
|
||||
lat = float(point["lat"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
return {
|
||||
"type": "Feature",
|
||||
"properties": properties or {},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [lon, lat],
|
||||
},
|
||||
}
|
||||
|
||||
def _build_stack_geographic_coverage(self, stack_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
scenes = stack_manifest.get("scenes") or []
|
||||
usable_scenes = [
|
||||
scene for scene in scenes
|
||||
if isinstance(scene, dict) and isinstance(scene.get("bbox"), dict)
|
||||
]
|
||||
bbox_union = self._stack_bbox_union(stack_manifest)
|
||||
bbox_intersection = self._bbox_intersection([scene.get("bbox") for scene in usable_scenes])
|
||||
center = self._stack_center(stack_manifest)
|
||||
union_feature = self._bbox_to_geojson_feature(
|
||||
bbox_union,
|
||||
properties={
|
||||
"role": "stack_bbox_union",
|
||||
"source": "lt1_scene_metadata",
|
||||
"scene_count": len(usable_scenes),
|
||||
},
|
||||
)
|
||||
intersection_feature = self._bbox_to_geojson_feature(
|
||||
bbox_intersection,
|
||||
properties={
|
||||
"role": "stack_bbox_intersection",
|
||||
"source": "lt1_scene_metadata",
|
||||
"scene_count": len(usable_scenes),
|
||||
},
|
||||
)
|
||||
center_feature = self._point_to_geojson_feature(
|
||||
center,
|
||||
properties={"role": "stack_center", "source": "scene_centers_or_bbox"},
|
||||
)
|
||||
scene_features: list[dict[str, Any]] = []
|
||||
for scene in usable_scenes:
|
||||
feature = self._bbox_to_geojson_feature(
|
||||
scene.get("bbox"),
|
||||
properties={
|
||||
"role": "scene_bbox",
|
||||
"scene_name": scene.get("scene_name"),
|
||||
"date": scene.get("date"),
|
||||
"satellite": scene.get("satellite"),
|
||||
"relative_orbit": scene.get("relative_orbit"),
|
||||
},
|
||||
)
|
||||
if feature:
|
||||
scene_features.append(feature)
|
||||
overview_features = [
|
||||
item for item in [union_feature, intersection_feature, center_feature]
|
||||
if item
|
||||
]
|
||||
return {
|
||||
"schema": "insar.sbas-geographic-coverage/v1",
|
||||
"crs": "EPSG:4326",
|
||||
"source": "lt1_scene_metadata",
|
||||
"bbox": bbox_union,
|
||||
"bbox_intersection": bbox_intersection,
|
||||
"center": center,
|
||||
"admin_region": lookup_admin_region_for_point(
|
||||
(center or {}).get("lon"),
|
||||
(center or {}).get("lat"),
|
||||
),
|
||||
"scene_bbox_count": len(scene_features),
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": overview_features,
|
||||
},
|
||||
"scene_footprints_geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": scene_features,
|
||||
},
|
||||
}
|
||||
|
||||
def _build_run_geographic_coverage(self, run_dir: Path, run_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
stack_manifest = self._read_optional_json(run_dir / "stack_manifest.json")
|
||||
if not stack_manifest:
|
||||
stack_manifest_path = Path(str(run_manifest.get("stack_manifest_path") or ""))
|
||||
if stack_manifest_path.is_file():
|
||||
stack_manifest = self._read_optional_json(stack_manifest_path)
|
||||
stack_manifest = stack_manifest or {}
|
||||
coverage = self._build_stack_geographic_coverage(stack_manifest)
|
||||
rdc_dem = run_manifest.get("rdc_dem") or {}
|
||||
rdc_dem_summary = (
|
||||
(rdc_dem.get("summary") if isinstance(rdc_dem, dict) else None)
|
||||
or self._read_optional_json(run_dir / "rdc_dem_summary.json")
|
||||
or {}
|
||||
)
|
||||
dem_source = rdc_dem_summary.get("dem_source") or (rdc_dem.get("dem_source") if isinstance(rdc_dem, dict) else None) or {}
|
||||
dem_coverage = self._normalize_bbox(dem_source.get("coverage")) if isinstance(dem_source, dict) else None
|
||||
monitor_summary = (
|
||||
(run_manifest.get("monitor_point_products") or {}).get("summary")
|
||||
or self._read_optional_json(run_dir / "monitor_points_summary.json")
|
||||
or {}
|
||||
)
|
||||
monitor_points: list[dict[str, Any]] = []
|
||||
for item in monitor_summary.get("monitor_outputs") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
metadata = item.get("metadata") or {}
|
||||
lonlat = metadata.get("approx_lonlat") or {}
|
||||
try:
|
||||
lon = float(lonlat["lon"])
|
||||
lat = float(lonlat["lat"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
monitor_points.append(
|
||||
{
|
||||
"point_id": item.get("point_id") or metadata.get("point_id"),
|
||||
"lon": lon,
|
||||
"lat": lat,
|
||||
"selection": metadata.get("selection"),
|
||||
"los_rate_toward_mm_per_year": metadata.get("los_rate_toward_mm_per_year"),
|
||||
"los_sigma_mm_per_year": metadata.get("los_sigma_mm_per_year"),
|
||||
"source": "monitor_points_summary",
|
||||
}
|
||||
)
|
||||
if not monitor_points:
|
||||
for item in monitor_summary.get("monitor_points") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
lon = float(item["lon"])
|
||||
lat = float(item["lat"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
monitor_points.append(
|
||||
{
|
||||
"point_id": item.get("point_id"),
|
||||
"lon": lon,
|
||||
"lat": lat,
|
||||
"selection": item.get("selection"),
|
||||
"los_rate_toward_mm_per_year": item.get("los_rate_toward_mm_per_year"),
|
||||
"los_sigma_mm_per_year": item.get("los_sigma_mm_per_year"),
|
||||
"source": "monitor_points_summary",
|
||||
}
|
||||
)
|
||||
monitor_features = [
|
||||
feature for feature in (
|
||||
self._point_to_geojson_feature(
|
||||
{"lon": point["lon"], "lat": point["lat"]},
|
||||
properties={
|
||||
"role": "monitor_point",
|
||||
"point_id": point.get("point_id"),
|
||||
"selection": point.get("selection"),
|
||||
"los_rate_toward_mm_per_year": point.get("los_rate_toward_mm_per_year"),
|
||||
"los_sigma_mm_per_year": point.get("los_sigma_mm_per_year"),
|
||||
},
|
||||
)
|
||||
for point in monitor_points
|
||||
)
|
||||
if feature
|
||||
]
|
||||
dem_feature = self._bbox_to_geojson_feature(
|
||||
dem_coverage,
|
||||
properties={
|
||||
"role": "dem_coverage",
|
||||
"source": "rdc_dem_summary",
|
||||
"covers_stack_bbox": dem_source.get("covers_stack_bbox"),
|
||||
"covers_stack_center": dem_source.get("covers_stack_center"),
|
||||
},
|
||||
)
|
||||
features = list((coverage.get("geojson") or {}).get("features") or [])
|
||||
if dem_feature:
|
||||
features.append(dem_feature)
|
||||
features.extend(monitor_features)
|
||||
coverage.update(
|
||||
{
|
||||
"source": "run_stack_manifest",
|
||||
"run_id": run_manifest.get("run_id") or run_dir.name,
|
||||
"stack_id": run_manifest.get("stack_id") or stack_manifest.get("stack_id"),
|
||||
"stack": stack_manifest.get("stack") or run_manifest.get("stack") or {},
|
||||
"date_start": min(self._stack_dates(stack_manifest), default=None),
|
||||
"date_end": max(self._stack_dates(stack_manifest), default=None),
|
||||
"dem_coverage": dem_coverage,
|
||||
"dem_covers_stack_bbox": dem_source.get("covers_stack_bbox"),
|
||||
"dem_covers_stack_center": dem_source.get("covers_stack_center"),
|
||||
"monitor_points": monitor_points,
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": features,
|
||||
},
|
||||
}
|
||||
)
|
||||
return coverage
|
||||
|
||||
@staticmethod
|
||||
def _file_record(path: Path) -> dict[str, Any]:
|
||||
exists = path.is_file()
|
||||
@@ -4478,12 +4828,157 @@ class SbasInsarProductionService:
|
||||
]
|
||||
return "|".join(str(part or "") for part in parts)
|
||||
|
||||
@staticmethod
|
||||
def _aoi_stack_group_key(scene: dict[str, Any]) -> str:
|
||||
parts = [
|
||||
scene.get("satellite"),
|
||||
scene.get("satellite_mode"),
|
||||
scene.get("relative_orbit"),
|
||||
scene.get("orbit_direction"),
|
||||
scene.get("imaging_mode"),
|
||||
scene.get("polarization"),
|
||||
]
|
||||
return "|".join(str(part or "") for part in parts)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_discovery_mode(value: str | None) -> str:
|
||||
text = str(value or "").strip().lower()
|
||||
return "aoi" if text == "aoi" else "strict"
|
||||
|
||||
def _build_discovery_aoi(
|
||||
self,
|
||||
*,
|
||||
admin_region: str | None,
|
||||
aoi_bbox: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
bbox = self._normalize_bbox(aoi_bbox)
|
||||
if bbox:
|
||||
geometry = shapely_box(
|
||||
bbox["min_lon"],
|
||||
bbox["min_lat"],
|
||||
bbox["max_lon"],
|
||||
bbox["max_lat"],
|
||||
)
|
||||
return {
|
||||
"geometry": geometry,
|
||||
"summary": {
|
||||
"match_status": "matched",
|
||||
"source": "bbox",
|
||||
"bbox": bbox,
|
||||
"display_name": "Custom AOI bbox",
|
||||
},
|
||||
}
|
||||
|
||||
region = lookup_admin_region_geometry(admin_region)
|
||||
if not region:
|
||||
return {"geometry": None, "summary": None}
|
||||
geometry = region.get("geometry")
|
||||
summary = {key: value for key, value in region.items() if key != "geometry"}
|
||||
if geometry is None or getattr(geometry, "is_empty", False):
|
||||
return {"geometry": None, "summary": summary}
|
||||
return {"geometry": geometry, "summary": summary}
|
||||
|
||||
def _scene_with_aoi_metrics(self, scene: dict[str, Any], aoi_geometry: Any) -> dict[str, Any]:
|
||||
bbox = self._normalize_bbox(scene.get("bbox"))
|
||||
if not bbox:
|
||||
return {**scene, "aoi_intersects": False, "aoi_overlap_ratio": 0.0}
|
||||
scene_geometry = shapely_box(
|
||||
bbox["min_lon"],
|
||||
bbox["min_lat"],
|
||||
bbox["max_lon"],
|
||||
bbox["max_lat"],
|
||||
)
|
||||
try:
|
||||
intersects = bool(scene_geometry.intersects(aoi_geometry))
|
||||
except Exception:
|
||||
return {**scene, "aoi_intersects": False, "aoi_overlap_ratio": 0.0}
|
||||
if not intersects:
|
||||
return {**scene, "aoi_intersects": False, "aoi_overlap_ratio": 0.0}
|
||||
try:
|
||||
intersection_area = float(scene_geometry.intersection(aoi_geometry).area or 0.0)
|
||||
scene_area = float(scene_geometry.area or 0.0)
|
||||
aoi_area = float(getattr(aoi_geometry, "area", 0.0) or 0.0)
|
||||
except Exception:
|
||||
intersection_area = 0.0
|
||||
scene_area = 0.0
|
||||
aoi_area = 0.0
|
||||
return {
|
||||
**scene,
|
||||
"aoi_intersects": True,
|
||||
"aoi_overlap_ratio": intersection_area / scene_area if scene_area > 0 else 0.0,
|
||||
"aoi_covered_ratio": intersection_area / aoi_area if aoi_area > 0 else None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _bbox_area(value: dict[str, Any] | None) -> float:
|
||||
if not value:
|
||||
return 0.0
|
||||
try:
|
||||
width = float(value["max_lon"]) - float(value["min_lon"])
|
||||
height = float(value["max_lat"]) - float(value["min_lat"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return 0.0
|
||||
return width * height if width > 0 and height > 0 else 0.0
|
||||
|
||||
def _cluster_aoi_scenes(self, scenes: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
|
||||
sorted_scenes = sorted(
|
||||
scenes,
|
||||
key=lambda item: (
|
||||
str(item.get("date") or ""),
|
||||
float(item.get("center_lon") or 0.0),
|
||||
float(item.get("center_lat") or 0.0),
|
||||
),
|
||||
)
|
||||
clusters: list[dict[str, Any]] = []
|
||||
for scene in sorted_scenes:
|
||||
scene_bbox = self._normalize_bbox(scene.get("bbox"))
|
||||
if not scene_bbox:
|
||||
continue
|
||||
best_index: int | None = None
|
||||
best_score = -1.0
|
||||
for index, cluster in enumerate(clusters):
|
||||
candidate_intersection = self._bbox_intersection(
|
||||
[cluster.get("bbox_intersection"), scene_bbox]
|
||||
)
|
||||
if not candidate_intersection:
|
||||
continue
|
||||
score = self._bbox_area(candidate_intersection)
|
||||
if score > best_score:
|
||||
best_index = index
|
||||
best_score = score
|
||||
if best_index is None:
|
||||
clusters.append({"bbox_intersection": scene_bbox, "scenes": [scene]})
|
||||
continue
|
||||
cluster = clusters[best_index]
|
||||
cluster["bbox_intersection"] = self._bbox_intersection(
|
||||
[cluster.get("bbox_intersection"), scene_bbox]
|
||||
)
|
||||
cluster["scenes"].append(scene)
|
||||
|
||||
return [cluster["scenes"] for cluster in clusters if cluster.get("scenes")]
|
||||
|
||||
def _aoi_cluster_key(self, observation_key: str, scenes: list[dict[str, Any]]) -> str:
|
||||
bbox = self._bbox_intersection([scene.get("bbox") for scene in scenes])
|
||||
if bbox:
|
||||
lon = (bbox["min_lon"] + bbox["max_lon"]) / 2
|
||||
lat = (bbox["min_lat"] + bbox["max_lat"]) / 2
|
||||
spatial_key = f"overlap_E{lon:.2f}_N{lat:.2f}"
|
||||
else:
|
||||
center = self._stack_center({"scenes": scenes}) or {}
|
||||
lon = self._as_float(center.get("lon"))
|
||||
lat = self._as_float(center.get("lat"))
|
||||
spatial_key = f"center_{self._center_bucket(lon, lat)}"
|
||||
return f"{observation_key}|{spatial_key}"
|
||||
|
||||
def _build_stack_candidate(
|
||||
self,
|
||||
scenes: list[dict[str, Any]],
|
||||
*,
|
||||
min_scenes: int,
|
||||
require_orbits: bool,
|
||||
discovery_mode: str = "strict",
|
||||
aoi_summary: dict[str, Any] | None = None,
|
||||
min_common_overlap_ratio: float = 0.0,
|
||||
) -> dict[str, Any]:
|
||||
scenes = sorted(scenes, key=lambda item: str(item.get("date") or ""))
|
||||
first = scenes[0]
|
||||
@@ -4491,7 +4986,11 @@ class SbasInsarProductionService:
|
||||
usable = orbit_ready if require_orbits else scenes
|
||||
dates = [scene.get("date") for scene in scenes if scene.get("date")]
|
||||
usable_dates = [scene.get("date") for scene in usable if scene.get("date")]
|
||||
group_key = self._stack_group_key(first)
|
||||
mode = self._normalize_discovery_mode(discovery_mode)
|
||||
group_key = (
|
||||
str(first.get("aoi_cluster_key") or "")
|
||||
or (self._aoi_stack_group_key(first) if mode == "aoi" else self._stack_group_key(first))
|
||||
)
|
||||
stack_id = self._stable_id(group_key)
|
||||
temporal_gaps = self._temporal_gaps(usable_dates)
|
||||
blockers: list[str] = []
|
||||
@@ -4499,11 +4998,55 @@ class SbasInsarProductionService:
|
||||
blockers.append(f"usable_scene_count {len(usable)} < min_scenes {min_scenes}")
|
||||
if require_orbits and len(orbit_ready) < len(scenes):
|
||||
blockers.append("missing precise orbit for one or more scenes")
|
||||
usable_stack = {"scenes": usable}
|
||||
bbox_intersection = self._bbox_intersection([scene.get("bbox") for scene in usable])
|
||||
bbox_union = self._stack_bbox_union(usable_stack)
|
||||
common_overlap_ratio = (
|
||||
self._bbox_area(bbox_intersection) / self._bbox_area(bbox_union)
|
||||
if bbox_intersection and bbox_union and self._bbox_area(bbox_union) > 0
|
||||
else 0.0
|
||||
)
|
||||
if mode == "aoi" and usable and not bbox_intersection:
|
||||
blockers.append("no common overlap across usable scenes")
|
||||
if mode == "aoi" and min_common_overlap_ratio > 0 and common_overlap_ratio < min_common_overlap_ratio:
|
||||
blockers.append(
|
||||
f"common_overlap_ratio {common_overlap_ratio:.3f} < min_common_overlap_ratio {min_common_overlap_ratio:.3f}"
|
||||
)
|
||||
center = self._stack_center(usable_stack)
|
||||
admin_region = lookup_admin_region_for_point(
|
||||
(center or {}).get("lon"),
|
||||
(center or {}).get("lat"),
|
||||
)
|
||||
aoi_overlap_values = [
|
||||
float(scene.get("aoi_overlap_ratio") or 0.0)
|
||||
for scene in usable
|
||||
if scene.get("aoi_overlap_ratio") is not None
|
||||
]
|
||||
return {
|
||||
"stack_id": stack_id,
|
||||
"status": "READY" if not blockers else "BLOCKED",
|
||||
"blockers": blockers,
|
||||
"discovery_mode": mode,
|
||||
"aoi": aoi_summary,
|
||||
"group_key": group_key,
|
||||
"hard_group_fields": [
|
||||
"satellite",
|
||||
"satellite_mode",
|
||||
"relative_orbit",
|
||||
"orbit_direction",
|
||||
"imaging_mode",
|
||||
"polarization",
|
||||
] if mode == "aoi" else [
|
||||
"satellite",
|
||||
"satellite_mode",
|
||||
"receiving_station",
|
||||
"relative_orbit",
|
||||
"orbit_direction",
|
||||
"imaging_mode",
|
||||
"polarization",
|
||||
"center_bucket",
|
||||
],
|
||||
"soft_group_fields": ["receiving_station", "center_bucket"] if mode == "aoi" else [],
|
||||
"satellite": first.get("satellite"),
|
||||
"satellite_mode": first.get("satellite_mode"),
|
||||
"receiving_station": first.get("receiving_station"),
|
||||
@@ -4523,7 +5066,17 @@ class SbasInsarProductionService:
|
||||
"reference_date": usable_dates[len(usable_dates) // 2] if usable_dates else None,
|
||||
"temporal_gaps_days": temporal_gaps,
|
||||
"max_temporal_gap_days": max(temporal_gaps) if temporal_gaps else 0,
|
||||
"bbox_intersection": self._bbox_intersection([scene.get("bbox") for scene in usable]),
|
||||
"bbox": bbox_union,
|
||||
"bbox_intersection": bbox_intersection,
|
||||
"common_overlap_ratio": common_overlap_ratio,
|
||||
"aoi_overlap_ratio_min": min(aoi_overlap_values) if aoi_overlap_values else None,
|
||||
"aoi_overlap_ratio_max": max(aoi_overlap_values) if aoi_overlap_values else None,
|
||||
"aoi_overlap_ratio_mean": (
|
||||
sum(aoi_overlap_values) / len(aoi_overlap_values)
|
||||
if aoi_overlap_values else None
|
||||
),
|
||||
"center": center,
|
||||
"admin_region": admin_region,
|
||||
"scenes": scenes,
|
||||
}
|
||||
|
||||
@@ -5072,19 +5625,22 @@ class SbasInsarProductionService:
|
||||
mode = "manual_lonlat"
|
||||
note = "Manual monitoring points are stored for extraction after geocoded products are available."
|
||||
else:
|
||||
mode = strategy or "auto_low_sigma_high_rate"
|
||||
mode = strategy or "auto_representative_points"
|
||||
if mode == "auto_low_sigma_high_rate":
|
||||
mode = "auto_representative_points"
|
||||
note = (
|
||||
"Automatic point is only a production placeholder until users provide a point layer "
|
||||
"or approve a quality-filtered sampler."
|
||||
"Automatic representative points are report-preview candidates until users provide "
|
||||
"a point layer or approve final monitoring locations."
|
||||
)
|
||||
return {
|
||||
"schema": "insar.sbas-monitor-points/v1",
|
||||
"mode": mode,
|
||||
"points": normalized_points,
|
||||
"auto_count": 5,
|
||||
"default_auto_strategy": {
|
||||
"key": "auto_low_sigma_high_rate",
|
||||
"selection": "low LOS sigma, high absolute LOS velocity, non-edge valid pixel",
|
||||
"usage": "debug/sample only; not a business monitoring network",
|
||||
"key": "auto_representative_points",
|
||||
"selection": "away/toward/high-absolute-rate/stable/center valid pixels with low sigma and non-edge constraints",
|
||||
"usage": "preview candidates only; not a business monitoring network",
|
||||
},
|
||||
"reference_date": (stack_manifest.get("stack") or {}).get("reference_date"),
|
||||
"coordinate_system": "EPSG:4326 for manual lon/lat points; radar coordinates are derived during publishing",
|
||||
@@ -6062,6 +6618,14 @@ class SbasInsarProductionService:
|
||||
python_bin = settings.GAMMA_SBAS_PYTHON or settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python"
|
||||
tool_script = Path(settings.PROJECT_ROOT) / "deploy" / "wsl" / "runners" / "gamma_sbas_product_tools.py"
|
||||
phase_to_los = wavelength / (4.0 * math.pi)
|
||||
stack_manifest = self._read_optional_json(run_dir / "stack_manifest.json")
|
||||
stack_dates = self._stack_dates(stack_manifest)
|
||||
date_start = min(stack_dates, default="")
|
||||
date_end = max(stack_dates, default="")
|
||||
coverage = self._build_stack_geographic_coverage(stack_manifest)
|
||||
admin_region = coverage.get("admin_region") or {}
|
||||
admin_province = str(admin_region.get("province") or "").strip()
|
||||
admin_city = str(admin_region.get("city") or "").strip()
|
||||
lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
@@ -6081,9 +6645,14 @@ class SbasInsarProductionService:
|
||||
f'RLKS="{rlks}"',
|
||||
f'WAVELENGTH="{wavelength:.12g}"',
|
||||
f'PHASE_TO_LOS="{phase_to_los:.12g}"',
|
||||
f'DATE_START="{date_start}"',
|
||||
f'DATE_END="{date_end}"',
|
||||
f'ADMIN_PROVINCE="{admin_province}"',
|
||||
f'ADMIN_CITY="{admin_city}"',
|
||||
"",
|
||||
f'source "{env_script}" >/dev/null 2>&1',
|
||||
'mkdir -p "${EXPORT_DIR}" "${LOG_DIR}"',
|
||||
'VECTOR_DIR="${RUN_ROOT}/publish/vectors"',
|
||||
'mkdir -p "${EXPORT_DIR}" "${VECTOR_DIR}" "${LOG_DIR}"',
|
||||
"",
|
||||
'RDC_WIDTH="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_PAR}")"',
|
||||
'GEO_WIDTH="$(awk \'$1 == "width:" {print $2; exit}\' "${DEM_PAR}")"',
|
||||
@@ -6218,7 +6787,21 @@ class SbasInsarProductionService:
|
||||
"",
|
||||
' make_preview "${EXPORT_DIR}/los_rate_toward_mm_per_year.tif" "${RATE_CMAP}" "${EXPORT_DIR}/los_rate_toward_mm_per_year.geo_preview.png"',
|
||||
' make_preview "${EXPORT_DIR}/los_sigma_mm_per_year.tif" "${SIGMA_CMAP}" "${EXPORT_DIR}/los_sigma_mm_per_year.geo_preview.png"',
|
||||
"",
|
||||
' "${PYTHON_BIN}" "${TOOL_SCRIPT}" export-points-geojson \\',
|
||||
' --toward-tif "${EXPORT_DIR}/los_rate_toward_mm_per_year.tif" \\',
|
||||
' --away-tif "${EXPORT_DIR}/los_rate_away_mm_per_year.tif" \\',
|
||||
' --sigma-tif "${EXPORT_DIR}/los_sigma_mm_per_year.tif" \\',
|
||||
' --output "${VECTOR_DIR}/los_rate_points.geojson.gz" \\',
|
||||
' --summary-path "${VECTOR_DIR}/los_rate_points_summary.json" \\',
|
||||
' --run-id "${RUN_ROOT##*/}" \\',
|
||||
' --date-start "${DATE_START}" \\',
|
||||
' --date-end "${DATE_END}" \\',
|
||||
' --reference-date "${REF_DATE}" \\',
|
||||
' --admin-province "${ADMIN_PROVINCE}" \\',
|
||||
' --admin-city "${ADMIN_CITY}"',
|
||||
' ls -lh "${EXPORT_DIR}"',
|
||||
' ls -lh "${VECTOR_DIR}"',
|
||||
'} >"${LOG_DIR}/publish_products.log" 2>&1',
|
||||
"",
|
||||
'echo "Published Gamma SBAS products: ${EXPORT_DIR}"',
|
||||
@@ -6853,6 +7436,12 @@ class SbasInsarProductionService:
|
||||
"los_rate_m_per_year_tif": export_dir / "los_rate_m_per_year.tif",
|
||||
"los_rate_away_m_per_year_hls_bmp": export_dir / "los_rate_away_m_per_year.hls.bmp",
|
||||
}
|
||||
vector_dir = run_dir / "publish" / "vectors"
|
||||
vector_outputs = {
|
||||
"point_vector_geojson_gz": vector_dir / "los_rate_points.geojson.gz",
|
||||
"point_vector_summary": vector_dir / "los_rate_points_summary.json",
|
||||
}
|
||||
point_vector_summary = self._read_optional_json(vector_outputs["point_vector_summary"]) or {}
|
||||
missing_outputs = [
|
||||
name for name, path in required_outputs.items()
|
||||
if not path.is_file() or path.stat().st_size <= 0
|
||||
@@ -6961,8 +7550,10 @@ class SbasInsarProductionService:
|
||||
},
|
||||
"outputs": {
|
||||
"export_dir": str(export_dir),
|
||||
**{name: self._file_record(path) for name, path in {**required_outputs, **optional_outputs}.items()},
|
||||
"vector_dir": str(vector_dir),
|
||||
**{name: self._file_record(path) for name, path in {**required_outputs, **optional_outputs, **vector_outputs}.items()},
|
||||
},
|
||||
"point_vector_summary": point_vector_summary,
|
||||
"rdc_size_checks": rdc_size_checks,
|
||||
"quality_summary": quality_stats,
|
||||
"product_summary": product_summary,
|
||||
@@ -7489,6 +8080,10 @@ class SbasInsarProductionService:
|
||||
|
||||
def _build_run_card(self, run_dir: Path, manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
stack = manifest.get("stack") or {}
|
||||
try:
|
||||
coverage = self._build_run_geographic_coverage(run_dir, manifest)
|
||||
except Exception:
|
||||
coverage = {}
|
||||
return {
|
||||
"run_id": manifest.get("run_id") or run_dir.name,
|
||||
"run_label": manifest.get("run_label"),
|
||||
@@ -7501,12 +8096,19 @@ class SbasInsarProductionService:
|
||||
"scene_count": manifest.get("scene_count"),
|
||||
"pair_count": manifest.get("pair_count"),
|
||||
"next_stage": manifest.get("next_stage"),
|
||||
"discovery_mode": manifest.get("discovery_mode"),
|
||||
"aoi": manifest.get("aoi"),
|
||||
"common_overlap_ratio": manifest.get("common_overlap_ratio"),
|
||||
"platform": stack.get("satellite"),
|
||||
"relative_orbit": stack.get("relative_orbit"),
|
||||
"direction": stack.get("orbit_direction"),
|
||||
"polarization": stack.get("polarization"),
|
||||
"center_bucket": stack.get("center_bucket"),
|
||||
"reference_date": stack.get("reference_date"),
|
||||
"date_start": coverage.get("date_start"),
|
||||
"date_end": coverage.get("date_end"),
|
||||
"center": coverage.get("center"),
|
||||
"admin_region": coverage.get("admin_region"),
|
||||
"run_dir": str(run_dir),
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user