283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""Backfill XML refRow/refColumn corner mappings for archive-managed radar records."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import gzip
|
|
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(PROJECT_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
from geoalchemy2.shape import from_shape # noqa: E402
|
|
from shapely.geometry import Polygon # noqa: E402
|
|
from sqlalchemy import or_, select # noqa: E402
|
|
from sqlalchemy.ext.asyncio import AsyncSession # noqa: E402
|
|
|
|
from backend.app import database # noqa: E402
|
|
from backend.app.config import settings # noqa: E402
|
|
from backend.app.models import RadarDataORM, SourceMetadataDocumentORM, SourceProductAssetORM # noqa: E402
|
|
from backend.app.services.asset_inventory_service import _parse_radar_xml_metadata_bytes # noqa: E402
|
|
from backend.app.utils import build_corner_pixel_mapping # noqa: E402
|
|
|
|
|
|
SOURCE_FORMATS = {"LT1_ARCHIVE"}
|
|
DOCUMENT_TYPES = {"LT1_META"}
|
|
|
|
|
|
def _metadata_dict(value: Any) -> Dict[str, Any]:
|
|
return dict(value or {}) if isinstance(value, dict) else {}
|
|
|
|
|
|
def _normalize_families(values: Optional[Iterable[str]]) -> Sequence[str]:
|
|
families = []
|
|
for value in values or ["LT1"]:
|
|
for item in str(value or "").split(","):
|
|
family = item.strip().upper()
|
|
if family and family not in families:
|
|
families.append(family)
|
|
supported = {"LT1"}
|
|
invalid = [item for item in families if item not in supported]
|
|
if invalid:
|
|
raise SystemExit(f"Unsupported family: {', '.join(invalid)}")
|
|
return families or ["LT1"]
|
|
|
|
|
|
def _decode_document(doc: SourceMetadataDocumentORM) -> Optional[bytes]:
|
|
payload = bytes(doc.content_bytes or b"")
|
|
if not payload:
|
|
return None
|
|
if str(doc.content_encoding or "").lower() == "gzip":
|
|
return gzip.decompress(payload)
|
|
return payload
|
|
|
|
|
|
def _mapping_from_xml_document(doc: SourceMetadataDocumentORM) -> Tuple[Optional[Dict[str, Any]], Optional[Any]]:
|
|
payload = _decode_document(doc)
|
|
if not payload:
|
|
return None, None
|
|
coverage_polygon, xml_meta = _parse_radar_xml_metadata_bytes(payload)
|
|
mapping = xml_meta.get("corner_pixel_mapping") if isinstance(xml_meta, dict) else None
|
|
return mapping if isinstance(mapping, dict) else None, coverage_polygon
|
|
|
|
|
|
def _mapping_from_metadata(metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
|
existing = metadata.get("corner_pixel_mapping")
|
|
if isinstance(existing, dict):
|
|
return existing
|
|
corners = metadata.get("corner_details")
|
|
if isinstance(corners, dict):
|
|
mapping = build_corner_pixel_mapping(corners)
|
|
if isinstance(mapping, dict):
|
|
return mapping
|
|
return None
|
|
|
|
|
|
def _bbox(points: Any) -> Optional[Tuple[float, float, float, float]]:
|
|
try:
|
|
ring = [(float(item[0]), float(item[1])) for item in points or []]
|
|
except (TypeError, ValueError, IndexError):
|
|
return None
|
|
if len(ring) < 3:
|
|
return None
|
|
lons = [item[0] for item in ring]
|
|
lats = [item[1] for item in ring]
|
|
return min(lons), min(lats), max(lons), max(lats)
|
|
|
|
|
|
async def _first_metadata_document(
|
|
db: AsyncSession,
|
|
*,
|
|
source_asset_id: Optional[int],
|
|
radar_data_id: Optional[int],
|
|
) -> Optional[SourceMetadataDocumentORM]:
|
|
clauses = []
|
|
if source_asset_id:
|
|
clauses.append(SourceMetadataDocumentORM.source_asset_id == int(source_asset_id))
|
|
if radar_data_id:
|
|
clauses.append(SourceMetadataDocumentORM.radar_data_id == int(radar_data_id))
|
|
if not clauses:
|
|
return None
|
|
stmt = (
|
|
select(SourceMetadataDocumentORM)
|
|
.where(or_(*clauses))
|
|
.where(SourceMetadataDocumentORM.document_type.in_(sorted(DOCUMENT_TYPES)))
|
|
.order_by(SourceMetadataDocumentORM.id.asc())
|
|
.limit(1)
|
|
)
|
|
return (await db.execute(stmt)).scalar_one_or_none()
|
|
|
|
|
|
async def _load_source_assets(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[SourceProductAssetORM]:
|
|
stmt = (
|
|
select(SourceProductAssetORM)
|
|
.where(SourceProductAssetORM.satellite_family.in_(families))
|
|
.where(SourceProductAssetORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
|
.where(SourceProductAssetORM.is_active.is_(True))
|
|
.order_by(SourceProductAssetORM.id.asc())
|
|
)
|
|
if limit > 0:
|
|
stmt = stmt.limit(limit)
|
|
return list((await db.execute(stmt)).scalars().all())
|
|
|
|
|
|
async def _load_radar_records(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[RadarDataORM]:
|
|
stmt = (
|
|
select(RadarDataORM)
|
|
.where(RadarDataORM.satellite_family.in_(families))
|
|
.where(RadarDataORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
|
.order_by(RadarDataORM.id.asc())
|
|
)
|
|
if limit > 0:
|
|
stmt = stmt.limit(limit)
|
|
return list((await db.execute(stmt)).scalars().all())
|
|
|
|
|
|
async def backfill(args: argparse.Namespace) -> Dict[str, int]:
|
|
families = _normalize_families(args.family)
|
|
database.init_db(settings.DATABASE_URL)
|
|
stats = {
|
|
"source_seen": 0,
|
|
"source_updated": 0,
|
|
"source_existing": 0,
|
|
"source_missing_mapping": 0,
|
|
"radar_seen": 0,
|
|
"radar_updated": 0,
|
|
"radar_existing": 0,
|
|
"radar_missing_mapping": 0,
|
|
"cache_invalidated": 0,
|
|
"document_parse_failed": 0,
|
|
}
|
|
|
|
async with database.AsyncSessionLocal() as db:
|
|
source_assets = await _load_source_assets(db, families, int(args.limit or 0))
|
|
source_mappings: Dict[int, Dict[str, Any]] = {}
|
|
source_polygons: Dict[int, Any] = {}
|
|
now = datetime.utcnow()
|
|
|
|
for asset in source_assets:
|
|
stats["source_seen"] += 1
|
|
metadata = _metadata_dict(asset.metadata_json)
|
|
mapping = _mapping_from_metadata(metadata)
|
|
coverage_polygon = metadata.get("coverage_polygon")
|
|
if mapping:
|
|
stats["source_existing"] += 1
|
|
else:
|
|
doc = await _first_metadata_document(db, source_asset_id=asset.id, radar_data_id=None)
|
|
if doc:
|
|
try:
|
|
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
|
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
|
except Exception:
|
|
stats["document_parse_failed"] += 1
|
|
mapping = None
|
|
if not mapping:
|
|
stats["source_missing_mapping"] += 1
|
|
continue
|
|
metadata["corner_pixel_mapping"] = mapping
|
|
if coverage_polygon:
|
|
metadata["coverage_polygon"] = coverage_polygon
|
|
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
|
asset.metadata_json = metadata
|
|
asset.updated_at = now
|
|
db.add(asset)
|
|
stats["source_updated"] += 1
|
|
|
|
if mapping and asset.id is not None:
|
|
source_mappings[int(asset.id)] = mapping
|
|
if coverage_polygon:
|
|
source_polygons[int(asset.id)] = coverage_polygon
|
|
|
|
radar_records = await _load_radar_records(db, families, int(args.limit or 0))
|
|
for radar in radar_records:
|
|
stats["radar_seen"] += 1
|
|
metadata = _metadata_dict(radar.metadata_json)
|
|
mapping = _mapping_from_metadata(metadata)
|
|
coverage_polygon = radar.coverage_polygon or metadata.get("coverage_polygon")
|
|
if mapping:
|
|
stats["radar_existing"] += 1
|
|
else:
|
|
source_id = int(radar.source_product_ref_id or 0)
|
|
mapping = source_mappings.get(source_id)
|
|
if source_id and source_id in source_polygons:
|
|
coverage_polygon = coverage_polygon or source_polygons[source_id]
|
|
if not mapping:
|
|
doc = await _first_metadata_document(
|
|
db,
|
|
source_asset_id=radar.source_product_ref_id,
|
|
radar_data_id=radar.id,
|
|
)
|
|
if doc:
|
|
try:
|
|
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
|
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
|
except Exception:
|
|
stats["document_parse_failed"] += 1
|
|
mapping = None
|
|
if not mapping:
|
|
stats["radar_missing_mapping"] += 1
|
|
continue
|
|
|
|
metadata["corner_pixel_mapping"] = mapping
|
|
if coverage_polygon:
|
|
metadata["coverage_polygon"] = coverage_polygon
|
|
radar.coverage_polygon = coverage_polygon
|
|
bbox = _bbox(coverage_polygon)
|
|
if bbox:
|
|
radar.min_lon, radar.min_lat, radar.max_lon, radar.max_lat = bbox
|
|
try:
|
|
polygon = Polygon(coverage_polygon)
|
|
if not polygon.is_valid:
|
|
polygon = polygon.buffer(0)
|
|
if not polygon.is_empty:
|
|
radar.geom = from_shape(polygon, srid=4326)
|
|
except Exception:
|
|
pass
|
|
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
|
radar.metadata_json = metadata
|
|
db.add(radar)
|
|
stats["radar_updated"] += 1
|
|
|
|
if args.invalidate_preview and mapping:
|
|
radar.preview_cache_status = "NONE"
|
|
radar.preview_cache_version = None
|
|
radar.preview_cache_path = None
|
|
radar.preview_cache_error = "corner_pixel_mapping_backfilled"
|
|
radar.preview_cache_updated_at = now
|
|
db.add(radar)
|
|
stats["cache_invalidated"] += 1
|
|
|
|
if args.apply:
|
|
await db.commit()
|
|
else:
|
|
await db.rollback()
|
|
|
|
return stats
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Backfill corner_pixel_mapping into source/radar metadata_json.")
|
|
parser.add_argument("--apply", action="store_true", help="commit database changes")
|
|
parser.add_argument("--family", action="append", help="family to process: LT1 or GF3; defaults to LT1")
|
|
parser.add_argument("--limit", type=int, default=0, help="maximum source/radar rows per table; 0 means all")
|
|
parser.add_argument(
|
|
"--no-invalidate-preview",
|
|
dest="invalidate_preview",
|
|
action="store_false",
|
|
help="do not mark old geocorrected WebP caches stale",
|
|
)
|
|
parser.set_defaults(invalidate_preview=True)
|
|
args = parser.parse_args()
|
|
stats = asyncio.run(backfill(args))
|
|
print(json.dumps({"apply": bool(args.apply), **stats}, ensure_ascii=False, indent=2, default=str))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|