Add LandSAR cluster worker deployment
This commit is contained in:
@@ -0,0 +1,282 @@
|
||||
"""Backfill XML refRow/refColumn corner mappings for archive-managed radar records."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from geoalchemy2.shape import from_shape # noqa: E402
|
||||
from shapely.geometry import Polygon # noqa: E402
|
||||
from sqlalchemy import or_, select # noqa: E402
|
||||
from sqlalchemy.ext.asyncio import AsyncSession # noqa: E402
|
||||
|
||||
from backend.app import database # noqa: E402
|
||||
from backend.app.config import settings # noqa: E402
|
||||
from backend.app.models import RadarDataORM, SourceMetadataDocumentORM, SourceProductAssetORM # noqa: E402
|
||||
from backend.app.services.asset_inventory_service import _parse_radar_xml_metadata_bytes # noqa: E402
|
||||
from backend.app.utils import build_corner_pixel_mapping # noqa: E402
|
||||
|
||||
|
||||
SOURCE_FORMATS = {"LT1_ARCHIVE"}
|
||||
DOCUMENT_TYPES = {"LT1_META"}
|
||||
|
||||
|
||||
def _metadata_dict(value: Any) -> Dict[str, Any]:
|
||||
return dict(value or {}) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _normalize_families(values: Optional[Iterable[str]]) -> Sequence[str]:
|
||||
families = []
|
||||
for value in values or ["LT1"]:
|
||||
for item in str(value or "").split(","):
|
||||
family = item.strip().upper()
|
||||
if family and family not in families:
|
||||
families.append(family)
|
||||
supported = {"LT1"}
|
||||
invalid = [item for item in families if item not in supported]
|
||||
if invalid:
|
||||
raise SystemExit(f"Unsupported family: {', '.join(invalid)}")
|
||||
return families or ["LT1"]
|
||||
|
||||
|
||||
def _decode_document(doc: SourceMetadataDocumentORM) -> Optional[bytes]:
|
||||
payload = bytes(doc.content_bytes or b"")
|
||||
if not payload:
|
||||
return None
|
||||
if str(doc.content_encoding or "").lower() == "gzip":
|
||||
return gzip.decompress(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _mapping_from_xml_document(doc: SourceMetadataDocumentORM) -> Tuple[Optional[Dict[str, Any]], Optional[Any]]:
|
||||
payload = _decode_document(doc)
|
||||
if not payload:
|
||||
return None, None
|
||||
coverage_polygon, xml_meta = _parse_radar_xml_metadata_bytes(payload)
|
||||
mapping = xml_meta.get("corner_pixel_mapping") if isinstance(xml_meta, dict) else None
|
||||
return mapping if isinstance(mapping, dict) else None, coverage_polygon
|
||||
|
||||
|
||||
def _mapping_from_metadata(metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
existing = metadata.get("corner_pixel_mapping")
|
||||
if isinstance(existing, dict):
|
||||
return existing
|
||||
corners = metadata.get("corner_details")
|
||||
if isinstance(corners, dict):
|
||||
mapping = build_corner_pixel_mapping(corners)
|
||||
if isinstance(mapping, dict):
|
||||
return mapping
|
||||
return None
|
||||
|
||||
|
||||
def _bbox(points: Any) -> Optional[Tuple[float, float, float, float]]:
|
||||
try:
|
||||
ring = [(float(item[0]), float(item[1])) for item in points or []]
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return None
|
||||
if len(ring) < 3:
|
||||
return None
|
||||
lons = [item[0] for item in ring]
|
||||
lats = [item[1] for item in ring]
|
||||
return min(lons), min(lats), max(lons), max(lats)
|
||||
|
||||
|
||||
async def _first_metadata_document(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source_asset_id: Optional[int],
|
||||
radar_data_id: Optional[int],
|
||||
) -> Optional[SourceMetadataDocumentORM]:
|
||||
clauses = []
|
||||
if source_asset_id:
|
||||
clauses.append(SourceMetadataDocumentORM.source_asset_id == int(source_asset_id))
|
||||
if radar_data_id:
|
||||
clauses.append(SourceMetadataDocumentORM.radar_data_id == int(radar_data_id))
|
||||
if not clauses:
|
||||
return None
|
||||
stmt = (
|
||||
select(SourceMetadataDocumentORM)
|
||||
.where(or_(*clauses))
|
||||
.where(SourceMetadataDocumentORM.document_type.in_(sorted(DOCUMENT_TYPES)))
|
||||
.order_by(SourceMetadataDocumentORM.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def _load_source_assets(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[SourceProductAssetORM]:
|
||||
stmt = (
|
||||
select(SourceProductAssetORM)
|
||||
.where(SourceProductAssetORM.satellite_family.in_(families))
|
||||
.where(SourceProductAssetORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
||||
.where(SourceProductAssetORM.is_active.is_(True))
|
||||
.order_by(SourceProductAssetORM.id.asc())
|
||||
)
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def _load_radar_records(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[RadarDataORM]:
|
||||
stmt = (
|
||||
select(RadarDataORM)
|
||||
.where(RadarDataORM.satellite_family.in_(families))
|
||||
.where(RadarDataORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
||||
.order_by(RadarDataORM.id.asc())
|
||||
)
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def backfill(args: argparse.Namespace) -> Dict[str, int]:
|
||||
families = _normalize_families(args.family)
|
||||
database.init_db(settings.DATABASE_URL)
|
||||
stats = {
|
||||
"source_seen": 0,
|
||||
"source_updated": 0,
|
||||
"source_existing": 0,
|
||||
"source_missing_mapping": 0,
|
||||
"radar_seen": 0,
|
||||
"radar_updated": 0,
|
||||
"radar_existing": 0,
|
||||
"radar_missing_mapping": 0,
|
||||
"cache_invalidated": 0,
|
||||
"document_parse_failed": 0,
|
||||
}
|
||||
|
||||
async with database.AsyncSessionLocal() as db:
|
||||
source_assets = await _load_source_assets(db, families, int(args.limit or 0))
|
||||
source_mappings: Dict[int, Dict[str, Any]] = {}
|
||||
source_polygons: Dict[int, Any] = {}
|
||||
now = datetime.utcnow()
|
||||
|
||||
for asset in source_assets:
|
||||
stats["source_seen"] += 1
|
||||
metadata = _metadata_dict(asset.metadata_json)
|
||||
mapping = _mapping_from_metadata(metadata)
|
||||
coverage_polygon = metadata.get("coverage_polygon")
|
||||
if mapping:
|
||||
stats["source_existing"] += 1
|
||||
else:
|
||||
doc = await _first_metadata_document(db, source_asset_id=asset.id, radar_data_id=None)
|
||||
if doc:
|
||||
try:
|
||||
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
||||
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
||||
except Exception:
|
||||
stats["document_parse_failed"] += 1
|
||||
mapping = None
|
||||
if not mapping:
|
||||
stats["source_missing_mapping"] += 1
|
||||
continue
|
||||
metadata["corner_pixel_mapping"] = mapping
|
||||
if coverage_polygon:
|
||||
metadata["coverage_polygon"] = coverage_polygon
|
||||
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
||||
asset.metadata_json = metadata
|
||||
asset.updated_at = now
|
||||
db.add(asset)
|
||||
stats["source_updated"] += 1
|
||||
|
||||
if mapping and asset.id is not None:
|
||||
source_mappings[int(asset.id)] = mapping
|
||||
if coverage_polygon:
|
||||
source_polygons[int(asset.id)] = coverage_polygon
|
||||
|
||||
radar_records = await _load_radar_records(db, families, int(args.limit or 0))
|
||||
for radar in radar_records:
|
||||
stats["radar_seen"] += 1
|
||||
metadata = _metadata_dict(radar.metadata_json)
|
||||
mapping = _mapping_from_metadata(metadata)
|
||||
coverage_polygon = radar.coverage_polygon or metadata.get("coverage_polygon")
|
||||
if mapping:
|
||||
stats["radar_existing"] += 1
|
||||
else:
|
||||
source_id = int(radar.source_product_ref_id or 0)
|
||||
mapping = source_mappings.get(source_id)
|
||||
if source_id and source_id in source_polygons:
|
||||
coverage_polygon = coverage_polygon or source_polygons[source_id]
|
||||
if not mapping:
|
||||
doc = await _first_metadata_document(
|
||||
db,
|
||||
source_asset_id=radar.source_product_ref_id,
|
||||
radar_data_id=radar.id,
|
||||
)
|
||||
if doc:
|
||||
try:
|
||||
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
||||
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
||||
except Exception:
|
||||
stats["document_parse_failed"] += 1
|
||||
mapping = None
|
||||
if not mapping:
|
||||
stats["radar_missing_mapping"] += 1
|
||||
continue
|
||||
|
||||
metadata["corner_pixel_mapping"] = mapping
|
||||
if coverage_polygon:
|
||||
metadata["coverage_polygon"] = coverage_polygon
|
||||
radar.coverage_polygon = coverage_polygon
|
||||
bbox = _bbox(coverage_polygon)
|
||||
if bbox:
|
||||
radar.min_lon, radar.min_lat, radar.max_lon, radar.max_lat = bbox
|
||||
try:
|
||||
polygon = Polygon(coverage_polygon)
|
||||
if not polygon.is_valid:
|
||||
polygon = polygon.buffer(0)
|
||||
if not polygon.is_empty:
|
||||
radar.geom = from_shape(polygon, srid=4326)
|
||||
except Exception:
|
||||
pass
|
||||
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
||||
radar.metadata_json = metadata
|
||||
db.add(radar)
|
||||
stats["radar_updated"] += 1
|
||||
|
||||
if args.invalidate_preview and mapping:
|
||||
radar.preview_cache_status = "NONE"
|
||||
radar.preview_cache_version = None
|
||||
radar.preview_cache_path = None
|
||||
radar.preview_cache_error = "corner_pixel_mapping_backfilled"
|
||||
radar.preview_cache_updated_at = now
|
||||
db.add(radar)
|
||||
stats["cache_invalidated"] += 1
|
||||
|
||||
if args.apply:
|
||||
await db.commit()
|
||||
else:
|
||||
await db.rollback()
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Backfill corner_pixel_mapping into source/radar metadata_json.")
|
||||
parser.add_argument("--apply", action="store_true", help="commit database changes")
|
||||
parser.add_argument("--family", action="append", help="family to process: LT1 or GF3; defaults to LT1")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum source/radar rows per table; 0 means all")
|
||||
parser.add_argument(
|
||||
"--no-invalidate-preview",
|
||||
dest="invalidate_preview",
|
||||
action="store_false",
|
||||
help="do not mark old geocorrected WebP caches stale",
|
||||
)
|
||||
parser.set_defaults(invalidate_preview=True)
|
||||
args = parser.parse_args()
|
||||
stats = asyncio.run(backfill(args))
|
||||
print(json.dumps({"apply": bool(args.apply), **stats}, ensure_ascii=False, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user