chore: initialize insar management system v2
This commit is contained in:
@@ -0,0 +1,482 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, text, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import PairingCacheStateORM, PairingDirtySceneORM, PairingMetricCacheORM, RadarDataORM
|
||||
from .pairing_state_service import (
|
||||
PAIRING_CACHE_SCOPE_GLOBAL,
|
||||
pairing_state_service,
|
||||
)
|
||||
|
||||
|
||||
FULL_REBUILD_DIRTY_SCENE_THRESHOLD = 64
|
||||
FULL_REBUILD_DIRTY_SCENE_RATIO = 0.25
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.utcnow()
|
||||
|
||||
|
||||
def _scene_uid_expr(alias: str) -> str:
|
||||
return (
|
||||
f"COALESCE(NULLIF({alias}.unique_id, ''), NULLIF({alias}.file_path, ''), "
|
||||
f"'scene:' || {alias}.id::text)"
|
||||
)
|
||||
|
||||
|
||||
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_uid = _scene_uid_expr(left_alias)
|
||||
right_uid = _scene_uid_expr(right_alias)
|
||||
return (
|
||||
f"({left_alias}.imaging_date < {right_alias}.imaging_date "
|
||||
f"OR ({left_alias}.imaging_date = {right_alias}.imaging_date AND "
|
||||
f"({left_uid} < {right_uid} OR ({left_uid} = {right_uid} AND {left_alias}.id < {right_alias}.id))))"
|
||||
)
|
||||
|
||||
|
||||
def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
|
||||
return (
|
||||
f"{left_alias}.id <> {right_alias}.id "
|
||||
f"AND {left_alias}.geom IS NOT NULL "
|
||||
f"AND {right_alias}.geom IS NOT NULL "
|
||||
f"AND {left_alias}.imaging_date ~ '^[0-9]{{8}}$' "
|
||||
f"AND {right_alias}.imaging_date ~ '^[0-9]{{8}}$' "
|
||||
f"AND {left_alias}.orbit_direction IS NOT NULL "
|
||||
f"AND {right_alias}.orbit_direction IS NOT NULL "
|
||||
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
|
||||
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
|
||||
)
|
||||
|
||||
|
||||
def _full_rebuild_insert_sql() -> str:
|
||||
master_uid = _scene_uid_expr("m")
|
||||
slave_uid = _scene_uid_expr("s")
|
||||
return f"""
|
||||
INSERT INTO pairing_metric_cache (
|
||||
master_scene_ref_id,
|
||||
slave_scene_ref_id,
|
||||
master_scene_uid,
|
||||
slave_scene_uid,
|
||||
pair_uid,
|
||||
metric_version,
|
||||
orientation_rule_version,
|
||||
time_baseline_days,
|
||||
spatial_baseline_meters,
|
||||
scene_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_satellite,
|
||||
same_imaging_mode,
|
||||
same_polarization,
|
||||
master_imaging_date,
|
||||
slave_imaging_date,
|
||||
master_satellite,
|
||||
slave_satellite,
|
||||
master_imaging_mode,
|
||||
slave_imaging_mode,
|
||||
master_polarization,
|
||||
slave_polarization,
|
||||
master_file_path,
|
||||
slave_file_path,
|
||||
status,
|
||||
computed_at
|
||||
)
|
||||
SELECT
|
||||
m.id AS master_scene_ref_id,
|
||||
s.id AS slave_scene_ref_id,
|
||||
{master_uid} AS master_scene_uid,
|
||||
{slave_uid} AS slave_scene_uid,
|
||||
md5({master_uid} || '|' || {slave_uid}) AS pair_uid,
|
||||
:metric_version AS metric_version,
|
||||
:orientation_rule_version AS orientation_rule_version,
|
||||
ABS(to_date(s.imaging_date, 'YYYYMMDD') - to_date(m.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
|
||||
ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision AS spatial_baseline_meters,
|
||||
(
|
||||
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
m.orbit_direction,
|
||||
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
|
||||
(
|
||||
NULLIF(m.imaging_mode, '') IS NOT NULL
|
||||
AND NULLIF(s.imaging_mode, '') IS NOT NULL
|
||||
AND m.imaging_mode = s.imaging_mode
|
||||
) AS same_imaging_mode,
|
||||
(
|
||||
NULLIF(m.polarization, '') IS NOT NULL
|
||||
AND NULLIF(s.polarization, '') IS NOT NULL
|
||||
AND m.polarization = s.polarization
|
||||
) AS same_polarization,
|
||||
m.imaging_date AS master_imaging_date,
|
||||
s.imaging_date AS slave_imaging_date,
|
||||
m.satellite AS master_satellite,
|
||||
s.satellite AS slave_satellite,
|
||||
m.imaging_mode AS master_imaging_mode,
|
||||
s.imaging_mode AS slave_imaging_mode,
|
||||
m.polarization AS master_polarization,
|
||||
s.polarization AS slave_polarization,
|
||||
m.file_path AS master_file_path,
|
||||
s.file_path AS slave_file_path,
|
||||
'READY' AS status,
|
||||
NOW() AS computed_at
|
||||
FROM radar_data m
|
||||
JOIN radar_data s
|
||||
ON { _hard_constraints_expr('m', 's') }
|
||||
AND { _orientation_is_left_master_expr('m', 's') }
|
||||
"""
|
||||
|
||||
|
||||
def _incremental_insert_sql() -> str:
|
||||
dirty_uid = _scene_uid_expr("d")
|
||||
other_uid = _scene_uid_expr("o")
|
||||
dirty_is_master = _orientation_is_left_master_expr("d", "o")
|
||||
return f"""
|
||||
INSERT INTO pairing_metric_cache (
|
||||
master_scene_ref_id,
|
||||
slave_scene_ref_id,
|
||||
master_scene_uid,
|
||||
slave_scene_uid,
|
||||
pair_uid,
|
||||
metric_version,
|
||||
orientation_rule_version,
|
||||
time_baseline_days,
|
||||
spatial_baseline_meters,
|
||||
scene_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_satellite,
|
||||
same_imaging_mode,
|
||||
same_polarization,
|
||||
master_imaging_date,
|
||||
slave_imaging_date,
|
||||
master_satellite,
|
||||
slave_satellite,
|
||||
master_imaging_mode,
|
||||
slave_imaging_mode,
|
||||
master_polarization,
|
||||
slave_polarization,
|
||||
master_file_path,
|
||||
slave_file_path,
|
||||
status,
|
||||
computed_at
|
||||
)
|
||||
SELECT
|
||||
CASE WHEN {dirty_is_master} THEN d.id ELSE o.id END AS master_scene_ref_id,
|
||||
CASE WHEN {dirty_is_master} THEN o.id ELSE d.id END AS slave_scene_ref_id,
|
||||
CASE WHEN {dirty_is_master} THEN {dirty_uid} ELSE {other_uid} END AS master_scene_uid,
|
||||
CASE WHEN {dirty_is_master} THEN {other_uid} ELSE {dirty_uid} END AS slave_scene_uid,
|
||||
md5(
|
||||
CASE WHEN {dirty_is_master}
|
||||
THEN {dirty_uid} || '|' || {other_uid}
|
||||
ELSE {other_uid} || '|' || {dirty_uid}
|
||||
END
|
||||
) AS pair_uid,
|
||||
:metric_version AS metric_version,
|
||||
:orientation_rule_version AS orientation_rule_version,
|
||||
ABS(to_date(o.imaging_date, 'YYYYMMDD') - to_date(d.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
|
||||
ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision AS spatial_baseline_meters,
|
||||
(
|
||||
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
d.orbit_direction,
|
||||
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
|
||||
(
|
||||
NULLIF(d.imaging_mode, '') IS NOT NULL
|
||||
AND NULLIF(o.imaging_mode, '') IS NOT NULL
|
||||
AND d.imaging_mode = o.imaging_mode
|
||||
) AS same_imaging_mode,
|
||||
(
|
||||
NULLIF(d.polarization, '') IS NOT NULL
|
||||
AND NULLIF(o.polarization, '') IS NOT NULL
|
||||
AND d.polarization = o.polarization
|
||||
) AS same_polarization,
|
||||
CASE WHEN {dirty_is_master} THEN d.imaging_date ELSE o.imaging_date END AS master_imaging_date,
|
||||
CASE WHEN {dirty_is_master} THEN o.imaging_date ELSE d.imaging_date END AS slave_imaging_date,
|
||||
CASE WHEN {dirty_is_master} THEN d.satellite ELSE o.satellite END AS master_satellite,
|
||||
CASE WHEN {dirty_is_master} THEN o.satellite ELSE d.satellite END AS slave_satellite,
|
||||
CASE WHEN {dirty_is_master} THEN d.imaging_mode ELSE o.imaging_mode END AS master_imaging_mode,
|
||||
CASE WHEN {dirty_is_master} THEN o.imaging_mode ELSE d.imaging_mode END AS slave_imaging_mode,
|
||||
CASE WHEN {dirty_is_master} THEN d.polarization ELSE o.polarization END AS master_polarization,
|
||||
CASE WHEN {dirty_is_master} THEN o.polarization ELSE d.polarization END AS slave_polarization,
|
||||
CASE WHEN {dirty_is_master} THEN d.file_path ELSE o.file_path END AS master_file_path,
|
||||
CASE WHEN {dirty_is_master} THEN o.file_path ELSE d.file_path END AS slave_file_path,
|
||||
'READY' AS status,
|
||||
NOW() AS computed_at
|
||||
FROM radar_data d
|
||||
JOIN radar_data o
|
||||
ON d.id = :dirty_scene_id
|
||||
AND { _hard_constraints_expr('d', 'o') }
|
||||
ON CONFLICT (master_scene_ref_id, slave_scene_ref_id, metric_version)
|
||||
DO NOTHING
|
||||
"""
|
||||
|
||||
|
||||
class PairingCacheService:
|
||||
async def _get_state_row(self, db: AsyncSession) -> PairingCacheStateORM:
|
||||
payload = await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
|
||||
result = await db.execute(
|
||||
select(PairingCacheStateORM).where(
|
||||
PairingCacheStateORM.cache_scope == PAIRING_CACHE_SCOPE_GLOBAL
|
||||
)
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
if state is None:
|
||||
raise RuntimeError(f"Pairing cache state row missing after bootstrap: {payload}")
|
||||
return state
|
||||
|
||||
async def _count_pair_rows(self, db: AsyncSession) -> int:
|
||||
result = await db.execute(select(func.count(PairingMetricCacheORM.id)))
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
async def _count_scene_rows(self, db: AsyncSession) -> int:
|
||||
result = await db.execute(select(func.count(RadarDataORM.id)))
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
async def _count_pending_dirty_rows(self, db: AsyncSession) -> int:
|
||||
result = await db.execute(
|
||||
select(func.count(PairingDirtySceneORM.id)).where(
|
||||
PairingDirtySceneORM.status == "PENDING"
|
||||
)
|
||||
)
|
||||
return int(result.scalar_one() or 0)
|
||||
|
||||
async def _resolve_dirty_rows(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
scene_ids: Optional[Sequence[int]] = None,
|
||||
) -> int:
|
||||
stmt = (
|
||||
update(PairingDirtySceneORM)
|
||||
.where(PairingDirtySceneORM.status == "PENDING")
|
||||
.values(status="RESOLVED", resolved_at=_utcnow())
|
||||
)
|
||||
if scene_ids is not None:
|
||||
normalized_ids = [int(value) for value in scene_ids if value is not None]
|
||||
if not normalized_ids:
|
||||
return 0
|
||||
stmt = stmt.where(PairingDirtySceneORM.scene_ref_id.in_(normalized_ids))
|
||||
result = await db.execute(stmt)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def _set_state_rebuilding(self, db: AsyncSession, *, last_error: Optional[str] = None) -> None:
|
||||
state = await self._get_state_row(db)
|
||||
state.status = "REBUILDING"
|
||||
state.last_error = last_error
|
||||
await db.flush()
|
||||
|
||||
async def _finalize_state_success(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
full_rebuild: bool,
|
||||
) -> Dict[str, Any]:
|
||||
state = await self._get_state_row(db)
|
||||
pair_count = await self._count_pair_rows(db)
|
||||
scene_count = await self._count_scene_rows(db)
|
||||
dirty_scene_count = await self._count_pending_dirty_rows(db)
|
||||
state.metric_version = pairing_state_service.metric_version
|
||||
state.scene_count = scene_count
|
||||
state.pair_count = pair_count
|
||||
state.dirty_scene_count = dirty_scene_count
|
||||
state.status = (
|
||||
"READY"
|
||||
if dirty_scene_count == 0 and not (scene_count > 1 and pair_count == 0)
|
||||
else "DIRTY"
|
||||
)
|
||||
state.last_error = None
|
||||
if full_rebuild:
|
||||
state.last_full_rebuild_at = _utcnow()
|
||||
else:
|
||||
state.last_incremental_reconcile_at = _utcnow()
|
||||
await db.flush()
|
||||
return {
|
||||
"status": state.status,
|
||||
"scene_count": scene_count,
|
||||
"pair_count": pair_count,
|
||||
"dirty_scene_count": dirty_scene_count,
|
||||
"metric_version": state.metric_version,
|
||||
"cache_ready": state.status == "READY",
|
||||
"needs_rebuild": dirty_scene_count > 0 or (scene_count > 1 and pair_count == 0),
|
||||
}
|
||||
|
||||
async def _finalize_state_failure(self, db: AsyncSession, *, error: Exception) -> None:
|
||||
state = await self._get_state_row(db)
|
||||
state.status = "FAILED"
|
||||
state.last_error = str(error)
|
||||
await db.flush()
|
||||
|
||||
def _should_full_rebuild(
|
||||
self,
|
||||
*,
|
||||
dirty_scene_count: int,
|
||||
scene_count: int,
|
||||
pair_count: int,
|
||||
force_full: bool,
|
||||
) -> bool:
|
||||
if force_full:
|
||||
return True
|
||||
if pair_count == 0 and scene_count > 1:
|
||||
return True
|
||||
if dirty_scene_count >= FULL_REBUILD_DIRTY_SCENE_THRESHOLD:
|
||||
return True
|
||||
if scene_count > 0 and dirty_scene_count / max(scene_count, 1) >= FULL_REBUILD_DIRTY_SCENE_RATIO:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def rebuild_metric_cache(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
commit: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
|
||||
await self._set_state_rebuilding(db)
|
||||
try:
|
||||
delete_result = await db.execute(delete(PairingMetricCacheORM))
|
||||
await db.execute(
|
||||
text(_full_rebuild_insert_sql()),
|
||||
{
|
||||
"metric_version": pairing_state_service.metric_version,
|
||||
"orientation_rule_version": pairing_state_service.orientation_rule_version,
|
||||
},
|
||||
)
|
||||
resolved_dirty = await self._resolve_dirty_rows(db)
|
||||
summary = await self._finalize_state_success(db, full_rebuild=True)
|
||||
if commit:
|
||||
await db.commit()
|
||||
else:
|
||||
await db.flush()
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "full_rebuild",
|
||||
"deleted_pair_rows": int(delete_result.rowcount or 0),
|
||||
"resolved_dirty_rows": resolved_dirty,
|
||||
**summary,
|
||||
}
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
await self._finalize_state_failure(db, error=exc)
|
||||
if commit:
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
async def reconcile_dirty_scenes(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
force_full: bool = False,
|
||||
commit: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
|
||||
dirty_result = await db.execute(
|
||||
select(PairingDirtySceneORM.scene_ref_id)
|
||||
.where(PairingDirtySceneORM.status == "PENDING")
|
||||
.order_by(PairingDirtySceneORM.marked_at.asc(), PairingDirtySceneORM.id.asc())
|
||||
)
|
||||
dirty_scene_ids = [int(value) for value in dirty_result.scalars().all() if value is not None]
|
||||
dirty_scene_count = len(dirty_scene_ids)
|
||||
pair_count = await self._count_pair_rows(db)
|
||||
scene_count = await self._count_scene_rows(db)
|
||||
|
||||
if dirty_scene_count == 0:
|
||||
summary = await self._finalize_state_success(db, full_rebuild=False)
|
||||
if commit:
|
||||
await db.commit()
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "noop",
|
||||
"dirty_scene_count": 0,
|
||||
"deleted_pair_rows": 0,
|
||||
"resolved_dirty_rows": 0,
|
||||
"insert_attempts": 0,
|
||||
**summary,
|
||||
}
|
||||
|
||||
if self._should_full_rebuild(
|
||||
dirty_scene_count=dirty_scene_count,
|
||||
scene_count=scene_count,
|
||||
pair_count=pair_count,
|
||||
force_full=force_full,
|
||||
):
|
||||
result = await self.rebuild_metric_cache(db, commit=commit)
|
||||
result["trigger_dirty_scene_count"] = dirty_scene_count
|
||||
result["forced"] = force_full
|
||||
return result
|
||||
|
||||
await self._set_state_rebuilding(db)
|
||||
try:
|
||||
delete_result = await db.execute(
|
||||
delete(PairingMetricCacheORM).where(
|
||||
or_(
|
||||
PairingMetricCacheORM.master_scene_ref_id.in_(dirty_scene_ids),
|
||||
PairingMetricCacheORM.slave_scene_ref_id.in_(dirty_scene_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
insert_attempts = 0
|
||||
insert_sql = text(_incremental_insert_sql())
|
||||
for dirty_scene_id in dirty_scene_ids:
|
||||
insert_result = await db.execute(
|
||||
insert_sql,
|
||||
{
|
||||
"dirty_scene_id": int(dirty_scene_id),
|
||||
"metric_version": pairing_state_service.metric_version,
|
||||
"orientation_rule_version": pairing_state_service.orientation_rule_version,
|
||||
},
|
||||
)
|
||||
insert_attempts += int(insert_result.rowcount or 0)
|
||||
|
||||
resolved_dirty = await self._resolve_dirty_rows(db, scene_ids=dirty_scene_ids)
|
||||
summary = await self._finalize_state_success(db, full_rebuild=False)
|
||||
if commit:
|
||||
await db.commit()
|
||||
else:
|
||||
await db.flush()
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "incremental_reconcile",
|
||||
"dirty_scene_count": dirty_scene_count,
|
||||
"deleted_pair_rows": int(delete_result.rowcount or 0),
|
||||
"resolved_dirty_rows": resolved_dirty,
|
||||
"insert_attempts": insert_attempts,
|
||||
**summary,
|
||||
}
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
await self._finalize_state_failure(db, error=exc)
|
||||
if commit:
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
async def get_admin_summary(self, db: AsyncSession) -> Dict[str, Any]:
|
||||
payload = await pairing_state_service.get_pairing_system_status(db)
|
||||
pending_result = await db.execute(
|
||||
select(
|
||||
PairingDirtySceneORM.scene_ref_id,
|
||||
PairingDirtySceneORM.scene_uid,
|
||||
PairingDirtySceneORM.reason,
|
||||
PairingDirtySceneORM.marked_at,
|
||||
)
|
||||
.where(PairingDirtySceneORM.status == "PENDING")
|
||||
.order_by(PairingDirtySceneORM.marked_at.asc(), PairingDirtySceneORM.id.asc())
|
||||
.limit(20)
|
||||
)
|
||||
pending_rows = pending_result.all()
|
||||
payload["pending_dirty_scenes"] = [
|
||||
{
|
||||
"scene_ref_id": int(scene_ref_id),
|
||||
"scene_uid": scene_uid,
|
||||
"reason": reason,
|
||||
"marked_at": marked_at,
|
||||
}
|
||||
for scene_ref_id, scene_uid, reason, marked_at in pending_rows
|
||||
]
|
||||
return payload
|
||||
|
||||
|
||||
pairing_cache_service = PairingCacheService()
|
||||
Reference in New Issue
Block a user