Refactor local InSAR asset and production workflows
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
"""Reset LT1/Sentinel-1 source catalog rows and radar preview caches.
|
||||
|
||||
This maintenance script is intentionally scoped to source-scene registration.
|
||||
It does not delete source archives, unpacked source directories, orbit assets,
|
||||
D-InSAR/SBAS/GF3 product catalogs, task logs, users, or flood products.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from backend.app import database # noqa: E402
|
||||
from backend.app.config import settings # noqa: E402
|
||||
from backend.app.models import ( # noqa: E402
|
||||
AssetInventoryIssueORM,
|
||||
AssetInventoryStateORM,
|
||||
PairingCacheStateORM,
|
||||
PairingDirtySceneORM,
|
||||
PairingMetricCacheORM,
|
||||
PairingNetworkEdgeORM,
|
||||
PairingNetworkRunORM,
|
||||
RadarDataORM,
|
||||
SARSceneGeoORM,
|
||||
SceneOrbitBindingORM,
|
||||
ScanStateORM,
|
||||
SourceProductAssetORM,
|
||||
TimeseriesStackPlanEdgeORM,
|
||||
TimeseriesStackPlanItemORM,
|
||||
)
|
||||
|
||||
|
||||
TARGET_FAMILIES = ("LT1", "S1")
|
||||
TARGET_SOURCE_FORMATS = ("LT1_DIR", "LT1_ARCHIVE", "S1_SAFE_DIR", "S1_ZIP")
|
||||
RADAR_CACHE_DIRS = (
|
||||
settings.RADAR_RAW_CACHE_DIR,
|
||||
settings.RADAR_GEO_CACHE_DIR,
|
||||
os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources"),
|
||||
)
|
||||
|
||||
|
||||
def _target_radar_filter() -> Any:
|
||||
return or_(
|
||||
RadarDataORM.satellite_family.in_(TARGET_FAMILIES),
|
||||
RadarDataORM.source_format.in_(TARGET_SOURCE_FORMATS),
|
||||
)
|
||||
|
||||
|
||||
def _target_asset_filter() -> Any:
|
||||
return or_(
|
||||
SourceProductAssetORM.satellite_family.in_(TARGET_FAMILIES),
|
||||
SourceProductAssetORM.source_format.in_(TARGET_SOURCE_FORMATS),
|
||||
)
|
||||
|
||||
|
||||
async def _count(db, stmt) -> int:
|
||||
return int((await db.execute(stmt)).scalar_one() or 0)
|
||||
|
||||
|
||||
def _cache_dir_summary(paths: Iterable[str]) -> List[Dict[str, Any]]:
|
||||
payload = []
|
||||
for raw_path in paths:
|
||||
path = Path(raw_path)
|
||||
files = 0
|
||||
bytes_total = 0
|
||||
if path.exists():
|
||||
for item in path.rglob("*"):
|
||||
if not item.is_file():
|
||||
continue
|
||||
try:
|
||||
files += 1
|
||||
bytes_total += item.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
payload.append(
|
||||
{
|
||||
"path": str(path),
|
||||
"exists": path.exists(),
|
||||
"files": files,
|
||||
"bytes": bytes_total,
|
||||
}
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _clear_cache_dirs(paths: Iterable[str]) -> List[Dict[str, Any]]:
|
||||
results = []
|
||||
for raw_path in paths:
|
||||
path = Path(raw_path)
|
||||
before = _cache_dir_summary([str(path)])[0]
|
||||
if path.exists():
|
||||
for item in path.iterdir():
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
after = _cache_dir_summary([str(path)])[0]
|
||||
results.append({"before": before, "after": after})
|
||||
return results
|
||||
|
||||
|
||||
async def _summary(db) -> Dict[str, Any]:
|
||||
radar_filter = _target_radar_filter()
|
||||
asset_filter = _target_asset_filter()
|
||||
radar_ids = select(RadarDataORM.id).where(radar_filter)
|
||||
asset_ids = select(SourceProductAssetORM.id).where(asset_filter)
|
||||
|
||||
by_source_format = {}
|
||||
radar_format_rows = await db.execute(
|
||||
select(RadarDataORM.source_format, func.count(RadarDataORM.id))
|
||||
.where(radar_filter)
|
||||
.group_by(RadarDataORM.source_format)
|
||||
.order_by(RadarDataORM.source_format.asc())
|
||||
)
|
||||
for source_format, count in radar_format_rows.all():
|
||||
by_source_format[str(source_format or "NULL")] = int(count or 0)
|
||||
|
||||
asset_by_source_format = {}
|
||||
asset_format_rows = await db.execute(
|
||||
select(SourceProductAssetORM.source_format, func.count(SourceProductAssetORM.id))
|
||||
.where(asset_filter)
|
||||
.group_by(SourceProductAssetORM.source_format)
|
||||
.order_by(SourceProductAssetORM.source_format.asc())
|
||||
)
|
||||
for source_format, count in asset_format_rows.all():
|
||||
asset_by_source_format[str(source_format or "NULL")] = int(count or 0)
|
||||
|
||||
return {
|
||||
"radar_data_count": await _count(db, select(func.count(RadarDataORM.id)).where(radar_filter)),
|
||||
"radar_data_by_source_format": by_source_format,
|
||||
"source_product_asset_count": await _count(db, select(func.count(SourceProductAssetORM.id)).where(asset_filter)),
|
||||
"source_product_assets_by_source_format": asset_by_source_format,
|
||||
"scene_orbit_binding_count": await _count(
|
||||
db,
|
||||
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.radar_data_id.in_(radar_ids)),
|
||||
),
|
||||
"asset_inventory_issue_count": await _count(
|
||||
db,
|
||||
select(func.count(AssetInventoryIssueORM.id)).where(
|
||||
or_(
|
||||
AssetInventoryIssueORM.radar_data_id.in_(radar_ids),
|
||||
AssetInventoryIssueORM.asset_ref_id.in_(asset_ids),
|
||||
)
|
||||
),
|
||||
),
|
||||
"sar_scene_geo_blocker_count": await _count(
|
||||
db,
|
||||
select(func.count(SARSceneGeoORM.id)).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)),
|
||||
),
|
||||
"timeseries_plan_item_refs": await _count(
|
||||
db,
|
||||
select(func.count(TimeseriesStackPlanItemORM.id)).where(
|
||||
TimeseriesStackPlanItemORM.radar_data_ref_id.in_(radar_ids)
|
||||
),
|
||||
),
|
||||
"timeseries_plan_edge_refs": await _count(
|
||||
db,
|
||||
select(func.count(TimeseriesStackPlanEdgeORM.id)).where(
|
||||
or_(
|
||||
TimeseriesStackPlanEdgeORM.master_scene_ref_id.in_(radar_ids),
|
||||
TimeseriesStackPlanEdgeORM.slave_scene_ref_id.in_(radar_ids),
|
||||
)
|
||||
),
|
||||
),
|
||||
"pairing_metric_cache_count": await _count(db, select(func.count(PairingMetricCacheORM.id))),
|
||||
"pairing_dirty_scene_count": await _count(db, select(func.count(PairingDirtySceneORM.id))),
|
||||
"pairing_network_run_count": await _count(db, select(func.count(PairingNetworkRunORM.id))),
|
||||
"pairing_network_edge_count": await _count(db, select(func.count(PairingNetworkEdgeORM.id))),
|
||||
"radar_cache_dirs": _cache_dir_summary(RADAR_CACHE_DIRS),
|
||||
}
|
||||
|
||||
|
||||
async def _apply_reset(db, *, allow_sar_scene_geo: bool) -> Dict[str, Any]:
|
||||
radar_filter = _target_radar_filter()
|
||||
asset_filter = _target_asset_filter()
|
||||
radar_ids = select(RadarDataORM.id).where(radar_filter)
|
||||
asset_ids = select(SourceProductAssetORM.id).where(asset_filter)
|
||||
|
||||
sar_blockers = await _count(
|
||||
db,
|
||||
select(func.count(SARSceneGeoORM.id)).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)),
|
||||
)
|
||||
if sar_blockers and not allow_sar_scene_geo:
|
||||
raise RuntimeError(
|
||||
f"Refusing to delete radar_data: {sar_blockers} SAR analysis rows reference LT1/S1 scenes. "
|
||||
"Re-run with --allow-sar-scene-geo only if you intend to clear those analysis rows."
|
||||
)
|
||||
|
||||
counts: Dict[str, int] = {}
|
||||
|
||||
if allow_sar_scene_geo:
|
||||
result = await db.execute(delete(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)))
|
||||
counts["sar_scene_geo_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
result = await db.execute(delete(SceneOrbitBindingORM).where(SceneOrbitBindingORM.radar_data_id.in_(radar_ids)))
|
||||
counts["scene_orbit_bindings_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
result = await db.execute(
|
||||
delete(AssetInventoryIssueORM).where(
|
||||
or_(
|
||||
AssetInventoryIssueORM.radar_data_id.in_(radar_ids),
|
||||
AssetInventoryIssueORM.asset_ref_id.in_(asset_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
counts["asset_inventory_issues_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
await db.execute(
|
||||
update(TimeseriesStackPlanItemORM)
|
||||
.where(TimeseriesStackPlanItemORM.radar_data_ref_id.in_(radar_ids))
|
||||
.values(radar_data_ref_id=None)
|
||||
)
|
||||
await db.execute(
|
||||
update(TimeseriesStackPlanEdgeORM)
|
||||
.where(
|
||||
or_(
|
||||
TimeseriesStackPlanEdgeORM.master_scene_ref_id.in_(radar_ids),
|
||||
TimeseriesStackPlanEdgeORM.slave_scene_ref_id.in_(radar_ids),
|
||||
)
|
||||
)
|
||||
.values(master_scene_ref_id=None, slave_scene_ref_id=None, metric_cache_ref_id=None)
|
||||
)
|
||||
|
||||
result = await db.execute(delete(PairingNetworkEdgeORM))
|
||||
counts["pairing_network_edges_deleted"] = int(result.rowcount or 0)
|
||||
result = await db.execute(delete(PairingNetworkRunORM))
|
||||
counts["pairing_network_runs_deleted"] = int(result.rowcount or 0)
|
||||
result = await db.execute(delete(PairingMetricCacheORM))
|
||||
counts["pairing_metric_cache_deleted"] = int(result.rowcount or 0)
|
||||
result = await db.execute(delete(PairingDirtySceneORM))
|
||||
counts["pairing_dirty_scenes_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
result = await db.execute(delete(RadarDataORM).where(radar_filter))
|
||||
counts["radar_data_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
result = await db.execute(delete(SourceProductAssetORM).where(asset_filter))
|
||||
counts["source_product_assets_deleted"] = int(result.rowcount or 0)
|
||||
|
||||
await db.execute(delete(ScanStateORM).where(ScanStateORM.data_type == "radar"))
|
||||
await db.execute(
|
||||
update(AssetInventoryStateORM)
|
||||
.where(AssetInventoryStateORM.inventory_type == "source_product")
|
||||
.values(
|
||||
status="NEVER_SCANNED",
|
||||
last_scan_started_at=None,
|
||||
last_scan_finished_at=None,
|
||||
last_seen_entry_count=None,
|
||||
last_asset_count=None,
|
||||
last_issue_count=None,
|
||||
needs_rescan=True,
|
||||
last_error=None,
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
)
|
||||
await db.execute(
|
||||
update(PairingCacheStateORM).values(
|
||||
status="READY",
|
||||
scene_count=0,
|
||||
pair_count=0,
|
||||
dirty_scene_count=0,
|
||||
last_error=None,
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
return counts
|
||||
|
||||
|
||||
async def _run(args: argparse.Namespace) -> Dict[str, Any]:
|
||||
if database.AsyncSessionLocal is None:
|
||||
database.init_db(settings.DATABASE_URL)
|
||||
if database.AsyncSessionLocal is None:
|
||||
raise RuntimeError("Database session factory is not initialized")
|
||||
|
||||
async with database.AsyncSessionLocal() as db:
|
||||
before = await _summary(db)
|
||||
payload: Dict[str, Any] = {
|
||||
"apply": bool(args.apply),
|
||||
"before": before,
|
||||
"scope": {
|
||||
"families": TARGET_FAMILIES,
|
||||
"source_formats": TARGET_SOURCE_FORMATS,
|
||||
"clears_preview_cache": bool(args.clear_preview_cache),
|
||||
},
|
||||
}
|
||||
if args.apply:
|
||||
payload["database_changes"] = await _apply_reset(db, allow_sar_scene_geo=bool(args.allow_sar_scene_geo))
|
||||
if args.clear_preview_cache:
|
||||
payload["preview_cache_changes"] = _clear_cache_dirs(RADAR_CACHE_DIRS)
|
||||
else:
|
||||
payload["planned_actions"] = [
|
||||
"delete LT1/S1 radar_data rows",
|
||||
"delete LT1/S1 source_product_assets rows",
|
||||
"delete source inventory issues and scene-orbit bindings for those rows",
|
||||
"clear pairing cache/network cache",
|
||||
"reset source inventory scan state",
|
||||
"clear radar preview cache directories" if args.clear_preview_cache else "keep radar preview cache directories",
|
||||
]
|
||||
payload["after"] = await _summary(db)
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--apply", action="store_true", help="perform the reset; default only prints a dry-run plan")
|
||||
parser.add_argument("--clear-preview-cache", action="store_true", default=True, help="clear radar preview cache dirs")
|
||||
parser.add_argument("--keep-preview-cache", action="store_false", dest="clear_preview_cache")
|
||||
parser.add_argument("--allow-sar-scene-geo", action="store_true", help="also delete SAR analysis scene rows if they block reset")
|
||||
args = parser.parse_args()
|
||||
|
||||
import json
|
||||
|
||||
print(json.dumps(asyncio.run(_run(args)), ensure_ascii=False, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user