Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor local InSAR asset and production workflows

Browse Source
This commit is contained in:
Harmon
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build_archive_preview_caches.py
+62
View File
@@ -0,0 +1,62 @@
"""Build WebP preview caches for archive-managed LT1 and Sentinel-1 assets."""
from __future__ import annotations
import argparse
import asyncio
import json
import sys
from pathlib import Path
from typing import Iterable, List, Optional
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from backend.app import database # noqa: E402
from backend.app.config import settings # noqa: E402
from backend.app.services.asset_inventory_service import asset_inventory_service # noqa: E402
def _normalize_families(values: Optional[Iterable[str]]) -> List[str]:
families: List[str] = []
for value in values or ["LT1", "S1"]:
for item in str(value or "").split(","):
family = item.strip().upper()
if family and family not in families:
families.append(family)
invalid = [item for item in families if item not in {"LT1", "S1"}]
if invalid:
raise SystemExit(f"Unsupported family: {', '.join(invalid)}")
return families or ["LT1", "S1"]
async def _run(args: argparse.Namespace) -> dict:
families = _normalize_families(args.family)
database.init_db(settings.DATABASE_URL)
summary = await asset_inventory_service.build_archive_preview_caches(
families=families,
limit=args.limit,
force=args.force,
apply=bool(args.apply),
progress_start=0,
progress_end=100,
)
return {"apply": bool(args.apply), **summary}
def main() -> int:
parser = argparse.ArgumentParser(description="Build LT1/Sentinel-1 archive preview WebP caches.")
parser.add_argument("--apply", action="store_true", help="write preview cache files and update database rows")
parser.add_argument("--family", action="append", help="family to process: LT1, S1, or comma-separated values")
parser.add_argument("--limit", type=int, default=0, help="maximum rows to build; 0 means all pending rows")
parser.add_argument("--force", action="store_true", help="rebuild even if preview_cache_status is READY")
args = parser.parse_args()
payload = asyncio.run(_run(args))
print(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/migrate_source_archives_to_zip.py
+594
View File
@@ -0,0 +1,594 @@
"""Migrate LT1/Sentinel-1 source records from unpacked pools to ZIP archives.
Default mode is a dry run. Use --apply to sync configured roots and scan ZIP
source pools. Use --quarantine-old-pools only after the database and preview
checks pass.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import shutil
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Sequence
from sqlalchemy import and_, case, exists, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from backend.app import database # noqa: E402
from backend.app.config import settings, split_env_paths # noqa: E402
from backend.app.models import ManagedRootORM, RadarDataORM, SourceProductAssetORM # noqa: E402
from backend.app.services.asset_inventory_service import asset_inventory_service # noqa: E402
from backend.app.services.data_service import DataService # noqa: E402
from backend.app.services.image_service import image_service # noqa: E402
from backend.app.services.root_registry_service import root_registry_service # noqa: E402
OLD_LT1_POOL = r"D:\LuTan1_Image_Pool"
OLD_S1_POOL = r"D:\Sentinel1_Image_Pool"
def _norm(path: str | Path) -> str:
text = str(path or "").strip()
if not text:
return ""
return os.path.normpath(os.path.abspath(text))
def _normcase(path: str | Path) -> str:
return os.path.normcase(_norm(path))
def _path_under(path: str | Path, root: str | Path) -> bool:
path_norm = _normcase(path)
root_norm = _normcase(root)
return path_norm == root_norm or path_norm.startswith(root_norm + os.sep)
def _human_bytes(value: int | None) -> str:
size = float(value or 0)
for unit in ("B", "KB", "MB", "GB", "TB"):
if size < 1024 or unit == "TB":
return f"{size:.2f} {unit}"
size /= 1024
return f"{size:.2f} TB"
def _measure_top_level(path: str) -> Dict[str, Any]:
root = Path(path)
if not root.exists():
return {
"path": path,
"exists": False,
"top_level_items": 0,
"sample": [],
}
sample = []
top_level_items = 0
try:
for item in root.iterdir():
top_level_items += 1
if len(sample) < 10:
sample.append(str(item))
except OSError as exc:
return {
"path": path,
"exists": True,
"error": str(exc),
"top_level_items": top_level_items,
"sample": sample,
}
return {
"path": path,
"exists": True,
"top_level_items": top_level_items,
"sample": sample,
}
async def _count_scalar(db: AsyncSession, stmt) -> int:
return int((await db.execute(stmt)).scalar_one() or 0)
def _path_prefix_clauses(column: Any, roots: Sequence[str]) -> List[Any]:
clauses: List[Any] = []
for root in roots:
text = str(root or "").strip()
if not text:
continue
clauses.append(
and_(
func.lower(func.substr(column, 1, len(text))) == text.lower(),
or_(
func.length(column) == len(text),
func.substr(column, len(text) + 1, 1).in_(["\\", "/"]),
),
)
)
return clauses
def _path_prefix_filter(column: Any, roots: Sequence[str]) -> Any:
clauses = _path_prefix_clauses(column, roots)
if not clauses:
return None
return or_(*clauses)
async def _db_summary(db: AsyncSession, old_pools: Sequence[str], zip_roots: Sequence[str]) -> Dict[str, Any]:
old_filter = _path_prefix_filter(RadarDataORM.file_path, old_pools)
zip_filter = _path_prefix_filter(RadarDataORM.file_path, zip_roots)
radar_by_family = {}
family_rows = await db.execute(
select(RadarDataORM.satellite_family, func.count(RadarDataORM.id))
.where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
.group_by(RadarDataORM.satellite_family)
)
for family, count in family_rows.all():
radar_by_family[str(family or "")] = int(count or 0)
radar_by_source_format = {}
format_rows = await db.execute(
select(RadarDataORM.source_format, func.count(RadarDataORM.id))
.where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
.group_by(RadarDataORM.source_format)
)
for source_format, count in format_rows.all():
radar_by_source_format[str(source_format or "NULL")] = int(count or 0)
assets_by_format = {}
asset_rows = await db.execute(
select(SourceProductAssetORM.source_format, func.count(SourceProductAssetORM.id))
.where(SourceProductAssetORM.satellite_family.in_(["LT1", "S1"]))
.group_by(SourceProductAssetORM.source_format)
)
for source_format, count in asset_rows.all():
assets_by_format[str(source_format or "NULL")] = int(count or 0)
preview_rows = await db.execute(
select(
func.count(RadarDataORM.id),
func.sum(case((RadarDataORM.preview_cache_status == "READY", 1), else_=0)),
func.sum(case((RadarDataORM.preview_cache_path.is_not(None), 1), else_=0)),
).where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
)
total_previews, ready_previews, path_previews = preview_rows.one()
summary: Dict[str, Any] = {
"radar_by_family": radar_by_family,
"radar_by_source_format": radar_by_source_format,
"source_assets_by_format": assets_by_format,
"preview_rows": {
"total": int(total_previews or 0),
"ready": int(ready_previews or 0),
"has_cache_path": int(path_previews or 0),
},
}
if old_filter is not None:
summary["radar_file_paths_under_old_pools"] = await _count_scalar(
db,
select(func.count(RadarDataORM.id)).where(old_filter),
)
if zip_filter is not None:
summary["radar_file_paths_under_zip_roots"] = await _count_scalar(
db,
select(func.count(RadarDataORM.id)).where(zip_filter),
)
return summary
async def _archive_migration_gap_summary(db: AsyncSession, old_pools: Sequence[str]) -> Dict[str, Any]:
old_filter = _path_prefix_filter(RadarDataORM.file_path, old_pools)
if old_filter is None:
return {"old_pool_records": 0}
def _archive_exists(family: str, source_format: str) -> Any:
return exists(
select(SourceProductAssetORM.id).where(
SourceProductAssetORM.satellite_family == family,
SourceProductAssetORM.source_format == source_format,
SourceProductAssetORM.logical_product_uid == RadarDataORM.product_unique_id,
SourceProductAssetORM.is_active == True, # noqa: E712
)
)
lt1_archive_exists = _archive_exists("LT1", "LT1_ARCHIVE")
s1_archive_exists = _archive_exists("S1", "S1_ZIP")
base_filter = and_(
old_filter,
RadarDataORM.satellite_family.in_(["LT1", "S1"]),
RadarDataORM.source_format.in_(["LT1_DIR", "S1_SAFE_DIR"]),
)
migrateable_filter = and_(
base_filter,
or_(
and_(RadarDataORM.satellite_family == "LT1", lt1_archive_exists),
and_(RadarDataORM.satellite_family == "S1", s1_archive_exists),
),
)
missing_filter = and_(
base_filter,
or_(
and_(RadarDataORM.satellite_family == "LT1", ~lt1_archive_exists),
and_(RadarDataORM.satellite_family == "S1", ~s1_archive_exists),
),
)
samples = await db.execute(
select(
RadarDataORM.id,
RadarDataORM.satellite_family,
RadarDataORM.product_unique_id,
RadarDataORM.file_path,
)
.where(missing_filter)
.order_by(RadarDataORM.id.asc())
.limit(10)
)
return {
"old_pool_records": await _count_scalar(db, select(func.count(RadarDataORM.id)).where(base_filter)),
"migrateable_with_archive_asset": await _count_scalar(db, select(func.count(RadarDataORM.id)).where(migrateable_filter)),
"missing_archive_asset": await _count_scalar(db, select(func.count(RadarDataORM.id)).where(missing_filter)),
"missing_archive_samples": [
{
"radar_id": radar_id,
"family": family,
"product_unique_id": product_unique_id,
"file_path": file_path,
}
for radar_id, family, product_unique_id, file_path in samples.all()
],
}
async def _list_zip_roots(db: AsyncSession) -> List[Dict[str, Any]]:
result = await db.execute(
select(ManagedRootORM)
.where(ManagedRootORM.root_role == "source_product_pool")
.order_by(ManagedRootORM.id.asc())
)
roots = []
for root in result.scalars().all():
roots.append(
{
"id": root.id,
"path": root.path,
"enabled": bool(root.enabled),
"exists": bool(root.exists_flag),
"source_ref": root.source_ref,
}
)
return roots
async def _target_scan_root_ids(db: AsyncSession, *, bind_orbits: bool) -> List[int]:
source_result = await db.execute(
select(ManagedRootORM.id).where(
ManagedRootORM.enabled == True, # noqa: E712
ManagedRootORM.root_role == "source_product_pool",
ManagedRootORM.source_ref.like("SOURCE_PRODUCT_DIRS%"),
)
)
root_ids = [int(item) for item in source_result.scalars().all()]
if bind_orbits:
orbit_result = await db.execute(
select(ManagedRootORM.id).where(
ManagedRootORM.enabled == True, # noqa: E712
ManagedRootORM.root_role == "orbit_asset_pool",
)
)
root_ids.extend(int(item) for item in orbit_result.scalars().all())
return root_ids
async def _sample_preview_archive_sources(db: AsyncSession, limit: int) -> List[Dict[str, Any]]:
result = await db.execute(
select(RadarDataORM)
.where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
.where(RadarDataORM.source_format.in_(["LT1_ARCHIVE", "S1_ZIP"]))
.order_by(RadarDataORM.id.asc())
.limit(max(0, int(limit)))
)
samples: List[Dict[str, Any]] = []
for record in result.scalars().all():
preview_source = DataService.find_radar_preview_source(record.file_path)
geo_path = DataService.get_radar_geo_cache_path(record.unique_id or record.file_path, record.file_path)
samples.append(
{
"radar_id": record.id,
"family": record.satellite_family,
"source_format": record.source_format,
"file_path": record.file_path,
"preview_source_found": bool(preview_source),
"preview_source": preview_source,
"db_preview_ready": (record.preview_cache_status or "") == "READY",
"db_preview_cache_exists": bool(record.preview_cache_path and os.path.exists(record.preview_cache_path)),
"expected_geo_cache_exists": os.path.exists(geo_path),
}
)
return samples
async def _build_archive_previews(
db: AsyncSession,
*,
limit: int,
force: bool,
) -> Dict[str, Any]:
stmt = (
select(RadarDataORM)
.where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
.where(RadarDataORM.source_format.in_(["LT1_ARCHIVE", "S1_ZIP"]))
.order_by(RadarDataORM.id.asc())
)
if not force:
stmt = stmt.where(
or_(
RadarDataORM.preview_cache_status != "READY",
RadarDataORM.preview_cache_status.is_(None),
RadarDataORM.preview_cache_path.is_(None),
)
)
if limit > 0:
stmt = stmt.limit(limit)
result = await db.execute(stmt)
records = result.scalars().all()
summary = {
"candidate_count": len(records),
"ready": 0,
"failed": 0,
"missing_source": 0,
"skipped_invalid_geometry": 0,
"items": [],
}
thumb_size = (settings.RADAR_THUMBNAIL_MAX_SIZE, settings.RADAR_THUMBNAIL_MAX_SIZE)
for record in records:
unique_id = record.unique_id or record.file_path
raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path)
geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path)
preview_source = DataService.find_radar_preview_source(record.file_path)
item: Dict[str, Any] = {
"radar_id": record.id,
"family": record.satellite_family,
"source_format": record.source_format,
"file_path": record.file_path,
"preview_source": preview_source,
}
if not preview_source:
record.preview_cache_status = "NONE"
record.preview_cache_path = None
record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION
record.preview_cache_updated_at = datetime.utcnow()
record.preview_cache_error = "preview_source_not_found"
db.add(record)
summary["missing_source"] += 1
item["status"] = "missing_source"
summary["items"].append(item)
continue
coverage_polygon = DataService._normalize_coverage_polygon(record.coverage_polygon)
try:
bbox = (
float(record.min_lon),
float(record.min_lat),
float(record.max_lon),
float(record.max_lat),
)
except (TypeError, ValueError):
bbox = None
if not coverage_polygon or not bbox:
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION
record.preview_cache_updated_at = datetime.utcnow()
record.preview_cache_error = "invalid_coverage_polygon" if not coverage_polygon else "invalid_bbox"
db.add(record)
summary["skipped_invalid_geometry"] += 1
item["status"] = record.preview_cache_error
summary["items"].append(item)
continue
source_corner_mapping = DataService.get_radar_source_corner_mapping(record.file_path)
ok_geo, geo_error = image_service.create_geocorrected_radar_cached_image(
preview_source,
geo_cache_path,
coverage_polygon,
bbox,
source_corner_mapping,
thumb_size,
settings.RADAR_GEO_CACHE_QUALITY,
)
ok_raw = image_service.create_radar_cached_image(preview_source, raw_cache_path, thumb_size)
if ok_geo and os.path.exists(geo_cache_path):
record.preview_cache_status = "READY"
record.preview_cache_path = geo_cache_path
record.preview_cache_error = None
summary["ready"] += 1
item["status"] = "ready"
item["geo_cache_path"] = geo_cache_path
else:
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = geo_error or ("raw_cache_ready_only" if ok_raw else "preview_cache_build_failed")
summary["failed"] += 1
item["status"] = "failed"
item["error"] = record.preview_cache_error
record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION
record.preview_cache_updated_at = datetime.utcnow()
db.add(record)
summary["items"].append(item)
await db.commit()
return summary
async def _count_duplicate_products(db: AsyncSession) -> List[Dict[str, Any]]:
result = await db.execute(
select(
RadarDataORM.satellite_family,
RadarDataORM.product_unique_id,
func.count(RadarDataORM.id).label("count"),
)
.where(RadarDataORM.satellite_family.in_(["LT1", "S1"]))
.where(RadarDataORM.product_unique_id.is_not(None))
.group_by(RadarDataORM.satellite_family, RadarDataORM.product_unique_id)
.having(func.count(RadarDataORM.id) > 1)
.order_by(func.count(RadarDataORM.id).desc())
.limit(50)
)
return [
{
"family": family,
"product_unique_id": product_unique_id,
"count": int(count or 0),
}
for family, product_unique_id, count in result.all()
]
async def _mark_old_assets_inactive(db: AsyncSession, old_pools: Sequence[str]) -> Dict[str, int]:
if not old_pools:
return {"source_assets": 0}
old_filter = _path_prefix_filter(SourceProductAssetORM.file_path, old_pools)
if old_filter is None:
return {"source_assets": 0}
result = await db.execute(
update(SourceProductAssetORM)
.where(old_filter)
.where(SourceProductAssetORM.source_format.in_(["LT1_DIR", "S1_SAFE_DIR"]))
.values(is_active=False, missing_since=datetime.utcnow())
.execution_options(synchronize_session=False)
)
return {"source_assets": int(result.rowcount or 0)}
def _quarantine_old_pools(old_pools: Iterable[str], suffix: str) -> List[Dict[str, Any]]:
results: List[Dict[str, Any]] = []
for raw_path in old_pools:
path = _norm(raw_path)
if not path:
continue
target = f"{path}.{suffix}"
result: Dict[str, Any] = {
"source": path,
"target": target,
"source_exists": os.path.exists(path),
"target_exists": os.path.exists(target),
}
if not os.path.exists(path):
results.append(result)
continue
if os.path.exists(target):
result["status"] = "skipped_target_exists"
results.append(result)
continue
if Path(path).anchor == path:
raise ValueError(f"Refusing to quarantine drive root: {path}")
shutil.move(path, target)
result["status"] = "moved"
results.append(result)
return results
async def _run(args: argparse.Namespace) -> Dict[str, Any]:
if database.AsyncSessionLocal is None:
database.init_db(settings.DATABASE_URL)
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized")
old_pools = [_norm(item) for item in (args.old_pool or []) if str(item or "").strip()]
zip_roots = [_norm(item) for item in split_env_paths(settings.SOURCE_PRODUCT_DIRS)]
async with database.AsyncSessionLocal() as db:
before = await _db_summary(db, old_pools, zip_roots)
result: Dict[str, Any] = {
"apply": bool(args.apply),
"old_pools": [_measure_top_level(path) for path in old_pools],
"source_product_dirs": zip_roots,
"before": before,
"archive_migration_gap_before": await _archive_migration_gap_summary(db, old_pools),
}
if args.apply:
result["root_registry_sync"] = await root_registry_service.sync_from_settings(db)
if args.scan_archives:
root_ids = await _target_scan_root_ids(db, bind_orbits=bool(args.bind_orbits))
result["asset_inventory_scan"] = await asset_inventory_service.scan_configured_roots(
db,
inventory_types=["source_product", "orbit_asset"] if args.bind_orbits else ["source_product"],
root_ids=root_ids,
bind_orbits=bool(args.bind_orbits),
)
if args.build_archive_previews:
result["archive_preview_build"] = await _build_archive_previews(
db,
limit=max(0, int(args.preview_build_limit)),
force=bool(args.force_preview_rebuild),
)
if args.mark_old_assets_inactive:
result["marked_old_assets_inactive"] = await _mark_old_assets_inactive(db, old_pools)
await db.commit()
else:
result["planned_actions"] = [
"sync managed roots from .env",
"scan SOURCE_PRODUCT_DIRS for LT1_ARCHIVE/S1_ZIP",
"upsert radar_data by logical product id",
]
if args.mark_old_assets_inactive:
result["planned_actions"].append("mark old LT1_DIR/S1_SAFE_DIR source assets inactive")
result["roots"] = await _list_zip_roots(db)
result["after"] = await _db_summary(db, old_pools, zip_roots)
result["archive_migration_gap_after"] = await _archive_migration_gap_summary(db, old_pools)
result["duplicate_products"] = await _count_duplicate_products(db)
if args.preview_sample > 0:
result["preview_archive_samples"] = await _sample_preview_archive_sources(db, args.preview_sample)
if args.quarantine_old_pools:
if not args.apply:
result["quarantine"] = {"skipped": True, "reason": "requires --apply"}
elif result["after"].get("radar_file_paths_under_old_pools", 0) > 0:
result["quarantine"] = {
"skipped": True,
"reason": "database still has radar_data.file_path under old pools",
}
else:
result["quarantine"] = _quarantine_old_pools(old_pools, args.quarantine_suffix)
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apply", action="store_true", help="write database changes and run scans")
parser.add_argument("--scan-archives", action="store_true", default=True, help="scan configured source product roots")
parser.add_argument("--no-scan-archives", action="store_false", dest="scan_archives")
parser.add_argument("--bind-orbits", action="store_true", default=True, help="bind scenes to orbit assets after scan")
parser.add_argument("--no-bind-orbits", action="store_false", dest="bind_orbits")
parser.add_argument("--mark-old-assets-inactive", action="store_true", help="mark old LT1_DIR/S1_SAFE_DIR assets inactive")
parser.add_argument("--preview-sample", type=int, default=5, help="try archive preview extraction on N migrated records")
parser.add_argument("--build-archive-previews", action="store_true", help="build WebP preview caches for LT1_ARCHIVE/S1_ZIP records")
parser.add_argument("--preview-build-limit", type=int, default=0, help="limit preview cache builds; 0 means no limit")
parser.add_argument("--force-preview-rebuild", action="store_true", help="rebuild archive preview caches even if READY")
parser.add_argument("--quarantine-old-pools", action="store_true", help="rename old unpacked pools after migration checks pass")
parser.add_argument("--quarantine-suffix", default="__quarantine_20260616", help="suffix appended to old pool directory names")
parser.add_argument("--old-pool", action="append", default=[OLD_LT1_POOL, OLD_S1_POOL], help="old unpacked source pool path")
args = parser.parse_args()
payload = asyncio.run(_run(args))
print(json.dumps(payload, ensure_ascii=False, indent=2, default=str))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/prepare_landsar_dem_int16.py
+106 -30
View File
@@ -2,16 +2,21 @@
"""
Prepare reusable LandSAR DEM GeoTIFFs.
The script converts large DEM rasters to uncompressed Int16 GeoTIFFs with a
stable nodata value. It streams data by windows, so it can process the 10 m
Heilongjiang DEM and the COPDEM China DEM without loading them into memory.
The script has two explicit modes:
1. Convert a large DEM raster once to an uncompressed Int16 GeoTIFF.
2. Crop-copy a regional DEM from that prepared Int16 GeoTIFF without changing
pixel values.
Both modes stream data by windows, so they do not load large DEMs into memory.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import tempfile
from pathlib import Path
from typing import Iterable, Optional
@@ -160,21 +165,34 @@ def convert_dem(
*,
dem_root: Path,
output_root: Path,
target_path: Optional[Path],
bbox: Optional[tuple[float, float, float, float]],
suffix: str,
nodata: int,
block_size: int,
overwrite: bool,
dry_run: bool,
crop_only: bool,
) -> Path:
source = _source_path(source_text, dem_root)
stem = _safe_stem(source_text, source)
if suffix:
stem = f"{stem}_{suffix.strip('_')}"
target = output_root / f"{stem}_int16.tif"
if target_path:
target = target_path if target_path.is_absolute() else output_root / target_path
else:
target = output_root / f"{stem}_int16.tif"
if crop_only and not bbox:
raise ValueError("--crop-only requires --bbox because full-size crop-copy is not useful")
with rasterio.open(source) as src:
src_crs = src.crs or DEFAULT_CRS
source_dtype = str(src.dtypes[0]).lower()
if crop_only and source_dtype != "int16":
raise ValueError(
f"--crop-only requires an already prepared Int16 GeoTIFF; got dtype={src.dtypes[0]} from {source}"
)
if bbox:
window = _align_window(from_bounds(*bbox, transform=src.transform), src.width, src.height)
else:
@@ -182,17 +200,21 @@ def convert_dem(
window = Window(int(window.col_off), int(window.row_off), int(window.width), int(window.height))
transform = src.window_transform(window)
bounds = array_bounds(int(window.height), int(window.width), transform)
estimated_bytes = int(window.width) * int(window.height) * np.dtype("int16").itemsize
target_dtype = source_dtype if crop_only else "int16"
target_nodata = src.nodata if crop_only else nodata
mode = "crop-copy-int16" if crop_only else "convert-int16"
estimated_bytes = int(window.width) * int(window.height) * np.dtype(target_dtype).itemsize
print(f"Source: {source}")
print(f" mode={mode}")
print(f" driver={src.driver} dtype={src.dtypes[0]} size={src.width}x{src.height} crs={src.crs or 'EPSG:4326 assumed'}")
print(f" output window={int(window.width)}x{int(window.height)} bounds={tuple(round(v, 8) for v in bounds)}")
print(f" target={target}")
print(f" estimated raw int16 size={_format_gib(estimated_bytes)}")
print(f" estimated raw {target_dtype} size={_format_gib(estimated_bytes)}")
if dry_run:
return target
output_root.mkdir(parents=True, exist_ok=True)
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and not overwrite:
raise FileExistsError(f"target exists; pass --overwrite to replace it: {target}")
@@ -202,10 +224,10 @@ def convert_dem(
height=int(window.height),
width=int(window.width),
count=1,
dtype="int16",
dtype=target_dtype,
crs=src_crs,
transform=transform,
nodata=nodata,
nodata=target_nodata,
compress="NONE",
tiled=True,
blockxsize=512,
@@ -216,27 +238,66 @@ def convert_dem(
profile.pop("photometric", None)
profile.pop("predictor", None)
if target.exists():
target.unlink()
temp_path: Optional[Path] = None
with rasterio.open(target, "w", **profile) as dst:
total_pixels = int(window.width) * int(window.height)
done_pixels = 0
last_percent = -1
for rel_window in _iter_windows(int(window.width), int(window.height), block_size):
src_window = Window(
window.col_off + rel_window.col_off,
window.row_off + rel_window.row_off,
rel_window.width,
rel_window.height,
)
data = src.read(1, window=src_window, masked=True)
dst.write(_convert_array(data, nodata), 1, window=rel_window)
done_pixels += int(rel_window.width) * int(rel_window.height)
percent = int(done_pixels * 100 / max(1, total_pixels))
if percent != last_percent and (percent % 5 == 0 or percent == 100):
print(f" progress={percent}%")
last_percent = percent
try:
with tempfile.NamedTemporaryFile(
prefix=f"{target.stem}.",
suffix=".tmp.tif",
dir=str(target.parent),
delete=False,
) as tmp:
temp_path = Path(tmp.name)
with rasterio.open(temp_path, "w", **profile) as dst:
total_pixels = int(window.width) * int(window.height)
done_pixels = 0
last_percent = -1
for rel_window in _iter_windows(int(window.width), int(window.height), block_size):
src_window = Window(
window.col_off + rel_window.col_off,
window.row_off + rel_window.row_off,
rel_window.width,
rel_window.height,
)
if crop_only:
data = src.read(1, window=src_window, masked=False)
else:
data = _convert_array(src.read(1, window=src_window, masked=True), nodata)
dst.write(data, 1, window=rel_window)
done_pixels += int(rel_window.width) * int(rel_window.height)
percent = int(done_pixels * 100 / max(1, total_pixels))
if percent != last_percent and (percent % 5 == 0 or percent == 100):
print(f" progress={percent}%")
last_percent = percent
os.replace(temp_path, target)
temp_path = None
manifest = target.with_suffix(target.suffix + ".json")
manifest.write_text(
json.dumps(
{
"source": str(source),
"target": str(target),
"mode": mode,
"bbox": list(bbox) if bbox else None,
"bounds": [float(value) for value in bounds],
"width": int(window.width),
"height": int(window.height),
"source_dtype": src.dtypes[0],
"target_dtype": target_dtype,
"source_nodata": src.nodata,
"target_nodata": target_nodata,
"big_tiff": True,
"compress": "NONE",
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
finally:
if temp_path and temp_path.exists():
temp_path.unlink(missing_ok=True)
actual_size = target.stat().st_size if target.exists() else 0
print(f"Done: {target} ({_format_gib(actual_size)})")
@@ -245,7 +306,7 @@ def convert_dem(
def main() -> int:
parser = argparse.ArgumentParser(
description="Convert large DEMs to reusable LandSAR Int16 GeoTIFFs."
description="Prepare reusable LandSAR Int16 GeoTIFFs and regional crop copies."
)
parser.add_argument(
"--source",
@@ -258,6 +319,11 @@ def main() -> int:
)
parser.add_argument("--dem-root", default=str(DEFAULT_DEM_ROOT))
parser.add_argument("--output-root", default=str(DEFAULT_OUTPUT_ROOT))
parser.add_argument(
"--target",
default="",
help="Optional exact target path. Only valid with one --source.",
)
parser.add_argument(
"--bbox",
default="",
@@ -268,21 +334,31 @@ def main() -> int:
parser.add_argument("--block-size", type=int, default=2048)
parser.add_argument("--overwrite", action="store_true")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--crop-only",
action="store_true",
help="Copy a bbox window from an already prepared Int16 GeoTIFF without value conversion.",
)
args = parser.parse_args()
sources = args.source or ["HeiLongJiang10M_DEM", "COPDEM_GLO30_China_4326_DEM"]
if args.target and len(sources) != 1:
raise ValueError("--target can only be used with exactly one --source")
bbox = _parse_bbox(args.bbox)
target_path = Path(args.target) if args.target else None
for source in sources:
convert_dem(
source,
dem_root=Path(args.dem_root),
output_root=Path(args.output_root),
target_path=target_path,
bbox=bbox,
suffix=args.suffix,
nodata=args.nodata,
block_size=args.block_size,
overwrite=args.overwrite,
dry_run=args.dry_run,
crop_only=args.crop_only,
)
return 0
scripts/repack_unpacked_sources_to_archives.py
+189
View File
@@ -0,0 +1,189 @@
"""Repack unpacked LT1/Sentinel-1 source directories back to archives.
This script never deletes the unpacked source directories. It only creates
missing archives in the configured local ZIP pools, then can optionally run the
source-archive migration script.
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from datetime import datetime
from pathlib import Path
from typing import Iterable, List, Sequence
PROJECT_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_LT1_SRC = Path(r"D:\LuTan1_Image_Pool")
DEFAULT_LT1_DST = Path(r"D:\LuTan1_Image_Pool_Zip")
DEFAULT_S1_SRC = Path(r"D:\Sentinel1_Image_Pool")
DEFAULT_S1_DST = Path(r"D:\Sentinel1_Image_Pool_ZIP")
def _ts() -> str:
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def _log(message: str) -> None:
print(f"[{_ts()}] {message}", flush=True)
def _existing_lt1_archive(dst: Path, name: str) -> Path | None:
for suffix in (".tar.gz", ".tgz", ".tar", ".zip"):
candidate = dst / f"{name}{suffix}"
if candidate.is_file() and candidate.stat().st_size > 0:
return candidate
return None
def _run_tar(args: Sequence[str]) -> None:
command = ["tar.exe", *args]
result = subprocess.run(command, cwd=str(PROJECT_ROOT))
if result.returncode != 0:
raise RuntimeError(f"tar.exe failed with code {result.returncode}: {' '.join(command)}")
def _pack_with_tar(src_root: Path, item_name: str, target_tmp: Path, target_final: Path, *, mode: str) -> None:
if target_tmp.exists():
target_tmp.unlink()
if target_final.exists():
target_final.unlink()
if mode == "tar.gz":
_run_tar(["-czf", str(target_tmp), "-C", str(src_root), item_name])
elif mode == "zip":
_run_tar(["-a", "-cf", str(target_tmp), "-C", str(src_root), item_name])
else:
raise ValueError(f"Unsupported archive mode: {mode}")
if not target_tmp.is_file() or target_tmp.stat().st_size <= 0:
raise RuntimeError(f"Archive was not created: {target_tmp}")
target_tmp.replace(target_final)
def _iter_dirs(root: Path, pattern: str = "*") -> List[Path]:
if not root.is_dir():
raise FileNotFoundError(f"Source directory not found: {root}")
return sorted([item for item in root.glob(pattern) if item.is_dir()], key=lambda p: p.name)
def repack_lt1(src: Path, dst: Path, *, limit: int = 0, dry_run: bool = False) -> tuple[int, int]:
dst.mkdir(parents=True, exist_ok=True)
dirs = [item for item in _iter_dirs(src) if _existing_lt1_archive(dst, item.name) is None]
if limit > 0:
dirs = dirs[:limit]
_log(f"LT1 missing archives: {len(dirs)}")
packed = 0
for index, item in enumerate(dirs, start=1):
final_path = dst / f"{item.name}.tar.gz"
tmp_path = dst / f"{item.name}.tmp.tar.gz"
_log(f"PACK LT1 [{index}/{len(dirs)}] {item.name}")
if dry_run:
continue
try:
_pack_with_tar(src, item.name, tmp_path, final_path, mode="tar.gz")
except Exception:
if tmp_path.exists():
tmp_path.unlink()
raise
packed += 1
return packed, len(dirs)
def repack_s1(src: Path, dst: Path, *, limit: int = 0, dry_run: bool = False) -> tuple[int, int]:
dst.mkdir(parents=True, exist_ok=True)
missing = []
for item in _iter_dirs(src, "*.SAFE"):
base = item.name[:-5] if item.name.upper().endswith(".SAFE") else item.name
final_path = dst / f"{base}.zip"
if not final_path.is_file() or final_path.stat().st_size <= 0:
missing.append(item)
if limit > 0:
missing = missing[:limit]
_log(f"Sentinel-1 missing ZIP archives: {len(missing)}")
packed = 0
for index, item in enumerate(missing, start=1):
base = item.name[:-5] if item.name.upper().endswith(".SAFE") else item.name
final_path = dst / f"{base}.zip"
tmp_path = dst / f"{base}.tmp.zip"
_log(f"PACK S1 [{index}/{len(missing)}] {item.name}")
if dry_run:
continue
try:
_pack_with_tar(src, item.name, tmp_path, final_path, mode="zip")
except Exception:
if tmp_path.exists():
tmp_path.unlink()
raise
packed += 1
return packed, len(missing)
def run_migration(python_exe: str) -> None:
migration_script = PROJECT_ROOT / "scripts" / "migrate_source_archives_to_zip.py"
commands = [
[
python_exe,
str(migration_script),
"--apply",
"--no-bind-orbits",
"--preview-sample",
"5",
],
[
python_exe,
str(migration_script),
"--apply",
"--no-scan-archives",
"--no-bind-orbits",
"--build-archive-previews",
"--preview-build-limit",
"0",
"--preview-sample",
"5",
],
[
python_exe,
str(migration_script),
"--preview-sample",
"5",
],
]
for command in commands:
_log(f"RUN {' '.join(command)}")
subprocess.run(command, cwd=str(PROJECT_ROOT), check=True)
def main(argv: Iterable[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--lt-src", default=str(DEFAULT_LT1_SRC))
parser.add_argument("--lt-dst", default=str(DEFAULT_LT1_DST))
parser.add_argument("--s1-src", default=str(DEFAULT_S1_SRC))
parser.add_argument("--s1-dst", default=str(DEFAULT_S1_DST))
parser.add_argument("--only", choices=["all", "lt1", "s1"], default="all")
parser.add_argument("--limit", type=int, default=0, help="limit each sensor; 0 means no limit")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--run-migration", action="store_true")
parser.add_argument("--python", default=sys.executable)
args = parser.parse_args(list(argv) if argv is not None else None)
_log("Repack started. Existing archives are skipped. Source directories are never deleted.")
if args.only in {"all", "lt1"}:
packed, total = repack_lt1(Path(args.lt_src), Path(args.lt_dst), limit=args.limit, dry_run=args.dry_run)
_log(f"LT1 done: packed={packed}, planned={total}")
if args.only in {"all", "s1"}:
packed, total = repack_s1(Path(args.s1_src), Path(args.s1_dst), limit=args.limit, dry_run=args.dry_run)
_log(f"Sentinel-1 done: packed={packed}, planned={total}")
if args.run_migration and not args.dry_run:
run_migration(args.python)
_log("Repack finished.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/repair_lt1_archive_metadata.py
+292
View File
@@ -0,0 +1,292 @@
"""Repair LT1 archive metadata after XML imageDataType/product_type mix-up."""
from __future__ import annotations
import argparse
import asyncio
import math
import os
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Optional
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from geoalchemy2.shape import from_shape # noqa: E402
from shapely.geometry import Polygon # noqa: E402
from sqlalchemy import func, or_, select # noqa: E402
from backend.app import database # noqa: E402
from backend.app.config import settings # noqa: E402
from backend.app.models import RadarDataORM, SourceProductAssetORM # noqa: E402
from backend.app.utils import parse_lt1_radar_filename # noqa: E402
COMPLEX_TOKENS = {"COMPLEX", "SLC", "SSC"}
def _strip_known_suffix(name: str) -> str:
lower = name.lower()
for suffix in (".tar.gz", ".tgz", ".zip", ".tar"):
if lower.endswith(suffix):
return name[: -len(suffix)]
return os.path.splitext(name)[0]
def _scene_name_from_path(path: Optional[str], fallback: Optional[str] = None) -> str:
name = os.path.basename(str(path or "").strip())
if not name:
name = str(fallback or "").strip()
return _strip_known_suffix(name)
def _metadata_dict(value: Any) -> Dict[str, Any]:
return dict(value or {}) if isinstance(value, dict) else {}
def _ordered_closed_polygon(points: Any) -> Optional[list[tuple[float, float]]]:
unique: list[tuple[float, float]] = []
for point in points or []:
try:
lon = float(point[0])
lat = float(point[1])
except (TypeError, ValueError, IndexError):
continue
current = (lon, lat)
if unique and abs(unique[-1][0] - lon) < 1e-12 and abs(unique[-1][1] - lat) < 1e-12:
continue
if unique and abs(unique[0][0] - lon) < 1e-12 and abs(unique[0][1] - lat) < 1e-12:
continue
if current not in unique:
unique.append(current)
if len(unique) < 3:
return None
if len(unique) == 4:
center_lon = sum(item[0] for item in unique) / len(unique)
center_lat = sum(item[1] for item in unique) / len(unique)
ordered = sorted(
unique,
key=lambda item: math.atan2(item[1] - center_lat, item[0] - center_lon),
)
else:
ordered = unique
if ordered[0] != ordered[-1]:
ordered.append(ordered[0])
try:
polygon = Polygon(ordered)
if polygon.is_valid and not polygon.is_empty and polygon.area > 0:
return ordered
except Exception:
return None
return None
def _normalize_lt1_metadata(metadata: Dict[str, Any], parsed: Dict[str, Any]) -> Dict[str, Any]:
updated = dict(metadata)
previous_product_type = str(updated.get("product_type") or "").strip().upper()
if previous_product_type == "COMPLEX" and not updated.get("image_data_type"):
updated["image_data_type"] = "COMPLEX"
for key, value in parsed.items():
if value not in (None, ""):
updated[key] = value
if parsed.get("source_product_token"):
updated["filename_class_token"] = parsed.get("source_product_token")
if previous_product_type == "COMPLEX":
updated["xml_image_data_type_repaired_from_product_type"] = True
return updated
def _ready_reason(row: Dict[str, Any], metadata: Dict[str, Any], coverage_polygon: Any) -> tuple[bool, Optional[str]]:
reasons = []
if not coverage_polygon or len(coverage_polygon) < 3:
reasons.append("missing_footprint")
if not row.get("imaging_date"):
reasons.append("missing_date")
if not row.get("orbit_direction"):
reasons.append("missing_orbit_direction")
if not row.get("imaging_mode"):
reasons.append("missing_imaging_mode")
if not row.get("polarization"):
reasons.append("missing_polarization")
tokens = {
str(row.get("product_type") or "").strip().upper(),
str(row.get("image_data_type") or "").strip().upper(),
str(row.get("source_product_token") or "").strip().upper(),
str(row.get("product_variant") or "").strip().upper(),
str(metadata.get("image_data_type") or "").strip().upper(),
str(metadata.get("product_variant") or "").strip().upper(),
str(metadata.get("filename_class_token") or "").strip().upper(),
str(metadata.get("source_product_token") or "").strip().upper(),
}
if not tokens.intersection(COMPLEX_TOKENS):
reasons.append("not_complex_source")
if reasons:
return False, ";".join(reasons)
return True, None
def _bbox(points: Any) -> Optional[tuple[float, float, float, float]]:
ordered = _ordered_closed_polygon(points)
if not ordered or len(ordered) < 4:
return None
try:
lons = [float(item[0]) for item in ordered]
lats = [float(item[1]) for item in ordered]
except (TypeError, ValueError, IndexError):
return None
return min(lons), min(lats), max(lons), max(lats)
async def repair(apply: bool) -> Dict[str, int]:
database.init_db(settings.DATABASE_URL)
stats = {
"source_seen": 0,
"source_repaired": 0,
"radar_seen": 0,
"radar_repaired": 0,
"radar_ready": 0,
"radar_not_ready": 0,
"source_polygon_repaired": 0,
"radar_polygon_repaired": 0,
"skipped_unparsed": 0,
}
async with database.AsyncSessionLocal() as db:
assets = (
await db.execute(
select(SourceProductAssetORM).where(
SourceProductAssetORM.satellite_family == "LT1",
SourceProductAssetORM.source_format == "LT1_ARCHIVE",
)
)
).scalars().all()
for asset in assets:
stats["source_seen"] += 1
scene_name = _scene_name_from_path(asset.file_path, asset.logical_product_uid)
parsed = parse_lt1_radar_filename(scene_name)
if not parsed:
stats["skipped_unparsed"] += 1
continue
metadata = _normalize_lt1_metadata(_metadata_dict(asset.metadata_json), parsed)
ordered_polygon = _ordered_closed_polygon(metadata.get("coverage_polygon"))
if ordered_polygon:
metadata["coverage_polygon"] = ordered_polygon
metadata["coverage_bbox"] = _bbox(ordered_polygon)
stats["source_polygon_repaired"] += 1
asset.product_type = parsed.get("product_type") or asset.product_type
asset.product_level = parsed.get("product_level") or asset.product_level
asset.imaging_mode = parsed.get("imaging_mode") or asset.imaging_mode
asset.polarization = parsed.get("polarization") or asset.polarization
asset.absolute_orbit = parsed.get("orbit_circle") or asset.absolute_orbit
asset.imaging_date = parsed.get("imaging_date") or asset.imaging_date
asset.satellite = parsed.get("satellite") or asset.satellite
asset.logical_product_uid = scene_name
asset.metadata_json = metadata
asset.parser_version = "asset_inventory_v2"
asset.updated_at = datetime.utcnow()
stats["source_repaired"] += 1
radars = (
await db.execute(
select(RadarDataORM).where(
RadarDataORM.satellite_family == "LT1",
or_(
RadarDataORM.source_format == "LT1_ARCHIVE",
RadarDataORM.file_path.ilike("%.tar.gz"),
RadarDataORM.file_path.ilike("%.tgz"),
RadarDataORM.file_path.ilike("%.tar"),
RadarDataORM.file_path.ilike("%.zip"),
),
)
)
).scalars().all()
for radar in radars:
stats["radar_seen"] += 1
scene_name = _scene_name_from_path(radar.file_path, radar.product_unique_id)
parsed = parse_lt1_radar_filename(scene_name)
if not parsed:
stats["skipped_unparsed"] += 1
continue
metadata = _normalize_lt1_metadata(_metadata_dict(radar.metadata_json), parsed)
ordered_polygon = _ordered_closed_polygon(radar.coverage_polygon or metadata.get("coverage_polygon"))
if ordered_polygon:
metadata["coverage_polygon"] = ordered_polygon
metadata["coverage_bbox"] = _bbox(ordered_polygon)
radar.coverage_polygon = ordered_polygon
stats["radar_polygon_repaired"] += 1
product_type = parsed.get("product_type") or radar.product_type
source_token = parsed.get("source_product_token") or radar.source_product_token
row = {
"product_type": product_type,
"image_data_type": radar.image_data_type or metadata.get("image_data_type") or "COMPLEX",
"source_product_token": source_token,
"product_variant": radar.product_variant or metadata.get("product_variant"),
"imaging_date": parsed.get("imaging_date") or radar.imaging_date,
"orbit_direction": radar.orbit_direction,
"imaging_mode": parsed.get("imaging_mode") or radar.imaging_mode,
"polarization": parsed.get("polarization") or radar.polarization,
}
ready, reason = _ready_reason(row, metadata, radar.coverage_polygon)
radar.product_type = product_type
radar.source_product_token = source_token
radar.product_level = parsed.get("product_level") or radar.product_level
radar.imaging_mode = parsed.get("imaging_mode") or radar.imaging_mode
radar.polarization = parsed.get("polarization") or radar.polarization
radar.orbit_circle = parsed.get("orbit_circle") or radar.orbit_circle
radar.absolute_orbit = parsed.get("orbit_circle") or radar.absolute_orbit
radar.imaging_date = parsed.get("imaging_date") or radar.imaging_date
radar.satellite = parsed.get("satellite") or radar.satellite
radar.product_unique_id = scene_name
radar.image_data_type = row["image_data_type"]
radar.image_data_format = radar.image_data_format or "ARCHIVE"
radar.metadata_json = metadata
radar.insar_source_ready = ready
radar.insar_source_reason = reason
bbox = _bbox(radar.coverage_polygon)
if bbox:
radar.min_lon, radar.min_lat, radar.max_lon, radar.max_lat = bbox
try:
poly = Polygon(radar.coverage_polygon)
if not poly.is_valid:
poly = poly.buffer(0)
if not poly.is_empty:
radar.geom = from_shape(poly, srid=4326)
except Exception:
pass
stats["radar_repaired"] += 1
if ready:
stats["radar_ready"] += 1
else:
stats["radar_not_ready"] += 1
if apply:
await db.commit()
else:
await db.rollback()
ready_count = (
await db.execute(
select(func.count(RadarDataORM.id)).where(
RadarDataORM.satellite_family == "LT1",
RadarDataORM.insar_source_ready.is_(True),
)
)
).scalar_one()
stats["db_lt1_ready_after"] = int(ready_count or 0)
return stats
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="commit database changes")
args = parser.parse_args()
stats = asyncio.run(repair(apply=args.apply))
mode = "APPLY" if args.apply else "DRY-RUN"
print(f"{mode} LT1 archive metadata repair")
for key in sorted(stats):
print(f"{key}: {stats[key]}")
if __name__ == "__main__":
main()
scripts/reset_lt1_s1_source_catalog.py
+332
View File
@@ -0,0 +1,332 @@
"""Reset LT1/Sentinel-1 source catalog rows and radar preview caches.
This maintenance script is intentionally scoped to source-scene registration.
It does not delete source archives, unpacked source directories, orbit assets,
D-InSAR/SBAS/GF3 product catalogs, task logs, users, or flood products.
"""
from __future__ import annotations
import argparse
import asyncio
import os
import shutil
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List
from sqlalchemy import delete, func, or_, select, update
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from backend.app import database # noqa: E402
from backend.app.config import settings # noqa: E402
from backend.app.models import ( # noqa: E402
AssetInventoryIssueORM,
AssetInventoryStateORM,
PairingCacheStateORM,
PairingDirtySceneORM,
PairingMetricCacheORM,
PairingNetworkEdgeORM,
PairingNetworkRunORM,
RadarDataORM,
SARSceneGeoORM,
SceneOrbitBindingORM,
ScanStateORM,
SourceProductAssetORM,
TimeseriesStackPlanEdgeORM,
TimeseriesStackPlanItemORM,
)
TARGET_FAMILIES = ("LT1", "S1")
TARGET_SOURCE_FORMATS = ("LT1_DIR", "LT1_ARCHIVE", "S1_SAFE_DIR", "S1_ZIP")
RADAR_CACHE_DIRS = (
settings.RADAR_RAW_CACHE_DIR,
settings.RADAR_GEO_CACHE_DIR,
os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources"),
)
def _target_radar_filter() -> Any:
return or_(
RadarDataORM.satellite_family.in_(TARGET_FAMILIES),
RadarDataORM.source_format.in_(TARGET_SOURCE_FORMATS),
)
def _target_asset_filter() -> Any:
return or_(
SourceProductAssetORM.satellite_family.in_(TARGET_FAMILIES),
SourceProductAssetORM.source_format.in_(TARGET_SOURCE_FORMATS),
)
async def _count(db, stmt) -> int:
return int((await db.execute(stmt)).scalar_one() or 0)
def _cache_dir_summary(paths: Iterable[str]) -> List[Dict[str, Any]]:
payload = []
for raw_path in paths:
path = Path(raw_path)
files = 0
bytes_total = 0
if path.exists():
for item in path.rglob("*"):
if not item.is_file():
continue
try:
files += 1
bytes_total += item.stat().st_size
except OSError:
continue
payload.append(
{
"path": str(path),
"exists": path.exists(),
"files": files,
"bytes": bytes_total,
}
)
return payload
def _clear_cache_dirs(paths: Iterable[str]) -> List[Dict[str, Any]]:
results = []
for raw_path in paths:
path = Path(raw_path)
before = _cache_dir_summary([str(path)])[0]
if path.exists():
for item in path.iterdir():
if item.is_dir():
shutil.rmtree(item)
else:
item.unlink()
path.mkdir(parents=True, exist_ok=True)
after = _cache_dir_summary([str(path)])[0]
results.append({"before": before, "after": after})
return results
async def _summary(db) -> Dict[str, Any]:
radar_filter = _target_radar_filter()
asset_filter = _target_asset_filter()
radar_ids = select(RadarDataORM.id).where(radar_filter)
asset_ids = select(SourceProductAssetORM.id).where(asset_filter)
by_source_format = {}
radar_format_rows = await db.execute(
select(RadarDataORM.source_format, func.count(RadarDataORM.id))
.where(radar_filter)
.group_by(RadarDataORM.source_format)
.order_by(RadarDataORM.source_format.asc())
)
for source_format, count in radar_format_rows.all():
by_source_format[str(source_format or "NULL")] = int(count or 0)
asset_by_source_format = {}
asset_format_rows = await db.execute(
select(SourceProductAssetORM.source_format, func.count(SourceProductAssetORM.id))
.where(asset_filter)
.group_by(SourceProductAssetORM.source_format)
.order_by(SourceProductAssetORM.source_format.asc())
)
for source_format, count in asset_format_rows.all():
asset_by_source_format[str(source_format or "NULL")] = int(count or 0)
return {
"radar_data_count": await _count(db, select(func.count(RadarDataORM.id)).where(radar_filter)),
"radar_data_by_source_format": by_source_format,
"source_product_asset_count": await _count(db, select(func.count(SourceProductAssetORM.id)).where(asset_filter)),
"source_product_assets_by_source_format": asset_by_source_format,
"scene_orbit_binding_count": await _count(
db,
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.radar_data_id.in_(radar_ids)),
),
"asset_inventory_issue_count": await _count(
db,
select(func.count(AssetInventoryIssueORM.id)).where(
or_(
AssetInventoryIssueORM.radar_data_id.in_(radar_ids),
AssetInventoryIssueORM.asset_ref_id.in_(asset_ids),
)
),
),
"sar_scene_geo_blocker_count": await _count(
db,
select(func.count(SARSceneGeoORM.id)).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)),
),
"timeseries_plan_item_refs": await _count(
db,
select(func.count(TimeseriesStackPlanItemORM.id)).where(
TimeseriesStackPlanItemORM.radar_data_ref_id.in_(radar_ids)
),
),
"timeseries_plan_edge_refs": await _count(
db,
select(func.count(TimeseriesStackPlanEdgeORM.id)).where(
or_(
TimeseriesStackPlanEdgeORM.master_scene_ref_id.in_(radar_ids),
TimeseriesStackPlanEdgeORM.slave_scene_ref_id.in_(radar_ids),
)
),
),
"pairing_metric_cache_count": await _count(db, select(func.count(PairingMetricCacheORM.id))),
"pairing_dirty_scene_count": await _count(db, select(func.count(PairingDirtySceneORM.id))),
"pairing_network_run_count": await _count(db, select(func.count(PairingNetworkRunORM.id))),
"pairing_network_edge_count": await _count(db, select(func.count(PairingNetworkEdgeORM.id))),
"radar_cache_dirs": _cache_dir_summary(RADAR_CACHE_DIRS),
}
async def _apply_reset(db, *, allow_sar_scene_geo: bool) -> Dict[str, Any]:
radar_filter = _target_radar_filter()
asset_filter = _target_asset_filter()
radar_ids = select(RadarDataORM.id).where(radar_filter)
asset_ids = select(SourceProductAssetORM.id).where(asset_filter)
sar_blockers = await _count(
db,
select(func.count(SARSceneGeoORM.id)).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)),
)
if sar_blockers and not allow_sar_scene_geo:
raise RuntimeError(
f"Refusing to delete radar_data: {sar_blockers} SAR analysis rows reference LT1/S1 scenes. "
"Re-run with --allow-sar-scene-geo only if you intend to clear those analysis rows."
)
counts: Dict[str, int] = {}
if allow_sar_scene_geo:
result = await db.execute(delete(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id.in_(radar_ids)))
counts["sar_scene_geo_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(SceneOrbitBindingORM).where(SceneOrbitBindingORM.radar_data_id.in_(radar_ids)))
counts["scene_orbit_bindings_deleted"] = int(result.rowcount or 0)
result = await db.execute(
delete(AssetInventoryIssueORM).where(
or_(
AssetInventoryIssueORM.radar_data_id.in_(radar_ids),
AssetInventoryIssueORM.asset_ref_id.in_(asset_ids),
)
)
)
counts["asset_inventory_issues_deleted"] = int(result.rowcount or 0)
await db.execute(
update(TimeseriesStackPlanItemORM)
.where(TimeseriesStackPlanItemORM.radar_data_ref_id.in_(radar_ids))
.values(radar_data_ref_id=None)
)
await db.execute(
update(TimeseriesStackPlanEdgeORM)
.where(
or_(
TimeseriesStackPlanEdgeORM.master_scene_ref_id.in_(radar_ids),
TimeseriesStackPlanEdgeORM.slave_scene_ref_id.in_(radar_ids),
)
)
.values(master_scene_ref_id=None, slave_scene_ref_id=None, metric_cache_ref_id=None)
)
result = await db.execute(delete(PairingNetworkEdgeORM))
counts["pairing_network_edges_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(PairingNetworkRunORM))
counts["pairing_network_runs_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(PairingMetricCacheORM))
counts["pairing_metric_cache_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(PairingDirtySceneORM))
counts["pairing_dirty_scenes_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(RadarDataORM).where(radar_filter))
counts["radar_data_deleted"] = int(result.rowcount or 0)
result = await db.execute(delete(SourceProductAssetORM).where(asset_filter))
counts["source_product_assets_deleted"] = int(result.rowcount or 0)
await db.execute(delete(ScanStateORM).where(ScanStateORM.data_type == "radar"))
await db.execute(
update(AssetInventoryStateORM)
.where(AssetInventoryStateORM.inventory_type == "source_product")
.values(
status="NEVER_SCANNED",
last_scan_started_at=None,
last_scan_finished_at=None,
last_seen_entry_count=None,
last_asset_count=None,
last_issue_count=None,
needs_rescan=True,
last_error=None,
updated_at=datetime.utcnow(),
)
)
await db.execute(
update(PairingCacheStateORM).values(
status="READY",
scene_count=0,
pair_count=0,
dirty_scene_count=0,
last_error=None,
updated_at=datetime.utcnow(),
)
)
await db.commit()
return counts
async def _run(args: argparse.Namespace) -> Dict[str, Any]:
if database.AsyncSessionLocal is None:
database.init_db(settings.DATABASE_URL)
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized")
async with database.AsyncSessionLocal() as db:
before = await _summary(db)
payload: Dict[str, Any] = {
"apply": bool(args.apply),
"before": before,
"scope": {
"families": TARGET_FAMILIES,
"source_formats": TARGET_SOURCE_FORMATS,
"clears_preview_cache": bool(args.clear_preview_cache),
},
}
if args.apply:
payload["database_changes"] = await _apply_reset(db, allow_sar_scene_geo=bool(args.allow_sar_scene_geo))
if args.clear_preview_cache:
payload["preview_cache_changes"] = _clear_cache_dirs(RADAR_CACHE_DIRS)
else:
payload["planned_actions"] = [
"delete LT1/S1 radar_data rows",
"delete LT1/S1 source_product_assets rows",
"delete source inventory issues and scene-orbit bindings for those rows",
"clear pairing cache/network cache",
"reset source inventory scan state",
"clear radar preview cache directories" if args.clear_preview_cache else "keep radar preview cache directories",
]
payload["after"] = await _summary(db)
return payload
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--apply", action="store_true", help="perform the reset; default only prints a dry-run plan")
parser.add_argument("--clear-preview-cache", action="store_true", default=True, help="clear radar preview cache dirs")
parser.add_argument("--keep-preview-cache", action="store_false", dest="clear_preview_cache")
parser.add_argument("--allow-sar-scene-geo", action="store_true", help="also delete SAR analysis scene rows if they block reset")
args = parser.parse_args()
import json
print(json.dumps(asyncio.run(_run(args)), ensure_ascii=False, indent=2, default=str))
return 0
if __name__ == "__main__":
raise SystemExit(main())
scripts/unpack_archives_parallel.py
+7 -4
View File
@@ -457,6 +457,12 @@ def _process_archive(
target_root = _resolve_target_root(archive_path, source_dirs, target_dirs) if target_dirs else None
if not target_root:
target_root = os.path.dirname(archive_path)
if delete_archive:
log_fn(
logging.WARNING,
"ignoring delete_archive=true; local archives are the source of record: %s",
archive_path,
)
base_name = _strip_archive_extension(os.path.basename(archive_path), extensions)
output_dir = os.path.join(target_root, base_name)
@@ -489,9 +495,6 @@ def _process_archive(
if not extracted:
return {"status": "skipped", "archive_path": archive_path}
if delete_archive:
os.remove(archive_path)
progress_store.mark_processed(archive_path)
return {"status": "processed", "archive_path": archive_path}
@@ -559,7 +562,7 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None, con
or env.get("UNPACK_STORAGE_DIRS")
)
min_disk_gb = float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50"))
delete_archive = parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true"))
delete_archive = False
tmp_suffix = env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp")
extensions = parse_dirs(env.get("UNPACK_ARCHIVE_EXTS", ".tar.gz"))
scan_workers = parse_int(