Refactor local InSAR asset and production workflows

This commit is contained in:
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
+737 -9
View File
@@ -15,12 +15,13 @@ from shapely.geometry import Polygon
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..config import settings, split_env_paths
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
from .data_service import extract_geotiff_bounds
from .data_service import DataService, extract_geotiff_bounds
from .gf3_native_inventory_service import (
NATIVE_MANIFEST_NAME,
POLARIZATION_PRIORITY,
SKIP_DIR_NAMES,
scan_gf3_sarscape_native_roots,
)
from .image_service import image_service
@@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
CONVERTER_VERSION = "v1"
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK"
SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2"
def _utc_now() -> str:
@@ -72,6 +76,112 @@ def _path_kind(path: str) -> str:
return "relative"
def _path_key(path: str | Path) -> str:
return os.path.normcase(os.path.normpath(str(path))).lower()
def _has_native_geo_assets(path: Path) -> bool:
try:
return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir())
except OSError:
return False
def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]:
"""Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning.
Operators usually configure a durable GF3 root rather than a date folder.
Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer
direct ``*_geo`` children and fall back to recursive discovery.
"""
roots: list[str] = []
seen: set[str] = set()
def add(path: str | Path) -> None:
text = os.path.normpath(str(path))
key = _path_key(text)
if key and key not in seen:
seen.add(key)
roots.append(text)
for raw in native_dirs or []:
text = str(raw or "").strip()
if not text:
continue
if any(ch in text for ch in "*?"):
glob_root = Path(os.path.normpath(text))
try:
matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower())
except OSError:
matches = []
if matches:
for match in matches:
if match.is_dir():
add(match)
continue
add(glob_root)
continue
root = Path(os.path.normpath(text))
try:
is_dir = root.is_dir()
except OSError:
is_dir = False
if not is_dir:
add(root)
continue
if root.name.lower().endswith("_geo"):
add(root)
try:
immediate_geo_roots = sorted(
(
child
for child in root.iterdir()
if child.is_dir() and child.name.lower().endswith("_geo")
),
key=lambda item: item.name.lower(),
)
except OSError:
immediate_geo_roots = []
if immediate_geo_roots:
for child in immediate_geo_roots:
add(child)
continue
if _has_native_geo_assets(root):
add(root)
continue
found_recursive = 0
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
and not name.startswith(".gf3_")
]
selected: list[str] = []
remaining: list[str] = []
for name in dir_names:
if name.lower().endswith("_geo"):
selected.append(name)
else:
remaining.append(name)
for name in selected:
add(Path(current_dir) / name)
found_recursive += len(selected)
dir_names[:] = remaining
if found_recursive == 0:
add(root)
return roots
def _source_asset_uid(path: str) -> str:
normalized = os.path.normpath(str(path or "").strip())
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
@@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool:
return False
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None:
polygon = extract_geotiff_bounds(str(path))
if polygon and len(polygon) >= 4:
return polygon
@@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
return None
def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None:
if not asset:
return None
for key in ("path", "quicklook"):
text = _path_text(asset.get(key))
if not text:
continue
path = Path(text)
if not path.is_file():
continue
polygon = _polygon_from_raster(path)
if polygon and len(polygon) >= 4:
return polygon
return None
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
if not polygon:
return None, None
@@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None
return assets[0] if assets else None
def _is_quicklook_raster_path(path: Any) -> bool:
text = _path_text(path).lower()
return text.endswith("_ql.tif") or text.endswith("_ql.tiff")
def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
candidates = [
asset
for asset in assets
if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path"))
]
complete = [asset for asset in candidates if asset.get("complete") is not False]
return _select_default_asset(complete or candidates)
def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]:
output_assets: list[dict[str, Any]] = []
for asset in scene_manifest.get("assets") or []:
native_path = _path_text(asset.get("path"))
if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True:
continue
output_assets.append(
{
"polarization": str(asset.get("polarization") or "UNKNOWN").upper(),
"role": "native_geo",
"path": native_path,
"hdr": _path_text(asset.get("hdr")) or None,
"sml": _path_text(asset.get("sml")) or None,
"quicklook": _path_text(asset.get("quicklook")) or None,
"complete": bool(asset.get("complete")),
"status": "registered_native_geo",
}
)
return output_assets
def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]:
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
out_dir = _standard_scene_dir(scene_manifest, root)
manifest_path = out_dir / "gf3_native_preview_manifest.json"
assets = _quicklook_assets_from_scene_manifest(scene_manifest)
manifest = {
"schema": "gf3_sarscape_native_preview.v1",
"generated_at": _utc_now(),
"scene_name": scene_manifest.get("scene_name"),
"batch_name": scene_manifest.get("batch_name"),
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
"native_dir": scene_manifest.get("native_dir"),
"standard_dir": str(out_dir),
"manifest_path": str(manifest_path),
"status": "NATIVE_READY" if assets else "FAILED",
"assets": assets,
"summary": {
"native_assets": len(assets),
"quicklook_assets": 0,
"full_raster_materialized": True,
"webp_source": "native_geo",
},
"errors": [] if assets else [{"error": "no native _geo assets found"}],
}
_write_json(manifest_path, manifest)
return manifest
def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool:
"""Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``."""
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except Exception:
return False
try:
target = Path(target_path)
target.parent.mkdir(parents=True, exist_ok=True)
limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600)
with rasterio.open(source_path) as src:
scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
band = src.read(
1,
out_shape=(out_height, out_width),
masked=True,
resampling=Resampling.bilinear,
)
data = band.filled(0)
if data.dtype != np.uint8:
valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)]
if valid.size:
p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98])
scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1)
data = (scaled * 255).astype("uint8")
else:
data = np.zeros(data.shape, dtype="uint8")
else:
data = data.astype("uint8", copy=False)
mask = getattr(band, "mask", None)
if mask is None or np.ndim(mask) == 0:
alpha = np.full(data.shape, 255, dtype="uint8")
else:
alpha = np.where(mask, 0, 255).astype("uint8")
rgba = np.stack([data, data, data, alpha], axis=-1)
image = Image.fromarray(rgba, "RGBA")
image = image_service.make_edge_dark_transparent(image)
image_service.save_image_as_webp(image, target_path, quality=82)
return target.exists() and target.stat().st_size > 0
except Exception:
return False
def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str:
candidates: list[Any] = [metadata.get("default_native_path")]
for key in ("native_assets", "quicklook_assets"):
for asset in metadata.get(key) or []:
if isinstance(asset, dict):
candidates.append(asset.get("path"))
for candidate in candidates:
text = _path_text(candidate)
if text and not _is_quicklook_raster_path(text):
return text
return ""
def _path_is_under_any(path: str, roots: list[str]) -> bool:
target = _path_key(path)
if not target:
return False
for root in roots:
root_key = _path_key(root)
if root_key and (target == root_key or target.startswith(root_key + os.sep)):
return True
return False
def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]:
roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if not roots:
roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
return [os.path.normpath(root) for root in roots if str(root or "").strip()]
def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str:
return str(
scene_manifest.get("native_fingerprint")
or (scene_manifest.get("metadata") or {}).get("native_fingerprint")
or ""
).strip()
def _record_has_bounds(record: RadarDataORM) -> bool:
return (
record.coverage_polygon is not None
and record.min_lon is not None
and record.min_lat is not None
and record.max_lon is not None
and record.max_lat is not None
)
async def _find_existing_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> RadarDataORM | None:
native_dir = str(scene_manifest.get("native_dir") or "")
scene_name = scene_manifest.get("scene_name") or Path(native_dir).name
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
return result.scalars().first()
async def _can_skip_unchanged_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> tuple[bool, int | None, str]:
native_fingerprint = _scene_native_fingerprint(scene_manifest)
if not native_fingerprint:
return False, None, "missing_native_fingerprint"
record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest)
if record is None:
return False, None, "missing_record"
metadata = record.metadata_json or {}
if str(metadata.get("native_fingerprint") or "") != native_fingerprint:
return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed"
if str(metadata.get("registration_mode") or "") != "native_preview":
return False, int(record.id) if record.id is not None else None, "registration_mode_changed"
if not _record_has_bounds(record):
return False, int(record.id) if record.id is not None else None, "missing_bounds"
return True, int(record.id) if record.id is not None else None, "unchanged"
async def generate_gf3_quicklook_webp_cache(
db: AsyncSession,
*,
force: bool = False,
max_records: int | None = None,
native_dirs: list[str] | tuple[str, ...] | None = None,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters."""
stmt = (
select(RadarDataORM)
.where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT)
.order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc())
)
result = await db.execute(stmt)
configured_roots = _configured_gf3_native_roots(native_dirs)
records = [
record
for record in result.scalars().all()
if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots)
]
if max_records and max_records > 0:
records = records[: int(max_records)]
total = len(records)
generated = 0
skipped = 0
failed = 0
record_results: list[dict[str, Any]] = []
for idx, record in enumerate(records):
metadata = record.metadata_json or {}
source_path = _native_preview_source_from_metadata(metadata)
cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path)
if progress_callback:
pct = 5 + int((idx / max(total, 1)) * 90)
progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}")
if (
not force
and (record.preview_cache_status or "NONE") == "READY"
and record.preview_cache_path
and os.path.exists(record.preview_cache_path)
):
skipped += 1
record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path})
continue
if not source_path:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "default_native_path_missing"
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
continue
ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path)
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
if ok and os.path.exists(cache_path):
generated += 1
record.preview_cache_status = "READY"
record.preview_cache_path = cache_path
record.preview_cache_error = None
record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path})
else:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "native_webp_build_failed"
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
return {
"ok": failed == 0,
"mode": "gf3_native_webp",
"total": total,
"generated": generated,
"skipped": skipped,
"failed": failed,
"records": record_results,
}
async def _upsert_quicklook_source_product_asset(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
) -> int | None:
native_dir_text = _path_text(scene_manifest.get("native_dir"))
if not native_dir_text:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name
now = _db_now()
root = await _find_managed_root_for_path(db, native_dir_text)
asset_metadata = _json_safe(
{
"source": "GF3 SARscape native _geo",
"native_dir": native_dir_text,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": quicklook_manifest.get("assets") or [],
"full_raster_materialized": True,
"analysis_engine": "gf3_sarscape",
}
)
data = {
"asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"),
"logical_product_uid": scene_name,
"satellite_family": "GF3",
"satellite": "GF3",
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"product_type": metadata.get("product_type") or "SARSCAPE_NATIVE",
"product_level": "L2_NATIVE",
"imaging_mode": metadata.get("imaging_mode"),
"polarization": metadata.get("polarization"),
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"orbit_direction": metadata.get("orbit_direction"),
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"imaging_date": imaging_date,
"root_ref_id": root.id if root else None,
"root_path": root.path if root else str(Path(native_dir_text).parent),
"file_path": native_dir_text,
"archive_path": scene_manifest.get("source_archive"),
"path_kind": _path_kind(native_dir_text),
"file_name": Path(native_dir_text).name,
"file_stem": Path(native_dir_text).name,
"file_ext": "",
"size_bytes": None,
"mtime_epoch": None,
"checksum_status": "NOT_COMPUTED",
"parser_name": "gf3_sarscape_native_preview_manifest",
"parser_version": CONVERTER_VERSION,
"parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED",
"parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None,
"parsed_at": now,
"metadata_json": asset_metadata,
"is_active": True,
"missing_since": None,
"updated_at": now,
}
result = await db.execute(
select(SourceProductAssetORM).where(
or_(
SourceProductAssetORM.asset_uid == data["asset_uid"],
SourceProductAssetORM.file_path == native_dir_text,
)
)
)
asset = result.scalars().first()
if asset is None:
asset = SourceProductAssetORM(**data)
db.add(asset)
else:
for key, value in data.items():
setattr(asset, key, value)
await db.flush()
return int(asset.id) if asset.id is not None else None
async def _upsert_quicklook_radar_data(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
source_product_ref_id: int | None = None,
) -> int | None:
assets = quicklook_manifest.get("assets") or []
default_asset = _select_default_native_asset(assets)
if not default_asset:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
native_dir = str(scene_manifest.get("native_dir") or "")
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
default_native_path = str(default_asset.get("path") or "")
default_quicklook_path = str(default_asset.get("quicklook") or "")
polygon = _polygon_from_native_asset(default_asset)
min_lon = min_lat = max_lon = max_lat = None
geom = None
if polygon:
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
else:
center_lon = metadata.get("scene_center_lon")
center_lat = metadata.get("scene_center_lat")
preview_cache_path = None
preview_cache_kind = "deferred"
preview_cache_error = "native_webp_not_generated"
radar_metadata = _json_safe(
{
**metadata,
"native_dir": native_dir,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": assets,
"quicklook_assets": [],
"default_native_path": default_native_path,
"default_quicklook_path": default_quicklook_path,
"coverage_source": "native_geo_or_quicklook" if polygon else "scene_name",
"preview_cache_kind": preview_cache_kind,
"analysis_engine": "gf3_sarscape",
"registration_mode": "native_preview",
"full_raster_materialized": True,
}
)
data_to_upsert = {
"unique_id": unique_id,
"satellite": "GF3",
"satellite_family": "GF3",
"imaging_date": imaging_date,
"imaging_mode": metadata.get("imaging_mode"),
"polarization": ",".join(
pol
for pol in POLARIZATION_PRIORITY
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
)
or metadata.get("polarization"),
"scene_center_lon": center_lon,
"scene_center_lat": center_lat,
"acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None,
"product_level": "L2_NATIVE",
"product_unique_id": metadata.get("product_unique_id") or scene_name,
"source_product_token": scene_name,
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"source_product_ref_id": source_product_ref_id,
"image_data_format": "ENVI_NATIVE",
"geocoded_flag": True,
"metadata_json": radar_metadata,
"file_path": native_dir,
"has_orbit_data": False,
"orbit_file_path": None,
"is_envi_processed": True,
"coverage_polygon": polygon,
"geom": geom,
"min_lon": min_lon,
"min_lat": min_lat,
"max_lon": max_lon,
"max_lat": max_lat,
"preview_cache_status": "NONE",
"preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION,
"preview_cache_path": preview_cache_path,
"preview_cache_updated_at": _db_now(),
"preview_cache_error": preview_cache_error,
}
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
radar = result.scalars().first()
if (
radar is not None
and (radar.preview_cache_status or "").upper() == "READY"
and radar.preview_cache_path
and os.path.exists(radar.preview_cache_path)
and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path
):
data_to_upsert["preview_cache_status"] = radar.preview_cache_status
data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION
data_to_upsert["preview_cache_path"] = radar.preview_cache_path
data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at
data_to_upsert["preview_cache_error"] = radar.preview_cache_error
if radar is None:
radar = RadarDataORM(**data_to_upsert)
db.add(radar)
else:
for key, value in data_to_upsert.items():
setattr(radar, key, value)
await db.flush()
return int(radar.id) if radar.id is not None else None
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
metadata = dict(scene_manifest.get("metadata") or {})
metadata.update(
@@ -635,7 +1260,7 @@ async def _upsert_radar_data(
if not default_asset:
return None
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path"))))
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
@@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots(
storage_root: str | None = None,
force: bool = False,
register: bool = True,
quicklook_only: bool = False,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Scan native roots, convert complete assets, and register standard scenes."""
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
native_dirs,
write_manifest=True,
)
requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if quicklook_only:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
else:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
scenes = inventory.get("scenes") or []
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
if quicklook_only:
registered = 0
quicklook_assets = 0
failed_scenes = 0
skipped_unchanged = 0
scene_results: list[dict[str, Any]] = []
total = len(ready_scenes)
for idx, scene_manifest in enumerate(ready_scenes):
if progress_callback:
pct = 10 + int((idx / max(total, 1)) * 80)
progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
quicklook_manifest = await asyncio.to_thread(
_quicklook_manifest_for_scene,
scene_manifest,
storage_root,
)
assets = quicklook_manifest.get("assets") or []
quicklook_assets += len(assets)
source_asset_id = None
radar_id = None
skipped_reason = None
if register and assets:
can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar(
db,
scene_manifest=scene_manifest,
)
if can_skip:
skipped_unchanged += 1
radar_id = existing_radar_id
skipped_reason = skip_reason
if progress_callback:
pct = 10 + int(((idx + 1) / max(total, 1)) * 80)
progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
else:
skipped_reason = skip_reason
source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest)
radar_id = await _upsert_quicklook_radar_data(
db,
scene_manifest,
quicklook_manifest,
source_product_ref_id=source_asset_id,
)
if radar_id:
registered += 1
await db.commit()
if not assets:
failed_scenes += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"native_status": scene_manifest.get("status"),
"standard_status": quicklook_manifest.get("status"),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"source_asset_id": source_asset_id,
"radar_id": radar_id,
"skipped_unchanged": bool(skipped_reason == "unchanged"),
"skip_reason": skipped_reason,
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"summary": quicklook_manifest.get("summary") or {},
"errors": quicklook_manifest.get("errors") or [],
}
)
return {
"ok": failed_scenes == 0,
"mode": "native_preview",
"inventory": {
key: value
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": 0,
"partial_scenes": 0,
"failed_scenes": failed_scenes,
"converted_assets": 0,
"skipped_assets": skipped_unchanged,
"skipped_unchanged": skipped_unchanged,
"failed_assets": failed_scenes,
"quicklook_assets": quicklook_assets,
"native_assets": quicklook_assets,
"registered": registered,
"analysis_ready": 0,
"scenes": scene_results,
}
converted_scenes = 0
partial_scenes = 0
failed_scenes = 0
@@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots(
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": converted_scenes,