Add Gamma SBAS production workflow
This commit is contained in:
@@ -0,0 +1,967 @@
|
||||
"""Flood-analysis service layer.
|
||||
|
||||
This module owns the flood business API implementation used by
|
||||
``backend.app.routers.flood``. It intentionally does not import the legacy
|
||||
water router; the old router remains only as a compatibility surface.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from ..models import FloodDetectionORM, RadarDataORM, SARSceneGeoORM, WaterDetectionORM, WaterExtractionORM
|
||||
from ..services.job_handlers import (
|
||||
JOB_TYPE_FLOOD_DETECTION,
|
||||
JOB_TYPE_SAR_SCENE_PREPROCESS,
|
||||
JOB_TYPE_WATER_DETECT,
|
||||
)
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.task_service import task_service
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
_FLOOD_JOB_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
async def _queue_flood_job(
|
||||
*,
|
||||
job_type: str,
|
||||
task_type: str,
|
||||
task_name: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
task_type=task_type,
|
||||
task_name=task_name,
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
job_type=job_type,
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
max_attempts=_FLOOD_JOB_MAX_ATTEMPTS,
|
||||
)
|
||||
return {"task_id": task_id, "job_id": job_id, "job_type": job_type, "message": "Job queued."}
|
||||
except ValueError as exc:
|
||||
message = str(exc)
|
||||
raise HTTPException(status_code=409 if "conflict" in message.lower() else 400, detail=message) from exc
|
||||
|
||||
|
||||
def _overlap_ratio(poly_a: Any, poly_b: Any) -> float:
|
||||
try:
|
||||
import json
|
||||
from shapely.geometry import Polygon, shape
|
||||
|
||||
def _to_geom(poly: Any):
|
||||
if isinstance(poly, str):
|
||||
poly = json.loads(poly)
|
||||
if isinstance(poly, list):
|
||||
return Polygon(poly)
|
||||
return shape(poly)
|
||||
|
||||
a = _to_geom(poly_a)
|
||||
b = _to_geom(poly_b)
|
||||
if not a.is_valid or not b.is_valid:
|
||||
return 0.0
|
||||
intersection_area = a.intersection(b).area
|
||||
smaller_area = min(a.area, b.area)
|
||||
return intersection_area / smaller_area if smaller_area > 0 else 0.0
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _parse_ymd(value: str | None, *, field: str) -> datetime:
|
||||
try:
|
||||
normalized = str(value or "").replace("-", "").strip()
|
||||
return datetime.strptime(normalized, "%Y%m%d")
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=f"{field} must be YYYYMMDD") from exc
|
||||
|
||||
|
||||
def _format_ymd(value: datetime) -> str:
|
||||
return value.strftime("%Y%m%d")
|
||||
|
||||
|
||||
def _to_float(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
if value is None:
|
||||
return default
|
||||
return float(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _same_text(left: Any, right: Any) -> bool:
|
||||
left_text = str(left or "").strip().lower()
|
||||
right_text = str(right or "").strip().lower()
|
||||
if not left_text or not right_text:
|
||||
return True
|
||||
return left_text == right_text
|
||||
|
||||
|
||||
def _feature_collection_name(feature_collection: dict[str, Any]) -> str | None:
|
||||
try:
|
||||
features = feature_collection.get("features") or []
|
||||
properties = features[0].get("properties") or {}
|
||||
return properties.get("name") or properties.get("NAME") or properties.get("treeID")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _preprocess_engine_for_radar(radar: RadarDataORM) -> str | None:
|
||||
family = str(normalize_satellite_family(radar.satellite_family or radar.satellite) or "").upper()
|
||||
if family == "GF3":
|
||||
return "gf3_gdal"
|
||||
if family == "LT1":
|
||||
return "lt_gamma"
|
||||
return None
|
||||
|
||||
|
||||
def _scene_analysis_path(scene: SARSceneGeoORM | None) -> str | None:
|
||||
if not scene:
|
||||
return None
|
||||
return scene.analysis_tif_path
|
||||
|
||||
|
||||
def _resolve_aoi_wkt_from_request(req: Any) -> tuple[str, dict[str, Any], dict[str, Any]]:
|
||||
"""Resolve region/GeoJSON AOI using the same parser as the management search page."""
|
||||
aoi_geojson = getattr(req, "aoi_geojson", None)
|
||||
region_tree_id = getattr(req, "region_tree_id", None)
|
||||
|
||||
if aoi_geojson:
|
||||
feature_collection = aoi_geojson
|
||||
source = "geojson"
|
||||
elif region_tree_id:
|
||||
from ..routers.dependencies import _resolve_region_aoi_payload
|
||||
|
||||
payload = _resolve_region_aoi_payload(str(region_tree_id))
|
||||
feature_collection = payload.get("aoi_geojson") or payload
|
||||
source = "region"
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="region_tree_id or aoi_geojson is required")
|
||||
|
||||
from ..routers.dependencies import _parse_aoi_geojson_form_value
|
||||
|
||||
parsed = _parse_aoi_geojson_form_value(json.dumps(feature_collection, ensure_ascii=False))
|
||||
if not parsed:
|
||||
raise HTTPException(status_code=400, detail="AOI geometry is empty")
|
||||
aoi_wkt, normalized_feature_collection = parsed
|
||||
meta = {
|
||||
"source": source,
|
||||
"region_tree_id": region_tree_id,
|
||||
"name": _feature_collection_name(normalized_feature_collection),
|
||||
}
|
||||
return aoi_wkt, normalized_feature_collection, meta
|
||||
|
||||
|
||||
def _radar_scene_item(scene: SARSceneGeoORM, radar: RadarDataORM, *, aoi_coverage_ratio: float | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": scene.id,
|
||||
"scene_id": scene.id,
|
||||
"radar_data_id": scene.radar_data_id,
|
||||
"satellite": radar.satellite,
|
||||
"imaging_date": radar.imaging_date,
|
||||
"acquisition_time_utc": radar.acquisition_time_utc,
|
||||
"imaging_mode": radar.imaging_mode,
|
||||
"product_level": radar.product_level,
|
||||
"polarization": radar.polarization,
|
||||
"orbit_direction": radar.orbit_direction,
|
||||
"geo_path": scene.geo_path,
|
||||
"analysis_tif_path": scene.analysis_tif_path,
|
||||
"analysis_dir": scene.analysis_dir,
|
||||
"analysis_preview_path": scene.analysis_preview_path,
|
||||
"analysis_engine": scene.analysis_engine,
|
||||
"analysis_profile": scene.analysis_profile,
|
||||
"analysis_backscatter_unit": scene.analysis_backscatter_unit,
|
||||
"analysis_quality_json": scene.analysis_quality_json,
|
||||
"coverage_polygon": radar.coverage_polygon,
|
||||
"min_lat": radar.min_lat,
|
||||
"max_lat": radar.max_lat,
|
||||
"min_lon": radar.min_lon,
|
||||
"max_lon": radar.max_lon,
|
||||
"aoi_coverage_ratio": aoi_coverage_ratio,
|
||||
}
|
||||
|
||||
|
||||
async def _query_disaster_scene_pool(
|
||||
*,
|
||||
db: AsyncSession,
|
||||
req: Any,
|
||||
aoi_wkt: str,
|
||||
start_ymd: str,
|
||||
end_ymd: str,
|
||||
min_aoi_coverage_ratio: float,
|
||||
descending: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
|
||||
aoi_area = func.ST_Area(func.Geography(aoi_geom))
|
||||
coverage_expr = (
|
||||
func.ST_Area(func.Geography(func.ST_Intersection(RadarDataORM.geom, aoi_geom)))
|
||||
/ func.nullif(aoi_area, 0)
|
||||
).label("aoi_coverage_ratio")
|
||||
|
||||
filters = [
|
||||
SARSceneGeoORM.status == "DONE",
|
||||
SARSceneGeoORM.analysis_tif_path.isnot(None),
|
||||
RadarDataORM.geom.isnot(None),
|
||||
RadarDataORM.imaging_date.isnot(None),
|
||||
RadarDataORM.imaging_date >= start_ymd,
|
||||
RadarDataORM.imaging_date <= end_ymd,
|
||||
func.ST_Intersects(RadarDataORM.geom, aoi_geom),
|
||||
]
|
||||
|
||||
satellites = [str(item).strip() for item in (getattr(req, "satellites", None) or []) if str(item).strip()]
|
||||
if satellites:
|
||||
filters.append(RadarDataORM.satellite.in_(satellites))
|
||||
|
||||
polarization = str(getattr(req, "polarization", "") or "").strip()
|
||||
if polarization:
|
||||
filters.append(RadarDataORM.polarization.ilike(f"%{polarization}%"))
|
||||
|
||||
imaging_mode = str(getattr(req, "imaging_mode", "") or "").strip()
|
||||
if imaging_mode:
|
||||
filters.append(RadarDataORM.imaging_mode == imaging_mode)
|
||||
|
||||
product_level = str(getattr(req, "product_level", "") or "").strip()
|
||||
if product_level:
|
||||
filters.append(RadarDataORM.product_level == product_level)
|
||||
|
||||
order_by = RadarDataORM.imaging_date.desc() if descending else RadarDataORM.imaging_date.asc()
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM, RadarDataORM, coverage_expr)
|
||||
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
|
||||
.where(*filters)
|
||||
.order_by(order_by, SARSceneGeoORM.id.desc())
|
||||
)
|
||||
|
||||
pool: list[dict[str, Any]] = []
|
||||
for scene, radar, aoi_coverage_ratio in result.all():
|
||||
coverage_ratio = max(0.0, min(1.0, _to_float(aoi_coverage_ratio)))
|
||||
if coverage_ratio < min_aoi_coverage_ratio:
|
||||
continue
|
||||
pool.append(_radar_scene_item(scene, radar, aoi_coverage_ratio=round(coverage_ratio, 4)))
|
||||
return pool
|
||||
|
||||
|
||||
async def submit_geocode_job(req: Any, db: AsyncSession) -> dict[str, Any]:
|
||||
radar = await db.get(RadarDataORM, req.radar_data_id)
|
||||
if not radar:
|
||||
raise HTTPException(status_code=404, detail=f"RadarData id={req.radar_data_id} not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM)
|
||||
.where(SARSceneGeoORM.radar_data_id == req.radar_data_id)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
scene = result.scalar_one_or_none()
|
||||
if scene and scene.status in ("PENDING", "RUNNING"):
|
||||
raise HTTPException(status_code=409, detail="Scene already has an active geocode job")
|
||||
if not scene:
|
||||
scene = SARSceneGeoORM(radar_data_id=req.radar_data_id, status="PENDING")
|
||||
db.add(scene)
|
||||
await db.flush()
|
||||
else:
|
||||
scene.status = "PENDING"
|
||||
scene.error_msg = None
|
||||
await db.flush()
|
||||
scene_id = scene.id
|
||||
await db.commit()
|
||||
|
||||
engine = _preprocess_engine_for_radar(radar)
|
||||
if not engine:
|
||||
async with db.begin():
|
||||
failed_scene = await db.get(SARSceneGeoORM, scene_id)
|
||||
if failed_scene and failed_scene.status == "PENDING":
|
||||
failed_scene.status = "FAILED"
|
||||
failed_scene.error_msg = "No analysis-ready GeoTIFF preprocessor configured for this satellite"
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="洪涝模块不再使用 ENVI 兜底预处理;该卫星暂未配置 analysis-ready GeoTIFF 预处理器",
|
||||
)
|
||||
|
||||
job_type = JOB_TYPE_SAR_SCENE_PREPROCESS
|
||||
task_type = f"FLOOD_SCENE_PREPROCESS_{scene_id}"
|
||||
task_name = f"Flood analysis-ready preprocess radar_id={req.radar_data_id} engine={engine}"
|
||||
payload = {"scene_id": scene_id, "radar_data_id": req.radar_data_id}
|
||||
payload["engine"] = engine
|
||||
|
||||
try:
|
||||
return await _queue_flood_job(
|
||||
job_type=job_type,
|
||||
task_type=task_type,
|
||||
task_name=task_name,
|
||||
payload=payload,
|
||||
)
|
||||
except HTTPException:
|
||||
async with db.begin():
|
||||
failed_scene = await db.get(SARSceneGeoORM, scene_id)
|
||||
if failed_scene and failed_scene.status == "PENDING":
|
||||
failed_scene.status = "FAILED"
|
||||
failed_scene.error_msg = "Job queue failed"
|
||||
raise
|
||||
|
||||
|
||||
async def reset_scene_status(scene_id: int, db: AsyncSession) -> dict[str, Any]:
|
||||
scene = await db.get(SARSceneGeoORM, scene_id)
|
||||
if not scene:
|
||||
raise HTTPException(status_code=404, detail=f"Scene id={scene_id} not found")
|
||||
if scene.status not in ("PENDING", "RUNNING"):
|
||||
raise HTTPException(status_code=400, detail=f"Scene status is {scene.status}; reset is not needed")
|
||||
scene.status = "FAILED"
|
||||
scene.error_msg = "Manually reset"
|
||||
await db.commit()
|
||||
return {"id": scene_id, "status": "FAILED", "message": "Scene reset"}
|
||||
|
||||
|
||||
async def list_done_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]:
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM.radar_data_id).where(
|
||||
SARSceneGeoORM.status == "DONE",
|
||||
SARSceneGeoORM.analysis_tif_path.isnot(None),
|
||||
)
|
||||
)
|
||||
return {"ids": [row for (row,) in result.all()]}
|
||||
|
||||
|
||||
async def list_active_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]:
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM.radar_data_id).where(SARSceneGeoORM.status.in_(["PENDING", "RUNNING"]))
|
||||
)
|
||||
return {"ids": [row for (row,) in result.all()]}
|
||||
|
||||
|
||||
async def list_scenes(limit: int, offset: int, db: AsyncSession) -> dict[str, Any]:
|
||||
total_result = await db.execute(select(func.count()).select_from(SARSceneGeoORM))
|
||||
total = total_result.scalar_one()
|
||||
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM, RadarDataORM)
|
||||
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
|
||||
.order_by(SARSceneGeoORM.id.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
|
||||
items = []
|
||||
for scene, radar in result.all():
|
||||
items.append(
|
||||
{
|
||||
"id": scene.id,
|
||||
"radar_data_id": scene.radar_data_id,
|
||||
"satellite": radar.satellite,
|
||||
"imaging_date": radar.imaging_date,
|
||||
"acquisition_time_utc": radar.acquisition_time_utc,
|
||||
"imaging_mode": radar.imaging_mode,
|
||||
"product_level": radar.product_level,
|
||||
"polarization": radar.polarization,
|
||||
"orbit_direction": radar.orbit_direction,
|
||||
"geo_path": scene.geo_path,
|
||||
"analysis_tif_path": scene.analysis_tif_path,
|
||||
"analysis_dir": scene.analysis_dir,
|
||||
"analysis_preview_path": scene.analysis_preview_path,
|
||||
"analysis_engine": scene.analysis_engine,
|
||||
"analysis_profile": scene.analysis_profile,
|
||||
"analysis_backscatter_unit": scene.analysis_backscatter_unit,
|
||||
"analysis_nodata_value": scene.analysis_nodata_value,
|
||||
"analysis_metadata_json": scene.analysis_metadata_json,
|
||||
"analysis_quality_json": scene.analysis_quality_json,
|
||||
"pixel_size_m": scene.pixel_size_m,
|
||||
"status": scene.status,
|
||||
"error_msg": scene.error_msg,
|
||||
"created_at": scene.created_at.isoformat() if scene.created_at else None,
|
||||
"coverage_polygon": radar.coverage_polygon,
|
||||
"min_lat": radar.min_lat,
|
||||
"max_lat": radar.max_lat,
|
||||
"min_lon": radar.min_lon,
|
||||
"max_lon": radar.max_lon,
|
||||
}
|
||||
)
|
||||
return {"items": items, "total": total}
|
||||
|
||||
|
||||
async def submit_water_extraction(req: Any, db: AsyncSession) -> dict[str, Any]:
|
||||
input_path = req.input_path
|
||||
scene_id = req.scene_id
|
||||
|
||||
if scene_id:
|
||||
scene = await db.get(SARSceneGeoORM, scene_id)
|
||||
if not scene:
|
||||
raise HTTPException(status_code=404, detail=f"SARSceneGeoORM id={scene_id} not found")
|
||||
input_path = _scene_analysis_path(scene)
|
||||
if not input_path:
|
||||
raise HTTPException(status_code=400, detail="Scene has no analysis-ready GeoTIFF")
|
||||
|
||||
if not input_path:
|
||||
raise HTTPException(status_code=400, detail="scene_id or input_path is required")
|
||||
|
||||
extraction = WaterExtractionORM(
|
||||
scene_id=scene_id,
|
||||
processor=getattr(req, "processor", None) or "otsu",
|
||||
input_path=input_path,
|
||||
status="PENDING",
|
||||
)
|
||||
db.add(extraction)
|
||||
await db.flush()
|
||||
extraction_id = extraction.id
|
||||
await db.commit()
|
||||
|
||||
try:
|
||||
queued = await _queue_flood_job(
|
||||
job_type=JOB_TYPE_WATER_DETECT,
|
||||
task_type=f"FLOOD_WATER_EXTRACTION_{extraction_id}",
|
||||
task_name=f"Flood water extraction id={extraction_id}",
|
||||
payload={"extraction_id": extraction_id, "processor": extraction.processor},
|
||||
)
|
||||
async with db.begin():
|
||||
queued_extraction = await db.get(WaterExtractionORM, extraction_id)
|
||||
if queued_extraction:
|
||||
queued_extraction.task_id = queued.get("task_id")
|
||||
return queued
|
||||
except HTTPException:
|
||||
async with db.begin():
|
||||
failed_extraction = await db.get(WaterExtractionORM, extraction_id)
|
||||
if failed_extraction and failed_extraction.status == "PENDING":
|
||||
failed_extraction.status = "FAILED"
|
||||
failed_extraction.error_msg = "Job queue failed"
|
||||
raise
|
||||
|
||||
|
||||
async def list_water_extractions(
|
||||
*,
|
||||
limit: int,
|
||||
offset: int,
|
||||
status: str | None,
|
||||
db: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
count_query = select(func.count()).select_from(WaterExtractionORM)
|
||||
if status:
|
||||
count_query = count_query.where(WaterExtractionORM.status == status)
|
||||
total = (await db.execute(count_query)).scalar_one()
|
||||
|
||||
query = (
|
||||
select(WaterExtractionORM, SARSceneGeoORM, RadarDataORM)
|
||||
.outerjoin(SARSceneGeoORM, WaterExtractionORM.scene_id == SARSceneGeoORM.id)
|
||||
.outerjoin(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
|
||||
.order_by(WaterExtractionORM.id.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
if status:
|
||||
query = query.where(WaterExtractionORM.status == status)
|
||||
rows = (await db.execute(query)).all()
|
||||
|
||||
items = []
|
||||
for detection, scene, radar in rows:
|
||||
items.append(
|
||||
{
|
||||
"id": detection.id,
|
||||
"scene_id": detection.scene_id,
|
||||
"processor": detection.processor,
|
||||
"task_id": detection.task_id,
|
||||
"radar_data_id": scene.radar_data_id if scene else None,
|
||||
"satellite": radar.satellite if radar else None,
|
||||
"imaging_date": radar.imaging_date if radar else None,
|
||||
"acquisition_time_utc": radar.acquisition_time_utc if radar else None,
|
||||
"imaging_mode": radar.imaging_mode if radar else None,
|
||||
"product_level": radar.product_level if radar else None,
|
||||
"polarization": radar.polarization if radar else None,
|
||||
"orbit_direction": radar.orbit_direction if radar else None,
|
||||
"coverage_polygon": radar.coverage_polygon if radar else None,
|
||||
"min_lat": radar.min_lat if radar else None,
|
||||
"max_lat": radar.max_lat if radar else None,
|
||||
"min_lon": radar.min_lon if radar else None,
|
||||
"max_lon": radar.max_lon if radar else None,
|
||||
"input_path": detection.input_path,
|
||||
"output_path": detection.output_path,
|
||||
"preview_path": detection.preview_path,
|
||||
"vector_path": detection.vector_path,
|
||||
"water_area_km2": detection.water_area_km2,
|
||||
"water_pixel_count": detection.water_pixel_count,
|
||||
"otsu_threshold_db": detection.threshold_value,
|
||||
"threshold_value": detection.threshold_value,
|
||||
"metadata_json": detection.metadata_json,
|
||||
"status": detection.status,
|
||||
"error_msg": detection.error_msg,
|
||||
"created_at": detection.created_at.isoformat() if detection.created_at else None,
|
||||
"updated_at": detection.updated_at.isoformat() if detection.updated_at else None,
|
||||
}
|
||||
)
|
||||
return {"items": items, "total": total}
|
||||
|
||||
|
||||
async def submit_flood_detection(req: Any, db: AsyncSession) -> dict[str, Any]:
|
||||
pre_scene = await db.get(SARSceneGeoORM, req.pre_scene_id)
|
||||
post_scene = await db.get(SARSceneGeoORM, req.post_scene_id)
|
||||
if not pre_scene:
|
||||
raise HTTPException(status_code=404, detail=f"Pre-scene id={req.pre_scene_id} not found")
|
||||
if not post_scene:
|
||||
raise HTTPException(status_code=404, detail=f"Post-scene id={req.post_scene_id} not found")
|
||||
if pre_scene.status != "DONE":
|
||||
raise HTTPException(status_code=400, detail=f"Pre-scene is not DONE: {pre_scene.status}")
|
||||
if post_scene.status != "DONE":
|
||||
raise HTTPException(status_code=400, detail=f"Post-scene is not DONE: {post_scene.status}")
|
||||
if not pre_scene.analysis_tif_path:
|
||||
raise HTTPException(status_code=400, detail="Pre-scene has no analysis-ready GeoTIFF")
|
||||
if not post_scene.analysis_tif_path:
|
||||
raise HTTPException(status_code=400, detail="Post-scene has no analysis-ready GeoTIFF")
|
||||
|
||||
result = await db.execute(
|
||||
select(FloodDetectionORM)
|
||||
.where(
|
||||
FloodDetectionORM.pre_scene_id == req.pre_scene_id,
|
||||
FloodDetectionORM.post_scene_id == req.post_scene_id,
|
||||
)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
detection = result.scalar_one_or_none()
|
||||
if detection and detection.status in ("PENDING", "RUNNING"):
|
||||
raise HTTPException(status_code=409, detail="Pair already has an active flood-detection job")
|
||||
if not detection:
|
||||
detection = FloodDetectionORM(
|
||||
pre_scene_id=req.pre_scene_id,
|
||||
post_scene_id=req.post_scene_id,
|
||||
status="PENDING",
|
||||
)
|
||||
db.add(detection)
|
||||
await db.flush()
|
||||
else:
|
||||
detection.status = "PENDING"
|
||||
detection.error_msg = None
|
||||
await db.flush()
|
||||
detection_id = detection.id
|
||||
await db.commit()
|
||||
|
||||
try:
|
||||
return await _queue_flood_job(
|
||||
job_type=JOB_TYPE_FLOOD_DETECTION,
|
||||
task_type=f"FLOOD_DETECTION_{detection_id}",
|
||||
task_name=f"GeoTIFF flood detection pre={req.pre_scene_id} post={req.post_scene_id}",
|
||||
payload={"detection_id": detection_id, "refine": req.refine},
|
||||
)
|
||||
except HTTPException:
|
||||
async with db.begin():
|
||||
failed_detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if failed_detection and failed_detection.status == "PENDING":
|
||||
failed_detection.status = "FAILED"
|
||||
failed_detection.error_msg = "Job queue failed"
|
||||
raise
|
||||
|
||||
|
||||
async def list_flood_detections(db: AsyncSession) -> dict[str, Any]:
|
||||
result = await db.execute(
|
||||
select(FloodDetectionORM)
|
||||
.options(
|
||||
selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data),
|
||||
selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data),
|
||||
)
|
||||
.order_by(FloodDetectionORM.id.desc())
|
||||
)
|
||||
detections = result.scalars().all()
|
||||
|
||||
items = []
|
||||
for detection in detections:
|
||||
pre_radar = detection.pre_scene.radar_data if detection.pre_scene else None
|
||||
post_radar = detection.post_scene.radar_data if detection.post_scene else None
|
||||
items.append(
|
||||
{
|
||||
"id": detection.id,
|
||||
"pre_scene_id": detection.pre_scene_id,
|
||||
"post_scene_id": detection.post_scene_id,
|
||||
"pre_imaging_date": pre_radar.imaging_date if pre_radar else None,
|
||||
"post_imaging_date": post_radar.imaging_date if post_radar else None,
|
||||
"pre_satellite": pre_radar.satellite if pre_radar else None,
|
||||
"post_satellite": post_radar.satellite if post_radar else None,
|
||||
"pre_geo_path": _scene_analysis_path(detection.pre_scene),
|
||||
"post_geo_path": _scene_analysis_path(detection.post_scene),
|
||||
"pre_analysis_tif_path": _scene_analysis_path(detection.pre_scene),
|
||||
"post_analysis_tif_path": _scene_analysis_path(detection.post_scene),
|
||||
"classified_path": detection.classified_path,
|
||||
"flood_area_km2": detection.flood_area_km2,
|
||||
"stable_water_area_km2": detection.stable_water_area_km2,
|
||||
"status": detection.status,
|
||||
"error_msg": detection.error_msg,
|
||||
"created_at": detection.created_at.isoformat() if detection.created_at else None,
|
||||
"updated_at": detection.updated_at.isoformat() if detection.updated_at else None,
|
||||
}
|
||||
)
|
||||
return {"items": items, "total": len(items)}
|
||||
|
||||
|
||||
async def search_pairs(req: Any, db: AsyncSession) -> dict[str, Any]:
|
||||
pre_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)]
|
||||
if req.pre_start:
|
||||
pre_filters.append(RadarDataORM.imaging_date >= req.pre_start)
|
||||
if req.pre_end:
|
||||
pre_filters.append(RadarDataORM.imaging_date <= req.pre_end)
|
||||
pre_result = await db.execute(
|
||||
select(SARSceneGeoORM, RadarDataORM)
|
||||
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
|
||||
.where(*pre_filters)
|
||||
)
|
||||
|
||||
post_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)]
|
||||
if req.post_start:
|
||||
post_filters.append(RadarDataORM.imaging_date >= req.post_start)
|
||||
if req.post_end:
|
||||
post_filters.append(RadarDataORM.imaging_date <= req.post_end)
|
||||
post_result = await db.execute(
|
||||
select(SARSceneGeoORM, RadarDataORM)
|
||||
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
|
||||
.where(*post_filters)
|
||||
)
|
||||
|
||||
candidates = []
|
||||
for pre_scene, pre_radar in pre_result.all():
|
||||
for post_scene, post_radar in post_result.all():
|
||||
if pre_scene.id == post_scene.id:
|
||||
continue
|
||||
ratio = 0.0
|
||||
if pre_radar.coverage_polygon and post_radar.coverage_polygon:
|
||||
ratio = _overlap_ratio(pre_radar.coverage_polygon, post_radar.coverage_polygon)
|
||||
if ratio < req.overlap_threshold:
|
||||
continue
|
||||
try:
|
||||
pre_date = datetime.strptime(pre_radar.imaging_date, "%Y%m%d")
|
||||
post_date = datetime.strptime(post_radar.imaging_date, "%Y%m%d")
|
||||
time_diff = abs((post_date - pre_date).days)
|
||||
except Exception:
|
||||
time_diff = None
|
||||
candidates.append(
|
||||
{
|
||||
"pre": {
|
||||
"id": pre_scene.id,
|
||||
"imaging_date": pre_radar.imaging_date,
|
||||
"satellite": pre_radar.satellite,
|
||||
"geo_path": _scene_analysis_path(pre_scene),
|
||||
"analysis_tif_path": _scene_analysis_path(pre_scene),
|
||||
},
|
||||
"post": {
|
||||
"id": post_scene.id,
|
||||
"imaging_date": post_radar.imaging_date,
|
||||
"satellite": post_radar.satellite,
|
||||
"geo_path": _scene_analysis_path(post_scene),
|
||||
"analysis_tif_path": _scene_analysis_path(post_scene),
|
||||
},
|
||||
"overlap_ratio": round(ratio, 4),
|
||||
"time_diff_days": time_diff,
|
||||
}
|
||||
)
|
||||
|
||||
candidates.sort(key=lambda item: item["overlap_ratio"], reverse=True)
|
||||
used_pre: set[int] = set()
|
||||
used_post: set[int] = set()
|
||||
pairs = []
|
||||
for candidate in candidates:
|
||||
pre_id = candidate["pre"]["id"]
|
||||
post_id = candidate["post"]["id"]
|
||||
if pre_id in used_pre or post_id in used_post:
|
||||
continue
|
||||
used_pre.add(pre_id)
|
||||
used_post.add(post_id)
|
||||
pairs.append(candidate)
|
||||
|
||||
pairs.sort(key=lambda item: item["overlap_ratio"], reverse=True)
|
||||
return {"pairs": pairs, "total": len(pairs)}
|
||||
|
||||
|
||||
async def search_disaster_pairs(req: Any, db: AsyncSession) -> dict[str, Any]:
|
||||
disaster_date = _parse_ymd(req.disaster_date, field="disaster_date")
|
||||
pre_window_days = max(1, int(getattr(req, "pre_window_days", 30) or 30))
|
||||
post_window_days = max(1, int(getattr(req, "post_window_days", 30) or 30))
|
||||
min_aoi_coverage_ratio = max(0.0, min(1.0, float(getattr(req, "min_aoi_coverage_ratio", 0.2) or 0.0)))
|
||||
min_pair_overlap_ratio = max(0.0, min(1.0, float(getattr(req, "min_pair_overlap_ratio", 0.3) or 0.0)))
|
||||
max_pairs = max(1, min(200, int(getattr(req, "max_pairs", 50) or 50)))
|
||||
|
||||
aoi_wkt, aoi_geojson, aoi_meta = _resolve_aoi_wkt_from_request(req)
|
||||
pre_start = disaster_date - timedelta(days=pre_window_days)
|
||||
pre_end = disaster_date - timedelta(days=1)
|
||||
post_start = disaster_date
|
||||
post_end = disaster_date + timedelta(days=post_window_days)
|
||||
|
||||
pre_pool = await _query_disaster_scene_pool(
|
||||
db=db,
|
||||
req=req,
|
||||
aoi_wkt=aoi_wkt,
|
||||
start_ymd=_format_ymd(pre_start),
|
||||
end_ymd=_format_ymd(pre_end),
|
||||
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
|
||||
descending=True,
|
||||
)
|
||||
post_pool = await _query_disaster_scene_pool(
|
||||
db=db,
|
||||
req=req,
|
||||
aoi_wkt=aoi_wkt,
|
||||
start_ymd=_format_ymd(post_start),
|
||||
end_ymd=_format_ymd(post_end),
|
||||
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
|
||||
descending=False,
|
||||
)
|
||||
|
||||
candidates: list[dict[str, Any]] = []
|
||||
require_same_polarization = bool(getattr(req, "require_same_polarization", True))
|
||||
require_same_imaging_mode = bool(getattr(req, "require_same_imaging_mode", False))
|
||||
total_window = max(1, pre_window_days + post_window_days)
|
||||
|
||||
for pre in pre_pool:
|
||||
pre_date = _parse_ymd(pre.get("imaging_date"), field="pre.imaging_date")
|
||||
for post in post_pool:
|
||||
if pre["id"] == post["id"]:
|
||||
continue
|
||||
if require_same_polarization and not _same_text(pre.get("polarization"), post.get("polarization")):
|
||||
continue
|
||||
if require_same_imaging_mode and not _same_text(pre.get("imaging_mode"), post.get("imaging_mode")):
|
||||
continue
|
||||
|
||||
post_date = _parse_ymd(post.get("imaging_date"), field="post.imaging_date")
|
||||
scene_overlap = _overlap_ratio(pre.get("coverage_polygon"), post.get("coverage_polygon"))
|
||||
if scene_overlap < min_pair_overlap_ratio:
|
||||
continue
|
||||
|
||||
pre_delta_days = max(0, (disaster_date - pre_date).days)
|
||||
post_delta_days = max(0, (post_date - disaster_date).days)
|
||||
time_score = max(0.0, 1.0 - ((pre_delta_days + post_delta_days) / total_window))
|
||||
aoi_score = min(_to_float(pre.get("aoi_coverage_ratio")), _to_float(post.get("aoi_coverage_ratio")))
|
||||
score = (scene_overlap * 0.45) + (aoi_score * 0.35) + (time_score * 0.20)
|
||||
candidates.append(
|
||||
{
|
||||
"pre": pre,
|
||||
"post": post,
|
||||
"overlap_ratio": round(scene_overlap, 4),
|
||||
"aoi_coverage_ratio": round(aoi_score, 4),
|
||||
"time_score": round(time_score, 4),
|
||||
"score": round(score, 4),
|
||||
"pre_delta_days": pre_delta_days,
|
||||
"post_delta_days": post_delta_days,
|
||||
"time_diff_days": max(0, (post_date - pre_date).days),
|
||||
"same_polarization": _same_text(pre.get("polarization"), post.get("polarization")),
|
||||
"same_imaging_mode": _same_text(pre.get("imaging_mode"), post.get("imaging_mode")),
|
||||
}
|
||||
)
|
||||
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
item["score"],
|
||||
item["overlap_ratio"],
|
||||
item["aoi_coverage_ratio"],
|
||||
-item["time_diff_days"],
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
selected_pairs = candidates[:max_pairs]
|
||||
|
||||
warnings: list[str] = []
|
||||
if not pre_pool:
|
||||
warnings.append("No pre-disaster DONE scenes match the disaster AOI and time window")
|
||||
if not post_pool:
|
||||
warnings.append("No post-disaster DONE scenes match the disaster AOI and time window")
|
||||
if pre_pool and post_pool and not selected_pairs:
|
||||
warnings.append("Pre/post scene pools exist, but no pair meets the overlap/polarization constraints")
|
||||
|
||||
return {
|
||||
"disaster": {
|
||||
"name": getattr(req, "disaster_name", None),
|
||||
"date": _format_ymd(disaster_date),
|
||||
"pre_start": _format_ymd(pre_start),
|
||||
"pre_end": _format_ymd(pre_end),
|
||||
"post_start": _format_ymd(post_start),
|
||||
"post_end": _format_ymd(post_end),
|
||||
},
|
||||
"aoi": {
|
||||
**aoi_meta,
|
||||
"geojson": aoi_geojson,
|
||||
},
|
||||
"pre_pool": pre_pool,
|
||||
"post_pool": post_pool,
|
||||
"candidate_pairs": selected_pairs,
|
||||
"pairs": selected_pairs,
|
||||
"total": len(selected_pairs),
|
||||
"summary": {
|
||||
"pre_pool_count": len(pre_pool),
|
||||
"post_pool_count": len(post_pool),
|
||||
"candidate_count": len(selected_pairs),
|
||||
"min_aoi_coverage_ratio": min_aoi_coverage_ratio,
|
||||
"min_pair_overlap_ratio": min_pair_overlap_ratio,
|
||||
},
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
|
||||
def _open_envi_rasterio(path: str):
|
||||
import rasterio
|
||||
|
||||
normalized_path = path.replace("\\", "/")
|
||||
try:
|
||||
return rasterio.open(normalized_path)
|
||||
except Exception:
|
||||
pass
|
||||
for ext in (".bin", ".img", ".tif", ".tiff"):
|
||||
try:
|
||||
return rasterio.open(normalized_path + ext)
|
||||
except Exception:
|
||||
pass
|
||||
raise FileNotFoundError(f"Raster file cannot be opened: {path}")
|
||||
|
||||
|
||||
def _raster_to_png_bytes(path: str, colormap: dict[int, tuple[int, int, int, int]]) -> tuple[bytes, list[float]]:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
with _open_envi_rasterio(path) as ds:
|
||||
data = ds.read(1)
|
||||
bounds = ds.bounds
|
||||
geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right]
|
||||
|
||||
rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8)
|
||||
for value, color in colormap.items():
|
||||
rgba[data == value] = color
|
||||
image = Image.fromarray(rgba, "RGBA")
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
return buffer.getvalue(), geo_bounds
|
||||
|
||||
|
||||
def _geo_raster_to_png_bytes(path: str) -> tuple[bytes, list[float]]:
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
with _open_envi_rasterio(path) as ds:
|
||||
data = ds.read(1).astype("float32")
|
||||
nodata = ds.nodata
|
||||
bounds = ds.bounds
|
||||
geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right]
|
||||
|
||||
if nodata is not None:
|
||||
nodata_mask = (data == nodata) | ~np.isfinite(data)
|
||||
else:
|
||||
nodata_mask = ~np.isfinite(data)
|
||||
|
||||
valid = data[~nodata_mask]
|
||||
if valid.size == 0:
|
||||
normalized = np.zeros_like(data, dtype=np.uint8)
|
||||
else:
|
||||
p2, p98 = np.percentile(valid, 2), np.percentile(valid, 98)
|
||||
clipped = np.clip(data, p2, p98)
|
||||
normalized = ((clipped - p2) / max(p98 - p2, 1e-9) * 255).astype(np.uint8)
|
||||
|
||||
rgba = np.stack([normalized, normalized, normalized, np.full_like(normalized, 200)], axis=-1)
|
||||
rgba[nodata_mask, 3] = 0
|
||||
image = Image.fromarray(rgba, "RGBA")
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
return buffer.getvalue(), geo_bounds
|
||||
|
||||
|
||||
_FLOOD_COLORMAP = {
|
||||
1: (24, 144, 255, 200),
|
||||
2: (255, 77, 79, 220),
|
||||
3: (250, 173, 20, 180),
|
||||
4: (80, 80, 80, 80),
|
||||
}
|
||||
|
||||
|
||||
async def get_flood_detection_preview(detection_id: int, layer: str, db: AsyncSession):
|
||||
normalized_layer = layer.strip().lower()
|
||||
if normalized_layer == "classified":
|
||||
detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if not detection or not detection.classified_path:
|
||||
raise HTTPException(status_code=404, detail="Classified result not found")
|
||||
path = detection.classified_path.replace("\\", "/")
|
||||
png_bytes, geo_bounds = await _render_classified_preview(path)
|
||||
return JSONResponse(
|
||||
{
|
||||
"image_b64": base64.b64encode(png_bytes).decode(),
|
||||
"bounds": geo_bounds,
|
||||
"legend": {
|
||||
"stable_water": "#1890ff",
|
||||
"flood": "#ff4d4f",
|
||||
"high_backscatter": "#faad14",
|
||||
"non_water": "#505050",
|
||||
},
|
||||
}
|
||||
)
|
||||
if normalized_layer in ("pre", "post"):
|
||||
return await _get_scene_preview_for_detection(detection_id, normalized_layer, db)
|
||||
raise HTTPException(status_code=404, detail=f"Unsupported preview layer: {layer}")
|
||||
|
||||
|
||||
async def _render_classified_preview(path: str) -> tuple[bytes, list[float]]:
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(status_code=404, detail="Requested file does not exist")
|
||||
try:
|
||||
return await asyncio.to_thread(_raster_to_png_bytes, path, _FLOOD_COLORMAP)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc
|
||||
|
||||
|
||||
async def _get_scene_preview_for_detection(detection_id: int, layer: str, db: AsyncSession):
|
||||
detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if not detection:
|
||||
raise HTTPException(status_code=404, detail="Flood detection not found")
|
||||
scene_id = detection.pre_scene_id if layer == "pre" else detection.post_scene_id
|
||||
scene = await db.get(SARSceneGeoORM, scene_id)
|
||||
scene_path = _scene_analysis_path(scene)
|
||||
if not scene_path:
|
||||
raise HTTPException(status_code=404, detail="Scene analysis-ready GeoTIFF not found")
|
||||
path = scene_path.replace("\\", "/")
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(status_code=404, detail="Requested file does not exist")
|
||||
try:
|
||||
png_bytes, geo_bounds = await asyncio.to_thread(_geo_raster_to_png_bytes, path)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc
|
||||
return JSONResponse({"image_b64": base64.b64encode(png_bytes).decode(), "bounds": geo_bounds})
|
||||
|
||||
|
||||
async def get_water_extraction_preview(extraction_id: int, db: AsyncSession) -> dict[str, Any]:
|
||||
detection = await db.get(WaterExtractionORM, extraction_id)
|
||||
if not detection:
|
||||
detection = await db.get(WaterDetectionORM, extraction_id)
|
||||
if not detection:
|
||||
raise HTTPException(status_code=404, detail=f"Water extraction id={extraction_id} not found")
|
||||
if not detection.output_path or not os.path.isfile(detection.output_path):
|
||||
raise HTTPException(status_code=404, detail="Output file does not exist")
|
||||
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
|
||||
with rasterio.open(detection.output_path) as src:
|
||||
data = src.read(1)
|
||||
transform = src.transform
|
||||
height, width = data.shape
|
||||
min_lon = transform.c
|
||||
max_lon = transform.c + width * transform.a
|
||||
max_lat = transform.f
|
||||
min_lat = transform.f + height * transform.e
|
||||
|
||||
rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8)
|
||||
rgba[data > 0] = [24, 144, 255, 160]
|
||||
|
||||
image = Image.fromarray(rgba, "RGBA")
|
||||
max_dim = 1024
|
||||
if max(width, height) > max_dim:
|
||||
ratio = max_dim / max(width, height)
|
||||
image = image.resize((int(width * ratio), int(height * ratio)), Image.NEAREST)
|
||||
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
return {
|
||||
"png_base64": base64.b64encode(buffer.getvalue()).decode(),
|
||||
"bounds": {
|
||||
"min_lon": min_lon,
|
||||
"min_lat": min_lat,
|
||||
"max_lon": max_lon,
|
||||
"max_lat": max_lat,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
"""Pure GeoTIFF flood detection for the flood-analysis module.
|
||||
|
||||
This service deliberately does not depend on ENVI/SARscape. Satellite-specific
|
||||
preprocessors are responsible only for producing analysis-ready GeoTIFFs; the
|
||||
flood classification below operates on those GeoTIFFs with Python/rasterio.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
def _valid_mask(data: np.ndarray, nodata: float | int | None) -> np.ndarray:
|
||||
valid = np.isfinite(data)
|
||||
if nodata is not None and np.isfinite(float(nodata)):
|
||||
valid &= data != float(nodata)
|
||||
return valid
|
||||
|
||||
|
||||
def _sample_valid(values: np.ndarray, max_samples: int = 1_000_000) -> np.ndarray:
|
||||
flat = values[np.isfinite(values)]
|
||||
if flat.size <= max_samples:
|
||||
return flat
|
||||
step = max(1, int(math.ceil(flat.size / max_samples)))
|
||||
return flat[::step]
|
||||
|
||||
|
||||
def _otsu_threshold(values: np.ndarray) -> float:
|
||||
sample = _sample_valid(values)
|
||||
if sample.size < 100:
|
||||
raise ValueError("Too few valid pixels for thresholding")
|
||||
manual_threshold = _manual_otsu_threshold(sample)
|
||||
try:
|
||||
from skimage.filters import threshold_otsu
|
||||
|
||||
skimage_threshold = float(threshold_otsu(sample))
|
||||
if np.isfinite(skimage_threshold):
|
||||
p05, p95 = np.nanpercentile(sample, [5, 95])
|
||||
if p05 < skimage_threshold < p95:
|
||||
return skimage_threshold
|
||||
return manual_threshold
|
||||
except Exception:
|
||||
return manual_threshold
|
||||
|
||||
|
||||
def _manual_otsu_threshold(values: np.ndarray) -> float:
|
||||
sample = _sample_valid(values)
|
||||
if sample.size < 100:
|
||||
raise ValueError("Too few valid pixels for thresholding")
|
||||
vmin = float(np.nanmin(sample))
|
||||
vmax = float(np.nanmax(sample))
|
||||
if not np.isfinite(vmin) or not np.isfinite(vmax):
|
||||
raise ValueError("Input pixels are not finite")
|
||||
if math.isclose(vmin, vmax):
|
||||
return vmin
|
||||
|
||||
hist, edges = np.histogram(sample, bins=256, range=(vmin, vmax))
|
||||
hist = hist.astype("float64")
|
||||
centers = (edges[:-1] + edges[1:]) / 2.0
|
||||
total = hist.sum()
|
||||
if total <= 0:
|
||||
return float(np.nanpercentile(sample, 10))
|
||||
|
||||
weight_background = np.cumsum(hist)
|
||||
weight_foreground = total - weight_background
|
||||
mean_background = np.cumsum(hist * centers) / np.maximum(weight_background, 1e-12)
|
||||
mean_foreground = (
|
||||
np.cumsum((hist * centers)[::-1]) / np.maximum(np.cumsum(hist[::-1]), 1e-12)
|
||||
)[::-1]
|
||||
variance = weight_background[:-1] * weight_foreground[:-1] * (
|
||||
mean_background[:-1] - mean_foreground[1:]
|
||||
) ** 2
|
||||
if variance.size == 0 or not np.isfinite(variance).any():
|
||||
return float(np.nanpercentile(sample, 10))
|
||||
idx = int(np.nanargmax(variance))
|
||||
return float(edges[idx + 1])
|
||||
|
||||
|
||||
def _pixel_area_km2(transform: Any, crs: Any, bounds: Any) -> float:
|
||||
px_w = abs(float(transform.a))
|
||||
px_h = abs(float(transform.e))
|
||||
if crs and getattr(crs, "is_geographic", False):
|
||||
lat_center = (float(bounds.top) + float(bounds.bottom)) / 2.0
|
||||
px_w_m = px_w * math.cos(math.radians(lat_center)) * 111_320.0
|
||||
px_h_m = px_h * 111_320.0
|
||||
else:
|
||||
px_w_m, px_h_m = px_w, px_h
|
||||
return max(0.0, (px_w_m * px_h_m) / 1_000_000.0)
|
||||
|
||||
|
||||
def _clean_mask(mask: np.ndarray, min_pixels: int) -> np.ndarray:
|
||||
try:
|
||||
from scipy.ndimage import binary_closing, binary_opening, generate_binary_structure, label
|
||||
except Exception:
|
||||
return mask
|
||||
|
||||
structure = generate_binary_structure(2, 2)
|
||||
cleaned = binary_closing(mask, structure=structure, iterations=1)
|
||||
cleaned = binary_opening(cleaned, structure=structure, iterations=1)
|
||||
if min_pixels <= 1:
|
||||
return cleaned
|
||||
|
||||
labels, count = label(cleaned)
|
||||
if count <= 0:
|
||||
return cleaned
|
||||
component_sizes = np.bincount(labels.ravel())
|
||||
keep = component_sizes >= int(min_pixels)
|
||||
keep[0] = False
|
||||
return keep[labels]
|
||||
|
||||
|
||||
def _read_pre_on_post_grid(pre_path: str, post_profile: dict[str, Any]) -> tuple[np.ndarray, dict[str, Any]]:
|
||||
import rasterio
|
||||
from rasterio.enums import Resampling
|
||||
from rasterio.warp import reproject
|
||||
|
||||
with rasterio.open(pre_path) as pre_ds:
|
||||
pre_data = pre_ds.read(1).astype("float32")
|
||||
pre_nodata = pre_ds.nodata
|
||||
same_grid = (
|
||||
pre_ds.width == int(post_profile["width"])
|
||||
and pre_ds.height == int(post_profile["height"])
|
||||
and pre_ds.transform == post_profile["transform"]
|
||||
and str(pre_ds.crs or "") == str(post_profile["crs"] or "")
|
||||
)
|
||||
metadata = {
|
||||
"path": pre_path,
|
||||
"crs": pre_ds.crs.to_string() if pre_ds.crs else None,
|
||||
"width": pre_ds.width,
|
||||
"height": pre_ds.height,
|
||||
"nodata": pre_nodata,
|
||||
"reprojected_to_post_grid": not same_grid,
|
||||
}
|
||||
if same_grid:
|
||||
data = pre_data.astype("float32")
|
||||
if pre_nodata is not None and np.isfinite(float(pre_nodata)):
|
||||
data[data == float(pre_nodata)] = np.nan
|
||||
return data, metadata
|
||||
if not pre_ds.crs or not post_profile["crs"]:
|
||||
raise ValueError("Pre/post GeoTIFF CRS is required when grids differ")
|
||||
destination = np.full(
|
||||
(int(post_profile["height"]), int(post_profile["width"])),
|
||||
np.nan,
|
||||
dtype="float32",
|
||||
)
|
||||
reproject(
|
||||
source=pre_data,
|
||||
destination=destination,
|
||||
src_transform=pre_ds.transform,
|
||||
src_crs=pre_ds.crs,
|
||||
src_nodata=pre_nodata,
|
||||
dst_transform=post_profile["transform"],
|
||||
dst_crs=post_profile["crs"],
|
||||
dst_nodata=np.nan,
|
||||
resampling=Resampling.bilinear,
|
||||
)
|
||||
return destination, metadata
|
||||
|
||||
|
||||
def run_geotiff_flood_detection(
|
||||
*,
|
||||
pre_tif_path: str,
|
||||
post_tif_path: str,
|
||||
output_dir: str,
|
||||
job_id: str | None = None,
|
||||
refine: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Classify stable water and new flood extent from two analysis-ready GeoTIFFs."""
|
||||
import rasterio
|
||||
|
||||
pre_path = Path(os.path.normpath(str(pre_tif_path or "").strip()))
|
||||
post_path = Path(os.path.normpath(str(post_tif_path or "").strip()))
|
||||
out_dir = Path(os.path.normpath(str(output_dir or "").strip()))
|
||||
if not pre_path.is_file():
|
||||
return {"ok": False, "error": f"Pre-event analysis GeoTIFF not found: {pre_path}"}
|
||||
if not post_path.is_file():
|
||||
return {"ok": False, "error": f"Post-event analysis GeoTIFF not found: {post_path}"}
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with rasterio.open(post_path) as post_ds:
|
||||
post_data = post_ds.read(1).astype("float32")
|
||||
post_nodata = post_ds.nodata
|
||||
post_profile = post_ds.profile.copy()
|
||||
post_grid = {
|
||||
"height": post_ds.height,
|
||||
"width": post_ds.width,
|
||||
"transform": post_ds.transform,
|
||||
"crs": post_ds.crs,
|
||||
}
|
||||
post_metadata = {
|
||||
"path": str(post_path),
|
||||
"crs": post_ds.crs.to_string() if post_ds.crs else None,
|
||||
"width": post_ds.width,
|
||||
"height": post_ds.height,
|
||||
"nodata": post_nodata,
|
||||
}
|
||||
pixel_area_km2 = _pixel_area_km2(post_ds.transform, post_ds.crs, post_ds.bounds)
|
||||
|
||||
pre_data, pre_metadata = _read_pre_on_post_grid(str(pre_path), post_grid)
|
||||
valid_pre = _valid_mask(pre_data, None)
|
||||
valid_post = _valid_mask(post_data, post_nodata)
|
||||
valid = valid_pre & valid_post
|
||||
if int(np.count_nonzero(valid)) < 100:
|
||||
return {"ok": False, "error": "Too few overlapping valid pixels between pre/post GeoTIFFs"}
|
||||
|
||||
pre_valid_values = pre_data[valid]
|
||||
post_valid_values = post_data[valid]
|
||||
pre_threshold = _otsu_threshold(pre_valid_values)
|
||||
post_threshold = _otsu_threshold(post_valid_values)
|
||||
|
||||
pre_water = (pre_data <= pre_threshold) & valid
|
||||
post_water = (post_data <= post_threshold) & valid
|
||||
stable_water = pre_water & post_water
|
||||
flood = post_water & ~pre_water
|
||||
|
||||
if refine:
|
||||
min_pixels = max(4, int(round(3_000.0 / max(pixel_area_km2 * 1_000_000.0, 1.0))))
|
||||
stable_water = _clean_mask(stable_water, min_pixels=min_pixels)
|
||||
flood = _clean_mask(flood, min_pixels=min_pixels)
|
||||
|
||||
high_threshold = float(np.nanpercentile(post_valid_values, 98))
|
||||
high_backscatter = (post_data >= high_threshold) & valid & ~(stable_water | flood)
|
||||
|
||||
classified = np.zeros(post_data.shape, dtype="uint8")
|
||||
classified[valid] = 4
|
||||
classified[high_backscatter] = 3
|
||||
classified[stable_water] = 1
|
||||
classified[flood] = 2
|
||||
|
||||
classified_path = out_dir / "classified.tif"
|
||||
flood_mask_path = out_dir / "flood_mask.tif"
|
||||
stable_mask_path = out_dir / "stable_water_mask.tif"
|
||||
|
||||
classified_profile = post_profile.copy()
|
||||
classified_profile.update(
|
||||
driver="GTiff",
|
||||
dtype="uint8",
|
||||
count=1,
|
||||
nodata=0,
|
||||
compress="deflate",
|
||||
)
|
||||
with rasterio.open(classified_path, "w", **classified_profile) as dst:
|
||||
dst.write(classified, 1)
|
||||
try:
|
||||
dst.write_colormap(
|
||||
1,
|
||||
{
|
||||
0: (0, 0, 0, 0),
|
||||
1: (24, 144, 255, 255),
|
||||
2: (255, 77, 79, 255),
|
||||
3: (250, 173, 20, 255),
|
||||
4: (80, 80, 80, 255),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
mask_profile = classified_profile.copy()
|
||||
mask_profile.update(nodata=0)
|
||||
with rasterio.open(flood_mask_path, "w", **mask_profile) as dst:
|
||||
dst.write(np.where(flood, 255, 0).astype("uint8"), 1)
|
||||
with rasterio.open(stable_mask_path, "w", **mask_profile) as dst:
|
||||
dst.write(np.where(stable_water, 255, 0).astype("uint8"), 1)
|
||||
|
||||
flood_pixels = int(np.count_nonzero(flood))
|
||||
stable_pixels = int(np.count_nonzero(stable_water))
|
||||
high_pixels = int(np.count_nonzero(high_backscatter))
|
||||
non_water_pixels = int(np.count_nonzero(classified == 4))
|
||||
metadata = {
|
||||
"schema": "flood_detection_geotiff.v1",
|
||||
"job_id": job_id,
|
||||
"processor": "python_geotiff_otsu_change",
|
||||
"refine": bool(refine),
|
||||
"pre": pre_metadata,
|
||||
"post": post_metadata,
|
||||
"thresholds": {
|
||||
"pre_water_threshold": pre_threshold,
|
||||
"post_water_threshold": post_threshold,
|
||||
"post_high_backscatter_threshold": high_threshold,
|
||||
},
|
||||
"pixel_area_km2": pixel_area_km2,
|
||||
"class_values": {
|
||||
"0": "nodata",
|
||||
"1": "stable_water",
|
||||
"2": "flood",
|
||||
"3": "high_backscatter",
|
||||
"4": "non_water",
|
||||
},
|
||||
"counts": {
|
||||
"valid_pixels": int(np.count_nonzero(valid)),
|
||||
"stable_water_pixels": stable_pixels,
|
||||
"flood_pixels": flood_pixels,
|
||||
"high_backscatter_pixels": high_pixels,
|
||||
"non_water_pixels": non_water_pixels,
|
||||
},
|
||||
"outputs": {
|
||||
"classified_path": str(classified_path),
|
||||
"flood_mask_path": str(flood_mask_path),
|
||||
"stable_water_mask_path": str(stable_mask_path),
|
||||
},
|
||||
}
|
||||
metadata_path = out_dir / "metadata.json"
|
||||
metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"classified_path": str(classified_path),
|
||||
"flood_mask_path": str(flood_mask_path),
|
||||
"stable_water_mask_path": str(stable_mask_path),
|
||||
"metadata_path": str(metadata_path),
|
||||
"flood_area_km2": round(flood_pixels * pixel_area_km2, 4),
|
||||
"stable_water_area_km2": round(stable_pixels * pixel_area_km2, 4),
|
||||
"flood_pixel_count": flood_pixels,
|
||||
"stable_water_pixel_count": stable_pixels,
|
||||
"processor": "python_geotiff_otsu_change",
|
||||
"log": [
|
||||
"pre/post analysis-ready GeoTIFFs loaded",
|
||||
"pre scene reprojected to post-event grid",
|
||||
f"thresholds pre={pre_threshold:.4f}, post={post_threshold:.4f}",
|
||||
f"flood_pixels={flood_pixels}, stable_water_pixels={stable_pixels}",
|
||||
],
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
"""Flood overlay and impact analysis service."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from geoalchemy2.functions import ST_Intersects
|
||||
from geoalchemy2.shape import from_shape
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from ..config import settings
|
||||
from ..models import FloodDetectionORM, FloodOverlayORM, HazardPointORM, ResultProductORM
|
||||
|
||||
|
||||
def _to_float(value: Any) -> float | None:
|
||||
try:
|
||||
if value is None:
|
||||
return None
|
||||
return float(value)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _hazard_point_to_dict(point: HazardPointORM, *, distance_m: float | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"id": point.id,
|
||||
"name": point.hazard_name,
|
||||
"type": point.hazard_type,
|
||||
"city": point.city,
|
||||
"county": point.county,
|
||||
"township": point.township,
|
||||
"longitude": _to_float(point.longitude),
|
||||
"latitude": _to_float(point.latitude),
|
||||
"distance_m": 0 if distance_m is None else round(float(distance_m), 2),
|
||||
}
|
||||
|
||||
|
||||
def _dinsar_product_to_dict(product: ResultProductORM) -> dict[str, Any]:
|
||||
summary = product.summary_json if isinstance(product.summary_json, dict) else {}
|
||||
deformation = (
|
||||
summary.get("deformation_mm")
|
||||
or summary.get("max_deformation_mm")
|
||||
or summary.get("mean_deformation_mm")
|
||||
or summary.get("deformation")
|
||||
)
|
||||
return {
|
||||
"id": product.id,
|
||||
"product_id": product.product_id,
|
||||
"display_name": product.display_name,
|
||||
"engine": product.engine_code,
|
||||
"status": product.status,
|
||||
"deformation_mm": deformation,
|
||||
"ai_score": product.ai_score,
|
||||
"manifest_path": product.manifest_path,
|
||||
"preview_path": product.preview_path,
|
||||
}
|
||||
|
||||
|
||||
def _open_raster(path: str):
|
||||
import rasterio
|
||||
|
||||
normalized_path = path.replace("\\", "/")
|
||||
try:
|
||||
return rasterio.open(normalized_path)
|
||||
except Exception:
|
||||
pass
|
||||
for ext in (".bin", ".img", ".tif", ".tiff"):
|
||||
candidate = normalized_path + ext
|
||||
try:
|
||||
return rasterio.open(candidate)
|
||||
except Exception:
|
||||
pass
|
||||
raise FileNotFoundError(f"Raster file cannot be opened: {path}")
|
||||
|
||||
|
||||
def _classified_flood_to_geojson(path: str) -> tuple[dict[str, Any], float | None, list[str]]:
|
||||
import rasterio.features
|
||||
from pyproj import CRS, Transformer
|
||||
from shapely.geometry import shape as shape_geojson
|
||||
from shapely.ops import transform
|
||||
|
||||
warnings: list[str] = []
|
||||
with _open_raster(path) as src:
|
||||
data = src.read(1)
|
||||
mask = data == 2
|
||||
if not mask.any():
|
||||
return {"type": "FeatureCollection", "features": []}, 0.0, warnings
|
||||
|
||||
polygons = []
|
||||
for geom, value in rasterio.features.shapes(data, mask=mask, transform=src.transform):
|
||||
if int(value) != 2:
|
||||
continue
|
||||
polygon = shape_geojson(geom)
|
||||
if not polygon.is_empty and polygon.is_valid:
|
||||
polygons.append(polygon)
|
||||
|
||||
if not polygons:
|
||||
return {"type": "FeatureCollection", "features": []}, 0.0, warnings
|
||||
|
||||
flood_geom = unary_union(polygons)
|
||||
source_crs = src.crs
|
||||
|
||||
area_km2 = None
|
||||
output_geom = flood_geom
|
||||
if source_crs:
|
||||
try:
|
||||
crs = CRS.from_user_input(source_crs)
|
||||
if not crs.is_geographic:
|
||||
area_km2 = float(flood_geom.area) / 1_000_000.0
|
||||
transformer = Transformer.from_crs(crs, CRS.from_epsg(4326), always_xy=True)
|
||||
output_geom = transform(transformer.transform, flood_geom)
|
||||
else:
|
||||
centroid = flood_geom.centroid
|
||||
zone = int((centroid.x + 180) // 6) + 1
|
||||
epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone
|
||||
transformer = Transformer.from_crs(crs, CRS.from_epsg(epsg), always_xy=True)
|
||||
projected = transform(transformer.transform, flood_geom)
|
||||
area_km2 = float(projected.area) / 1_000_000.0
|
||||
except Exception as exc:
|
||||
warnings.append(f"area calculation failed: {exc}")
|
||||
else:
|
||||
warnings.append("classified raster has no CRS; geometry is stored in source coordinates")
|
||||
|
||||
feature_collection = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"class": 2, "name": "flood"},
|
||||
"geometry": mapping(output_geom),
|
||||
}
|
||||
],
|
||||
}
|
||||
return feature_collection, area_km2, warnings
|
||||
|
||||
|
||||
def _write_geojson(detection_id: int, feature_collection: dict[str, Any]) -> str:
|
||||
out_dir = Path(settings.WATER_RESULTS_DIR or Path(settings.BACKEND_DIR) / "water_results") / "flood_overlays"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = out_dir / f"flood_detection_{detection_id}_overlay.geojson"
|
||||
target.write_text(json.dumps(feature_collection, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return str(target)
|
||||
|
||||
|
||||
def _read_geojson(path: str | None) -> dict[str, Any] | None:
|
||||
if not path or not os.path.isfile(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as stream:
|
||||
payload = json.load(stream)
|
||||
return payload if isinstance(payload, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _geometry_area_km2(geom) -> float:
|
||||
if geom.is_empty:
|
||||
return 0.0
|
||||
try:
|
||||
from pyproj import CRS, Transformer
|
||||
from shapely.ops import transform
|
||||
|
||||
centroid = geom.centroid
|
||||
zone = int((centroid.x + 180) // 6) + 1
|
||||
epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone
|
||||
transformer = Transformer.from_crs(CRS.from_epsg(4326), CRS.from_epsg(epsg), always_xy=True)
|
||||
projected = transform(transformer.transform, geom)
|
||||
return float(projected.area) / 1_000_000.0
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _calculate_affected_aois(flood_geom, warnings: list[str], *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
try:
|
||||
from ..routers import dependencies as deps
|
||||
|
||||
deps._load_region_index()
|
||||
deps._load_region_geometry_index()
|
||||
region_by_id = deps._REGION_BY_ID_CACHE or {}
|
||||
geometry_by_id = deps._REGION_GEOMETRY_BY_ID_CACHE or {}
|
||||
except Exception as exc:
|
||||
warnings.append(f"AOI overlay unavailable: {exc}")
|
||||
return []
|
||||
|
||||
affected: list[dict[str, Any]] = []
|
||||
for tree_id, features in geometry_by_id.items():
|
||||
node = region_by_id.get(tree_id) or {}
|
||||
level = node.get("level")
|
||||
if level in {"country", "province"}:
|
||||
continue
|
||||
try:
|
||||
geometries = [shape(feature["geometry"]) for feature in features if feature.get("geometry")]
|
||||
if not geometries:
|
||||
continue
|
||||
region_geom = unary_union(geometries)
|
||||
if region_geom.is_empty or not flood_geom.intersects(region_geom):
|
||||
continue
|
||||
intersection = flood_geom.intersection(region_geom)
|
||||
area_km2 = _geometry_area_km2(intersection)
|
||||
if area_km2 <= 0.0001:
|
||||
continue
|
||||
affected.append(
|
||||
{
|
||||
"tree_id": tree_id,
|
||||
"name": node.get("name") or tree_id,
|
||||
"level": level,
|
||||
"flood_area_km2": round(area_km2, 4),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
affected.sort(key=lambda item: item["flood_area_km2"], reverse=True)
|
||||
return affected[:limit]
|
||||
|
||||
|
||||
def _attach_overlay_payload(overlay: FloodOverlayORM) -> dict[str, Any]:
|
||||
payload = dict(overlay.summary_json) if isinstance(overlay.summary_json, dict) else {}
|
||||
payload["overlay_id"] = overlay.id
|
||||
payload["detection_id"] = overlay.detection_id
|
||||
payload["flood_vector_path"] = overlay.flood_vector_path
|
||||
payload["flood_vector_geojson"] = _read_geojson(overlay.flood_vector_path)
|
||||
return payload
|
||||
|
||||
|
||||
async def run_overlay(detection_id: int, db: AsyncSession, *, near_threshold_m: float = 500.0) -> dict[str, Any]:
|
||||
detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if not detection:
|
||||
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
|
||||
if not detection.classified_path:
|
||||
raise HTTPException(status_code=400, detail="Flood detection has no classified raster")
|
||||
path = detection.classified_path.replace("\\", "/")
|
||||
if not os.path.isfile(path):
|
||||
raise HTTPException(status_code=404, detail="Classified raster file does not exist")
|
||||
|
||||
feature_collection, flood_area_km2, warnings = _classified_flood_to_geojson(path)
|
||||
flood_vector_path = _write_geojson(detection_id, feature_collection)
|
||||
impact = await _query_impact_from_geojson(
|
||||
detection_id=detection_id,
|
||||
feature_collection=feature_collection,
|
||||
db=db,
|
||||
near_threshold_m=near_threshold_m,
|
||||
warnings=warnings,
|
||||
)
|
||||
if flood_area_km2 is not None:
|
||||
impact["flood_area_km2"] = round(flood_area_km2, 4)
|
||||
impact["flood_vector_path"] = flood_vector_path
|
||||
|
||||
overlay = FloodOverlayORM(
|
||||
detection_id=detection_id,
|
||||
flood_vector_path=flood_vector_path,
|
||||
hazard_points_hit=len(impact["hazard_points"]["inside_flood"]),
|
||||
hazard_points_near=len(impact["hazard_points"]["near_flood"]),
|
||||
hazard_points_total=impact["hazard_points"]["total_in_scene"],
|
||||
dinsar_products_intersecting=len(impact["dinsar_products"]),
|
||||
affected_area_km2=impact.get("flood_area_km2"),
|
||||
summary_json=impact,
|
||||
)
|
||||
db.add(overlay)
|
||||
await db.flush()
|
||||
impact["overlay_id"] = overlay.id
|
||||
overlay.summary_json = impact
|
||||
if flood_area_km2 is not None:
|
||||
detection.flood_area_km2 = round(flood_area_km2, 4)
|
||||
await db.commit()
|
||||
await db.refresh(overlay)
|
||||
|
||||
return {
|
||||
"id": overlay.id,
|
||||
"detection_id": detection_id,
|
||||
"flood_vector_path": overlay.flood_vector_path,
|
||||
"flood_vector_geojson": feature_collection,
|
||||
"summary": _attach_overlay_payload(overlay),
|
||||
}
|
||||
|
||||
|
||||
async def get_overlay_result(detection_id: int, db: AsyncSession) -> dict[str, Any]:
|
||||
overlay = (
|
||||
await db.execute(
|
||||
select(FloodOverlayORM)
|
||||
.where(FloodOverlayORM.detection_id == detection_id)
|
||||
.order_by(FloodOverlayORM.id.desc())
|
||||
)
|
||||
).scalars().first()
|
||||
if overlay and isinstance(overlay.summary_json, dict):
|
||||
return _attach_overlay_payload(overlay)
|
||||
|
||||
detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if not detection:
|
||||
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
|
||||
return {
|
||||
"detection_id": detection_id,
|
||||
"flood_area_km2": detection.flood_area_km2,
|
||||
"hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0},
|
||||
"dinsar_products": [],
|
||||
"affected_aois": [],
|
||||
"flood_vector_path": None,
|
||||
"flood_vector_geojson": None,
|
||||
"warnings": ["overlay has not been run"],
|
||||
}
|
||||
|
||||
|
||||
async def _query_impact_from_geojson(
|
||||
*,
|
||||
detection_id: int,
|
||||
feature_collection: dict[str, Any],
|
||||
db: AsyncSession,
|
||||
near_threshold_m: float,
|
||||
warnings: list[str],
|
||||
) -> dict[str, Any]:
|
||||
features = feature_collection.get("features") or []
|
||||
if not features:
|
||||
return {
|
||||
"detection_id": detection_id,
|
||||
"flood_area_km2": 0.0,
|
||||
"hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0},
|
||||
"dinsar_products": [],
|
||||
"affected_aois": [],
|
||||
"warnings": warnings,
|
||||
}
|
||||
|
||||
flood_geom = unary_union([shape(feature["geometry"]) for feature in features if feature.get("geometry")])
|
||||
if flood_geom.is_empty:
|
||||
warnings.append("flood geometry is empty")
|
||||
flood_wkt = flood_geom.wkt
|
||||
area_geom = func.ST_GeomFromText(flood_wkt, 4326)
|
||||
area_geog = func.Geography(area_geom)
|
||||
|
||||
inside_points: list[dict[str, Any]] = []
|
||||
near_points: list[dict[str, Any]] = []
|
||||
dinsar_products: list[dict[str, Any]] = []
|
||||
affected_aois = _calculate_affected_aois(flood_geom, warnings)
|
||||
|
||||
try:
|
||||
inside_rows = (
|
||||
await db.execute(
|
||||
select(HazardPointORM).where(ST_Intersects(HazardPointORM.geom, area_geom))
|
||||
)
|
||||
).scalars().all()
|
||||
inside_ids = {point.id for point in inside_rows}
|
||||
inside_points = [_hazard_point_to_dict(point, distance_m=0) for point in inside_rows]
|
||||
|
||||
near_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
HazardPointORM,
|
||||
func.ST_Distance(func.Geography(HazardPointORM.geom), area_geog).label("distance_m"),
|
||||
).where(func.ST_DWithin(func.Geography(HazardPointORM.geom), area_geog, near_threshold_m))
|
||||
)
|
||||
).all()
|
||||
for point, distance_m in near_rows:
|
||||
if point.id in inside_ids:
|
||||
continue
|
||||
near_points.append(_hazard_point_to_dict(point, distance_m=distance_m))
|
||||
except Exception as exc:
|
||||
warnings.append(f"hazard point overlay unavailable: {exc}")
|
||||
|
||||
try:
|
||||
products = (
|
||||
await db.execute(
|
||||
select(ResultProductORM).where(
|
||||
ResultProductORM.catalog_name == "dinsar",
|
||||
ResultProductORM.status == "READY",
|
||||
ST_Intersects(ResultProductORM.geom, area_geom),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
dinsar_products = [_dinsar_product_to_dict(product) for product in products]
|
||||
except Exception as exc:
|
||||
warnings.append(f"dinsar product overlay unavailable: {exc}")
|
||||
|
||||
return {
|
||||
"detection_id": detection_id,
|
||||
"flood_area_km2": None,
|
||||
"hazard_points": {
|
||||
"inside_flood": inside_points,
|
||||
"near_flood": near_points,
|
||||
"total_in_scene": len(inside_points) + len(near_points),
|
||||
},
|
||||
"dinsar_products": dinsar_products,
|
||||
"affected_aois": affected_aois,
|
||||
"warnings": warnings,
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Flood product listing and manifest helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from ..models import FloodDetectionORM, FloodOverlayORM, FloodProductORM, SARSceneGeoORM
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
return value.isoformat() if value else None
|
||||
|
||||
|
||||
def _product_to_dict(product: FloodProductORM) -> dict[str, Any]:
|
||||
summary = product.summary_json if isinstance(product.summary_json, dict) else {}
|
||||
detection = product.detection
|
||||
overlay = product.overlay
|
||||
return {
|
||||
"id": product.id,
|
||||
"product_id": product.product_id,
|
||||
"detection_id": product.detection_id,
|
||||
"overlay_id": product.overlay_id,
|
||||
"display_name": product.display_name,
|
||||
"status": product.status,
|
||||
"publish_dir": product.publish_dir,
|
||||
"manifest_path": product.manifest_path,
|
||||
"summary": summary,
|
||||
"created_at": _iso(product.created_at),
|
||||
"flood_area_km2": getattr(detection, "flood_area_km2", None),
|
||||
"affected_area_km2": getattr(overlay, "affected_area_km2", None),
|
||||
}
|
||||
|
||||
|
||||
async def list_flood_products(
|
||||
*,
|
||||
db: AsyncSession,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
count_query = select(func.count()).select_from(FloodProductORM)
|
||||
query = (
|
||||
select(FloodProductORM)
|
||||
.options(
|
||||
selectinload(FloodProductORM.detection),
|
||||
selectinload(FloodProductORM.overlay),
|
||||
)
|
||||
.order_by(FloodProductORM.id.desc())
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
)
|
||||
if status:
|
||||
count_query = count_query.where(FloodProductORM.status == status)
|
||||
query = query.where(FloodProductORM.status == status)
|
||||
|
||||
total = (await db.execute(count_query)).scalar_one()
|
||||
rows = (await db.execute(query)).scalars().all()
|
||||
return {"items": [_product_to_dict(row) for row in rows], "total": total}
|
||||
|
||||
|
||||
async def get_flood_product(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]:
|
||||
product = await _get_product(product_id_or_pk, db)
|
||||
return _product_to_dict(product)
|
||||
|
||||
|
||||
async def get_flood_product_manifest(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]:
|
||||
product = await _get_product(product_id_or_pk, db)
|
||||
if product.manifest_path and os.path.isfile(product.manifest_path):
|
||||
try:
|
||||
with open(product.manifest_path, "r", encoding="utf-8") as stream:
|
||||
payload = json.load(stream)
|
||||
if isinstance(payload, dict):
|
||||
return payload
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to read manifest: {exc}") from exc
|
||||
|
||||
return _build_manifest_from_db(product)
|
||||
|
||||
|
||||
async def create_flood_product_for_detection(detection_id: int, db: AsyncSession) -> dict[str, Any]:
|
||||
detection = await db.get(FloodDetectionORM, detection_id)
|
||||
if not detection:
|
||||
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
|
||||
if detection.status != "DONE":
|
||||
raise HTTPException(status_code=400, detail=f"Flood detection is not DONE: {detection.status}")
|
||||
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(FloodProductORM)
|
||||
.options(
|
||||
selectinload(FloodProductORM.detection),
|
||||
selectinload(FloodProductORM.overlay),
|
||||
)
|
||||
.where(FloodProductORM.detection_id == detection_id)
|
||||
.order_by(FloodProductORM.id.desc())
|
||||
)
|
||||
).scalars().first()
|
||||
if existing:
|
||||
return _product_to_dict(existing)
|
||||
|
||||
overlay = (
|
||||
await db.execute(
|
||||
select(FloodOverlayORM)
|
||||
.where(FloodOverlayORM.detection_id == detection_id)
|
||||
.order_by(FloodOverlayORM.id.desc())
|
||||
)
|
||||
).scalars().first()
|
||||
product = FloodProductORM(
|
||||
product_id=f"FLOOD-{detection_id:06d}",
|
||||
detection_id=detection_id,
|
||||
overlay_id=overlay.id if overlay else None,
|
||||
display_name=f"Flood detection #{detection_id}",
|
||||
status="READY",
|
||||
publish_dir=detection.output_dir,
|
||||
manifest_path=None,
|
||||
summary_json={
|
||||
"created_from": "flood_detection",
|
||||
"flood_area_km2": detection.flood_area_km2,
|
||||
"stable_water_area_km2": detection.stable_water_area_km2,
|
||||
"classified_path": detection.classified_path,
|
||||
"created_at": datetime.utcnow().isoformat(),
|
||||
},
|
||||
)
|
||||
db.add(product)
|
||||
await db.commit()
|
||||
return await get_flood_product(str(product.id), db)
|
||||
|
||||
|
||||
async def _get_product(product_id_or_pk: str, db: AsyncSession) -> FloodProductORM:
|
||||
value = str(product_id_or_pk).strip()
|
||||
query = select(FloodProductORM).options(
|
||||
selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data),
|
||||
selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data),
|
||||
selectinload(FloodProductORM.overlay),
|
||||
)
|
||||
if value.isdigit():
|
||||
query = query.where(FloodProductORM.id == int(value))
|
||||
else:
|
||||
query = query.where(FloodProductORM.product_id == value)
|
||||
product = (await db.execute(query)).scalars().first()
|
||||
if not product:
|
||||
raise HTTPException(status_code=404, detail=f"Flood product {product_id_or_pk} not found")
|
||||
return product
|
||||
|
||||
|
||||
def _build_manifest_from_db(product: FloodProductORM) -> dict[str, Any]:
|
||||
detection = product.detection
|
||||
overlay = product.overlay
|
||||
pre_scene = detection.pre_scene if detection else None
|
||||
post_scene = detection.post_scene if detection else None
|
||||
pre_radar = pre_scene.radar_data if pre_scene else None
|
||||
post_radar = post_scene.radar_data if post_scene else None
|
||||
return {
|
||||
"schema": "flood_product_manifest.v1",
|
||||
"product": _product_to_dict(product),
|
||||
"detection": {
|
||||
"id": detection.id if detection else None,
|
||||
"status": detection.status if detection else None,
|
||||
"classified_path": detection.classified_path if detection else None,
|
||||
"flood_area_km2": detection.flood_area_km2 if detection else None,
|
||||
"stable_water_area_km2": detection.stable_water_area_km2 if detection else None,
|
||||
"pre_scene": {
|
||||
"id": pre_scene.id if pre_scene else None,
|
||||
"radar_data_id": pre_scene.radar_data_id if pre_scene else None,
|
||||
"satellite": pre_radar.satellite if pre_radar else None,
|
||||
"imaging_date": pre_radar.imaging_date if pre_radar else None,
|
||||
"geo_path": pre_scene.geo_path if pre_scene else None,
|
||||
"analysis_tif_path": pre_scene.analysis_tif_path if pre_scene else None,
|
||||
"analysis_engine": pre_scene.analysis_engine if pre_scene else None,
|
||||
"analysis_profile": pre_scene.analysis_profile if pre_scene else None,
|
||||
},
|
||||
"post_scene": {
|
||||
"id": post_scene.id if post_scene else None,
|
||||
"radar_data_id": post_scene.radar_data_id if post_scene else None,
|
||||
"satellite": post_radar.satellite if post_radar else None,
|
||||
"imaging_date": post_radar.imaging_date if post_radar else None,
|
||||
"geo_path": post_scene.geo_path if post_scene else None,
|
||||
"analysis_tif_path": post_scene.analysis_tif_path if post_scene else None,
|
||||
"analysis_engine": post_scene.analysis_engine if post_scene else None,
|
||||
"analysis_profile": post_scene.analysis_profile if post_scene else None,
|
||||
},
|
||||
},
|
||||
"overlay": {
|
||||
"id": overlay.id if overlay else None,
|
||||
"flood_vector_path": overlay.flood_vector_path if overlay else None,
|
||||
"affected_area_km2": overlay.affected_area_km2 if overlay else None,
|
||||
"hazard_points_hit": overlay.hazard_points_hit if overlay else 0,
|
||||
"hazard_points_near": overlay.hazard_points_near if overlay else 0,
|
||||
"dinsar_products_intersecting": overlay.dinsar_products_intersecting if overlay else 0,
|
||||
"summary": overlay.summary_json if overlay else None,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tarfile
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from ..config import settings, split_env_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LogCallback = Callable[[str, str], None]
|
||||
ProgressCallback = Callable[[int, str], None]
|
||||
|
||||
DEFAULT_GF3_ARCHIVE_EXTS = (".zip", ".tar", ".tar.gz", ".tgz")
|
||||
|
||||
|
||||
def _parse_float(value: Any, default: float) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _normalize_paths(paths: Optional[Iterable[str]]) -> List[str]:
|
||||
ordered: List[str] = []
|
||||
for raw_path in paths or []:
|
||||
text = str(raw_path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
continue
|
||||
normalized = os.path.normpath(os.path.abspath(text))
|
||||
if normalized not in ordered:
|
||||
ordered.append(normalized)
|
||||
return ordered
|
||||
|
||||
|
||||
def _normalize_extensions(extensions: Optional[Iterable[str]]) -> List[str]:
|
||||
ordered: List[str] = []
|
||||
for raw_ext in extensions or DEFAULT_GF3_ARCHIVE_EXTS:
|
||||
ext = str(raw_ext or "").strip().lower()
|
||||
if not ext:
|
||||
continue
|
||||
if not ext.startswith("."):
|
||||
ext = f".{ext}"
|
||||
if ext not in ordered:
|
||||
ordered.append(ext)
|
||||
return sorted(ordered or list(DEFAULT_GF3_ARCHIVE_EXTS), key=len, reverse=True)
|
||||
|
||||
|
||||
def _strip_archive_extension(file_name: str, extensions: Iterable[str]) -> str:
|
||||
lower_name = file_name.lower()
|
||||
for ext in _normalize_extensions(extensions):
|
||||
if lower_name.endswith(ext):
|
||||
return file_name[: -len(ext)]
|
||||
return os.path.splitext(file_name)[0]
|
||||
|
||||
|
||||
def _resolve_target_root(archive_path: str, source_dirs: List[str], target_dirs: List[str]) -> str:
|
||||
if not target_dirs:
|
||||
raise ValueError("GF3_SOURCE_DIRS is not configured.")
|
||||
if len(target_dirs) == 1:
|
||||
return target_dirs[0]
|
||||
if source_dirs and len(source_dirs) == len(target_dirs):
|
||||
archive_norm = os.path.normcase(os.path.abspath(archive_path))
|
||||
matches: List[Tuple[int, int]] = []
|
||||
for index, source_dir in enumerate(source_dirs):
|
||||
source_norm = os.path.normcase(os.path.abspath(source_dir))
|
||||
if archive_norm == source_norm or archive_norm.startswith(source_norm + os.sep):
|
||||
matches.append((len(source_norm), index))
|
||||
if matches:
|
||||
_prefix_len, best_index = max(matches)
|
||||
return target_dirs[best_index]
|
||||
return target_dirs[0]
|
||||
|
||||
|
||||
def _validate_relative_member_name(member_name: str, archive_path: str) -> str:
|
||||
name = str(member_name or "").replace("\\", "/")
|
||||
if not name or name in {".", "./"}:
|
||||
return ""
|
||||
if name.startswith("/") or os.path.isabs(name) or os.path.splitdrive(name)[0]:
|
||||
raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}")
|
||||
parts = [part for part in name.split("/") if part not in ("", ".")]
|
||||
if any(part == ".." for part in parts):
|
||||
raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}")
|
||||
if not parts:
|
||||
return ""
|
||||
return os.path.join(*parts)
|
||||
|
||||
|
||||
def _safe_destination(root_dir: str, relative_name: str) -> str:
|
||||
root_abs = os.path.abspath(root_dir)
|
||||
destination = os.path.abspath(os.path.join(root_abs, relative_name))
|
||||
if destination != root_abs and not destination.startswith(root_abs + os.sep):
|
||||
raise ValueError(f"Unsafe extraction destination: {relative_name}")
|
||||
return destination
|
||||
|
||||
|
||||
def _validate_tar_members(members: Iterable[tarfile.TarInfo], archive_path: str) -> None:
|
||||
for member in members:
|
||||
_validate_relative_member_name(member.name, archive_path)
|
||||
if member.issym() or member.islnk():
|
||||
raise ValueError(f"Unsupported link entry in {archive_path}: {member.name}")
|
||||
if not (member.isdir() or member.isfile()):
|
||||
raise ValueError(f"Unsupported special entry in {archive_path}: {member.name}")
|
||||
|
||||
|
||||
def _validate_zip_members(infos: Iterable[zipfile.ZipInfo], archive_path: str) -> None:
|
||||
for info in infos:
|
||||
_validate_relative_member_name(info.filename, archive_path)
|
||||
mode = (info.external_attr >> 16) & 0o170000
|
||||
if stat.S_ISLNK(mode):
|
||||
raise ValueError(f"Unsupported symlink entry in {archive_path}: {info.filename}")
|
||||
|
||||
|
||||
def _estimate_archive_size(archive_path: str) -> int:
|
||||
if zipfile.is_zipfile(archive_path):
|
||||
with zipfile.ZipFile(archive_path, "r") as zip_obj:
|
||||
infos = zip_obj.infolist()
|
||||
_validate_zip_members(infos, archive_path)
|
||||
return sum(max(0, int(info.file_size or 0)) for info in infos if not info.is_dir())
|
||||
if tarfile.is_tarfile(archive_path):
|
||||
with tarfile.open(archive_path, "r:*") as tar_obj:
|
||||
members = tar_obj.getmembers()
|
||||
_validate_tar_members(members, archive_path)
|
||||
return sum(max(0, int(member.size or 0)) for member in members if member.isfile())
|
||||
raise ValueError(f"Unsupported GF3 archive format: {archive_path}")
|
||||
|
||||
|
||||
def _ensure_disk_space(target_root: str, required_bytes: int, min_disk_space_gb: float) -> None:
|
||||
os.makedirs(target_root, exist_ok=True)
|
||||
_total, _used, free_bytes = shutil.disk_usage(target_root)
|
||||
min_free_bytes = int(max(0.0, min_disk_space_gb) * (1024 ** 3))
|
||||
if free_bytes - max(0, int(required_bytes or 0)) < min_free_bytes:
|
||||
raise OSError(
|
||||
"GF3 L1A storage has insufficient free space: "
|
||||
f"needed {required_bytes / (1024 ** 3):.2f} GB, "
|
||||
f"free {free_bytes / (1024 ** 3):.2f} GB, "
|
||||
f"min free after {min_disk_space_gb:.2f} GB"
|
||||
)
|
||||
|
||||
|
||||
def _prepare_atomic_output(output_dir: str, tmp_suffix: str) -> Tuple[bool, str, str]:
|
||||
tmp_dir = output_dir + tmp_suffix
|
||||
lock_path = output_dir + ".unpacking"
|
||||
if os.path.exists(output_dir):
|
||||
return False, tmp_dir, lock_path
|
||||
if os.path.exists(tmp_dir):
|
||||
return False, tmp_dir, lock_path
|
||||
if os.path.exists(lock_path):
|
||||
return False, tmp_dir, lock_path
|
||||
os.makedirs(tmp_dir, exist_ok=False)
|
||||
with open(lock_path, "w", encoding="utf-8") as stream:
|
||||
stream.write(datetime.now().isoformat())
|
||||
return True, tmp_dir, lock_path
|
||||
|
||||
|
||||
def _cleanup_atomic_paths(tmp_dir: str, lock_path: str) -> None:
|
||||
if os.path.exists(lock_path):
|
||||
try:
|
||||
os.remove(lock_path)
|
||||
except OSError:
|
||||
pass
|
||||
if os.path.exists(tmp_dir):
|
||||
try:
|
||||
shutil.rmtree(tmp_dir)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _extract_tar_archive(archive_path: str, tmp_dir: str) -> int:
|
||||
extracted_files = 0
|
||||
with tarfile.open(archive_path, "r:*") as tar_obj:
|
||||
members = tar_obj.getmembers()
|
||||
_validate_tar_members(members, archive_path)
|
||||
for member in members:
|
||||
relative_name = _validate_relative_member_name(member.name, archive_path)
|
||||
if not relative_name:
|
||||
continue
|
||||
destination = _safe_destination(tmp_dir, relative_name)
|
||||
if member.isdir():
|
||||
os.makedirs(destination, exist_ok=True)
|
||||
continue
|
||||
if not member.isfile():
|
||||
continue
|
||||
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
||||
source = tar_obj.extractfile(member)
|
||||
if source is None:
|
||||
raise OSError(f"Failed to read tar member: {member.name}")
|
||||
with source, open(destination, "wb") as target:
|
||||
shutil.copyfileobj(source, target, length=1024 * 1024)
|
||||
extracted_files += 1
|
||||
return extracted_files
|
||||
|
||||
|
||||
def _extract_zip_archive(archive_path: str, tmp_dir: str) -> int:
|
||||
extracted_files = 0
|
||||
with zipfile.ZipFile(archive_path, "r") as zip_obj:
|
||||
infos = zip_obj.infolist()
|
||||
_validate_zip_members(infos, archive_path)
|
||||
for info in infos:
|
||||
relative_name = _validate_relative_member_name(info.filename, archive_path)
|
||||
if not relative_name:
|
||||
continue
|
||||
destination = _safe_destination(tmp_dir, relative_name)
|
||||
if info.is_dir():
|
||||
os.makedirs(destination, exist_ok=True)
|
||||
continue
|
||||
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
||||
with zip_obj.open(info, "r") as source, open(destination, "wb") as target:
|
||||
shutil.copyfileobj(source, target, length=1024 * 1024)
|
||||
extracted_files += 1
|
||||
return extracted_files
|
||||
|
||||
|
||||
def _extract_archive_atomic(archive_path: str, output_dir: str, tmp_suffix: str) -> Tuple[str, int]:
|
||||
prepared, tmp_dir, lock_path = _prepare_atomic_output(output_dir, tmp_suffix)
|
||||
if not prepared:
|
||||
return "EXISTS", 0
|
||||
|
||||
try:
|
||||
if zipfile.is_zipfile(archive_path):
|
||||
extracted_files = _extract_zip_archive(archive_path, tmp_dir)
|
||||
elif tarfile.is_tarfile(archive_path):
|
||||
extracted_files = _extract_tar_archive(archive_path, tmp_dir)
|
||||
else:
|
||||
raise ValueError(f"Unsupported GF3 archive format: {archive_path}")
|
||||
|
||||
if extracted_files <= 0:
|
||||
raise OSError("GF3 archive extraction produced no files.")
|
||||
os.replace(tmp_dir, output_dir)
|
||||
return "EXTRACTED", extracted_files
|
||||
finally:
|
||||
_cleanup_atomic_paths(tmp_dir, lock_path)
|
||||
|
||||
|
||||
def _discover_archives(source_dirs: List[str], extensions: List[str], log_callback: Optional[LogCallback]) -> List[str]:
|
||||
archives: List[str] = []
|
||||
normalized_extensions = _normalize_extensions(extensions)
|
||||
for source_dir in source_dirs:
|
||||
if not os.path.isdir(source_dir):
|
||||
message = f"GF3 archive source does not exist or is not a directory: {source_dir}"
|
||||
logger.warning(message)
|
||||
if log_callback:
|
||||
log_callback("WARNING", message)
|
||||
continue
|
||||
for root, _dirs, files in os.walk(source_dir):
|
||||
for file_name in files:
|
||||
lower_name = file_name.lower()
|
||||
if any(lower_name.endswith(ext) for ext in normalized_extensions):
|
||||
archives.append(os.path.join(root, file_name))
|
||||
return sorted(archives)
|
||||
|
||||
|
||||
def run_gf3_archive_unpack(
|
||||
*,
|
||||
source_dirs: Optional[Iterable[str]] = None,
|
||||
target_dirs: Optional[Iterable[str]] = None,
|
||||
archive_exts: Optional[Iterable[str]] = None,
|
||||
max_files_per_run: Optional[int] = None,
|
||||
delete_archive: Optional[bool] = None,
|
||||
min_disk_space_gb: Optional[float] = None,
|
||||
tmp_suffix: Optional[str] = None,
|
||||
log_callback: Optional[LogCallback] = None,
|
||||
progress_callback: Optional[ProgressCallback] = None,
|
||||
) -> Dict[str, Any]:
|
||||
configured_source_dirs = _normalize_paths(
|
||||
source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
|
||||
)
|
||||
configured_target_dirs = _normalize_paths(
|
||||
target_dirs if target_dirs is not None else split_env_paths(settings.GF3_SOURCE_DIRS)
|
||||
)
|
||||
extensions = _normalize_extensions(
|
||||
archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
|
||||
)
|
||||
should_delete_archive = settings.GF3_UNPACK_DELETE_ARCHIVE if delete_archive is None else bool(delete_archive)
|
||||
limit = max(0, int(max_files_per_run or 0))
|
||||
min_free_gb = (
|
||||
_parse_float(os.getenv("UNPACK_MIN_DISK_SPACE_GB"), 50.0)
|
||||
if min_disk_space_gb is None
|
||||
else float(min_disk_space_gb)
|
||||
)
|
||||
atomic_tmp_suffix = str(tmp_suffix or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp").strip() or ".unpack_tmp"
|
||||
|
||||
if not configured_source_dirs:
|
||||
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
if not configured_target_dirs:
|
||||
raise ValueError("GF3_SOURCE_DIRS is not configured.")
|
||||
|
||||
def _log(level: str, message: str) -> None:
|
||||
logger.log(getattr(logging, level.upper(), logging.INFO), message)
|
||||
if log_callback:
|
||||
log_callback(level.upper(), message)
|
||||
|
||||
def _progress(progress: int, message: str) -> None:
|
||||
if progress_callback:
|
||||
progress_callback(max(0, min(100, int(progress))), message)
|
||||
|
||||
_progress(2, "Scanning GF3 archive source directories...")
|
||||
archives = _discover_archives(configured_source_dirs, extensions, log_callback)
|
||||
if limit > 0:
|
||||
archives_to_process = archives[:limit]
|
||||
else:
|
||||
archives_to_process = archives
|
||||
|
||||
total = len(archives_to_process)
|
||||
summary: Dict[str, Any] = {
|
||||
"total": total,
|
||||
"found": len(archives),
|
||||
"processed": 0,
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"remaining": max(0, len(archives) - total),
|
||||
"source_dirs": configured_source_dirs,
|
||||
"target_dirs": configured_target_dirs,
|
||||
"archive_exts": extensions,
|
||||
"delete_archive": should_delete_archive,
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
if not archives_to_process:
|
||||
_progress(100, "No GF3 archives pending.")
|
||||
summary["message"] = "No GF3 archives found."
|
||||
return summary
|
||||
|
||||
_log("INFO", f"Found {len(archives)} GF3 archives; processing {total}.")
|
||||
|
||||
for index, archive_path in enumerate(archives_to_process, start=1):
|
||||
archive_name = os.path.basename(archive_path)
|
||||
progress_base = 5 + int(((index - 1) / max(1, total)) * 90)
|
||||
_progress(progress_base, f"Unpacking GF3 archive {index}/{total}: {archive_name}")
|
||||
|
||||
try:
|
||||
target_root = _resolve_target_root(archive_path, configured_source_dirs, configured_target_dirs)
|
||||
os.makedirs(target_root, exist_ok=True)
|
||||
output_name = _strip_archive_extension(os.path.basename(archive_path), extensions)
|
||||
output_dir = os.path.join(target_root, output_name)
|
||||
required_bytes = _estimate_archive_size(archive_path)
|
||||
_ensure_disk_space(target_root, required_bytes, min_free_gb)
|
||||
status, extracted_files = _extract_archive_atomic(archive_path, output_dir, atomic_tmp_suffix)
|
||||
|
||||
if status == "EXISTS":
|
||||
summary["skipped"] += 1
|
||||
_log("INFO", f"GF3 archive already unpacked, skipped: {archive_path}")
|
||||
continue
|
||||
|
||||
if should_delete_archive:
|
||||
os.remove(archive_path)
|
||||
_log("INFO", f"GF3 archive deleted after successful unpack: {archive_path}")
|
||||
summary["processed"] += 1
|
||||
_log("INFO", f"GF3 archive unpacked: {archive_path} -> {output_dir} ({extracted_files} files)")
|
||||
except Exception as exc:
|
||||
summary["failed"] += 1
|
||||
failure = {
|
||||
"archive_path": archive_path,
|
||||
"error": str(exc),
|
||||
}
|
||||
summary["failures"].append(failure)
|
||||
_log("ERROR", f"GF3 archive unpack failed: {archive_path}: {exc}")
|
||||
|
||||
_progress(100, "GF3 archive unpack completed.")
|
||||
summary["message"] = (
|
||||
f"GF3 unpack complete: processed {summary['processed']}, "
|
||||
f"skipped {summary['skipped']}, failed {summary['failed']}"
|
||||
)
|
||||
return summary
|
||||
@@ -17,6 +17,7 @@ from ..models import (
|
||||
OrbitAssetORM,
|
||||
ResultCatalogStateORM,
|
||||
ResultProductORM,
|
||||
SARSceneGeoORM,
|
||||
SceneOrbitBindingORM,
|
||||
SourceProductAssetORM,
|
||||
SystemWorkerHeartbeatORM,
|
||||
@@ -362,6 +363,19 @@ def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_sar_analysis_ready_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
scenes = payload.get("scenes", {}) or {}
|
||||
roots = payload.get("roots", {}) or {}
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
"configured_root_count": len(roots),
|
||||
"accessible_root_count": sum(1 for item in roots.values() if item.get("accessible")),
|
||||
"scene_count": int(scenes.get("scene_count") or 0),
|
||||
"analysis_scene_count": int(scenes.get("analysis_scene_count") or 0),
|
||||
"missing_file_count": int(scenes.get("missing_file_count") or 0),
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
@@ -460,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
psinsar_result_catalog = timeseries_result_catalog
|
||||
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
sar_analysis_ready = payload.get("sar_analysis_ready", {}) or {}
|
||||
product_packages = payload.get("product_packages", {}) or {}
|
||||
asset_inventory = payload.get("asset_inventory", {}) or {}
|
||||
wsl_runtime = payload.get("wsl_runtime", {}) or {}
|
||||
@@ -473,6 +488,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sanitized_psinsar_catalog = sanitized_timeseries_catalog
|
||||
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
|
||||
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
|
||||
sanitized_sar_analysis_ready = _sanitize_sar_analysis_ready_status(sar_analysis_ready)
|
||||
sanitized_product_packages = _sanitize_product_package_status(product_packages)
|
||||
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
|
||||
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
|
||||
@@ -502,6 +518,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
},
|
||||
"dinsar_bridge": sanitized_dinsar_bridge,
|
||||
"source_roots": sanitized_source_roots,
|
||||
"sar_analysis_ready": sanitized_sar_analysis_ready,
|
||||
"product_packages": sanitized_product_packages,
|
||||
"asset_inventory": sanitized_asset_inventory,
|
||||
"wsl_runtime": sanitized_wsl_runtime,
|
||||
@@ -869,6 +886,21 @@ async def _check_source_roots() -> Dict[str, Any]:
|
||||
status["role"] = "dinsar_source"
|
||||
items.append(status)
|
||||
|
||||
for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS):
|
||||
status = _probe_directory_status(path)
|
||||
status["role"] = "gf3_archive_source"
|
||||
items.append(status)
|
||||
|
||||
for path in split_env_paths(settings.GF3_SOURCE_DIRS):
|
||||
status = _probe_directory_status(path)
|
||||
status["role"] = "gf3_l1a_source"
|
||||
items.append(status)
|
||||
|
||||
for path in split_env_paths(settings.GF3_STORAGE_DIRS):
|
||||
status = _probe_directory_status(path)
|
||||
status["role"] = "gf3_l2_storage"
|
||||
items.append(status)
|
||||
|
||||
configured_count = len(items)
|
||||
accessible_count = sum(1 for item in items if item.get("accessible"))
|
||||
inaccessible_count = configured_count - accessible_count
|
||||
@@ -882,6 +914,56 @@ async def _check_source_roots() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
async def _check_sar_analysis_ready() -> Dict[str, Any]:
|
||||
roots = {
|
||||
"ready": _probe_directory_status(settings.SAR_ANALYSIS_READY_ROOT),
|
||||
"work": _probe_directory_status(settings.SAR_ANALYSIS_WORK_ROOT),
|
||||
"preview": _probe_directory_status(settings.SAR_ANALYSIS_PREVIEW_ROOT),
|
||||
}
|
||||
for role, payload in roots.items():
|
||||
payload["role"] = role
|
||||
|
||||
status: Dict[str, Any] = {
|
||||
"ok": False,
|
||||
"roots": roots,
|
||||
"scenes": {
|
||||
"scene_count": 0,
|
||||
"analysis_scene_count": 0,
|
||||
"missing_file_count": 0,
|
||||
"missing_files": [],
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
session_factory = _get_session_factory()
|
||||
async with session_factory() as db:
|
||||
scene_count_result = await db.execute(select(func.count(SARSceneGeoORM.id)))
|
||||
status["scenes"]["scene_count"] = int(scene_count_result.scalar_one() or 0)
|
||||
analysis_rows_result = await db.execute(
|
||||
select(SARSceneGeoORM.id, SARSceneGeoORM.analysis_tif_path)
|
||||
.where(SARSceneGeoORM.analysis_tif_path.is_not(None))
|
||||
.order_by(SARSceneGeoORM.id.desc())
|
||||
)
|
||||
analysis_rows = analysis_rows_result.all()
|
||||
status["scenes"]["analysis_scene_count"] = len(analysis_rows)
|
||||
|
||||
missing_files = []
|
||||
for scene_id, tif_path in analysis_rows:
|
||||
path_text = str(tif_path or "").strip()
|
||||
if path_text and not os.path.isfile(path_text):
|
||||
missing_files.append({"scene_id": scene_id, "path": path_text})
|
||||
status["scenes"]["missing_file_count"] = len(missing_files)
|
||||
status["scenes"]["missing_files"] = missing_files[:20]
|
||||
|
||||
status["ok"] = (
|
||||
all(item.get("accessible") for item in roots.values())
|
||||
and status["scenes"]["missing_file_count"] == 0
|
||||
)
|
||||
except Exception as exc:
|
||||
status["error"] = str(exc)
|
||||
return status
|
||||
|
||||
|
||||
async def _check_product_packages() -> Dict[str, Any]:
|
||||
status = {
|
||||
"ok": False,
|
||||
@@ -1256,6 +1338,7 @@ async def get_health_status(
|
||||
psinsar_result_catalog_status = timeseries_result_catalog_status
|
||||
dinsar_bridge_status = await _check_dinsar_bridge()
|
||||
source_roots_status = await _check_source_roots()
|
||||
sar_analysis_ready_status = await _check_sar_analysis_ready()
|
||||
product_packages_status = await _check_product_packages()
|
||||
asset_inventory_status = await _check_asset_inventory()
|
||||
wsl_runtime_status = await _check_wsl_runtime()
|
||||
@@ -1273,6 +1356,7 @@ async def get_health_status(
|
||||
result_catalog_status.get("ok"),
|
||||
dinsar_bridge_status.get("ok"),
|
||||
source_roots_status.get("ok"),
|
||||
sar_analysis_ready_status.get("ok"),
|
||||
product_packages_status.get("ok"),
|
||||
asset_inventory_status.get("ok"),
|
||||
wsl_runtime_status.get("ok"),
|
||||
@@ -1297,6 +1381,7 @@ async def get_health_status(
|
||||
},
|
||||
"dinsar_bridge": dinsar_bridge_status,
|
||||
"source_roots": source_roots_status,
|
||||
"sar_analysis_ready": sar_analysis_ready_status,
|
||||
"product_packages": product_packages_status,
|
||||
"asset_inventory": asset_inventory_status,
|
||||
"wsl_runtime": wsl_runtime_status,
|
||||
|
||||
@@ -19,7 +19,7 @@ from sqlalchemy import select
|
||||
|
||||
from .. import database
|
||||
from ..config import settings
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..scheduler import scan_data_job
|
||||
from .data_service import data_service
|
||||
from .asset_inventory_service import asset_inventory_service
|
||||
@@ -80,7 +80,10 @@ JOB_TYPE_IDL_RUN_DINSAR = "IDL_RUN_DINSAR"
|
||||
JOB_TYPE_WATER_GEOCODE = "WATER_GEOCODE"
|
||||
JOB_TYPE_WATER_FLOOD = "WATER_FLOOD"
|
||||
JOB_TYPE_WATER_DETECT = "WATER_DETECT"
|
||||
JOB_TYPE_SAR_SCENE_PREPROCESS = "SAR_SCENE_PREPROCESS"
|
||||
JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION"
|
||||
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
|
||||
JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
@@ -88,6 +91,7 @@ JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
|
||||
|
||||
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
|
||||
|
||||
@@ -3149,6 +3153,106 @@ async def _handle_water_geocode(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_sar_scene_preprocess(job: SystemJobORM) -> None:
|
||||
"""Build one analysis-ready GeoTIFF for flood/water algorithms."""
|
||||
payload = job.payload or {}
|
||||
scene_id = payload.get("scene_id")
|
||||
radar_data_id = payload.get("radar_data_id")
|
||||
engine = str(payload.get("engine") or "").strip().lower()
|
||||
if not scene_id and not radar_data_id:
|
||||
raise ValueError("SAR_SCENE_PREPROCESS requires scene_id or radar_data_id")
|
||||
if engine not in {"gf3_gdal", "lt_gamma"}:
|
||||
raise ValueError(f"Unsupported SAR scene preprocessing engine: {engine}")
|
||||
|
||||
await task_service.start_task(job.task_id, message="Preparing analysis-ready SAR GeoTIFF...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
scene: SARSceneGeoORM | None = None
|
||||
if scene_id:
|
||||
scene = await db.get(SARSceneGeoORM, int(scene_id))
|
||||
if not scene and radar_data_id:
|
||||
result = await db.execute(
|
||||
select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == int(radar_data_id))
|
||||
)
|
||||
scene = result.scalar_one_or_none()
|
||||
if not scene:
|
||||
scene = SARSceneGeoORM(radar_data_id=int(radar_data_id), status="PENDING")
|
||||
db.add(scene)
|
||||
await db.flush()
|
||||
radar = await db.get(RadarDataORM, int(scene.radar_data_id))
|
||||
if not radar:
|
||||
raise ValueError(f"RadarDataORM id={scene.radar_data_id} does not exist")
|
||||
scene.status = "RUNNING"
|
||||
scene.error_msg = None
|
||||
await db.commit()
|
||||
scene_id = int(scene.id)
|
||||
radar_data_id = int(radar.id)
|
||||
|
||||
try:
|
||||
if engine == "gf3_gdal":
|
||||
await task_service.update_task(job.task_id, progress=20, message="Standardizing GF3 L2 GeoTIFF...")
|
||||
from .sar_analysis_ready_service import standardize_gf3_l2_for_radar
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
manifest = await standardize_gf3_l2_for_radar(
|
||||
db=db,
|
||||
radar_id=int(radar_data_id),
|
||||
l2_path=payload.get("l2_path"),
|
||||
polarization=payload.get("polarization"),
|
||||
)
|
||||
else:
|
||||
await task_service.update_task(job.task_id, progress=15, message="Running LT Gamma single-scene preprocessing...")
|
||||
from .lt_gamma_scene_service import run_lt_gamma_scene_preprocess
|
||||
from .sar_analysis_ready_service import register_analysis_ready_tif
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
scene = await db.get(SARSceneGeoORM, int(scene_id))
|
||||
radar = await db.get(RadarDataORM, int(radar_data_id))
|
||||
if not scene or not radar:
|
||||
raise ValueError("Scene or radar record disappeared before LT Gamma preprocessing")
|
||||
|
||||
def _run_lt() -> Dict[str, Any]:
|
||||
return run_lt_gamma_scene_preprocess(radar=radar, scene=scene, job_id=job.job_id)
|
||||
|
||||
lt_manifest = await asyncio.to_thread(_run_lt)
|
||||
await task_service.update_task(job.task_id, progress=85, message="Registering LT analysis-ready GeoTIFF...")
|
||||
analysis_tif_path = str(lt_manifest.get("analysis_tif_path") or "").strip()
|
||||
if not analysis_tif_path:
|
||||
raise RuntimeError("LT Gamma preprocessing returned no analysis_tif_path")
|
||||
async with AsyncSessionLocal() as db:
|
||||
scene = await db.get(SARSceneGeoORM, int(scene_id))
|
||||
radar = await db.get(RadarDataORM, int(radar_data_id))
|
||||
if not scene or not radar:
|
||||
raise ValueError("Scene or radar record disappeared before analysis-ready registration")
|
||||
manifest = await register_analysis_ready_tif(
|
||||
db=db,
|
||||
scene=scene,
|
||||
radar=radar,
|
||||
source_tif_path=analysis_tif_path,
|
||||
engine="lt_gamma",
|
||||
profile="lt1_gamma_geocoded_mli",
|
||||
backscatter_unit=str(lt_manifest.get("backscatter_unit") or "gamma_mli_db"),
|
||||
polarization=radar.polarization,
|
||||
metadata=lt_manifest,
|
||||
)
|
||||
await db.commit()
|
||||
except Exception as exc:
|
||||
async with AsyncSessionLocal() as db:
|
||||
scene = await db.get(SARSceneGeoORM, int(scene_id))
|
||||
if scene:
|
||||
scene.status = "FAILED"
|
||||
scene.error_msg = str(exc)
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=f"Analysis-ready GeoTIFF ready: {manifest.get('analysis_tif_path')}",
|
||||
)
|
||||
|
||||
|
||||
async def _handle_water_flood(job: SystemJobORM) -> None:
|
||||
"""洪涝检测 job handler(灾前 + 灾后配对分类)。"""
|
||||
from .water_service import run_flood_detection, WATER_RESULTS_DIR
|
||||
@@ -3171,8 +3275,10 @@ async def _handle_water_flood(job: SystemJobORM) -> None:
|
||||
raise ValueError("灾前或灾后场景记录不存在")
|
||||
if pre_scene.status != "DONE" or post_scene.status != "DONE":
|
||||
raise ValueError("灾前或灾后场景尚未完成地理编码")
|
||||
pre_geo = pre_scene.geo_path
|
||||
post_geo = post_scene.geo_path
|
||||
pre_geo = pre_scene.analysis_tif_path or pre_scene.geo_path
|
||||
post_geo = post_scene.analysis_tif_path or post_scene.geo_path
|
||||
if not pre_geo or not post_geo:
|
||||
raise ValueError("Pre/post scenes must have analysis-ready GeoTIFF paths")
|
||||
|
||||
output_dir = os.path.join(WATER_RESULTS_DIR, f"flood_{detection_id}")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
@@ -3228,36 +3334,129 @@ async def _handle_water_flood(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
"""水体检测 job handler(Otsu + DEM + 形态学 + 连通分量)。"""
|
||||
from .water_detect_service import run_water_detection
|
||||
async def _handle_flood_detection(job: SystemJobORM) -> None:
|
||||
"""Flood-analysis detection job: pure Python GeoTIFF change classification."""
|
||||
from .flood_detection_service import run_geotiff_flood_detection
|
||||
|
||||
payload = job.payload or {}
|
||||
detection_id = payload.get("detection_id")
|
||||
if not detection_id:
|
||||
raise ValueError("WATER_DETECT job 缺少 detection_id")
|
||||
raise ValueError("FLOOD_DETECTION job requires detection_id")
|
||||
refine = bool(payload.get("refine", False))
|
||||
|
||||
await task_service.start_task(job.task_id, message="Reading flood-detection scene pair...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(FloodDetectionORM, int(detection_id))
|
||||
if not det:
|
||||
raise ValueError(f"FloodDetectionORM id={detection_id} does not exist")
|
||||
pre_scene = await db.get(SARSceneGeoORM, det.pre_scene_id)
|
||||
post_scene = await db.get(SARSceneGeoORM, det.post_scene_id)
|
||||
if not pre_scene or not post_scene:
|
||||
raise ValueError("Pre/post scene records do not exist")
|
||||
if pre_scene.status != "DONE" or post_scene.status != "DONE":
|
||||
raise ValueError("Pre/post scenes are not DONE")
|
||||
pre_tif = pre_scene.analysis_tif_path
|
||||
post_tif = post_scene.analysis_tif_path
|
||||
if not pre_tif or not post_tif:
|
||||
raise ValueError("Flood detection requires analysis-ready GeoTIFF paths for both scenes")
|
||||
|
||||
output_dir = os.path.join(settings.WATER_RESULTS_DIR, f"flood_{detection_id}")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
await task_service.update_task(job.task_id, progress=10, message="Running GeoTIFF flood classification...")
|
||||
|
||||
def _run() -> Dict[str, Any]:
|
||||
return run_geotiff_flood_detection(
|
||||
pre_tif_path=pre_tif,
|
||||
post_tif_path=post_tif,
|
||||
output_dir=output_dir,
|
||||
job_id=job.job_id,
|
||||
refine=refine,
|
||||
)
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(_run)
|
||||
except Exception as exc:
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(FloodDetectionORM, int(detection_id))
|
||||
if det:
|
||||
det.status = "FAILED"
|
||||
det.error_msg = str(exc)
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(FloodDetectionORM, int(detection_id))
|
||||
if det:
|
||||
if result.get("ok"):
|
||||
det.classified_path = result.get("classified_path")
|
||||
det.flood_area_km2 = result.get("flood_area_km2")
|
||||
det.stable_water_area_km2 = result.get("stable_water_area_km2")
|
||||
det.output_dir = output_dir
|
||||
det.status = "DONE"
|
||||
det.error_msg = None
|
||||
else:
|
||||
det.status = "FAILED"
|
||||
det.error_msg = result.get("error", "Unknown error")
|
||||
await db.commit()
|
||||
|
||||
if not result.get("ok"):
|
||||
raise RuntimeError(f"Flood detection failed: {result.get('error')}")
|
||||
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
f"GeoTIFF flood detection completed: flood_area={result.get('flood_area_km2')} km2, "
|
||||
f"stable_water={result.get('stable_water_area_km2')} km2"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
"""水体检测 job handler(Otsu + DEM + 形态学 + 连通分量)。"""
|
||||
from .water_extraction_service import run_otsu_water_extraction
|
||||
|
||||
payload = job.payload or {}
|
||||
extraction_id = payload.get("extraction_id")
|
||||
detection_id = payload.get("detection_id")
|
||||
record_id = extraction_id or detection_id
|
||||
if not record_id:
|
||||
raise ValueError("WATER_DETECT job 缺少 extraction_id/detection_id")
|
||||
use_extraction_table = extraction_id is not None
|
||||
|
||||
await task_service.start_task(job.task_id, message="读取检测任务信息...")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(WaterDetectionORM, int(detection_id))
|
||||
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
|
||||
if not det:
|
||||
raise ValueError(f"WaterDetectionORM id={detection_id} 不存在")
|
||||
model_name = "WaterExtractionORM" if use_extraction_table else "WaterDetectionORM"
|
||||
raise ValueError(f"{model_name} id={record_id} 不存在")
|
||||
input_path = det.input_path
|
||||
det.status = "RUNNING"
|
||||
if use_extraction_table and hasattr(det, "task_id"):
|
||||
det.task_id = job.task_id
|
||||
if not use_extraction_table:
|
||||
mirror = await db.get(WaterExtractionORM, int(record_id))
|
||||
if mirror:
|
||||
mirror.status = "RUNNING"
|
||||
mirror.task_id = job.task_id
|
||||
await db.commit()
|
||||
|
||||
if not input_path:
|
||||
raise ValueError("水体检测缺少输入路径 input_path")
|
||||
|
||||
output_dir = os.path.join(os.path.dirname(input_path), f"water_detect_{detection_id}")
|
||||
output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}"
|
||||
output_dir = os.path.join(os.path.dirname(input_path), output_name)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...")
|
||||
|
||||
def _run() -> Dict[str, Any]:
|
||||
return run_water_detection(
|
||||
geo_tiff_path=input_path,
|
||||
return run_otsu_water_extraction(
|
||||
input_path=input_path,
|
||||
output_dir=output_dir,
|
||||
job_id=job.job_id,
|
||||
)
|
||||
@@ -3266,26 +3465,56 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
|
||||
result = await asyncio.to_thread(_run)
|
||||
except Exception as exc:
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(WaterDetectionORM, int(detection_id))
|
||||
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
|
||||
if det:
|
||||
det.status = "FAILED"
|
||||
det.error_msg = str(exc)
|
||||
await db.commit()
|
||||
if not use_extraction_table:
|
||||
mirror = await db.get(WaterExtractionORM, int(record_id))
|
||||
if mirror:
|
||||
mirror.status = "FAILED"
|
||||
mirror.error_msg = str(exc)
|
||||
mirror.task_id = job.task_id
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
det = await db.get(WaterDetectionORM, int(detection_id))
|
||||
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
|
||||
if det:
|
||||
if result.get("ok"):
|
||||
det.output_path = result.get("output_path")
|
||||
det.water_area_km2 = result.get("water_area_km2")
|
||||
det.water_pixel_count = result.get("water_pixel_count")
|
||||
det.otsu_threshold_db = result.get("otsu_threshold_db")
|
||||
if use_extraction_table:
|
||||
det.processor = result.get("processor") or det.processor or "otsu"
|
||||
det.threshold_value = result.get("threshold_value")
|
||||
det.metadata_json = {
|
||||
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
|
||||
"job_id": job.job_id,
|
||||
}
|
||||
else:
|
||||
det.otsu_threshold_db = result.get("otsu_threshold_db")
|
||||
det.status = "DONE"
|
||||
det.error_msg = None
|
||||
else:
|
||||
det.status = "FAILED"
|
||||
det.error_msg = result.get("error", "Unknown error")
|
||||
if not use_extraction_table:
|
||||
mirror = await db.get(WaterExtractionORM, int(record_id))
|
||||
if mirror:
|
||||
mirror.output_path = det.output_path
|
||||
mirror.water_area_km2 = det.water_area_km2
|
||||
mirror.water_pixel_count = det.water_pixel_count
|
||||
mirror.threshold_value = result.get("threshold_value") or result.get("otsu_threshold_db")
|
||||
mirror.processor = result.get("processor") or mirror.processor or "otsu"
|
||||
mirror.status = det.status
|
||||
mirror.error_msg = det.error_msg
|
||||
mirror.task_id = job.task_id
|
||||
mirror.metadata_json = {
|
||||
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
|
||||
"legacy_detection_id": int(record_id),
|
||||
"job_id": job.job_id,
|
||||
}
|
||||
await db.commit()
|
||||
|
||||
if not result.get("ok"):
|
||||
@@ -3376,9 +3605,70 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_gf3_unpack(job: SystemJobORM) -> None:
|
||||
"""GF3 archive inbox -> persistent L1A source pool."""
|
||||
from .gf3_unpack_service import run_gf3_archive_unpack
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_UNPACK requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
await task_service.start_task(job.task_id, message="扫描 GF3 压缩包来源目录...")
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _submit(coro):
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(coro, loop)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
def _swallow_errors(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 Unpack] task callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_errors)
|
||||
|
||||
def _log_cb(level: str, message: str) -> None:
|
||||
_submit(task_service.add_log(job.task_id, level, message))
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
_submit(task_service.update_task(job.task_id, progress=progress, message=message))
|
||||
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
run_gf3_archive_unpack,
|
||||
source_dirs=payload.get("source_dirs"),
|
||||
target_dirs=payload.get("target_dirs"),
|
||||
archive_exts=payload.get("archive_exts"),
|
||||
max_files_per_run=payload.get("max_files_per_run"),
|
||||
delete_archive=payload.get("delete_archive") if "delete_archive" in payload else None,
|
||||
min_disk_space_gb=payload.get("min_disk_space_gb"),
|
||||
tmp_suffix=payload.get("tmp_suffix"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=f"GF3 解包失败: {exc}")
|
||||
raise
|
||||
|
||||
message = (
|
||||
f"GF3 解包完成: 成功 {int(result.get('processed') or 0)}, "
|
||||
f"跳过 {int(result.get('skipped') or 0)}, "
|
||||
f"失败 {int(result.get('failed') or 0)}"
|
||||
)
|
||||
remaining = int(result.get("remaining") or 0)
|
||||
if remaining > 0:
|
||||
message += f", 剩余 {remaining}"
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
"""批量 GF3 L1A→L2:扫描来源目录,逐个处理并自动入库到 radar_data。"""
|
||||
from .gf3_service import run_gf3_l1a_to_l2, register_l2_to_radar_data
|
||||
from .sar_analysis_ready_service import standardize_gf3_l2_for_radar
|
||||
|
||||
payload = job.payload or {}
|
||||
source_dirs = payload.get("source_dirs") or []
|
||||
@@ -3458,12 +3748,18 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
# Auto-register to radar_data
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
await register_l2_to_radar_data(
|
||||
radar_id = await register_l2_to_radar_data(
|
||||
l2_dir=result.get("output_dir", output_dir),
|
||||
input_dir_name=dir_name,
|
||||
polarizations=result.get("polarizations", []),
|
||||
db=db,
|
||||
)
|
||||
if radar_id:
|
||||
await standardize_gf3_l2_for_radar(
|
||||
db=db,
|
||||
radar_id=int(radar_id),
|
||||
l2_path=result.get("output_dir", output_dir),
|
||||
)
|
||||
except Exception as reg_err:
|
||||
logger.warning("[GF3 Batch] Auto-register failed for %s: %s", dir_name, reg_err)
|
||||
else:
|
||||
@@ -3888,6 +4184,85 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def _handle_sbas_coregistration(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("SBAS_COREGISTRATION requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
run_id = str(payload.get("run_id") or "").strip()
|
||||
if not run_id:
|
||||
raise ValueError("SBAS_COREGISTRATION requires run_id payload.")
|
||||
|
||||
rlks = _normalize_positive_int(payload.get("rlks")) or 8
|
||||
azlks = _normalize_positive_int(payload.get("azlks")) or 8
|
||||
timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200
|
||||
|
||||
await task_service.start_task(job.task_id, message="正在执行 SBAS-InSAR Gamma 共参考配准...")
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=5,
|
||||
message=f"准备运行 Gamma SLC_coreg.py: run_id={run_id}",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"SBAS coregistration queued: run_id={run_id}, rlks={rlks}, azlks={azlks}, timeout={timeout_seconds}s",
|
||||
)
|
||||
|
||||
from .sbas_insar_production_service import sbas_insar_production_service
|
||||
|
||||
async def _task_keepalive() -> None:
|
||||
progress = 12
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
progress = min(88, progress + 2)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress,
|
||||
message=f"Gamma 共参考配准仍在运行: run_id={run_id}",
|
||||
)
|
||||
|
||||
runner_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
sbas_insar_production_service.execute_coregistration,
|
||||
run_id,
|
||||
rlks=rlks,
|
||||
azlks=azlks,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
)
|
||||
keepalive_task = asyncio.create_task(_task_keepalive())
|
||||
try:
|
||||
result = await runner_task
|
||||
finally:
|
||||
keepalive_task.cancel()
|
||||
try:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
manifest = result.get("manifest") or {}
|
||||
run = result.get("run") or {}
|
||||
summary = (manifest.get("coregistration") or {}).get("summary") or {}
|
||||
status = str(run.get("status") or manifest.get("status") or "").strip()
|
||||
if status != "COREGISTRATION_READY":
|
||||
raise RuntimeError(
|
||||
"SBAS coregistration failed: "
|
||||
f"status={status or 'UNKNOWN'}, "
|
||||
f"missing_dates={summary.get('missing_dates') or []}, "
|
||||
f"missing_tabs={summary.get('missing_tabs') or []}"
|
||||
)
|
||||
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
"SBAS-InSAR 共参考配准完成: "
|
||||
f"{summary.get('ready_secondary_count', 0)}/{summary.get('expected_secondary_count', 0)} secondary scenes ready"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_HANDLERS = {
|
||||
JOB_TYPE_SCAN_DATA: _handle_scan_data,
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
|
||||
@@ -3918,10 +4293,14 @@ _HANDLERS = {
|
||||
JOB_TYPE_ISCE2_RUN: _handle_isce2_run,
|
||||
JOB_TYPE_PYINT_RUN: _handle_pyint_run,
|
||||
JOB_TYPE_WATER_GEOCODE: _handle_water_geocode,
|
||||
JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess,
|
||||
JOB_TYPE_WATER_FLOOD: _handle_water_flood,
|
||||
JOB_TYPE_FLOOD_DETECTION: _handle_flood_detection,
|
||||
JOB_TYPE_WATER_DETECT: _handle_water_detect,
|
||||
JOB_TYPE_GF3_PROCESS: _handle_gf3_process,
|
||||
JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack,
|
||||
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
|
||||
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
"""LT single-scene Gamma preprocessing service."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..config import settings
|
||||
from ..models import RadarDataORM, SARSceneGeoORM
|
||||
from .pyint_service import (
|
||||
DEFAULT_AZIMUTH_LOOKS,
|
||||
DEFAULT_RANGE_LOOKS,
|
||||
quote_shell,
|
||||
resolve_gamma_env_script,
|
||||
to_wsl_path,
|
||||
)
|
||||
from .wsl_service import run_wsl_exec
|
||||
|
||||
|
||||
def _safe_token(value: Any, *, default: str = "scene") -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
text = default
|
||||
text = re.sub(r"[^0-9A-Za-z._-]+", "_", text).strip("._-")
|
||||
return text or default
|
||||
|
||||
|
||||
def _scene_date(radar: RadarDataORM) -> str:
|
||||
for value in (radar.imaging_date, radar.file_path, radar.unique_id):
|
||||
text = str(value or "")
|
||||
match = re.search(r"(20\d{6})", re.sub(r"\D", "", text))
|
||||
if match:
|
||||
return match.group(1)
|
||||
match = re.search(r"(20\d{6})", text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
raise ValueError(f"Cannot infer LT scene date for radar_data id={radar.id}")
|
||||
|
||||
|
||||
def _prepared_dem_path() -> str:
|
||||
if str(settings.PYINT_DEM_MODE or "").strip().lower() == "prepared_file":
|
||||
return (
|
||||
settings.PYINT_PREPARED_DEM_PATH
|
||||
or settings.ISCE2_DEM_PATH
|
||||
or settings.IDL_DINSAR_DEM_BASE_FILE
|
||||
or ""
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _build_shell_command(parts: list[str]) -> str:
|
||||
return " ".join(quote_shell(part) for part in parts)
|
||||
|
||||
|
||||
def _runner_script() -> Path:
|
||||
return Path(settings.PROJECT_ROOT) / "backend" / "app" / "pyint_pipeline" / "run_gamma_scene_preprocess.py"
|
||||
|
||||
|
||||
def run_lt_gamma_scene_preprocess(
|
||||
*,
|
||||
radar: RadarDataORM,
|
||||
scene: SARSceneGeoORM,
|
||||
job_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if not settings.PYINT_ENABLED:
|
||||
raise RuntimeError("PYINT_ENABLED=false; LT Gamma scene preprocessing is disabled")
|
||||
if not radar.file_path:
|
||||
raise ValueError(f"RadarDataORM id={radar.id} has no file_path")
|
||||
|
||||
date = _scene_date(radar)
|
||||
token = _safe_token(radar.unique_id or Path(str(radar.file_path)).stem or f"radar_{radar.id}")
|
||||
run_name = _safe_token(f"lt_{date}_{token}_scene_{scene.id}_{job_id or 'manual'}")
|
||||
work_dir = Path(settings.SAR_ANALYSIS_WORK_ROOT) / "lt_gamma" / run_name
|
||||
output_dir = work_dir / "output"
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pyint_home = settings.PYINT_HOME
|
||||
if not pyint_home:
|
||||
raise RuntimeError("PYINT_HOME is not configured")
|
||||
pyint_python = settings.PYINT_WSL_PYTHON or settings.WSL_SHARED_PYTHON
|
||||
if not pyint_python:
|
||||
raise RuntimeError("PYINT_WSL_PYTHON is not configured")
|
||||
|
||||
runner = _runner_script()
|
||||
if not runner.is_file():
|
||||
raise FileNotFoundError(f"Gamma scene runner not found: {runner}")
|
||||
|
||||
args = [
|
||||
pyint_python,
|
||||
to_wsl_path(str(runner)),
|
||||
"--source-path",
|
||||
to_wsl_path(str(radar.file_path)),
|
||||
"--output-dir",
|
||||
to_wsl_path(str(output_dir)),
|
||||
"--work-dir",
|
||||
to_wsl_path(str(work_dir)),
|
||||
"--pyint-home",
|
||||
to_wsl_path(str(pyint_home)),
|
||||
"--dem-root",
|
||||
to_wsl_path(str(settings.PYINT_DEM_ROOT)),
|
||||
"--prepared-dem-path",
|
||||
to_wsl_path(_prepared_dem_path()),
|
||||
"--project-name",
|
||||
run_name,
|
||||
"--date",
|
||||
date,
|
||||
"--satellite-family",
|
||||
"LT1",
|
||||
"--range-looks",
|
||||
str(DEFAULT_RANGE_LOOKS),
|
||||
"--azimuth-looks",
|
||||
str(DEFAULT_AZIMUTH_LOOKS),
|
||||
"--geo-interp",
|
||||
str(settings.PYINT_GEO_INTERP or "1"),
|
||||
"--nodata-value",
|
||||
str(float(settings.SAR_ANALYSIS_NODATA_VALUE)),
|
||||
"--to-db",
|
||||
]
|
||||
|
||||
gamma_env_script = resolve_gamma_env_script(settings.PYINT_GAMMA_ENV_SCRIPT)
|
||||
prefix = ""
|
||||
if gamma_env_script:
|
||||
prefix = f". {quote_shell(to_wsl_path(gamma_env_script))} >/dev/null 2>&1 || exit 1; "
|
||||
command = prefix + f"export PYTHONPATH={quote_shell(to_wsl_path(str(pyint_home)))}:$PYTHONPATH; " + _build_shell_command(args)
|
||||
|
||||
rc, stdout, stderr = run_wsl_exec(
|
||||
["bash", "-lc", command],
|
||||
distro=settings.PYINT_WSL_DISTRO or settings.WSL_DISTRO,
|
||||
timeout=int(settings.PYINT_DEFAULT_TIMEOUT_SECONDS or 43200),
|
||||
)
|
||||
if rc != 0:
|
||||
detail = (stderr or stdout or "").strip()
|
||||
raise RuntimeError(f"LT Gamma scene preprocessing failed rc={rc}: {detail}")
|
||||
|
||||
manifest_path = output_dir / "manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
raise RuntimeError(f"LT Gamma scene preprocessing produced no manifest: {manifest_path}")
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"Cannot read LT Gamma manifest: {manifest_path}: {exc}") from exc
|
||||
|
||||
analysis_tif_path = manifest.get("analysis_tif_path")
|
||||
if analysis_tif_path and str(analysis_tif_path).startswith("/mnt/"):
|
||||
# The service registers the Windows-side path below.
|
||||
manifest["analysis_tif_path_wsl"] = analysis_tif_path
|
||||
manifest["analysis_tif_path"] = str(output_dir / "analysis_ready.tif")
|
||||
manifest["service"] = {
|
||||
"work_dir": str(work_dir),
|
||||
"output_dir": str(output_dir),
|
||||
"job_id": job_id,
|
||||
"stdout": stdout[-4000:] if stdout else "",
|
||||
"stderr": stderr[-4000:] if stderr else "",
|
||||
}
|
||||
return manifest
|
||||
@@ -235,6 +235,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="directory_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_ARCHIVE_SOURCE_DIRS",
|
||||
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
root_role="source_pool_gf3_archive",
|
||||
display_prefix="GF3 Archive Pool",
|
||||
scan_mode="archive_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_SOURCE_DIRS",
|
||||
@@ -253,6 +262,24 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="SAR_ANALYSIS_READY_ROOT",
|
||||
path=settings.SAR_ANALYSIS_READY_ROOT,
|
||||
root_role="sar_analysis_ready",
|
||||
display_name="SAR Analysis-ready GeoTIFF Root",
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="SAR_ANALYSIS_WORK_ROOT",
|
||||
path=settings.SAR_ANALYSIS_WORK_ROOT,
|
||||
root_role="sar_analysis_work",
|
||||
display_name="SAR Analysis Work Root",
|
||||
scan_mode="directory_walk",
|
||||
)
|
||||
)
|
||||
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
|
||||
if not orbit_source_paths:
|
||||
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Analysis-ready SAR GeoTIFF registration for flood/water algorithms.
|
||||
|
||||
This service owns the common contract between satellite-specific preprocessing
|
||||
and downstream flood/water algorithms: one geocoded, single-band GeoTIFF plus
|
||||
sidecar metadata under SAR_ANALYSIS_READY_ROOT.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import settings
|
||||
from ..models import RadarDataORM, SARSceneGeoORM
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
_POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
||||
|
||||
|
||||
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
text = default
|
||||
text = _SAFE_TEXT_RE.sub("_", text).strip("._-")
|
||||
return text or default
|
||||
|
||||
|
||||
def _scene_family(radar: RadarDataORM | None) -> str:
|
||||
family = normalize_satellite_family(
|
||||
getattr(radar, "satellite_family", None) or getattr(radar, "satellite", None)
|
||||
)
|
||||
return _safe_slug(family or "SAR").upper()
|
||||
|
||||
|
||||
def _scene_date(radar: RadarDataORM | None) -> str:
|
||||
text = str(getattr(radar, "imaging_date", None) or "").strip()
|
||||
match = re.search(r"(20\d{6})", re.sub(r"\D", "", text))
|
||||
if match:
|
||||
return match.group(1)
|
||||
return "unknown_date"
|
||||
|
||||
|
||||
def _scene_token(
|
||||
*,
|
||||
radar: RadarDataORM | None,
|
||||
scene: SARSceneGeoORM,
|
||||
polarization: str | None = None,
|
||||
) -> str:
|
||||
unique = getattr(radar, "unique_id", None) or f"radar_{getattr(radar, 'id', scene.radar_data_id)}"
|
||||
parts = [_scene_date(radar), _safe_slug(unique), f"scene_{scene.id}"]
|
||||
if polarization:
|
||||
parts.append(_safe_slug(polarization).upper())
|
||||
return "_".join(parts)
|
||||
|
||||
|
||||
def scene_analysis_dir(
|
||||
*,
|
||||
radar: RadarDataORM | None,
|
||||
scene: SARSceneGeoORM,
|
||||
engine: str,
|
||||
profile: str,
|
||||
polarization: str | None = None,
|
||||
) -> Path:
|
||||
return (
|
||||
Path(settings.SAR_ANALYSIS_READY_ROOT)
|
||||
/ _scene_family(radar)
|
||||
/ _safe_slug(engine)
|
||||
/ _safe_slug(profile)
|
||||
/ _scene_date(radar)
|
||||
/ _scene_token(radar=radar, scene=scene, polarization=polarization)
|
||||
)
|
||||
|
||||
|
||||
def _write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as stream:
|
||||
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
|
||||
|
||||
|
||||
def _link_or_copy(source: Path, target: Path) -> str:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if source.resolve() == target.resolve():
|
||||
return "same_path"
|
||||
if target.exists():
|
||||
target.unlink()
|
||||
try:
|
||||
os.link(source, target)
|
||||
return "hardlink"
|
||||
except OSError:
|
||||
shutil.copy2(source, target)
|
||||
return "copy"
|
||||
|
||||
|
||||
def _choose_gf3_l2_tif(l2_dir: str, polarization: str | None = None) -> Path:
|
||||
root = Path(os.path.normpath(str(l2_dir or "").strip()))
|
||||
if root.is_file():
|
||||
return root
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(f"GF3 L2 directory does not exist: {l2_dir}")
|
||||
|
||||
candidates = sorted(
|
||||
path
|
||||
for path in root.rglob("*")
|
||||
if path.is_file()
|
||||
and path.suffix.lower() in {".tif", ".tiff"}
|
||||
and "L2" in path.name.upper()
|
||||
)
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"No GF3 L2 GeoTIFF found in: {l2_dir}")
|
||||
|
||||
requested = str(polarization or "").strip().upper()
|
||||
if requested:
|
||||
for path in candidates:
|
||||
if requested in path.name.upper():
|
||||
return path
|
||||
|
||||
for pol in _POLARIZATION_PRIORITY:
|
||||
for path in candidates:
|
||||
if pol in path.name.upper():
|
||||
return path
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _infer_polarization_from_path(path: Path) -> str | None:
|
||||
upper_name = path.name.upper()
|
||||
for pol in _POLARIZATION_PRIORITY:
|
||||
if pol in upper_name:
|
||||
return pol
|
||||
return None
|
||||
|
||||
|
||||
def _raster_quality(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
except Exception as exc:
|
||||
return {"ok": False, "warning": f"rasterio unavailable: {exc}"}
|
||||
|
||||
with rasterio.open(path) as src:
|
||||
if src.height > 2048 or src.width > 2048:
|
||||
scale = min(1024 / src.width, 1024 / src.height)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
sampled = src.read(1, out_shape=(out_height, out_width), masked=True)
|
||||
else:
|
||||
sampled = src.read(1, masked=True)
|
||||
|
||||
valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)]
|
||||
bounds = src.bounds
|
||||
transform = src.transform
|
||||
quality: dict[str, Any] = {
|
||||
"ok": True,
|
||||
"driver": src.driver,
|
||||
"width": src.width,
|
||||
"height": src.height,
|
||||
"count": src.count,
|
||||
"dtype": str(src.dtypes[0]) if src.dtypes else None,
|
||||
"crs": src.crs.to_string() if src.crs else None,
|
||||
"bounds": {
|
||||
"left": bounds.left,
|
||||
"bottom": bounds.bottom,
|
||||
"right": bounds.right,
|
||||
"top": bounds.top,
|
||||
},
|
||||
"transform": list(transform)[:6],
|
||||
"nodata": src.nodata,
|
||||
"valid_sample_count": int(valid.size),
|
||||
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
|
||||
}
|
||||
if valid.size:
|
||||
quality.update(
|
||||
{
|
||||
"sample_min": float(np.nanmin(valid)),
|
||||
"sample_max": float(np.nanmax(valid)),
|
||||
"sample_mean": float(np.nanmean(valid)),
|
||||
"sample_p02": float(np.nanpercentile(valid, 2)),
|
||||
"sample_p98": float(np.nanpercentile(valid, 98)),
|
||||
}
|
||||
)
|
||||
return quality
|
||||
|
||||
|
||||
def _pixel_size_m_from_quality(quality: dict[str, Any]) -> float | None:
|
||||
try:
|
||||
transform = quality.get("transform") or []
|
||||
xres = abs(float(transform[0]))
|
||||
yres = abs(float(transform[4]))
|
||||
crs = str(quality.get("crs") or "").upper()
|
||||
if not xres or not yres:
|
||||
return None
|
||||
if crs and "4326" not in crs:
|
||||
return round((xres + yres) / 2.0, 3)
|
||||
bounds = quality.get("bounds") or {}
|
||||
lat = (float(bounds.get("bottom", 0.0)) + float(bounds.get("top", 0.0))) / 2.0
|
||||
meters_per_degree_lon = 111320.0 * max(0.01, math.cos(math.radians(lat)))
|
||||
x_m = xres * meters_per_degree_lon
|
||||
y_m = yres * 110540.0
|
||||
return round((x_m + y_m) / 2.0, 3)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_preview_png(source: Path, target: Path) -> str | None:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with rasterio.open(source) as src:
|
||||
if src.height > 1600 or src.width > 1600:
|
||||
scale = min(1600 / src.width, 1600 / src.height)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
band = src.read(1, out_shape=(out_height, out_width), masked=True)
|
||||
else:
|
||||
band = src.read(1, masked=True)
|
||||
data = band.filled(np.nan).astype("float32")
|
||||
|
||||
valid = data[np.isfinite(data)]
|
||||
if valid.size:
|
||||
p2, p98 = np.nanpercentile(valid, [2, 98])
|
||||
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
|
||||
gray = (normalized * 255).astype("uint8")
|
||||
else:
|
||||
gray = np.zeros(data.shape, dtype="uint8")
|
||||
alpha = np.where(np.isfinite(data), 255, 0).astype("uint8")
|
||||
rgba = np.stack([gray, gray, gray, alpha], axis=-1)
|
||||
Image.fromarray(rgba, "RGBA").save(target)
|
||||
return str(target)
|
||||
|
||||
|
||||
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
|
||||
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
|
||||
scene = result.scalar_one_or_none()
|
||||
if scene:
|
||||
return scene
|
||||
scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING")
|
||||
db.add(scene)
|
||||
await db.flush()
|
||||
return scene
|
||||
|
||||
|
||||
async def register_analysis_ready_tif(
|
||||
*,
|
||||
db: AsyncSession,
|
||||
scene: SARSceneGeoORM,
|
||||
radar: RadarDataORM | None,
|
||||
source_tif_path: str,
|
||||
engine: str,
|
||||
profile: str,
|
||||
backscatter_unit: str,
|
||||
polarization: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
copy_mode: str = "link_or_copy",
|
||||
) -> dict[str, Any]:
|
||||
source = Path(os.path.normpath(str(source_tif_path or "").strip()))
|
||||
if not source.is_file():
|
||||
raise FileNotFoundError(f"Analysis-ready source GeoTIFF does not exist: {source}")
|
||||
|
||||
out_dir = scene_analysis_dir(
|
||||
radar=radar,
|
||||
scene=scene,
|
||||
engine=engine,
|
||||
profile=profile,
|
||||
polarization=polarization,
|
||||
)
|
||||
target_tif = out_dir / "analysis_ready.tif"
|
||||
transfer = "none"
|
||||
if copy_mode == "reference":
|
||||
target_tif = source
|
||||
else:
|
||||
transfer = _link_or_copy(source, target_tif)
|
||||
|
||||
quality = _raster_quality(target_tif)
|
||||
preview_path = _build_preview_png(target_tif, out_dir / "preview.png")
|
||||
manifest = {
|
||||
"scene_id": scene.id,
|
||||
"radar_data_id": scene.radar_data_id,
|
||||
"source_tif_path": str(source),
|
||||
"analysis_tif_path": str(target_tif),
|
||||
"analysis_dir": str(out_dir),
|
||||
"analysis_preview_path": preview_path,
|
||||
"engine": engine,
|
||||
"profile": profile,
|
||||
"backscatter_unit": backscatter_unit,
|
||||
"polarization": polarization,
|
||||
"transfer": transfer,
|
||||
"metadata": metadata or {},
|
||||
"quality": quality,
|
||||
}
|
||||
_write_json(out_dir / "manifest.json", manifest)
|
||||
_write_json(out_dir / "quality.json", quality)
|
||||
|
||||
scene.geo_path = str(target_tif)
|
||||
scene.analysis_tif_path = str(target_tif)
|
||||
scene.analysis_dir = str(out_dir)
|
||||
scene.analysis_preview_path = preview_path
|
||||
scene.analysis_engine = engine
|
||||
scene.analysis_profile = profile
|
||||
scene.analysis_backscatter_unit = backscatter_unit
|
||||
nodata_value = quality.get("nodata")
|
||||
scene.analysis_nodata_value = (
|
||||
float(nodata_value)
|
||||
if nodata_value is not None
|
||||
else float(settings.SAR_ANALYSIS_NODATA_VALUE)
|
||||
)
|
||||
scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")}
|
||||
scene.analysis_quality_json = quality
|
||||
scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m
|
||||
scene.status = "DONE"
|
||||
scene.error_msg = None
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
async def standardize_gf3_l2_for_radar(
|
||||
*,
|
||||
db: AsyncSession,
|
||||
radar_id: int,
|
||||
l2_path: str | None = None,
|
||||
polarization: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
radar = await db.get(RadarDataORM, int(radar_id))
|
||||
if not radar:
|
||||
raise ValueError(f"RadarDataORM id={radar_id} does not exist")
|
||||
|
||||
scene = await _get_or_create_scene(db, int(radar_id))
|
||||
source_root = l2_path or radar.file_path
|
||||
selected_tif = _choose_gf3_l2_tif(source_root, polarization=polarization or radar.polarization)
|
||||
selected_pol = polarization or _infer_polarization_from_path(selected_tif)
|
||||
|
||||
manifest = await register_analysis_ready_tif(
|
||||
db=db,
|
||||
scene=scene,
|
||||
radar=radar,
|
||||
source_tif_path=str(selected_tif),
|
||||
engine="gf3_gdal",
|
||||
profile="gf3_l1a_l2_rpc",
|
||||
backscatter_unit="sigma0_db",
|
||||
polarization=selected_pol,
|
||||
metadata={
|
||||
"source": "GF3 L2",
|
||||
"source_l2_path": str(selected_tif),
|
||||
"source_l2_dir": str(Path(source_root).resolve()) if source_root else None,
|
||||
"available_polarization": radar.polarization,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
return manifest
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
"""Water extraction processors for the flood-analysis pipeline."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def run_otsu_water_extraction(
|
||||
*,
|
||||
input_path: str,
|
||||
output_dir: str,
|
||||
job_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run the fast Otsu water-extraction processor.
|
||||
|
||||
This wraps the legacy implementation while exposing the terminology used by
|
||||
the flood pipeline: extraction, processor and threshold_value.
|
||||
"""
|
||||
from .water_detect_service import run_water_detection
|
||||
|
||||
result = run_water_detection(
|
||||
geo_tiff_path=input_path,
|
||||
output_dir=output_dir,
|
||||
job_id=job_id,
|
||||
)
|
||||
result["processor"] = "otsu"
|
||||
if "threshold_value" not in result and "otsu_threshold_db" in result:
|
||||
result["threshold_value"] = result.get("otsu_threshold_db")
|
||||
return result
|
||||
|
||||
|
||||
def run_envi_water_extraction(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
"""Placeholder for the future ENVI/SARscape precise extractor."""
|
||||
raise NotImplementedError("ENVI/SARscape water extraction is not wired yet")
|
||||
Reference in New Issue
Block a user