Add Gamma SBAS production workflow

This commit is contained in:
2026-05-21 04:53:39 +08:00
parent 9000feeee8
commit cc22b9ac2d
46 changed files with 11386 additions and 332 deletions
@@ -0,0 +1,967 @@
"""Flood-analysis service layer.
This module owns the flood business API implementation used by
``backend.app.routers.flood``. It intentionally does not import the legacy
water router; the old router remains only as a compatibility surface.
"""
from __future__ import annotations
import asyncio
import base64
import io
import json
import os
from datetime import datetime, timedelta
from typing import Any
from fastapi import HTTPException
from fastapi.responses import JSONResponse
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..models import FloodDetectionORM, RadarDataORM, SARSceneGeoORM, WaterDetectionORM, WaterExtractionORM
from ..services.job_handlers import (
JOB_TYPE_FLOOD_DETECTION,
JOB_TYPE_SAR_SCENE_PREPROCESS,
JOB_TYPE_WATER_DETECT,
)
from ..services.job_queue_service import job_queue_service
from ..services.task_service import task_service
from ..utils import normalize_satellite_family
_FLOOD_JOB_MAX_ATTEMPTS = 3
async def _queue_flood_job(
*,
job_type: str,
task_type: str,
task_name: str,
payload: dict[str, Any],
) -> dict[str, Any]:
try:
task_id = await task_service.create_task(
task_type=task_type,
task_name=task_name,
params=payload,
)
job_id = await job_queue_service.create_job(
job_type=job_type,
payload=payload,
task_id=task_id,
max_attempts=_FLOOD_JOB_MAX_ATTEMPTS,
)
return {"task_id": task_id, "job_id": job_id, "job_type": job_type, "message": "Job queued."}
except ValueError as exc:
message = str(exc)
raise HTTPException(status_code=409 if "conflict" in message.lower() else 400, detail=message) from exc
def _overlap_ratio(poly_a: Any, poly_b: Any) -> float:
try:
import json
from shapely.geometry import Polygon, shape
def _to_geom(poly: Any):
if isinstance(poly, str):
poly = json.loads(poly)
if isinstance(poly, list):
return Polygon(poly)
return shape(poly)
a = _to_geom(poly_a)
b = _to_geom(poly_b)
if not a.is_valid or not b.is_valid:
return 0.0
intersection_area = a.intersection(b).area
smaller_area = min(a.area, b.area)
return intersection_area / smaller_area if smaller_area > 0 else 0.0
except Exception:
return 0.0
def _parse_ymd(value: str | None, *, field: str) -> datetime:
try:
normalized = str(value or "").replace("-", "").strip()
return datetime.strptime(normalized, "%Y%m%d")
except Exception as exc:
raise HTTPException(status_code=400, detail=f"{field} must be YYYYMMDD") from exc
def _format_ymd(value: datetime) -> str:
return value.strftime("%Y%m%d")
def _to_float(value: Any, default: float = 0.0) -> float:
try:
if value is None:
return default
return float(value)
except Exception:
return default
def _same_text(left: Any, right: Any) -> bool:
left_text = str(left or "").strip().lower()
right_text = str(right or "").strip().lower()
if not left_text or not right_text:
return True
return left_text == right_text
def _feature_collection_name(feature_collection: dict[str, Any]) -> str | None:
try:
features = feature_collection.get("features") or []
properties = features[0].get("properties") or {}
return properties.get("name") or properties.get("NAME") or properties.get("treeID")
except Exception:
return None
def _preprocess_engine_for_radar(radar: RadarDataORM) -> str | None:
family = str(normalize_satellite_family(radar.satellite_family or radar.satellite) or "").upper()
if family == "GF3":
return "gf3_gdal"
if family == "LT1":
return "lt_gamma"
return None
def _scene_analysis_path(scene: SARSceneGeoORM | None) -> str | None:
if not scene:
return None
return scene.analysis_tif_path
def _resolve_aoi_wkt_from_request(req: Any) -> tuple[str, dict[str, Any], dict[str, Any]]:
"""Resolve region/GeoJSON AOI using the same parser as the management search page."""
aoi_geojson = getattr(req, "aoi_geojson", None)
region_tree_id = getattr(req, "region_tree_id", None)
if aoi_geojson:
feature_collection = aoi_geojson
source = "geojson"
elif region_tree_id:
from ..routers.dependencies import _resolve_region_aoi_payload
payload = _resolve_region_aoi_payload(str(region_tree_id))
feature_collection = payload.get("aoi_geojson") or payload
source = "region"
else:
raise HTTPException(status_code=400, detail="region_tree_id or aoi_geojson is required")
from ..routers.dependencies import _parse_aoi_geojson_form_value
parsed = _parse_aoi_geojson_form_value(json.dumps(feature_collection, ensure_ascii=False))
if not parsed:
raise HTTPException(status_code=400, detail="AOI geometry is empty")
aoi_wkt, normalized_feature_collection = parsed
meta = {
"source": source,
"region_tree_id": region_tree_id,
"name": _feature_collection_name(normalized_feature_collection),
}
return aoi_wkt, normalized_feature_collection, meta
def _radar_scene_item(scene: SARSceneGeoORM, radar: RadarDataORM, *, aoi_coverage_ratio: float | None = None) -> dict[str, Any]:
return {
"id": scene.id,
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"satellite": radar.satellite,
"imaging_date": radar.imaging_date,
"acquisition_time_utc": radar.acquisition_time_utc,
"imaging_mode": radar.imaging_mode,
"product_level": radar.product_level,
"polarization": radar.polarization,
"orbit_direction": radar.orbit_direction,
"geo_path": scene.geo_path,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"analysis_engine": scene.analysis_engine,
"analysis_profile": scene.analysis_profile,
"analysis_backscatter_unit": scene.analysis_backscatter_unit,
"analysis_quality_json": scene.analysis_quality_json,
"coverage_polygon": radar.coverage_polygon,
"min_lat": radar.min_lat,
"max_lat": radar.max_lat,
"min_lon": radar.min_lon,
"max_lon": radar.max_lon,
"aoi_coverage_ratio": aoi_coverage_ratio,
}
async def _query_disaster_scene_pool(
*,
db: AsyncSession,
req: Any,
aoi_wkt: str,
start_ymd: str,
end_ymd: str,
min_aoi_coverage_ratio: float,
descending: bool,
) -> list[dict[str, Any]]:
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
aoi_area = func.ST_Area(func.Geography(aoi_geom))
coverage_expr = (
func.ST_Area(func.Geography(func.ST_Intersection(RadarDataORM.geom, aoi_geom)))
/ func.nullif(aoi_area, 0)
).label("aoi_coverage_ratio")
filters = [
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
RadarDataORM.geom.isnot(None),
RadarDataORM.imaging_date.isnot(None),
RadarDataORM.imaging_date >= start_ymd,
RadarDataORM.imaging_date <= end_ymd,
func.ST_Intersects(RadarDataORM.geom, aoi_geom),
]
satellites = [str(item).strip() for item in (getattr(req, "satellites", None) or []) if str(item).strip()]
if satellites:
filters.append(RadarDataORM.satellite.in_(satellites))
polarization = str(getattr(req, "polarization", "") or "").strip()
if polarization:
filters.append(RadarDataORM.polarization.ilike(f"%{polarization}%"))
imaging_mode = str(getattr(req, "imaging_mode", "") or "").strip()
if imaging_mode:
filters.append(RadarDataORM.imaging_mode == imaging_mode)
product_level = str(getattr(req, "product_level", "") or "").strip()
if product_level:
filters.append(RadarDataORM.product_level == product_level)
order_by = RadarDataORM.imaging_date.desc() if descending else RadarDataORM.imaging_date.asc()
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM, coverage_expr)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*filters)
.order_by(order_by, SARSceneGeoORM.id.desc())
)
pool: list[dict[str, Any]] = []
for scene, radar, aoi_coverage_ratio in result.all():
coverage_ratio = max(0.0, min(1.0, _to_float(aoi_coverage_ratio)))
if coverage_ratio < min_aoi_coverage_ratio:
continue
pool.append(_radar_scene_item(scene, radar, aoi_coverage_ratio=round(coverage_ratio, 4)))
return pool
async def submit_geocode_job(req: Any, db: AsyncSession) -> dict[str, Any]:
radar = await db.get(RadarDataORM, req.radar_data_id)
if not radar:
raise HTTPException(status_code=404, detail=f"RadarData id={req.radar_data_id} not found")
result = await db.execute(
select(SARSceneGeoORM)
.where(SARSceneGeoORM.radar_data_id == req.radar_data_id)
.with_for_update(skip_locked=True)
)
scene = result.scalar_one_or_none()
if scene and scene.status in ("PENDING", "RUNNING"):
raise HTTPException(status_code=409, detail="Scene already has an active geocode job")
if not scene:
scene = SARSceneGeoORM(radar_data_id=req.radar_data_id, status="PENDING")
db.add(scene)
await db.flush()
else:
scene.status = "PENDING"
scene.error_msg = None
await db.flush()
scene_id = scene.id
await db.commit()
engine = _preprocess_engine_for_radar(radar)
if not engine:
async with db.begin():
failed_scene = await db.get(SARSceneGeoORM, scene_id)
if failed_scene and failed_scene.status == "PENDING":
failed_scene.status = "FAILED"
failed_scene.error_msg = "No analysis-ready GeoTIFF preprocessor configured for this satellite"
raise HTTPException(
status_code=400,
detail="洪涝模块不再使用 ENVI 兜底预处理;该卫星暂未配置 analysis-ready GeoTIFF 预处理器",
)
job_type = JOB_TYPE_SAR_SCENE_PREPROCESS
task_type = f"FLOOD_SCENE_PREPROCESS_{scene_id}"
task_name = f"Flood analysis-ready preprocess radar_id={req.radar_data_id} engine={engine}"
payload = {"scene_id": scene_id, "radar_data_id": req.radar_data_id}
payload["engine"] = engine
try:
return await _queue_flood_job(
job_type=job_type,
task_type=task_type,
task_name=task_name,
payload=payload,
)
except HTTPException:
async with db.begin():
failed_scene = await db.get(SARSceneGeoORM, scene_id)
if failed_scene and failed_scene.status == "PENDING":
failed_scene.status = "FAILED"
failed_scene.error_msg = "Job queue failed"
raise
async def reset_scene_status(scene_id: int, db: AsyncSession) -> dict[str, Any]:
scene = await db.get(SARSceneGeoORM, scene_id)
if not scene:
raise HTTPException(status_code=404, detail=f"Scene id={scene_id} not found")
if scene.status not in ("PENDING", "RUNNING"):
raise HTTPException(status_code=400, detail=f"Scene status is {scene.status}; reset is not needed")
scene.status = "FAILED"
scene.error_msg = "Manually reset"
await db.commit()
return {"id": scene_id, "status": "FAILED", "message": "Scene reset"}
async def list_done_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]:
result = await db.execute(
select(SARSceneGeoORM.radar_data_id).where(
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
)
)
return {"ids": [row for (row,) in result.all()]}
async def list_active_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]:
result = await db.execute(
select(SARSceneGeoORM.radar_data_id).where(SARSceneGeoORM.status.in_(["PENDING", "RUNNING"]))
)
return {"ids": [row for (row,) in result.all()]}
async def list_scenes(limit: int, offset: int, db: AsyncSession) -> dict[str, Any]:
total_result = await db.execute(select(func.count()).select_from(SARSceneGeoORM))
total = total_result.scalar_one()
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.order_by(SARSceneGeoORM.id.desc())
.limit(limit)
.offset(offset)
)
items = []
for scene, radar in result.all():
items.append(
{
"id": scene.id,
"radar_data_id": scene.radar_data_id,
"satellite": radar.satellite,
"imaging_date": radar.imaging_date,
"acquisition_time_utc": radar.acquisition_time_utc,
"imaging_mode": radar.imaging_mode,
"product_level": radar.product_level,
"polarization": radar.polarization,
"orbit_direction": radar.orbit_direction,
"geo_path": scene.geo_path,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"analysis_engine": scene.analysis_engine,
"analysis_profile": scene.analysis_profile,
"analysis_backscatter_unit": scene.analysis_backscatter_unit,
"analysis_nodata_value": scene.analysis_nodata_value,
"analysis_metadata_json": scene.analysis_metadata_json,
"analysis_quality_json": scene.analysis_quality_json,
"pixel_size_m": scene.pixel_size_m,
"status": scene.status,
"error_msg": scene.error_msg,
"created_at": scene.created_at.isoformat() if scene.created_at else None,
"coverage_polygon": radar.coverage_polygon,
"min_lat": radar.min_lat,
"max_lat": radar.max_lat,
"min_lon": radar.min_lon,
"max_lon": radar.max_lon,
}
)
return {"items": items, "total": total}
async def submit_water_extraction(req: Any, db: AsyncSession) -> dict[str, Any]:
input_path = req.input_path
scene_id = req.scene_id
if scene_id:
scene = await db.get(SARSceneGeoORM, scene_id)
if not scene:
raise HTTPException(status_code=404, detail=f"SARSceneGeoORM id={scene_id} not found")
input_path = _scene_analysis_path(scene)
if not input_path:
raise HTTPException(status_code=400, detail="Scene has no analysis-ready GeoTIFF")
if not input_path:
raise HTTPException(status_code=400, detail="scene_id or input_path is required")
extraction = WaterExtractionORM(
scene_id=scene_id,
processor=getattr(req, "processor", None) or "otsu",
input_path=input_path,
status="PENDING",
)
db.add(extraction)
await db.flush()
extraction_id = extraction.id
await db.commit()
try:
queued = await _queue_flood_job(
job_type=JOB_TYPE_WATER_DETECT,
task_type=f"FLOOD_WATER_EXTRACTION_{extraction_id}",
task_name=f"Flood water extraction id={extraction_id}",
payload={"extraction_id": extraction_id, "processor": extraction.processor},
)
async with db.begin():
queued_extraction = await db.get(WaterExtractionORM, extraction_id)
if queued_extraction:
queued_extraction.task_id = queued.get("task_id")
return queued
except HTTPException:
async with db.begin():
failed_extraction = await db.get(WaterExtractionORM, extraction_id)
if failed_extraction and failed_extraction.status == "PENDING":
failed_extraction.status = "FAILED"
failed_extraction.error_msg = "Job queue failed"
raise
async def list_water_extractions(
*,
limit: int,
offset: int,
status: str | None,
db: AsyncSession,
) -> dict[str, Any]:
count_query = select(func.count()).select_from(WaterExtractionORM)
if status:
count_query = count_query.where(WaterExtractionORM.status == status)
total = (await db.execute(count_query)).scalar_one()
query = (
select(WaterExtractionORM, SARSceneGeoORM, RadarDataORM)
.outerjoin(SARSceneGeoORM, WaterExtractionORM.scene_id == SARSceneGeoORM.id)
.outerjoin(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.order_by(WaterExtractionORM.id.desc())
.limit(limit)
.offset(offset)
)
if status:
query = query.where(WaterExtractionORM.status == status)
rows = (await db.execute(query)).all()
items = []
for detection, scene, radar in rows:
items.append(
{
"id": detection.id,
"scene_id": detection.scene_id,
"processor": detection.processor,
"task_id": detection.task_id,
"radar_data_id": scene.radar_data_id if scene else None,
"satellite": radar.satellite if radar else None,
"imaging_date": radar.imaging_date if radar else None,
"acquisition_time_utc": radar.acquisition_time_utc if radar else None,
"imaging_mode": radar.imaging_mode if radar else None,
"product_level": radar.product_level if radar else None,
"polarization": radar.polarization if radar else None,
"orbit_direction": radar.orbit_direction if radar else None,
"coverage_polygon": radar.coverage_polygon if radar else None,
"min_lat": radar.min_lat if radar else None,
"max_lat": radar.max_lat if radar else None,
"min_lon": radar.min_lon if radar else None,
"max_lon": radar.max_lon if radar else None,
"input_path": detection.input_path,
"output_path": detection.output_path,
"preview_path": detection.preview_path,
"vector_path": detection.vector_path,
"water_area_km2": detection.water_area_km2,
"water_pixel_count": detection.water_pixel_count,
"otsu_threshold_db": detection.threshold_value,
"threshold_value": detection.threshold_value,
"metadata_json": detection.metadata_json,
"status": detection.status,
"error_msg": detection.error_msg,
"created_at": detection.created_at.isoformat() if detection.created_at else None,
"updated_at": detection.updated_at.isoformat() if detection.updated_at else None,
}
)
return {"items": items, "total": total}
async def submit_flood_detection(req: Any, db: AsyncSession) -> dict[str, Any]:
pre_scene = await db.get(SARSceneGeoORM, req.pre_scene_id)
post_scene = await db.get(SARSceneGeoORM, req.post_scene_id)
if not pre_scene:
raise HTTPException(status_code=404, detail=f"Pre-scene id={req.pre_scene_id} not found")
if not post_scene:
raise HTTPException(status_code=404, detail=f"Post-scene id={req.post_scene_id} not found")
if pre_scene.status != "DONE":
raise HTTPException(status_code=400, detail=f"Pre-scene is not DONE: {pre_scene.status}")
if post_scene.status != "DONE":
raise HTTPException(status_code=400, detail=f"Post-scene is not DONE: {post_scene.status}")
if not pre_scene.analysis_tif_path:
raise HTTPException(status_code=400, detail="Pre-scene has no analysis-ready GeoTIFF")
if not post_scene.analysis_tif_path:
raise HTTPException(status_code=400, detail="Post-scene has no analysis-ready GeoTIFF")
result = await db.execute(
select(FloodDetectionORM)
.where(
FloodDetectionORM.pre_scene_id == req.pre_scene_id,
FloodDetectionORM.post_scene_id == req.post_scene_id,
)
.with_for_update(skip_locked=True)
)
detection = result.scalar_one_or_none()
if detection and detection.status in ("PENDING", "RUNNING"):
raise HTTPException(status_code=409, detail="Pair already has an active flood-detection job")
if not detection:
detection = FloodDetectionORM(
pre_scene_id=req.pre_scene_id,
post_scene_id=req.post_scene_id,
status="PENDING",
)
db.add(detection)
await db.flush()
else:
detection.status = "PENDING"
detection.error_msg = None
await db.flush()
detection_id = detection.id
await db.commit()
try:
return await _queue_flood_job(
job_type=JOB_TYPE_FLOOD_DETECTION,
task_type=f"FLOOD_DETECTION_{detection_id}",
task_name=f"GeoTIFF flood detection pre={req.pre_scene_id} post={req.post_scene_id}",
payload={"detection_id": detection_id, "refine": req.refine},
)
except HTTPException:
async with db.begin():
failed_detection = await db.get(FloodDetectionORM, detection_id)
if failed_detection and failed_detection.status == "PENDING":
failed_detection.status = "FAILED"
failed_detection.error_msg = "Job queue failed"
raise
async def list_flood_detections(db: AsyncSession) -> dict[str, Any]:
result = await db.execute(
select(FloodDetectionORM)
.options(
selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data),
selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data),
)
.order_by(FloodDetectionORM.id.desc())
)
detections = result.scalars().all()
items = []
for detection in detections:
pre_radar = detection.pre_scene.radar_data if detection.pre_scene else None
post_radar = detection.post_scene.radar_data if detection.post_scene else None
items.append(
{
"id": detection.id,
"pre_scene_id": detection.pre_scene_id,
"post_scene_id": detection.post_scene_id,
"pre_imaging_date": pre_radar.imaging_date if pre_radar else None,
"post_imaging_date": post_radar.imaging_date if post_radar else None,
"pre_satellite": pre_radar.satellite if pre_radar else None,
"post_satellite": post_radar.satellite if post_radar else None,
"pre_geo_path": _scene_analysis_path(detection.pre_scene),
"post_geo_path": _scene_analysis_path(detection.post_scene),
"pre_analysis_tif_path": _scene_analysis_path(detection.pre_scene),
"post_analysis_tif_path": _scene_analysis_path(detection.post_scene),
"classified_path": detection.classified_path,
"flood_area_km2": detection.flood_area_km2,
"stable_water_area_km2": detection.stable_water_area_km2,
"status": detection.status,
"error_msg": detection.error_msg,
"created_at": detection.created_at.isoformat() if detection.created_at else None,
"updated_at": detection.updated_at.isoformat() if detection.updated_at else None,
}
)
return {"items": items, "total": len(items)}
async def search_pairs(req: Any, db: AsyncSession) -> dict[str, Any]:
pre_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)]
if req.pre_start:
pre_filters.append(RadarDataORM.imaging_date >= req.pre_start)
if req.pre_end:
pre_filters.append(RadarDataORM.imaging_date <= req.pre_end)
pre_result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*pre_filters)
)
post_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)]
if req.post_start:
post_filters.append(RadarDataORM.imaging_date >= req.post_start)
if req.post_end:
post_filters.append(RadarDataORM.imaging_date <= req.post_end)
post_result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*post_filters)
)
candidates = []
for pre_scene, pre_radar in pre_result.all():
for post_scene, post_radar in post_result.all():
if pre_scene.id == post_scene.id:
continue
ratio = 0.0
if pre_radar.coverage_polygon and post_radar.coverage_polygon:
ratio = _overlap_ratio(pre_radar.coverage_polygon, post_radar.coverage_polygon)
if ratio < req.overlap_threshold:
continue
try:
pre_date = datetime.strptime(pre_radar.imaging_date, "%Y%m%d")
post_date = datetime.strptime(post_radar.imaging_date, "%Y%m%d")
time_diff = abs((post_date - pre_date).days)
except Exception:
time_diff = None
candidates.append(
{
"pre": {
"id": pre_scene.id,
"imaging_date": pre_radar.imaging_date,
"satellite": pre_radar.satellite,
"geo_path": _scene_analysis_path(pre_scene),
"analysis_tif_path": _scene_analysis_path(pre_scene),
},
"post": {
"id": post_scene.id,
"imaging_date": post_radar.imaging_date,
"satellite": post_radar.satellite,
"geo_path": _scene_analysis_path(post_scene),
"analysis_tif_path": _scene_analysis_path(post_scene),
},
"overlap_ratio": round(ratio, 4),
"time_diff_days": time_diff,
}
)
candidates.sort(key=lambda item: item["overlap_ratio"], reverse=True)
used_pre: set[int] = set()
used_post: set[int] = set()
pairs = []
for candidate in candidates:
pre_id = candidate["pre"]["id"]
post_id = candidate["post"]["id"]
if pre_id in used_pre or post_id in used_post:
continue
used_pre.add(pre_id)
used_post.add(post_id)
pairs.append(candidate)
pairs.sort(key=lambda item: item["overlap_ratio"], reverse=True)
return {"pairs": pairs, "total": len(pairs)}
async def search_disaster_pairs(req: Any, db: AsyncSession) -> dict[str, Any]:
disaster_date = _parse_ymd(req.disaster_date, field="disaster_date")
pre_window_days = max(1, int(getattr(req, "pre_window_days", 30) or 30))
post_window_days = max(1, int(getattr(req, "post_window_days", 30) or 30))
min_aoi_coverage_ratio = max(0.0, min(1.0, float(getattr(req, "min_aoi_coverage_ratio", 0.2) or 0.0)))
min_pair_overlap_ratio = max(0.0, min(1.0, float(getattr(req, "min_pair_overlap_ratio", 0.3) or 0.0)))
max_pairs = max(1, min(200, int(getattr(req, "max_pairs", 50) or 50)))
aoi_wkt, aoi_geojson, aoi_meta = _resolve_aoi_wkt_from_request(req)
pre_start = disaster_date - timedelta(days=pre_window_days)
pre_end = disaster_date - timedelta(days=1)
post_start = disaster_date
post_end = disaster_date + timedelta(days=post_window_days)
pre_pool = await _query_disaster_scene_pool(
db=db,
req=req,
aoi_wkt=aoi_wkt,
start_ymd=_format_ymd(pre_start),
end_ymd=_format_ymd(pre_end),
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
descending=True,
)
post_pool = await _query_disaster_scene_pool(
db=db,
req=req,
aoi_wkt=aoi_wkt,
start_ymd=_format_ymd(post_start),
end_ymd=_format_ymd(post_end),
min_aoi_coverage_ratio=min_aoi_coverage_ratio,
descending=False,
)
candidates: list[dict[str, Any]] = []
require_same_polarization = bool(getattr(req, "require_same_polarization", True))
require_same_imaging_mode = bool(getattr(req, "require_same_imaging_mode", False))
total_window = max(1, pre_window_days + post_window_days)
for pre in pre_pool:
pre_date = _parse_ymd(pre.get("imaging_date"), field="pre.imaging_date")
for post in post_pool:
if pre["id"] == post["id"]:
continue
if require_same_polarization and not _same_text(pre.get("polarization"), post.get("polarization")):
continue
if require_same_imaging_mode and not _same_text(pre.get("imaging_mode"), post.get("imaging_mode")):
continue
post_date = _parse_ymd(post.get("imaging_date"), field="post.imaging_date")
scene_overlap = _overlap_ratio(pre.get("coverage_polygon"), post.get("coverage_polygon"))
if scene_overlap < min_pair_overlap_ratio:
continue
pre_delta_days = max(0, (disaster_date - pre_date).days)
post_delta_days = max(0, (post_date - disaster_date).days)
time_score = max(0.0, 1.0 - ((pre_delta_days + post_delta_days) / total_window))
aoi_score = min(_to_float(pre.get("aoi_coverage_ratio")), _to_float(post.get("aoi_coverage_ratio")))
score = (scene_overlap * 0.45) + (aoi_score * 0.35) + (time_score * 0.20)
candidates.append(
{
"pre": pre,
"post": post,
"overlap_ratio": round(scene_overlap, 4),
"aoi_coverage_ratio": round(aoi_score, 4),
"time_score": round(time_score, 4),
"score": round(score, 4),
"pre_delta_days": pre_delta_days,
"post_delta_days": post_delta_days,
"time_diff_days": max(0, (post_date - pre_date).days),
"same_polarization": _same_text(pre.get("polarization"), post.get("polarization")),
"same_imaging_mode": _same_text(pre.get("imaging_mode"), post.get("imaging_mode")),
}
)
candidates.sort(
key=lambda item: (
item["score"],
item["overlap_ratio"],
item["aoi_coverage_ratio"],
-item["time_diff_days"],
),
reverse=True,
)
selected_pairs = candidates[:max_pairs]
warnings: list[str] = []
if not pre_pool:
warnings.append("No pre-disaster DONE scenes match the disaster AOI and time window")
if not post_pool:
warnings.append("No post-disaster DONE scenes match the disaster AOI and time window")
if pre_pool and post_pool and not selected_pairs:
warnings.append("Pre/post scene pools exist, but no pair meets the overlap/polarization constraints")
return {
"disaster": {
"name": getattr(req, "disaster_name", None),
"date": _format_ymd(disaster_date),
"pre_start": _format_ymd(pre_start),
"pre_end": _format_ymd(pre_end),
"post_start": _format_ymd(post_start),
"post_end": _format_ymd(post_end),
},
"aoi": {
**aoi_meta,
"geojson": aoi_geojson,
},
"pre_pool": pre_pool,
"post_pool": post_pool,
"candidate_pairs": selected_pairs,
"pairs": selected_pairs,
"total": len(selected_pairs),
"summary": {
"pre_pool_count": len(pre_pool),
"post_pool_count": len(post_pool),
"candidate_count": len(selected_pairs),
"min_aoi_coverage_ratio": min_aoi_coverage_ratio,
"min_pair_overlap_ratio": min_pair_overlap_ratio,
},
"warnings": warnings,
}
def _open_envi_rasterio(path: str):
import rasterio
normalized_path = path.replace("\\", "/")
try:
return rasterio.open(normalized_path)
except Exception:
pass
for ext in (".bin", ".img", ".tif", ".tiff"):
try:
return rasterio.open(normalized_path + ext)
except Exception:
pass
raise FileNotFoundError(f"Raster file cannot be opened: {path}")
def _raster_to_png_bytes(path: str, colormap: dict[int, tuple[int, int, int, int]]) -> tuple[bytes, list[float]]:
import numpy as np
from PIL import Image
with _open_envi_rasterio(path) as ds:
data = ds.read(1)
bounds = ds.bounds
geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right]
rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8)
for value, color in colormap.items():
rgba[data == value] = color
image = Image.fromarray(rgba, "RGBA")
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return buffer.getvalue(), geo_bounds
def _geo_raster_to_png_bytes(path: str) -> tuple[bytes, list[float]]:
import numpy as np
from PIL import Image
with _open_envi_rasterio(path) as ds:
data = ds.read(1).astype("float32")
nodata = ds.nodata
bounds = ds.bounds
geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right]
if nodata is not None:
nodata_mask = (data == nodata) | ~np.isfinite(data)
else:
nodata_mask = ~np.isfinite(data)
valid = data[~nodata_mask]
if valid.size == 0:
normalized = np.zeros_like(data, dtype=np.uint8)
else:
p2, p98 = np.percentile(valid, 2), np.percentile(valid, 98)
clipped = np.clip(data, p2, p98)
normalized = ((clipped - p2) / max(p98 - p2, 1e-9) * 255).astype(np.uint8)
rgba = np.stack([normalized, normalized, normalized, np.full_like(normalized, 200)], axis=-1)
rgba[nodata_mask, 3] = 0
image = Image.fromarray(rgba, "RGBA")
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return buffer.getvalue(), geo_bounds
_FLOOD_COLORMAP = {
1: (24, 144, 255, 200),
2: (255, 77, 79, 220),
3: (250, 173, 20, 180),
4: (80, 80, 80, 80),
}
async def get_flood_detection_preview(detection_id: int, layer: str, db: AsyncSession):
normalized_layer = layer.strip().lower()
if normalized_layer == "classified":
detection = await db.get(FloodDetectionORM, detection_id)
if not detection or not detection.classified_path:
raise HTTPException(status_code=404, detail="Classified result not found")
path = detection.classified_path.replace("\\", "/")
png_bytes, geo_bounds = await _render_classified_preview(path)
return JSONResponse(
{
"image_b64": base64.b64encode(png_bytes).decode(),
"bounds": geo_bounds,
"legend": {
"stable_water": "#1890ff",
"flood": "#ff4d4f",
"high_backscatter": "#faad14",
"non_water": "#505050",
},
}
)
if normalized_layer in ("pre", "post"):
return await _get_scene_preview_for_detection(detection_id, normalized_layer, db)
raise HTTPException(status_code=404, detail=f"Unsupported preview layer: {layer}")
async def _render_classified_preview(path: str) -> tuple[bytes, list[float]]:
if not os.path.isfile(path):
raise HTTPException(status_code=404, detail="Requested file does not exist")
try:
return await asyncio.to_thread(_raster_to_png_bytes, path, _FLOOD_COLORMAP)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc
async def _get_scene_preview_for_detection(detection_id: int, layer: str, db: AsyncSession):
detection = await db.get(FloodDetectionORM, detection_id)
if not detection:
raise HTTPException(status_code=404, detail="Flood detection not found")
scene_id = detection.pre_scene_id if layer == "pre" else detection.post_scene_id
scene = await db.get(SARSceneGeoORM, scene_id)
scene_path = _scene_analysis_path(scene)
if not scene_path:
raise HTTPException(status_code=404, detail="Scene analysis-ready GeoTIFF not found")
path = scene_path.replace("\\", "/")
if not os.path.isfile(path):
raise HTTPException(status_code=404, detail="Requested file does not exist")
try:
png_bytes, geo_bounds = await asyncio.to_thread(_geo_raster_to_png_bytes, path)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc
return JSONResponse({"image_b64": base64.b64encode(png_bytes).decode(), "bounds": geo_bounds})
async def get_water_extraction_preview(extraction_id: int, db: AsyncSession) -> dict[str, Any]:
detection = await db.get(WaterExtractionORM, extraction_id)
if not detection:
detection = await db.get(WaterDetectionORM, extraction_id)
if not detection:
raise HTTPException(status_code=404, detail=f"Water extraction id={extraction_id} not found")
if not detection.output_path or not os.path.isfile(detection.output_path):
raise HTTPException(status_code=404, detail="Output file does not exist")
import numpy as np
import rasterio
from PIL import Image
with rasterio.open(detection.output_path) as src:
data = src.read(1)
transform = src.transform
height, width = data.shape
min_lon = transform.c
max_lon = transform.c + width * transform.a
max_lat = transform.f
min_lat = transform.f + height * transform.e
rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8)
rgba[data > 0] = [24, 144, 255, 160]
image = Image.fromarray(rgba, "RGBA")
max_dim = 1024
if max(width, height) > max_dim:
ratio = max_dim / max(width, height)
image = image.resize((int(width * ratio), int(height * ratio)), Image.NEAREST)
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return {
"png_base64": base64.b64encode(buffer.getvalue()).decode(),
"bounds": {
"min_lon": min_lon,
"min_lat": min_lat,
"max_lon": max_lon,
"max_lat": max_lat,
},
}
@@ -0,0 +1,327 @@
"""Pure GeoTIFF flood detection for the flood-analysis module.
This service deliberately does not depend on ENVI/SARscape. Satellite-specific
preprocessors are responsible only for producing analysis-ready GeoTIFFs; the
flood classification below operates on those GeoTIFFs with Python/rasterio.
"""
from __future__ import annotations
import json
import math
import os
from pathlib import Path
from typing import Any
import numpy as np
def _valid_mask(data: np.ndarray, nodata: float | int | None) -> np.ndarray:
valid = np.isfinite(data)
if nodata is not None and np.isfinite(float(nodata)):
valid &= data != float(nodata)
return valid
def _sample_valid(values: np.ndarray, max_samples: int = 1_000_000) -> np.ndarray:
flat = values[np.isfinite(values)]
if flat.size <= max_samples:
return flat
step = max(1, int(math.ceil(flat.size / max_samples)))
return flat[::step]
def _otsu_threshold(values: np.ndarray) -> float:
sample = _sample_valid(values)
if sample.size < 100:
raise ValueError("Too few valid pixels for thresholding")
manual_threshold = _manual_otsu_threshold(sample)
try:
from skimage.filters import threshold_otsu
skimage_threshold = float(threshold_otsu(sample))
if np.isfinite(skimage_threshold):
p05, p95 = np.nanpercentile(sample, [5, 95])
if p05 < skimage_threshold < p95:
return skimage_threshold
return manual_threshold
except Exception:
return manual_threshold
def _manual_otsu_threshold(values: np.ndarray) -> float:
sample = _sample_valid(values)
if sample.size < 100:
raise ValueError("Too few valid pixels for thresholding")
vmin = float(np.nanmin(sample))
vmax = float(np.nanmax(sample))
if not np.isfinite(vmin) or not np.isfinite(vmax):
raise ValueError("Input pixels are not finite")
if math.isclose(vmin, vmax):
return vmin
hist, edges = np.histogram(sample, bins=256, range=(vmin, vmax))
hist = hist.astype("float64")
centers = (edges[:-1] + edges[1:]) / 2.0
total = hist.sum()
if total <= 0:
return float(np.nanpercentile(sample, 10))
weight_background = np.cumsum(hist)
weight_foreground = total - weight_background
mean_background = np.cumsum(hist * centers) / np.maximum(weight_background, 1e-12)
mean_foreground = (
np.cumsum((hist * centers)[::-1]) / np.maximum(np.cumsum(hist[::-1]), 1e-12)
)[::-1]
variance = weight_background[:-1] * weight_foreground[:-1] * (
mean_background[:-1] - mean_foreground[1:]
) ** 2
if variance.size == 0 or not np.isfinite(variance).any():
return float(np.nanpercentile(sample, 10))
idx = int(np.nanargmax(variance))
return float(edges[idx + 1])
def _pixel_area_km2(transform: Any, crs: Any, bounds: Any) -> float:
px_w = abs(float(transform.a))
px_h = abs(float(transform.e))
if crs and getattr(crs, "is_geographic", False):
lat_center = (float(bounds.top) + float(bounds.bottom)) / 2.0
px_w_m = px_w * math.cos(math.radians(lat_center)) * 111_320.0
px_h_m = px_h * 111_320.0
else:
px_w_m, px_h_m = px_w, px_h
return max(0.0, (px_w_m * px_h_m) / 1_000_000.0)
def _clean_mask(mask: np.ndarray, min_pixels: int) -> np.ndarray:
try:
from scipy.ndimage import binary_closing, binary_opening, generate_binary_structure, label
except Exception:
return mask
structure = generate_binary_structure(2, 2)
cleaned = binary_closing(mask, structure=structure, iterations=1)
cleaned = binary_opening(cleaned, structure=structure, iterations=1)
if min_pixels <= 1:
return cleaned
labels, count = label(cleaned)
if count <= 0:
return cleaned
component_sizes = np.bincount(labels.ravel())
keep = component_sizes >= int(min_pixels)
keep[0] = False
return keep[labels]
def _read_pre_on_post_grid(pre_path: str, post_profile: dict[str, Any]) -> tuple[np.ndarray, dict[str, Any]]:
import rasterio
from rasterio.enums import Resampling
from rasterio.warp import reproject
with rasterio.open(pre_path) as pre_ds:
pre_data = pre_ds.read(1).astype("float32")
pre_nodata = pre_ds.nodata
same_grid = (
pre_ds.width == int(post_profile["width"])
and pre_ds.height == int(post_profile["height"])
and pre_ds.transform == post_profile["transform"]
and str(pre_ds.crs or "") == str(post_profile["crs"] or "")
)
metadata = {
"path": pre_path,
"crs": pre_ds.crs.to_string() if pre_ds.crs else None,
"width": pre_ds.width,
"height": pre_ds.height,
"nodata": pre_nodata,
"reprojected_to_post_grid": not same_grid,
}
if same_grid:
data = pre_data.astype("float32")
if pre_nodata is not None and np.isfinite(float(pre_nodata)):
data[data == float(pre_nodata)] = np.nan
return data, metadata
if not pre_ds.crs or not post_profile["crs"]:
raise ValueError("Pre/post GeoTIFF CRS is required when grids differ")
destination = np.full(
(int(post_profile["height"]), int(post_profile["width"])),
np.nan,
dtype="float32",
)
reproject(
source=pre_data,
destination=destination,
src_transform=pre_ds.transform,
src_crs=pre_ds.crs,
src_nodata=pre_nodata,
dst_transform=post_profile["transform"],
dst_crs=post_profile["crs"],
dst_nodata=np.nan,
resampling=Resampling.bilinear,
)
return destination, metadata
def run_geotiff_flood_detection(
*,
pre_tif_path: str,
post_tif_path: str,
output_dir: str,
job_id: str | None = None,
refine: bool = False,
) -> dict[str, Any]:
"""Classify stable water and new flood extent from two analysis-ready GeoTIFFs."""
import rasterio
pre_path = Path(os.path.normpath(str(pre_tif_path or "").strip()))
post_path = Path(os.path.normpath(str(post_tif_path or "").strip()))
out_dir = Path(os.path.normpath(str(output_dir or "").strip()))
if not pre_path.is_file():
return {"ok": False, "error": f"Pre-event analysis GeoTIFF not found: {pre_path}"}
if not post_path.is_file():
return {"ok": False, "error": f"Post-event analysis GeoTIFF not found: {post_path}"}
out_dir.mkdir(parents=True, exist_ok=True)
with rasterio.open(post_path) as post_ds:
post_data = post_ds.read(1).astype("float32")
post_nodata = post_ds.nodata
post_profile = post_ds.profile.copy()
post_grid = {
"height": post_ds.height,
"width": post_ds.width,
"transform": post_ds.transform,
"crs": post_ds.crs,
}
post_metadata = {
"path": str(post_path),
"crs": post_ds.crs.to_string() if post_ds.crs else None,
"width": post_ds.width,
"height": post_ds.height,
"nodata": post_nodata,
}
pixel_area_km2 = _pixel_area_km2(post_ds.transform, post_ds.crs, post_ds.bounds)
pre_data, pre_metadata = _read_pre_on_post_grid(str(pre_path), post_grid)
valid_pre = _valid_mask(pre_data, None)
valid_post = _valid_mask(post_data, post_nodata)
valid = valid_pre & valid_post
if int(np.count_nonzero(valid)) < 100:
return {"ok": False, "error": "Too few overlapping valid pixels between pre/post GeoTIFFs"}
pre_valid_values = pre_data[valid]
post_valid_values = post_data[valid]
pre_threshold = _otsu_threshold(pre_valid_values)
post_threshold = _otsu_threshold(post_valid_values)
pre_water = (pre_data <= pre_threshold) & valid
post_water = (post_data <= post_threshold) & valid
stable_water = pre_water & post_water
flood = post_water & ~pre_water
if refine:
min_pixels = max(4, int(round(3_000.0 / max(pixel_area_km2 * 1_000_000.0, 1.0))))
stable_water = _clean_mask(stable_water, min_pixels=min_pixels)
flood = _clean_mask(flood, min_pixels=min_pixels)
high_threshold = float(np.nanpercentile(post_valid_values, 98))
high_backscatter = (post_data >= high_threshold) & valid & ~(stable_water | flood)
classified = np.zeros(post_data.shape, dtype="uint8")
classified[valid] = 4
classified[high_backscatter] = 3
classified[stable_water] = 1
classified[flood] = 2
classified_path = out_dir / "classified.tif"
flood_mask_path = out_dir / "flood_mask.tif"
stable_mask_path = out_dir / "stable_water_mask.tif"
classified_profile = post_profile.copy()
classified_profile.update(
driver="GTiff",
dtype="uint8",
count=1,
nodata=0,
compress="deflate",
)
with rasterio.open(classified_path, "w", **classified_profile) as dst:
dst.write(classified, 1)
try:
dst.write_colormap(
1,
{
0: (0, 0, 0, 0),
1: (24, 144, 255, 255),
2: (255, 77, 79, 255),
3: (250, 173, 20, 255),
4: (80, 80, 80, 255),
},
)
except Exception:
pass
mask_profile = classified_profile.copy()
mask_profile.update(nodata=0)
with rasterio.open(flood_mask_path, "w", **mask_profile) as dst:
dst.write(np.where(flood, 255, 0).astype("uint8"), 1)
with rasterio.open(stable_mask_path, "w", **mask_profile) as dst:
dst.write(np.where(stable_water, 255, 0).astype("uint8"), 1)
flood_pixels = int(np.count_nonzero(flood))
stable_pixels = int(np.count_nonzero(stable_water))
high_pixels = int(np.count_nonzero(high_backscatter))
non_water_pixels = int(np.count_nonzero(classified == 4))
metadata = {
"schema": "flood_detection_geotiff.v1",
"job_id": job_id,
"processor": "python_geotiff_otsu_change",
"refine": bool(refine),
"pre": pre_metadata,
"post": post_metadata,
"thresholds": {
"pre_water_threshold": pre_threshold,
"post_water_threshold": post_threshold,
"post_high_backscatter_threshold": high_threshold,
},
"pixel_area_km2": pixel_area_km2,
"class_values": {
"0": "nodata",
"1": "stable_water",
"2": "flood",
"3": "high_backscatter",
"4": "non_water",
},
"counts": {
"valid_pixels": int(np.count_nonzero(valid)),
"stable_water_pixels": stable_pixels,
"flood_pixels": flood_pixels,
"high_backscatter_pixels": high_pixels,
"non_water_pixels": non_water_pixels,
},
"outputs": {
"classified_path": str(classified_path),
"flood_mask_path": str(flood_mask_path),
"stable_water_mask_path": str(stable_mask_path),
},
}
metadata_path = out_dir / "metadata.json"
metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
return {
"ok": True,
"classified_path": str(classified_path),
"flood_mask_path": str(flood_mask_path),
"stable_water_mask_path": str(stable_mask_path),
"metadata_path": str(metadata_path),
"flood_area_km2": round(flood_pixels * pixel_area_km2, 4),
"stable_water_area_km2": round(stable_pixels * pixel_area_km2, 4),
"flood_pixel_count": flood_pixels,
"stable_water_pixel_count": stable_pixels,
"processor": "python_geotiff_otsu_change",
"log": [
"pre/post analysis-ready GeoTIFFs loaded",
"pre scene reprojected to post-event grid",
f"thresholds pre={pre_threshold:.4f}, post={post_threshold:.4f}",
f"flood_pixels={flood_pixels}, stable_water_pixels={stable_pixels}",
],
}
@@ -0,0 +1,389 @@
"""Flood overlay and impact analysis service."""
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import Any
from fastapi import HTTPException
from geoalchemy2.functions import ST_Intersects
from geoalchemy2.shape import from_shape
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from shapely.geometry import mapping, shape
from shapely.ops import unary_union
from ..config import settings
from ..models import FloodDetectionORM, FloodOverlayORM, HazardPointORM, ResultProductORM
def _to_float(value: Any) -> float | None:
try:
if value is None:
return None
return float(value)
except Exception:
return None
def _hazard_point_to_dict(point: HazardPointORM, *, distance_m: float | None = None) -> dict[str, Any]:
return {
"id": point.id,
"name": point.hazard_name,
"type": point.hazard_type,
"city": point.city,
"county": point.county,
"township": point.township,
"longitude": _to_float(point.longitude),
"latitude": _to_float(point.latitude),
"distance_m": 0 if distance_m is None else round(float(distance_m), 2),
}
def _dinsar_product_to_dict(product: ResultProductORM) -> dict[str, Any]:
summary = product.summary_json if isinstance(product.summary_json, dict) else {}
deformation = (
summary.get("deformation_mm")
or summary.get("max_deformation_mm")
or summary.get("mean_deformation_mm")
or summary.get("deformation")
)
return {
"id": product.id,
"product_id": product.product_id,
"display_name": product.display_name,
"engine": product.engine_code,
"status": product.status,
"deformation_mm": deformation,
"ai_score": product.ai_score,
"manifest_path": product.manifest_path,
"preview_path": product.preview_path,
}
def _open_raster(path: str):
import rasterio
normalized_path = path.replace("\\", "/")
try:
return rasterio.open(normalized_path)
except Exception:
pass
for ext in (".bin", ".img", ".tif", ".tiff"):
candidate = normalized_path + ext
try:
return rasterio.open(candidate)
except Exception:
pass
raise FileNotFoundError(f"Raster file cannot be opened: {path}")
def _classified_flood_to_geojson(path: str) -> tuple[dict[str, Any], float | None, list[str]]:
import rasterio.features
from pyproj import CRS, Transformer
from shapely.geometry import shape as shape_geojson
from shapely.ops import transform
warnings: list[str] = []
with _open_raster(path) as src:
data = src.read(1)
mask = data == 2
if not mask.any():
return {"type": "FeatureCollection", "features": []}, 0.0, warnings
polygons = []
for geom, value in rasterio.features.shapes(data, mask=mask, transform=src.transform):
if int(value) != 2:
continue
polygon = shape_geojson(geom)
if not polygon.is_empty and polygon.is_valid:
polygons.append(polygon)
if not polygons:
return {"type": "FeatureCollection", "features": []}, 0.0, warnings
flood_geom = unary_union(polygons)
source_crs = src.crs
area_km2 = None
output_geom = flood_geom
if source_crs:
try:
crs = CRS.from_user_input(source_crs)
if not crs.is_geographic:
area_km2 = float(flood_geom.area) / 1_000_000.0
transformer = Transformer.from_crs(crs, CRS.from_epsg(4326), always_xy=True)
output_geom = transform(transformer.transform, flood_geom)
else:
centroid = flood_geom.centroid
zone = int((centroid.x + 180) // 6) + 1
epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone
transformer = Transformer.from_crs(crs, CRS.from_epsg(epsg), always_xy=True)
projected = transform(transformer.transform, flood_geom)
area_km2 = float(projected.area) / 1_000_000.0
except Exception as exc:
warnings.append(f"area calculation failed: {exc}")
else:
warnings.append("classified raster has no CRS; geometry is stored in source coordinates")
feature_collection = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {"class": 2, "name": "flood"},
"geometry": mapping(output_geom),
}
],
}
return feature_collection, area_km2, warnings
def _write_geojson(detection_id: int, feature_collection: dict[str, Any]) -> str:
out_dir = Path(settings.WATER_RESULTS_DIR or Path(settings.BACKEND_DIR) / "water_results") / "flood_overlays"
out_dir.mkdir(parents=True, exist_ok=True)
target = out_dir / f"flood_detection_{detection_id}_overlay.geojson"
target.write_text(json.dumps(feature_collection, ensure_ascii=False, indent=2), encoding="utf-8")
return str(target)
def _read_geojson(path: str | None) -> dict[str, Any] | None:
if not path or not os.path.isfile(path):
return None
try:
with open(path, "r", encoding="utf-8") as stream:
payload = json.load(stream)
return payload if isinstance(payload, dict) else None
except Exception:
return None
def _geometry_area_km2(geom) -> float:
if geom.is_empty:
return 0.0
try:
from pyproj import CRS, Transformer
from shapely.ops import transform
centroid = geom.centroid
zone = int((centroid.x + 180) // 6) + 1
epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone
transformer = Transformer.from_crs(CRS.from_epsg(4326), CRS.from_epsg(epsg), always_xy=True)
projected = transform(transformer.transform, geom)
return float(projected.area) / 1_000_000.0
except Exception:
return 0.0
def _calculate_affected_aois(flood_geom, warnings: list[str], *, limit: int = 50) -> list[dict[str, Any]]:
try:
from ..routers import dependencies as deps
deps._load_region_index()
deps._load_region_geometry_index()
region_by_id = deps._REGION_BY_ID_CACHE or {}
geometry_by_id = deps._REGION_GEOMETRY_BY_ID_CACHE or {}
except Exception as exc:
warnings.append(f"AOI overlay unavailable: {exc}")
return []
affected: list[dict[str, Any]] = []
for tree_id, features in geometry_by_id.items():
node = region_by_id.get(tree_id) or {}
level = node.get("level")
if level in {"country", "province"}:
continue
try:
geometries = [shape(feature["geometry"]) for feature in features if feature.get("geometry")]
if not geometries:
continue
region_geom = unary_union(geometries)
if region_geom.is_empty or not flood_geom.intersects(region_geom):
continue
intersection = flood_geom.intersection(region_geom)
area_km2 = _geometry_area_km2(intersection)
if area_km2 <= 0.0001:
continue
affected.append(
{
"tree_id": tree_id,
"name": node.get("name") or tree_id,
"level": level,
"flood_area_km2": round(area_km2, 4),
}
)
except Exception:
continue
affected.sort(key=lambda item: item["flood_area_km2"], reverse=True)
return affected[:limit]
def _attach_overlay_payload(overlay: FloodOverlayORM) -> dict[str, Any]:
payload = dict(overlay.summary_json) if isinstance(overlay.summary_json, dict) else {}
payload["overlay_id"] = overlay.id
payload["detection_id"] = overlay.detection_id
payload["flood_vector_path"] = overlay.flood_vector_path
payload["flood_vector_geojson"] = _read_geojson(overlay.flood_vector_path)
return payload
async def run_overlay(detection_id: int, db: AsyncSession, *, near_threshold_m: float = 500.0) -> dict[str, Any]:
detection = await db.get(FloodDetectionORM, detection_id)
if not detection:
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
if not detection.classified_path:
raise HTTPException(status_code=400, detail="Flood detection has no classified raster")
path = detection.classified_path.replace("\\", "/")
if not os.path.isfile(path):
raise HTTPException(status_code=404, detail="Classified raster file does not exist")
feature_collection, flood_area_km2, warnings = _classified_flood_to_geojson(path)
flood_vector_path = _write_geojson(detection_id, feature_collection)
impact = await _query_impact_from_geojson(
detection_id=detection_id,
feature_collection=feature_collection,
db=db,
near_threshold_m=near_threshold_m,
warnings=warnings,
)
if flood_area_km2 is not None:
impact["flood_area_km2"] = round(flood_area_km2, 4)
impact["flood_vector_path"] = flood_vector_path
overlay = FloodOverlayORM(
detection_id=detection_id,
flood_vector_path=flood_vector_path,
hazard_points_hit=len(impact["hazard_points"]["inside_flood"]),
hazard_points_near=len(impact["hazard_points"]["near_flood"]),
hazard_points_total=impact["hazard_points"]["total_in_scene"],
dinsar_products_intersecting=len(impact["dinsar_products"]),
affected_area_km2=impact.get("flood_area_km2"),
summary_json=impact,
)
db.add(overlay)
await db.flush()
impact["overlay_id"] = overlay.id
overlay.summary_json = impact
if flood_area_km2 is not None:
detection.flood_area_km2 = round(flood_area_km2, 4)
await db.commit()
await db.refresh(overlay)
return {
"id": overlay.id,
"detection_id": detection_id,
"flood_vector_path": overlay.flood_vector_path,
"flood_vector_geojson": feature_collection,
"summary": _attach_overlay_payload(overlay),
}
async def get_overlay_result(detection_id: int, db: AsyncSession) -> dict[str, Any]:
overlay = (
await db.execute(
select(FloodOverlayORM)
.where(FloodOverlayORM.detection_id == detection_id)
.order_by(FloodOverlayORM.id.desc())
)
).scalars().first()
if overlay and isinstance(overlay.summary_json, dict):
return _attach_overlay_payload(overlay)
detection = await db.get(FloodDetectionORM, detection_id)
if not detection:
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
return {
"detection_id": detection_id,
"flood_area_km2": detection.flood_area_km2,
"hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0},
"dinsar_products": [],
"affected_aois": [],
"flood_vector_path": None,
"flood_vector_geojson": None,
"warnings": ["overlay has not been run"],
}
async def _query_impact_from_geojson(
*,
detection_id: int,
feature_collection: dict[str, Any],
db: AsyncSession,
near_threshold_m: float,
warnings: list[str],
) -> dict[str, Any]:
features = feature_collection.get("features") or []
if not features:
return {
"detection_id": detection_id,
"flood_area_km2": 0.0,
"hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0},
"dinsar_products": [],
"affected_aois": [],
"warnings": warnings,
}
flood_geom = unary_union([shape(feature["geometry"]) for feature in features if feature.get("geometry")])
if flood_geom.is_empty:
warnings.append("flood geometry is empty")
flood_wkt = flood_geom.wkt
area_geom = func.ST_GeomFromText(flood_wkt, 4326)
area_geog = func.Geography(area_geom)
inside_points: list[dict[str, Any]] = []
near_points: list[dict[str, Any]] = []
dinsar_products: list[dict[str, Any]] = []
affected_aois = _calculate_affected_aois(flood_geom, warnings)
try:
inside_rows = (
await db.execute(
select(HazardPointORM).where(ST_Intersects(HazardPointORM.geom, area_geom))
)
).scalars().all()
inside_ids = {point.id for point in inside_rows}
inside_points = [_hazard_point_to_dict(point, distance_m=0) for point in inside_rows]
near_rows = (
await db.execute(
select(
HazardPointORM,
func.ST_Distance(func.Geography(HazardPointORM.geom), area_geog).label("distance_m"),
).where(func.ST_DWithin(func.Geography(HazardPointORM.geom), area_geog, near_threshold_m))
)
).all()
for point, distance_m in near_rows:
if point.id in inside_ids:
continue
near_points.append(_hazard_point_to_dict(point, distance_m=distance_m))
except Exception as exc:
warnings.append(f"hazard point overlay unavailable: {exc}")
try:
products = (
await db.execute(
select(ResultProductORM).where(
ResultProductORM.catalog_name == "dinsar",
ResultProductORM.status == "READY",
ST_Intersects(ResultProductORM.geom, area_geom),
)
)
).scalars().all()
dinsar_products = [_dinsar_product_to_dict(product) for product in products]
except Exception as exc:
warnings.append(f"dinsar product overlay unavailable: {exc}")
return {
"detection_id": detection_id,
"flood_area_km2": None,
"hazard_points": {
"inside_flood": inside_points,
"near_flood": near_points,
"total_in_scene": len(inside_points) + len(near_points),
},
"dinsar_products": dinsar_products,
"affected_aois": affected_aois,
"warnings": warnings,
}
@@ -0,0 +1,199 @@
"""Flood product listing and manifest helpers."""
from __future__ import annotations
import json
import os
from datetime import datetime
from typing import Any
from fastapi import HTTPException
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..models import FloodDetectionORM, FloodOverlayORM, FloodProductORM, SARSceneGeoORM
def _iso(value: Any) -> str | None:
return value.isoformat() if value else None
def _product_to_dict(product: FloodProductORM) -> dict[str, Any]:
summary = product.summary_json if isinstance(product.summary_json, dict) else {}
detection = product.detection
overlay = product.overlay
return {
"id": product.id,
"product_id": product.product_id,
"detection_id": product.detection_id,
"overlay_id": product.overlay_id,
"display_name": product.display_name,
"status": product.status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"summary": summary,
"created_at": _iso(product.created_at),
"flood_area_km2": getattr(detection, "flood_area_km2", None),
"affected_area_km2": getattr(overlay, "affected_area_km2", None),
}
async def list_flood_products(
*,
db: AsyncSession,
limit: int = 20,
offset: int = 0,
status: str | None = None,
) -> dict[str, Any]:
count_query = select(func.count()).select_from(FloodProductORM)
query = (
select(FloodProductORM)
.options(
selectinload(FloodProductORM.detection),
selectinload(FloodProductORM.overlay),
)
.order_by(FloodProductORM.id.desc())
.limit(limit)
.offset(offset)
)
if status:
count_query = count_query.where(FloodProductORM.status == status)
query = query.where(FloodProductORM.status == status)
total = (await db.execute(count_query)).scalar_one()
rows = (await db.execute(query)).scalars().all()
return {"items": [_product_to_dict(row) for row in rows], "total": total}
async def get_flood_product(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]:
product = await _get_product(product_id_or_pk, db)
return _product_to_dict(product)
async def get_flood_product_manifest(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]:
product = await _get_product(product_id_or_pk, db)
if product.manifest_path and os.path.isfile(product.manifest_path):
try:
with open(product.manifest_path, "r", encoding="utf-8") as stream:
payload = json.load(stream)
if isinstance(payload, dict):
return payload
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to read manifest: {exc}") from exc
return _build_manifest_from_db(product)
async def create_flood_product_for_detection(detection_id: int, db: AsyncSession) -> dict[str, Any]:
detection = await db.get(FloodDetectionORM, detection_id)
if not detection:
raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found")
if detection.status != "DONE":
raise HTTPException(status_code=400, detail=f"Flood detection is not DONE: {detection.status}")
existing = (
await db.execute(
select(FloodProductORM)
.options(
selectinload(FloodProductORM.detection),
selectinload(FloodProductORM.overlay),
)
.where(FloodProductORM.detection_id == detection_id)
.order_by(FloodProductORM.id.desc())
)
).scalars().first()
if existing:
return _product_to_dict(existing)
overlay = (
await db.execute(
select(FloodOverlayORM)
.where(FloodOverlayORM.detection_id == detection_id)
.order_by(FloodOverlayORM.id.desc())
)
).scalars().first()
product = FloodProductORM(
product_id=f"FLOOD-{detection_id:06d}",
detection_id=detection_id,
overlay_id=overlay.id if overlay else None,
display_name=f"Flood detection #{detection_id}",
status="READY",
publish_dir=detection.output_dir,
manifest_path=None,
summary_json={
"created_from": "flood_detection",
"flood_area_km2": detection.flood_area_km2,
"stable_water_area_km2": detection.stable_water_area_km2,
"classified_path": detection.classified_path,
"created_at": datetime.utcnow().isoformat(),
},
)
db.add(product)
await db.commit()
return await get_flood_product(str(product.id), db)
async def _get_product(product_id_or_pk: str, db: AsyncSession) -> FloodProductORM:
value = str(product_id_or_pk).strip()
query = select(FloodProductORM).options(
selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data),
selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data),
selectinload(FloodProductORM.overlay),
)
if value.isdigit():
query = query.where(FloodProductORM.id == int(value))
else:
query = query.where(FloodProductORM.product_id == value)
product = (await db.execute(query)).scalars().first()
if not product:
raise HTTPException(status_code=404, detail=f"Flood product {product_id_or_pk} not found")
return product
def _build_manifest_from_db(product: FloodProductORM) -> dict[str, Any]:
detection = product.detection
overlay = product.overlay
pre_scene = detection.pre_scene if detection else None
post_scene = detection.post_scene if detection else None
pre_radar = pre_scene.radar_data if pre_scene else None
post_radar = post_scene.radar_data if post_scene else None
return {
"schema": "flood_product_manifest.v1",
"product": _product_to_dict(product),
"detection": {
"id": detection.id if detection else None,
"status": detection.status if detection else None,
"classified_path": detection.classified_path if detection else None,
"flood_area_km2": detection.flood_area_km2 if detection else None,
"stable_water_area_km2": detection.stable_water_area_km2 if detection else None,
"pre_scene": {
"id": pre_scene.id if pre_scene else None,
"radar_data_id": pre_scene.radar_data_id if pre_scene else None,
"satellite": pre_radar.satellite if pre_radar else None,
"imaging_date": pre_radar.imaging_date if pre_radar else None,
"geo_path": pre_scene.geo_path if pre_scene else None,
"analysis_tif_path": pre_scene.analysis_tif_path if pre_scene else None,
"analysis_engine": pre_scene.analysis_engine if pre_scene else None,
"analysis_profile": pre_scene.analysis_profile if pre_scene else None,
},
"post_scene": {
"id": post_scene.id if post_scene else None,
"radar_data_id": post_scene.radar_data_id if post_scene else None,
"satellite": post_radar.satellite if post_radar else None,
"imaging_date": post_radar.imaging_date if post_radar else None,
"geo_path": post_scene.geo_path if post_scene else None,
"analysis_tif_path": post_scene.analysis_tif_path if post_scene else None,
"analysis_engine": post_scene.analysis_engine if post_scene else None,
"analysis_profile": post_scene.analysis_profile if post_scene else None,
},
},
"overlay": {
"id": overlay.id if overlay else None,
"flood_vector_path": overlay.flood_vector_path if overlay else None,
"affected_area_km2": overlay.affected_area_km2 if overlay else None,
"hazard_points_hit": overlay.hazard_points_hit if overlay else 0,
"hazard_points_near": overlay.hazard_points_near if overlay else 0,
"dinsar_products_intersecting": overlay.dinsar_products_intersecting if overlay else 0,
"summary": overlay.summary_json if overlay else None,
},
}
+369
View File
@@ -0,0 +1,369 @@
from __future__ import annotations
import logging
import os
import shutil
import stat
import tarfile
import zipfile
from datetime import datetime
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
from ..config import settings, split_env_paths
logger = logging.getLogger(__name__)
LogCallback = Callable[[str, str], None]
ProgressCallback = Callable[[int, str], None]
DEFAULT_GF3_ARCHIVE_EXTS = (".zip", ".tar", ".tar.gz", ".tgz")
def _parse_float(value: Any, default: float) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
def _normalize_paths(paths: Optional[Iterable[str]]) -> List[str]:
ordered: List[str] = []
for raw_path in paths or []:
text = str(raw_path or "").strip().strip('"').strip("'")
if not text:
continue
normalized = os.path.normpath(os.path.abspath(text))
if normalized not in ordered:
ordered.append(normalized)
return ordered
def _normalize_extensions(extensions: Optional[Iterable[str]]) -> List[str]:
ordered: List[str] = []
for raw_ext in extensions or DEFAULT_GF3_ARCHIVE_EXTS:
ext = str(raw_ext or "").strip().lower()
if not ext:
continue
if not ext.startswith("."):
ext = f".{ext}"
if ext not in ordered:
ordered.append(ext)
return sorted(ordered or list(DEFAULT_GF3_ARCHIVE_EXTS), key=len, reverse=True)
def _strip_archive_extension(file_name: str, extensions: Iterable[str]) -> str:
lower_name = file_name.lower()
for ext in _normalize_extensions(extensions):
if lower_name.endswith(ext):
return file_name[: -len(ext)]
return os.path.splitext(file_name)[0]
def _resolve_target_root(archive_path: str, source_dirs: List[str], target_dirs: List[str]) -> str:
if not target_dirs:
raise ValueError("GF3_SOURCE_DIRS is not configured.")
if len(target_dirs) == 1:
return target_dirs[0]
if source_dirs and len(source_dirs) == len(target_dirs):
archive_norm = os.path.normcase(os.path.abspath(archive_path))
matches: List[Tuple[int, int]] = []
for index, source_dir in enumerate(source_dirs):
source_norm = os.path.normcase(os.path.abspath(source_dir))
if archive_norm == source_norm or archive_norm.startswith(source_norm + os.sep):
matches.append((len(source_norm), index))
if matches:
_prefix_len, best_index = max(matches)
return target_dirs[best_index]
return target_dirs[0]
def _validate_relative_member_name(member_name: str, archive_path: str) -> str:
name = str(member_name or "").replace("\\", "/")
if not name or name in {".", "./"}:
return ""
if name.startswith("/") or os.path.isabs(name) or os.path.splitdrive(name)[0]:
raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}")
parts = [part for part in name.split("/") if part not in ("", ".")]
if any(part == ".." for part in parts):
raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}")
if not parts:
return ""
return os.path.join(*parts)
def _safe_destination(root_dir: str, relative_name: str) -> str:
root_abs = os.path.abspath(root_dir)
destination = os.path.abspath(os.path.join(root_abs, relative_name))
if destination != root_abs and not destination.startswith(root_abs + os.sep):
raise ValueError(f"Unsafe extraction destination: {relative_name}")
return destination
def _validate_tar_members(members: Iterable[tarfile.TarInfo], archive_path: str) -> None:
for member in members:
_validate_relative_member_name(member.name, archive_path)
if member.issym() or member.islnk():
raise ValueError(f"Unsupported link entry in {archive_path}: {member.name}")
if not (member.isdir() or member.isfile()):
raise ValueError(f"Unsupported special entry in {archive_path}: {member.name}")
def _validate_zip_members(infos: Iterable[zipfile.ZipInfo], archive_path: str) -> None:
for info in infos:
_validate_relative_member_name(info.filename, archive_path)
mode = (info.external_attr >> 16) & 0o170000
if stat.S_ISLNK(mode):
raise ValueError(f"Unsupported symlink entry in {archive_path}: {info.filename}")
def _estimate_archive_size(archive_path: str) -> int:
if zipfile.is_zipfile(archive_path):
with zipfile.ZipFile(archive_path, "r") as zip_obj:
infos = zip_obj.infolist()
_validate_zip_members(infos, archive_path)
return sum(max(0, int(info.file_size or 0)) for info in infos if not info.is_dir())
if tarfile.is_tarfile(archive_path):
with tarfile.open(archive_path, "r:*") as tar_obj:
members = tar_obj.getmembers()
_validate_tar_members(members, archive_path)
return sum(max(0, int(member.size or 0)) for member in members if member.isfile())
raise ValueError(f"Unsupported GF3 archive format: {archive_path}")
def _ensure_disk_space(target_root: str, required_bytes: int, min_disk_space_gb: float) -> None:
os.makedirs(target_root, exist_ok=True)
_total, _used, free_bytes = shutil.disk_usage(target_root)
min_free_bytes = int(max(0.0, min_disk_space_gb) * (1024 ** 3))
if free_bytes - max(0, int(required_bytes or 0)) < min_free_bytes:
raise OSError(
"GF3 L1A storage has insufficient free space: "
f"needed {required_bytes / (1024 ** 3):.2f} GB, "
f"free {free_bytes / (1024 ** 3):.2f} GB, "
f"min free after {min_disk_space_gb:.2f} GB"
)
def _prepare_atomic_output(output_dir: str, tmp_suffix: str) -> Tuple[bool, str, str]:
tmp_dir = output_dir + tmp_suffix
lock_path = output_dir + ".unpacking"
if os.path.exists(output_dir):
return False, tmp_dir, lock_path
if os.path.exists(tmp_dir):
return False, tmp_dir, lock_path
if os.path.exists(lock_path):
return False, tmp_dir, lock_path
os.makedirs(tmp_dir, exist_ok=False)
with open(lock_path, "w", encoding="utf-8") as stream:
stream.write(datetime.now().isoformat())
return True, tmp_dir, lock_path
def _cleanup_atomic_paths(tmp_dir: str, lock_path: str) -> None:
if os.path.exists(lock_path):
try:
os.remove(lock_path)
except OSError:
pass
if os.path.exists(tmp_dir):
try:
shutil.rmtree(tmp_dir)
except OSError:
pass
def _extract_tar_archive(archive_path: str, tmp_dir: str) -> int:
extracted_files = 0
with tarfile.open(archive_path, "r:*") as tar_obj:
members = tar_obj.getmembers()
_validate_tar_members(members, archive_path)
for member in members:
relative_name = _validate_relative_member_name(member.name, archive_path)
if not relative_name:
continue
destination = _safe_destination(tmp_dir, relative_name)
if member.isdir():
os.makedirs(destination, exist_ok=True)
continue
if not member.isfile():
continue
os.makedirs(os.path.dirname(destination), exist_ok=True)
source = tar_obj.extractfile(member)
if source is None:
raise OSError(f"Failed to read tar member: {member.name}")
with source, open(destination, "wb") as target:
shutil.copyfileobj(source, target, length=1024 * 1024)
extracted_files += 1
return extracted_files
def _extract_zip_archive(archive_path: str, tmp_dir: str) -> int:
extracted_files = 0
with zipfile.ZipFile(archive_path, "r") as zip_obj:
infos = zip_obj.infolist()
_validate_zip_members(infos, archive_path)
for info in infos:
relative_name = _validate_relative_member_name(info.filename, archive_path)
if not relative_name:
continue
destination = _safe_destination(tmp_dir, relative_name)
if info.is_dir():
os.makedirs(destination, exist_ok=True)
continue
os.makedirs(os.path.dirname(destination), exist_ok=True)
with zip_obj.open(info, "r") as source, open(destination, "wb") as target:
shutil.copyfileobj(source, target, length=1024 * 1024)
extracted_files += 1
return extracted_files
def _extract_archive_atomic(archive_path: str, output_dir: str, tmp_suffix: str) -> Tuple[str, int]:
prepared, tmp_dir, lock_path = _prepare_atomic_output(output_dir, tmp_suffix)
if not prepared:
return "EXISTS", 0
try:
if zipfile.is_zipfile(archive_path):
extracted_files = _extract_zip_archive(archive_path, tmp_dir)
elif tarfile.is_tarfile(archive_path):
extracted_files = _extract_tar_archive(archive_path, tmp_dir)
else:
raise ValueError(f"Unsupported GF3 archive format: {archive_path}")
if extracted_files <= 0:
raise OSError("GF3 archive extraction produced no files.")
os.replace(tmp_dir, output_dir)
return "EXTRACTED", extracted_files
finally:
_cleanup_atomic_paths(tmp_dir, lock_path)
def _discover_archives(source_dirs: List[str], extensions: List[str], log_callback: Optional[LogCallback]) -> List[str]:
archives: List[str] = []
normalized_extensions = _normalize_extensions(extensions)
for source_dir in source_dirs:
if not os.path.isdir(source_dir):
message = f"GF3 archive source does not exist or is not a directory: {source_dir}"
logger.warning(message)
if log_callback:
log_callback("WARNING", message)
continue
for root, _dirs, files in os.walk(source_dir):
for file_name in files:
lower_name = file_name.lower()
if any(lower_name.endswith(ext) for ext in normalized_extensions):
archives.append(os.path.join(root, file_name))
return sorted(archives)
def run_gf3_archive_unpack(
*,
source_dirs: Optional[Iterable[str]] = None,
target_dirs: Optional[Iterable[str]] = None,
archive_exts: Optional[Iterable[str]] = None,
max_files_per_run: Optional[int] = None,
delete_archive: Optional[bool] = None,
min_disk_space_gb: Optional[float] = None,
tmp_suffix: Optional[str] = None,
log_callback: Optional[LogCallback] = None,
progress_callback: Optional[ProgressCallback] = None,
) -> Dict[str, Any]:
configured_source_dirs = _normalize_paths(
source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
)
configured_target_dirs = _normalize_paths(
target_dirs if target_dirs is not None else split_env_paths(settings.GF3_SOURCE_DIRS)
)
extensions = _normalize_extensions(
archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
)
should_delete_archive = settings.GF3_UNPACK_DELETE_ARCHIVE if delete_archive is None else bool(delete_archive)
limit = max(0, int(max_files_per_run or 0))
min_free_gb = (
_parse_float(os.getenv("UNPACK_MIN_DISK_SPACE_GB"), 50.0)
if min_disk_space_gb is None
else float(min_disk_space_gb)
)
atomic_tmp_suffix = str(tmp_suffix or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp").strip() or ".unpack_tmp"
if not configured_source_dirs:
raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.")
if not configured_target_dirs:
raise ValueError("GF3_SOURCE_DIRS is not configured.")
def _log(level: str, message: str) -> None:
logger.log(getattr(logging, level.upper(), logging.INFO), message)
if log_callback:
log_callback(level.upper(), message)
def _progress(progress: int, message: str) -> None:
if progress_callback:
progress_callback(max(0, min(100, int(progress))), message)
_progress(2, "Scanning GF3 archive source directories...")
archives = _discover_archives(configured_source_dirs, extensions, log_callback)
if limit > 0:
archives_to_process = archives[:limit]
else:
archives_to_process = archives
total = len(archives_to_process)
summary: Dict[str, Any] = {
"total": total,
"found": len(archives),
"processed": 0,
"skipped": 0,
"failed": 0,
"remaining": max(0, len(archives) - total),
"source_dirs": configured_source_dirs,
"target_dirs": configured_target_dirs,
"archive_exts": extensions,
"delete_archive": should_delete_archive,
"failures": [],
}
if not archives_to_process:
_progress(100, "No GF3 archives pending.")
summary["message"] = "No GF3 archives found."
return summary
_log("INFO", f"Found {len(archives)} GF3 archives; processing {total}.")
for index, archive_path in enumerate(archives_to_process, start=1):
archive_name = os.path.basename(archive_path)
progress_base = 5 + int(((index - 1) / max(1, total)) * 90)
_progress(progress_base, f"Unpacking GF3 archive {index}/{total}: {archive_name}")
try:
target_root = _resolve_target_root(archive_path, configured_source_dirs, configured_target_dirs)
os.makedirs(target_root, exist_ok=True)
output_name = _strip_archive_extension(os.path.basename(archive_path), extensions)
output_dir = os.path.join(target_root, output_name)
required_bytes = _estimate_archive_size(archive_path)
_ensure_disk_space(target_root, required_bytes, min_free_gb)
status, extracted_files = _extract_archive_atomic(archive_path, output_dir, atomic_tmp_suffix)
if status == "EXISTS":
summary["skipped"] += 1
_log("INFO", f"GF3 archive already unpacked, skipped: {archive_path}")
continue
if should_delete_archive:
os.remove(archive_path)
_log("INFO", f"GF3 archive deleted after successful unpack: {archive_path}")
summary["processed"] += 1
_log("INFO", f"GF3 archive unpacked: {archive_path} -> {output_dir} ({extracted_files} files)")
except Exception as exc:
summary["failed"] += 1
failure = {
"archive_path": archive_path,
"error": str(exc),
}
summary["failures"].append(failure)
_log("ERROR", f"GF3 archive unpack failed: {archive_path}: {exc}")
_progress(100, "GF3 archive unpack completed.")
summary["message"] = (
f"GF3 unpack complete: processed {summary['processed']}, "
f"skipped {summary['skipped']}, failed {summary['failed']}"
)
return summary
+85
View File
@@ -17,6 +17,7 @@ from ..models import (
OrbitAssetORM,
ResultCatalogStateORM,
ResultProductORM,
SARSceneGeoORM,
SceneOrbitBindingORM,
SourceProductAssetORM,
SystemWorkerHeartbeatORM,
@@ -362,6 +363,19 @@ def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]:
}
def _sanitize_sar_analysis_ready_status(payload: Dict[str, Any]) -> Dict[str, Any]:
scenes = payload.get("scenes", {}) or {}
roots = payload.get("roots", {}) or {}
return {
"ok": bool(payload.get("ok")),
"configured_root_count": len(roots),
"accessible_root_count": sum(1 for item in roots.values() if item.get("accessible")),
"scene_count": int(scenes.get("scene_count") or 0),
"analysis_scene_count": int(scenes.get("analysis_scene_count") or 0),
"missing_file_count": int(scenes.get("missing_file_count") or 0),
}
def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
@@ -460,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
psinsar_result_catalog = timeseries_result_catalog
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
source_roots = payload.get("source_roots", {}) or {}
sar_analysis_ready = payload.get("sar_analysis_ready", {}) or {}
product_packages = payload.get("product_packages", {}) or {}
asset_inventory = payload.get("asset_inventory", {}) or {}
wsl_runtime = payload.get("wsl_runtime", {}) or {}
@@ -473,6 +488,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
sanitized_psinsar_catalog = sanitized_timeseries_catalog
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
sanitized_sar_analysis_ready = _sanitize_sar_analysis_ready_status(sar_analysis_ready)
sanitized_product_packages = _sanitize_product_package_status(product_packages)
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
@@ -502,6 +518,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
},
"dinsar_bridge": sanitized_dinsar_bridge,
"source_roots": sanitized_source_roots,
"sar_analysis_ready": sanitized_sar_analysis_ready,
"product_packages": sanitized_product_packages,
"asset_inventory": sanitized_asset_inventory,
"wsl_runtime": sanitized_wsl_runtime,
@@ -869,6 +886,21 @@ async def _check_source_roots() -> Dict[str, Any]:
status["role"] = "dinsar_source"
items.append(status)
for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_archive_source"
items.append(status)
for path in split_env_paths(settings.GF3_SOURCE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_l1a_source"
items.append(status)
for path in split_env_paths(settings.GF3_STORAGE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_l2_storage"
items.append(status)
configured_count = len(items)
accessible_count = sum(1 for item in items if item.get("accessible"))
inaccessible_count = configured_count - accessible_count
@@ -882,6 +914,56 @@ async def _check_source_roots() -> Dict[str, Any]:
}
async def _check_sar_analysis_ready() -> Dict[str, Any]:
roots = {
"ready": _probe_directory_status(settings.SAR_ANALYSIS_READY_ROOT),
"work": _probe_directory_status(settings.SAR_ANALYSIS_WORK_ROOT),
"preview": _probe_directory_status(settings.SAR_ANALYSIS_PREVIEW_ROOT),
}
for role, payload in roots.items():
payload["role"] = role
status: Dict[str, Any] = {
"ok": False,
"roots": roots,
"scenes": {
"scene_count": 0,
"analysis_scene_count": 0,
"missing_file_count": 0,
"missing_files": [],
},
"error": None,
}
try:
session_factory = _get_session_factory()
async with session_factory() as db:
scene_count_result = await db.execute(select(func.count(SARSceneGeoORM.id)))
status["scenes"]["scene_count"] = int(scene_count_result.scalar_one() or 0)
analysis_rows_result = await db.execute(
select(SARSceneGeoORM.id, SARSceneGeoORM.analysis_tif_path)
.where(SARSceneGeoORM.analysis_tif_path.is_not(None))
.order_by(SARSceneGeoORM.id.desc())
)
analysis_rows = analysis_rows_result.all()
status["scenes"]["analysis_scene_count"] = len(analysis_rows)
missing_files = []
for scene_id, tif_path in analysis_rows:
path_text = str(tif_path or "").strip()
if path_text and not os.path.isfile(path_text):
missing_files.append({"scene_id": scene_id, "path": path_text})
status["scenes"]["missing_file_count"] = len(missing_files)
status["scenes"]["missing_files"] = missing_files[:20]
status["ok"] = (
all(item.get("accessible") for item in roots.values())
and status["scenes"]["missing_file_count"] == 0
)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_product_packages() -> Dict[str, Any]:
status = {
"ok": False,
@@ -1256,6 +1338,7 @@ async def get_health_status(
psinsar_result_catalog_status = timeseries_result_catalog_status
dinsar_bridge_status = await _check_dinsar_bridge()
source_roots_status = await _check_source_roots()
sar_analysis_ready_status = await _check_sar_analysis_ready()
product_packages_status = await _check_product_packages()
asset_inventory_status = await _check_asset_inventory()
wsl_runtime_status = await _check_wsl_runtime()
@@ -1273,6 +1356,7 @@ async def get_health_status(
result_catalog_status.get("ok"),
dinsar_bridge_status.get("ok"),
source_roots_status.get("ok"),
sar_analysis_ready_status.get("ok"),
product_packages_status.get("ok"),
asset_inventory_status.get("ok"),
wsl_runtime_status.get("ok"),
@@ -1297,6 +1381,7 @@ async def get_health_status(
},
"dinsar_bridge": dinsar_bridge_status,
"source_roots": source_roots_status,
"sar_analysis_ready": sar_analysis_ready_status,
"product_packages": product_packages_status,
"asset_inventory": asset_inventory_status,
"wsl_runtime": wsl_runtime_status,
+396 -17
View File
@@ -19,7 +19,7 @@ from sqlalchemy import select
from .. import database
from ..config import settings
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job
from .data_service import data_service
from .asset_inventory_service import asset_inventory_service
@@ -80,7 +80,10 @@ JOB_TYPE_IDL_RUN_DINSAR = "IDL_RUN_DINSAR"
JOB_TYPE_WATER_GEOCODE = "WATER_GEOCODE"
JOB_TYPE_WATER_FLOOD = "WATER_FLOOD"
JOB_TYPE_WATER_DETECT = "WATER_DETECT"
JOB_TYPE_SAR_SCENE_PREPROCESS = "SAR_SCENE_PREPROCESS"
JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION"
JOB_TYPE_GF3_PROCESS = "GF3_PROCESS"
JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
@@ -88,6 +91,7 @@ JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
@@ -3149,6 +3153,106 @@ async def _handle_water_geocode(job: SystemJobORM) -> None:
)
async def _handle_sar_scene_preprocess(job: SystemJobORM) -> None:
"""Build one analysis-ready GeoTIFF for flood/water algorithms."""
payload = job.payload or {}
scene_id = payload.get("scene_id")
radar_data_id = payload.get("radar_data_id")
engine = str(payload.get("engine") or "").strip().lower()
if not scene_id and not radar_data_id:
raise ValueError("SAR_SCENE_PREPROCESS requires scene_id or radar_data_id")
if engine not in {"gf3_gdal", "lt_gamma"}:
raise ValueError(f"Unsupported SAR scene preprocessing engine: {engine}")
await task_service.start_task(job.task_id, message="Preparing analysis-ready SAR GeoTIFF...")
async with AsyncSessionLocal() as db:
scene: SARSceneGeoORM | None = None
if scene_id:
scene = await db.get(SARSceneGeoORM, int(scene_id))
if not scene and radar_data_id:
result = await db.execute(
select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == int(radar_data_id))
)
scene = result.scalar_one_or_none()
if not scene:
scene = SARSceneGeoORM(radar_data_id=int(radar_data_id), status="PENDING")
db.add(scene)
await db.flush()
radar = await db.get(RadarDataORM, int(scene.radar_data_id))
if not radar:
raise ValueError(f"RadarDataORM id={scene.radar_data_id} does not exist")
scene.status = "RUNNING"
scene.error_msg = None
await db.commit()
scene_id = int(scene.id)
radar_data_id = int(radar.id)
try:
if engine == "gf3_gdal":
await task_service.update_task(job.task_id, progress=20, message="Standardizing GF3 L2 GeoTIFF...")
from .sar_analysis_ready_service import standardize_gf3_l2_for_radar
async with AsyncSessionLocal() as db:
manifest = await standardize_gf3_l2_for_radar(
db=db,
radar_id=int(radar_data_id),
l2_path=payload.get("l2_path"),
polarization=payload.get("polarization"),
)
else:
await task_service.update_task(job.task_id, progress=15, message="Running LT Gamma single-scene preprocessing...")
from .lt_gamma_scene_service import run_lt_gamma_scene_preprocess
from .sar_analysis_ready_service import register_analysis_ready_tif
async with AsyncSessionLocal() as db:
scene = await db.get(SARSceneGeoORM, int(scene_id))
radar = await db.get(RadarDataORM, int(radar_data_id))
if not scene or not radar:
raise ValueError("Scene or radar record disappeared before LT Gamma preprocessing")
def _run_lt() -> Dict[str, Any]:
return run_lt_gamma_scene_preprocess(radar=radar, scene=scene, job_id=job.job_id)
lt_manifest = await asyncio.to_thread(_run_lt)
await task_service.update_task(job.task_id, progress=85, message="Registering LT analysis-ready GeoTIFF...")
analysis_tif_path = str(lt_manifest.get("analysis_tif_path") or "").strip()
if not analysis_tif_path:
raise RuntimeError("LT Gamma preprocessing returned no analysis_tif_path")
async with AsyncSessionLocal() as db:
scene = await db.get(SARSceneGeoORM, int(scene_id))
radar = await db.get(RadarDataORM, int(radar_data_id))
if not scene or not radar:
raise ValueError("Scene or radar record disappeared before analysis-ready registration")
manifest = await register_analysis_ready_tif(
db=db,
scene=scene,
radar=radar,
source_tif_path=analysis_tif_path,
engine="lt_gamma",
profile="lt1_gamma_geocoded_mli",
backscatter_unit=str(lt_manifest.get("backscatter_unit") or "gamma_mli_db"),
polarization=radar.polarization,
metadata=lt_manifest,
)
await db.commit()
except Exception as exc:
async with AsyncSessionLocal() as db:
scene = await db.get(SARSceneGeoORM, int(scene_id))
if scene:
scene.status = "FAILED"
scene.error_msg = str(exc)
await db.commit()
raise
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=f"Analysis-ready GeoTIFF ready: {manifest.get('analysis_tif_path')}",
)
async def _handle_water_flood(job: SystemJobORM) -> None:
"""洪涝检测 job handler(灾前 + 灾后配对分类)。"""
from .water_service import run_flood_detection, WATER_RESULTS_DIR
@@ -3171,8 +3275,10 @@ async def _handle_water_flood(job: SystemJobORM) -> None:
raise ValueError("灾前或灾后场景记录不存在")
if pre_scene.status != "DONE" or post_scene.status != "DONE":
raise ValueError("灾前或灾后场景尚未完成地理编码")
pre_geo = pre_scene.geo_path
post_geo = post_scene.geo_path
pre_geo = pre_scene.analysis_tif_path or pre_scene.geo_path
post_geo = post_scene.analysis_tif_path or post_scene.geo_path
if not pre_geo or not post_geo:
raise ValueError("Pre/post scenes must have analysis-ready GeoTIFF paths")
output_dir = os.path.join(WATER_RESULTS_DIR, f"flood_{detection_id}")
os.makedirs(output_dir, exist_ok=True)
@@ -3228,36 +3334,129 @@ async def _handle_water_flood(job: SystemJobORM) -> None:
)
async def _handle_water_detect(job: SystemJobORM) -> None:
"""水体检测 job handlerOtsu + DEM + 形态学 + 连通分量)。"""
from .water_detect_service import run_water_detection
async def _handle_flood_detection(job: SystemJobORM) -> None:
"""Flood-analysis detection job: pure Python GeoTIFF change classification."""
from .flood_detection_service import run_geotiff_flood_detection
payload = job.payload or {}
detection_id = payload.get("detection_id")
if not detection_id:
raise ValueError("WATER_DETECT job 缺少 detection_id")
raise ValueError("FLOOD_DETECTION job requires detection_id")
refine = bool(payload.get("refine", False))
await task_service.start_task(job.task_id, message="Reading flood-detection scene pair...")
async with AsyncSessionLocal() as db:
det = await db.get(FloodDetectionORM, int(detection_id))
if not det:
raise ValueError(f"FloodDetectionORM id={detection_id} does not exist")
pre_scene = await db.get(SARSceneGeoORM, det.pre_scene_id)
post_scene = await db.get(SARSceneGeoORM, det.post_scene_id)
if not pre_scene or not post_scene:
raise ValueError("Pre/post scene records do not exist")
if pre_scene.status != "DONE" or post_scene.status != "DONE":
raise ValueError("Pre/post scenes are not DONE")
pre_tif = pre_scene.analysis_tif_path
post_tif = post_scene.analysis_tif_path
if not pre_tif or not post_tif:
raise ValueError("Flood detection requires analysis-ready GeoTIFF paths for both scenes")
output_dir = os.path.join(settings.WATER_RESULTS_DIR, f"flood_{detection_id}")
os.makedirs(output_dir, exist_ok=True)
await task_service.update_task(job.task_id, progress=10, message="Running GeoTIFF flood classification...")
def _run() -> Dict[str, Any]:
return run_geotiff_flood_detection(
pre_tif_path=pre_tif,
post_tif_path=post_tif,
output_dir=output_dir,
job_id=job.job_id,
refine=refine,
)
try:
result = await asyncio.to_thread(_run)
except Exception as exc:
async with AsyncSessionLocal() as db:
det = await db.get(FloodDetectionORM, int(detection_id))
if det:
det.status = "FAILED"
det.error_msg = str(exc)
await db.commit()
raise
async with AsyncSessionLocal() as db:
det = await db.get(FloodDetectionORM, int(detection_id))
if det:
if result.get("ok"):
det.classified_path = result.get("classified_path")
det.flood_area_km2 = result.get("flood_area_km2")
det.stable_water_area_km2 = result.get("stable_water_area_km2")
det.output_dir = output_dir
det.status = "DONE"
det.error_msg = None
else:
det.status = "FAILED"
det.error_msg = result.get("error", "Unknown error")
await db.commit()
if not result.get("ok"):
raise RuntimeError(f"Flood detection failed: {result.get('error')}")
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
f"GeoTIFF flood detection completed: flood_area={result.get('flood_area_km2')} km2, "
f"stable_water={result.get('stable_water_area_km2')} km2"
),
)
async def _handle_water_detect(job: SystemJobORM) -> None:
"""水体检测 job handlerOtsu + DEM + 形态学 + 连通分量)。"""
from .water_extraction_service import run_otsu_water_extraction
payload = job.payload or {}
extraction_id = payload.get("extraction_id")
detection_id = payload.get("detection_id")
record_id = extraction_id or detection_id
if not record_id:
raise ValueError("WATER_DETECT job 缺少 extraction_id/detection_id")
use_extraction_table = extraction_id is not None
await task_service.start_task(job.task_id, message="读取检测任务信息...")
async with AsyncSessionLocal() as db:
det = await db.get(WaterDetectionORM, int(detection_id))
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
if not det:
raise ValueError(f"WaterDetectionORM id={detection_id} 不存在")
model_name = "WaterExtractionORM" if use_extraction_table else "WaterDetectionORM"
raise ValueError(f"{model_name} id={record_id} 不存在")
input_path = det.input_path
det.status = "RUNNING"
if use_extraction_table and hasattr(det, "task_id"):
det.task_id = job.task_id
if not use_extraction_table:
mirror = await db.get(WaterExtractionORM, int(record_id))
if mirror:
mirror.status = "RUNNING"
mirror.task_id = job.task_id
await db.commit()
if not input_path:
raise ValueError("水体检测缺少输入路径 input_path")
output_dir = os.path.join(os.path.dirname(input_path), f"water_detect_{detection_id}")
output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}"
output_dir = os.path.join(os.path.dirname(input_path), output_name)
os.makedirs(output_dir, exist_ok=True)
await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...")
def _run() -> Dict[str, Any]:
return run_water_detection(
geo_tiff_path=input_path,
return run_otsu_water_extraction(
input_path=input_path,
output_dir=output_dir,
job_id=job.job_id,
)
@@ -3266,26 +3465,56 @@ async def _handle_water_detect(job: SystemJobORM) -> None:
result = await asyncio.to_thread(_run)
except Exception as exc:
async with AsyncSessionLocal() as db:
det = await db.get(WaterDetectionORM, int(detection_id))
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
if det:
det.status = "FAILED"
det.error_msg = str(exc)
await db.commit()
if not use_extraction_table:
mirror = await db.get(WaterExtractionORM, int(record_id))
if mirror:
mirror.status = "FAILED"
mirror.error_msg = str(exc)
mirror.task_id = job.task_id
await db.commit()
raise
async with AsyncSessionLocal() as db:
det = await db.get(WaterDetectionORM, int(detection_id))
det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id))
if det:
if result.get("ok"):
det.output_path = result.get("output_path")
det.water_area_km2 = result.get("water_area_km2")
det.water_pixel_count = result.get("water_pixel_count")
det.otsu_threshold_db = result.get("otsu_threshold_db")
if use_extraction_table:
det.processor = result.get("processor") or det.processor or "otsu"
det.threshold_value = result.get("threshold_value")
det.metadata_json = {
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
"job_id": job.job_id,
}
else:
det.otsu_threshold_db = result.get("otsu_threshold_db")
det.status = "DONE"
det.error_msg = None
else:
det.status = "FAILED"
det.error_msg = result.get("error", "Unknown error")
if not use_extraction_table:
mirror = await db.get(WaterExtractionORM, int(record_id))
if mirror:
mirror.output_path = det.output_path
mirror.water_area_km2 = det.water_area_km2
mirror.water_pixel_count = det.water_pixel_count
mirror.threshold_value = result.get("threshold_value") or result.get("otsu_threshold_db")
mirror.processor = result.get("processor") or mirror.processor or "otsu"
mirror.status = det.status
mirror.error_msg = det.error_msg
mirror.task_id = job.task_id
mirror.metadata_json = {
"legacy_otsu_threshold_db": result.get("otsu_threshold_db"),
"legacy_detection_id": int(record_id),
"job_id": job.job_id,
}
await db.commit()
if not result.get("ok"):
@@ -3376,9 +3605,70 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
)
async def _handle_gf3_unpack(job: SystemJobORM) -> None:
"""GF3 archive inbox -> persistent L1A source pool."""
from .gf3_unpack_service import run_gf3_archive_unpack
if not job.task_id:
raise ValueError("GF3_UNPACK requires task_id for progress tracking.")
payload = job.payload or {}
await task_service.start_task(job.task_id, message="扫描 GF3 压缩包来源目录...")
loop = asyncio.get_running_loop()
def _submit(coro):
try:
future = asyncio.run_coroutine_threadsafe(coro, loop)
except RuntimeError:
return
def _swallow_errors(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 Unpack] task callback failed: %s", exc)
future.add_done_callback(_swallow_errors)
def _log_cb(level: str, message: str) -> None:
_submit(task_service.add_log(job.task_id, level, message))
def _progress_cb(progress: int, message: str) -> None:
_submit(task_service.update_task(job.task_id, progress=progress, message=message))
try:
result = await asyncio.to_thread(
run_gf3_archive_unpack,
source_dirs=payload.get("source_dirs"),
target_dirs=payload.get("target_dirs"),
archive_exts=payload.get("archive_exts"),
max_files_per_run=payload.get("max_files_per_run"),
delete_archive=payload.get("delete_archive") if "delete_archive" in payload else None,
min_disk_space_gb=payload.get("min_disk_space_gb"),
tmp_suffix=payload.get("tmp_suffix"),
log_callback=_log_cb,
progress_callback=_progress_cb,
)
except Exception as exc:
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=f"GF3 解包失败: {exc}")
raise
message = (
f"GF3 解包完成: 成功 {int(result.get('processed') or 0)}, "
f"跳过 {int(result.get('skipped') or 0)}, "
f"失败 {int(result.get('failed') or 0)}"
)
remaining = int(result.get("remaining") or 0)
if remaining > 0:
message += f", 剩余 {remaining}"
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
"""批量 GF3 L1A→L2:扫描来源目录,逐个处理并自动入库到 radar_data。"""
from .gf3_service import run_gf3_l1a_to_l2, register_l2_to_radar_data
from .sar_analysis_ready_service import standardize_gf3_l2_for_radar
payload = job.payload or {}
source_dirs = payload.get("source_dirs") or []
@@ -3458,12 +3748,18 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
# Auto-register to radar_data
try:
async with AsyncSessionLocal() as db:
await register_l2_to_radar_data(
radar_id = await register_l2_to_radar_data(
l2_dir=result.get("output_dir", output_dir),
input_dir_name=dir_name,
polarizations=result.get("polarizations", []),
db=db,
)
if radar_id:
await standardize_gf3_l2_for_radar(
db=db,
radar_id=int(radar_id),
l2_path=result.get("output_dir", output_dir),
)
except Exception as reg_err:
logger.warning("[GF3 Batch] Auto-register failed for %s: %s", dir_name, reg_err)
else:
@@ -3888,6 +4184,85 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
)
async def _handle_sbas_coregistration(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("SBAS_COREGISTRATION requires task_id for progress tracking.")
payload = job.payload or {}
run_id = str(payload.get("run_id") or "").strip()
if not run_id:
raise ValueError("SBAS_COREGISTRATION requires run_id payload.")
rlks = _normalize_positive_int(payload.get("rlks")) or 8
azlks = _normalize_positive_int(payload.get("azlks")) or 8
timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200
await task_service.start_task(job.task_id, message="正在执行 SBAS-InSAR Gamma 共参考配准...")
await task_service.update_task(
job.task_id,
progress=5,
message=f"准备运行 Gamma SLC_coreg.py: run_id={run_id}",
)
await task_service.add_log(
job.task_id,
"INFO",
f"SBAS coregistration queued: run_id={run_id}, rlks={rlks}, azlks={azlks}, timeout={timeout_seconds}s",
)
from .sbas_insar_production_service import sbas_insar_production_service
async def _task_keepalive() -> None:
progress = 12
while True:
await asyncio.sleep(60)
progress = min(88, progress + 2)
await task_service.update_task(
job.task_id,
progress=progress,
message=f"Gamma 共参考配准仍在运行: run_id={run_id}",
)
runner_task = asyncio.create_task(
asyncio.to_thread(
sbas_insar_production_service.execute_coregistration,
run_id,
rlks=rlks,
azlks=azlks,
timeout_seconds=timeout_seconds,
)
)
keepalive_task = asyncio.create_task(_task_keepalive())
try:
result = await runner_task
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
manifest = result.get("manifest") or {}
run = result.get("run") or {}
summary = (manifest.get("coregistration") or {}).get("summary") or {}
status = str(run.get("status") or manifest.get("status") or "").strip()
if status != "COREGISTRATION_READY":
raise RuntimeError(
"SBAS coregistration failed: "
f"status={status or 'UNKNOWN'}, "
f"missing_dates={summary.get('missing_dates') or []}, "
f"missing_tabs={summary.get('missing_tabs') or []}"
)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"SBAS-InSAR 共参考配准完成: "
f"{summary.get('ready_secondary_count', 0)}/{summary.get('expected_secondary_count', 0)} secondary scenes ready"
),
)
_HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
@@ -3918,10 +4293,14 @@ _HANDLERS = {
JOB_TYPE_ISCE2_RUN: _handle_isce2_run,
JOB_TYPE_PYINT_RUN: _handle_pyint_run,
JOB_TYPE_WATER_GEOCODE: _handle_water_geocode,
JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess,
JOB_TYPE_WATER_FLOOD: _handle_water_flood,
JOB_TYPE_FLOOD_DETECTION: _handle_flood_detection,
JOB_TYPE_WATER_DETECT: _handle_water_detect,
JOB_TYPE_GF3_PROCESS: _handle_gf3_process,
JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack,
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
}
@@ -0,0 +1,158 @@
"""LT single-scene Gamma preprocessing service."""
from __future__ import annotations
import json
import os
import re
from pathlib import Path
from typing import Any
from ..config import settings
from ..models import RadarDataORM, SARSceneGeoORM
from .pyint_service import (
DEFAULT_AZIMUTH_LOOKS,
DEFAULT_RANGE_LOOKS,
quote_shell,
resolve_gamma_env_script,
to_wsl_path,
)
from .wsl_service import run_wsl_exec
def _safe_token(value: Any, *, default: str = "scene") -> str:
text = str(value or "").strip()
if not text:
text = default
text = re.sub(r"[^0-9A-Za-z._-]+", "_", text).strip("._-")
return text or default
def _scene_date(radar: RadarDataORM) -> str:
for value in (radar.imaging_date, radar.file_path, radar.unique_id):
text = str(value or "")
match = re.search(r"(20\d{6})", re.sub(r"\D", "", text))
if match:
return match.group(1)
match = re.search(r"(20\d{6})", text)
if match:
return match.group(1)
raise ValueError(f"Cannot infer LT scene date for radar_data id={radar.id}")
def _prepared_dem_path() -> str:
if str(settings.PYINT_DEM_MODE or "").strip().lower() == "prepared_file":
return (
settings.PYINT_PREPARED_DEM_PATH
or settings.ISCE2_DEM_PATH
or settings.IDL_DINSAR_DEM_BASE_FILE
or ""
)
return ""
def _build_shell_command(parts: list[str]) -> str:
return " ".join(quote_shell(part) for part in parts)
def _runner_script() -> Path:
return Path(settings.PROJECT_ROOT) / "backend" / "app" / "pyint_pipeline" / "run_gamma_scene_preprocess.py"
def run_lt_gamma_scene_preprocess(
*,
radar: RadarDataORM,
scene: SARSceneGeoORM,
job_id: str | None = None,
) -> dict[str, Any]:
if not settings.PYINT_ENABLED:
raise RuntimeError("PYINT_ENABLED=false; LT Gamma scene preprocessing is disabled")
if not radar.file_path:
raise ValueError(f"RadarDataORM id={radar.id} has no file_path")
date = _scene_date(radar)
token = _safe_token(radar.unique_id or Path(str(radar.file_path)).stem or f"radar_{radar.id}")
run_name = _safe_token(f"lt_{date}_{token}_scene_{scene.id}_{job_id or 'manual'}")
work_dir = Path(settings.SAR_ANALYSIS_WORK_ROOT) / "lt_gamma" / run_name
output_dir = work_dir / "output"
work_dir.mkdir(parents=True, exist_ok=True)
output_dir.mkdir(parents=True, exist_ok=True)
pyint_home = settings.PYINT_HOME
if not pyint_home:
raise RuntimeError("PYINT_HOME is not configured")
pyint_python = settings.PYINT_WSL_PYTHON or settings.WSL_SHARED_PYTHON
if not pyint_python:
raise RuntimeError("PYINT_WSL_PYTHON is not configured")
runner = _runner_script()
if not runner.is_file():
raise FileNotFoundError(f"Gamma scene runner not found: {runner}")
args = [
pyint_python,
to_wsl_path(str(runner)),
"--source-path",
to_wsl_path(str(radar.file_path)),
"--output-dir",
to_wsl_path(str(output_dir)),
"--work-dir",
to_wsl_path(str(work_dir)),
"--pyint-home",
to_wsl_path(str(pyint_home)),
"--dem-root",
to_wsl_path(str(settings.PYINT_DEM_ROOT)),
"--prepared-dem-path",
to_wsl_path(_prepared_dem_path()),
"--project-name",
run_name,
"--date",
date,
"--satellite-family",
"LT1",
"--range-looks",
str(DEFAULT_RANGE_LOOKS),
"--azimuth-looks",
str(DEFAULT_AZIMUTH_LOOKS),
"--geo-interp",
str(settings.PYINT_GEO_INTERP or "1"),
"--nodata-value",
str(float(settings.SAR_ANALYSIS_NODATA_VALUE)),
"--to-db",
]
gamma_env_script = resolve_gamma_env_script(settings.PYINT_GAMMA_ENV_SCRIPT)
prefix = ""
if gamma_env_script:
prefix = f". {quote_shell(to_wsl_path(gamma_env_script))} >/dev/null 2>&1 || exit 1; "
command = prefix + f"export PYTHONPATH={quote_shell(to_wsl_path(str(pyint_home)))}:$PYTHONPATH; " + _build_shell_command(args)
rc, stdout, stderr = run_wsl_exec(
["bash", "-lc", command],
distro=settings.PYINT_WSL_DISTRO or settings.WSL_DISTRO,
timeout=int(settings.PYINT_DEFAULT_TIMEOUT_SECONDS or 43200),
)
if rc != 0:
detail = (stderr or stdout or "").strip()
raise RuntimeError(f"LT Gamma scene preprocessing failed rc={rc}: {detail}")
manifest_path = output_dir / "manifest.json"
if not manifest_path.is_file():
raise RuntimeError(f"LT Gamma scene preprocessing produced no manifest: {manifest_path}")
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except Exception as exc:
raise RuntimeError(f"Cannot read LT Gamma manifest: {manifest_path}: {exc}") from exc
analysis_tif_path = manifest.get("analysis_tif_path")
if analysis_tif_path and str(analysis_tif_path).startswith("/mnt/"):
# The service registers the Windows-side path below.
manifest["analysis_tif_path_wsl"] = analysis_tif_path
manifest["analysis_tif_path"] = str(output_dir / "analysis_ready.tif")
manifest["service"] = {
"work_dir": str(work_dir),
"output_dir": str(output_dir),
"job_id": job_id,
"stdout": stdout[-4000:] if stdout else "",
"stderr": stderr[-4000:] if stderr else "",
}
return manifest
@@ -235,6 +235,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="directory_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_ARCHIVE_SOURCE_DIRS",
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
root_role="source_pool_gf3_archive",
display_prefix="GF3 Archive Pool",
scan_mode="archive_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_SOURCE_DIRS",
@@ -253,6 +262,24 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="scene_directory",
)
)
specs.extend(
_iter_single_root_specs(
env_var="SAR_ANALYSIS_READY_ROOT",
path=settings.SAR_ANALYSIS_READY_ROOT,
root_role="sar_analysis_ready",
display_name="SAR Analysis-ready GeoTIFF Root",
scan_mode="scene_directory",
)
)
specs.extend(
_iter_single_root_specs(
env_var="SAR_ANALYSIS_WORK_ROOT",
path=settings.SAR_ANALYSIS_WORK_ROOT,
root_role="sar_analysis_work",
display_name="SAR Analysis Work Root",
scan_mode="directory_walk",
)
)
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
if not orbit_source_paths:
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
@@ -0,0 +1,360 @@
"""Analysis-ready SAR GeoTIFF registration for flood/water algorithms.
This service owns the common contract between satellite-specific preprocessing
and downstream flood/water algorithms: one geocoded, single-band GeoTIFF plus
sidecar metadata under SAR_ANALYSIS_READY_ROOT.
"""
from __future__ import annotations
import json
import math
import os
import re
import shutil
from pathlib import Path
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..models import RadarDataORM, SARSceneGeoORM
from ..utils import normalize_satellite_family
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
_POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
def _safe_slug(value: Any, *, default: str = "unknown") -> str:
text = str(value or "").strip()
if not text:
text = default
text = _SAFE_TEXT_RE.sub("_", text).strip("._-")
return text or default
def _scene_family(radar: RadarDataORM | None) -> str:
family = normalize_satellite_family(
getattr(radar, "satellite_family", None) or getattr(radar, "satellite", None)
)
return _safe_slug(family or "SAR").upper()
def _scene_date(radar: RadarDataORM | None) -> str:
text = str(getattr(radar, "imaging_date", None) or "").strip()
match = re.search(r"(20\d{6})", re.sub(r"\D", "", text))
if match:
return match.group(1)
return "unknown_date"
def _scene_token(
*,
radar: RadarDataORM | None,
scene: SARSceneGeoORM,
polarization: str | None = None,
) -> str:
unique = getattr(radar, "unique_id", None) or f"radar_{getattr(radar, 'id', scene.radar_data_id)}"
parts = [_scene_date(radar), _safe_slug(unique), f"scene_{scene.id}"]
if polarization:
parts.append(_safe_slug(polarization).upper())
return "_".join(parts)
def scene_analysis_dir(
*,
radar: RadarDataORM | None,
scene: SARSceneGeoORM,
engine: str,
profile: str,
polarization: str | None = None,
) -> Path:
return (
Path(settings.SAR_ANALYSIS_READY_ROOT)
/ _scene_family(radar)
/ _safe_slug(engine)
/ _safe_slug(profile)
/ _scene_date(radar)
/ _scene_token(radar=radar, scene=scene, polarization=polarization)
)
def _write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as stream:
json.dump(payload, stream, ensure_ascii=False, indent=2, default=str)
def _link_or_copy(source: Path, target: Path) -> str:
target.parent.mkdir(parents=True, exist_ok=True)
if source.resolve() == target.resolve():
return "same_path"
if target.exists():
target.unlink()
try:
os.link(source, target)
return "hardlink"
except OSError:
shutil.copy2(source, target)
return "copy"
def _choose_gf3_l2_tif(l2_dir: str, polarization: str | None = None) -> Path:
root = Path(os.path.normpath(str(l2_dir or "").strip()))
if root.is_file():
return root
if not root.is_dir():
raise FileNotFoundError(f"GF3 L2 directory does not exist: {l2_dir}")
candidates = sorted(
path
for path in root.rglob("*")
if path.is_file()
and path.suffix.lower() in {".tif", ".tiff"}
and "L2" in path.name.upper()
)
if not candidates:
raise FileNotFoundError(f"No GF3 L2 GeoTIFF found in: {l2_dir}")
requested = str(polarization or "").strip().upper()
if requested:
for path in candidates:
if requested in path.name.upper():
return path
for pol in _POLARIZATION_PRIORITY:
for path in candidates:
if pol in path.name.upper():
return path
return candidates[0]
def _infer_polarization_from_path(path: Path) -> str | None:
upper_name = path.name.upper()
for pol in _POLARIZATION_PRIORITY:
if pol in upper_name:
return pol
return None
def _raster_quality(path: Path) -> dict[str, Any]:
try:
import numpy as np
import rasterio
except Exception as exc:
return {"ok": False, "warning": f"rasterio unavailable: {exc}"}
with rasterio.open(path) as src:
if src.height > 2048 or src.width > 2048:
scale = min(1024 / src.width, 1024 / src.height)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
sampled = src.read(1, out_shape=(out_height, out_width), masked=True)
else:
sampled = src.read(1, masked=True)
valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)]
bounds = src.bounds
transform = src.transform
quality: dict[str, Any] = {
"ok": True,
"driver": src.driver,
"width": src.width,
"height": src.height,
"count": src.count,
"dtype": str(src.dtypes[0]) if src.dtypes else None,
"crs": src.crs.to_string() if src.crs else None,
"bounds": {
"left": bounds.left,
"bottom": bounds.bottom,
"right": bounds.right,
"top": bounds.top,
},
"transform": list(transform)[:6],
"nodata": src.nodata,
"valid_sample_count": int(valid.size),
"valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0,
}
if valid.size:
quality.update(
{
"sample_min": float(np.nanmin(valid)),
"sample_max": float(np.nanmax(valid)),
"sample_mean": float(np.nanmean(valid)),
"sample_p02": float(np.nanpercentile(valid, 2)),
"sample_p98": float(np.nanpercentile(valid, 98)),
}
)
return quality
def _pixel_size_m_from_quality(quality: dict[str, Any]) -> float | None:
try:
transform = quality.get("transform") or []
xres = abs(float(transform[0]))
yres = abs(float(transform[4]))
crs = str(quality.get("crs") or "").upper()
if not xres or not yres:
return None
if crs and "4326" not in crs:
return round((xres + yres) / 2.0, 3)
bounds = quality.get("bounds") or {}
lat = (float(bounds.get("bottom", 0.0)) + float(bounds.get("top", 0.0))) / 2.0
meters_per_degree_lon = 111320.0 * max(0.01, math.cos(math.radians(lat)))
x_m = xres * meters_per_degree_lon
y_m = yres * 110540.0
return round((x_m + y_m) / 2.0, 3)
except Exception:
return None
def _build_preview_png(source: Path, target: Path) -> str | None:
try:
import numpy as np
import rasterio
from PIL import Image
except Exception:
return None
target.parent.mkdir(parents=True, exist_ok=True)
with rasterio.open(source) as src:
if src.height > 1600 or src.width > 1600:
scale = min(1600 / src.width, 1600 / src.height)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
band = src.read(1, out_shape=(out_height, out_width), masked=True)
else:
band = src.read(1, masked=True)
data = band.filled(np.nan).astype("float32")
valid = data[np.isfinite(data)]
if valid.size:
p2, p98 = np.nanpercentile(valid, [2, 98])
normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1)
gray = (normalized * 255).astype("uint8")
else:
gray = np.zeros(data.shape, dtype="uint8")
alpha = np.where(np.isfinite(data), 255, 0).astype("uint8")
rgba = np.stack([gray, gray, gray, alpha], axis=-1)
Image.fromarray(rgba, "RGBA").save(target)
return str(target)
async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM:
result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id))
scene = result.scalar_one_or_none()
if scene:
return scene
scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING")
db.add(scene)
await db.flush()
return scene
async def register_analysis_ready_tif(
*,
db: AsyncSession,
scene: SARSceneGeoORM,
radar: RadarDataORM | None,
source_tif_path: str,
engine: str,
profile: str,
backscatter_unit: str,
polarization: str | None = None,
metadata: dict[str, Any] | None = None,
copy_mode: str = "link_or_copy",
) -> dict[str, Any]:
source = Path(os.path.normpath(str(source_tif_path or "").strip()))
if not source.is_file():
raise FileNotFoundError(f"Analysis-ready source GeoTIFF does not exist: {source}")
out_dir = scene_analysis_dir(
radar=radar,
scene=scene,
engine=engine,
profile=profile,
polarization=polarization,
)
target_tif = out_dir / "analysis_ready.tif"
transfer = "none"
if copy_mode == "reference":
target_tif = source
else:
transfer = _link_or_copy(source, target_tif)
quality = _raster_quality(target_tif)
preview_path = _build_preview_png(target_tif, out_dir / "preview.png")
manifest = {
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"source_tif_path": str(source),
"analysis_tif_path": str(target_tif),
"analysis_dir": str(out_dir),
"analysis_preview_path": preview_path,
"engine": engine,
"profile": profile,
"backscatter_unit": backscatter_unit,
"polarization": polarization,
"transfer": transfer,
"metadata": metadata or {},
"quality": quality,
}
_write_json(out_dir / "manifest.json", manifest)
_write_json(out_dir / "quality.json", quality)
scene.geo_path = str(target_tif)
scene.analysis_tif_path = str(target_tif)
scene.analysis_dir = str(out_dir)
scene.analysis_preview_path = preview_path
scene.analysis_engine = engine
scene.analysis_profile = profile
scene.analysis_backscatter_unit = backscatter_unit
nodata_value = quality.get("nodata")
scene.analysis_nodata_value = (
float(nodata_value)
if nodata_value is not None
else float(settings.SAR_ANALYSIS_NODATA_VALUE)
)
scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")}
scene.analysis_quality_json = quality
scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m
scene.status = "DONE"
scene.error_msg = None
return manifest
async def standardize_gf3_l2_for_radar(
*,
db: AsyncSession,
radar_id: int,
l2_path: str | None = None,
polarization: str | None = None,
) -> dict[str, Any]:
radar = await db.get(RadarDataORM, int(radar_id))
if not radar:
raise ValueError(f"RadarDataORM id={radar_id} does not exist")
scene = await _get_or_create_scene(db, int(radar_id))
source_root = l2_path or radar.file_path
selected_tif = _choose_gf3_l2_tif(source_root, polarization=polarization or radar.polarization)
selected_pol = polarization or _infer_polarization_from_path(selected_tif)
manifest = await register_analysis_ready_tif(
db=db,
scene=scene,
radar=radar,
source_tif_path=str(selected_tif),
engine="gf3_gdal",
profile="gf3_l1a_l2_rpc",
backscatter_unit="sigma0_db",
polarization=selected_pol,
metadata={
"source": "GF3 L2",
"source_l2_path": str(selected_tif),
"source_l2_dir": str(Path(source_root).resolve()) if source_root else None,
"available_polarization": radar.polarization,
},
)
await db.commit()
return manifest
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,33 @@
"""Water extraction processors for the flood-analysis pipeline."""
from __future__ import annotations
from typing import Any
def run_otsu_water_extraction(
*,
input_path: str,
output_dir: str,
job_id: str | None = None,
) -> dict[str, Any]:
"""Run the fast Otsu water-extraction processor.
This wraps the legacy implementation while exposing the terminology used by
the flood pipeline: extraction, processor and threshold_value.
"""
from .water_detect_service import run_water_detection
result = run_water_detection(
geo_tiff_path=input_path,
output_dir=output_dir,
job_id=job_id,
)
result["processor"] = "otsu"
if "threshold_value" not in result and "otsu_threshold_db" in result:
result["threshold_value"] = result.get("otsu_threshold_db")
return result
def run_envi_water_extraction(*args: Any, **kwargs: Any) -> dict[str, Any]:
"""Placeholder for the future ENVI/SARscape precise extractor."""
raise NotImplementedError("ENVI/SARscape water extraction is not wired yet")