1510 lines
63 KiB
Python
1510 lines
63 KiB
Python
"""
|
||
数据导入服务
|
||
|
||
提供数据扫描和导入功能：
|
||
- 扫描雷达数据目录
|
||
- 扫描 D-InSAR 结果目录
|
||
- 扫描灾害点数据
|
||
|
||
优化策略：
|
||
- 使用 geopandas/pyogrio 读取 Shapefile
|
||
- 利用 PostGIS 空间索引
|
||
- 支持增量扫描
|
||
"""
|
||
import os
|
||
import json
|
||
import asyncio
|
||
import hashlib
|
||
import re
|
||
from typing import List, Dict, Optional, Tuple, Any
|
||
from datetime import datetime
|
||
from shapely.geometry import Point, Polygon, shape, mapping
|
||
from shapely.ops import unary_union
|
||
import geopandas as gpd
|
||
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from sqlalchemy.future import select
|
||
from sqlalchemy import func, and_, text, update, delete
|
||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||
from geoalchemy2.shape import from_shape
|
||
from geoalchemy2.functions import ST_Intersects, ST_Intersection, ST_Area
|
||
|
||
from ..config import settings
|
||
from ..models import (
|
||
RadarDataORM, RadarData, DinsarResultORM, HazardPointORM, HazardPoint, ScanStateORM
|
||
)
|
||
from ..utils import (
|
||
get_parser,
|
||
RADAR_PARSERS,
|
||
find_xml_file,
|
||
normalize_satellite_family,
|
||
parse_xml_metadata,
|
||
)
|
||
from .task_service import task_service
|
||
from .image_service import image_service
|
||
from .orbit_converter import get_source_orbit_inventory, sync_orbit_pools
|
||
from .pairing_state_service import pairing_state_service
|
||
|
||
|
||
def _safe_mtime(path: str) -> float:
|
||
try:
|
||
return max(os.path.getmtime(path), os.path.getctime(path))
|
||
except OSError:
|
||
return 0.0
|
||
|
||
|
||
_DATE_RE = re.compile(r"^\d{8}$")
|
||
|
||
|
||
def _valid_imaging_date(value: Optional[str]) -> bool:
|
||
return bool(value and _DATE_RE.match(value))
|
||
|
||
|
||
def _extract_date_from_text(value: Optional[str]) -> Optional[str]:
|
||
if not value:
|
||
return None
|
||
digits = re.sub(r"\D", "", value)
|
||
if len(digits) >= 8:
|
||
return digits[:8]
|
||
return None
|
||
|
||
|
||
def _text_or_none(value: Any) -> Optional[str]:
|
||
if value is None:
|
||
return None
|
||
text_value = str(value).strip()
|
||
return text_value or None
|
||
|
||
|
||
def _bool_or_none(value: Any) -> Optional[bool]:
|
||
if value is None or isinstance(value, bool):
|
||
return value
|
||
text_value = str(value).strip().lower()
|
||
if text_value in {"1", "true", "yes", "y"}:
|
||
return True
|
||
if text_value in {"0", "false", "no", "n"}:
|
||
return False
|
||
return None
|
||
|
||
|
||
def _build_insar_source_readiness(
|
||
meta: Dict[str, Any],
|
||
coverage_polygon: Optional[List[Tuple[float, float]]],
|
||
) -> Tuple[bool, Optional[str]]:
|
||
reasons: List[str] = []
|
||
if not coverage_polygon or len(coverage_polygon) < 3:
|
||
reasons.append("missing_footprint")
|
||
if not _valid_imaging_date(_text_or_none(meta.get("imaging_date"))):
|
||
reasons.append("missing_date")
|
||
for field_name, reason in (
|
||
("orbit_direction", "missing_orbit_direction"),
|
||
("imaging_mode", "missing_imaging_mode"),
|
||
("polarization", "missing_polarization"),
|
||
("satellite_family", "missing_satellite_family"),
|
||
):
|
||
if not _text_or_none(meta.get(field_name)):
|
||
reasons.append(reason)
|
||
|
||
geocoded_flag = _bool_or_none(meta.get("geocoded_flag"))
|
||
if geocoded_flag is True:
|
||
reasons.append("geocoded_product")
|
||
|
||
complex_tokens = {
|
||
_text_or_none(meta.get("image_data_type")),
|
||
_text_or_none(meta.get("product_type")),
|
||
_text_or_none(meta.get("source_product_token")),
|
||
_text_or_none(meta.get("product_variant")),
|
||
}
|
||
normalized_tokens = {str(token).strip().upper() for token in complex_tokens if token}
|
||
is_complex_source = bool(normalized_tokens.intersection({"COMPLEX", "SLC", "SSC"}))
|
||
if not is_complex_source:
|
||
reasons.append("not_complex_source")
|
||
|
||
if reasons:
|
||
return False, ";".join(reasons)
|
||
return True, None
|
||
|
||
|
||
def _iter_dirs(root: str, last_mtime: float):
|
||
stack = [root]
|
||
while stack:
|
||
current = stack.pop()
|
||
try:
|
||
with os.scandir(current) as it:
|
||
for entry in it:
|
||
try:
|
||
if entry.is_dir(follow_symlinks=False):
|
||
try:
|
||
mtime = entry.stat().st_mtime
|
||
except OSError:
|
||
mtime = 0.0
|
||
if last_mtime and mtime <= last_mtime:
|
||
continue
|
||
stack.append(entry.path)
|
||
yield entry
|
||
except OSError:
|
||
continue
|
||
except OSError:
|
||
continue
|
||
|
||
|
||
def _iter_files(root: str, suffix: str, last_mtime: float):
|
||
stack = [root]
|
||
suffix = suffix.lower()
|
||
while stack:
|
||
current = stack.pop()
|
||
try:
|
||
with os.scandir(current) as it:
|
||
for entry in it:
|
||
try:
|
||
if entry.is_dir(follow_symlinks=False):
|
||
try:
|
||
mtime = entry.stat().st_mtime
|
||
except OSError:
|
||
mtime = 0.0
|
||
if last_mtime and mtime <= last_mtime:
|
||
continue
|
||
stack.append(entry.path)
|
||
elif entry.is_file(follow_symlinks=False):
|
||
if entry.name.lower().endswith(suffix):
|
||
yield entry
|
||
except OSError:
|
||
continue
|
||
except OSError:
|
||
continue
|
||
|
||
|
||
_DINSAR_ISCE2_DISP_RE = re.compile(r"^.+_disp\.(?:tif|tiff)$", re.IGNORECASE)
|
||
_DINSAR_ISCE2_SKIP_RE = re.compile(r"^.+_disp_full\.(?:tif|tiff)$", re.IGNORECASE)
|
||
|
||
|
||
def _iter_dinsar_result_candidates(root: str):
|
||
stack = [root]
|
||
while stack:
|
||
current = stack.pop()
|
||
try:
|
||
with os.scandir(current) as it:
|
||
for entry in it:
|
||
try:
|
||
if entry.is_dir(follow_symlinks=False):
|
||
stack.append(entry.path)
|
||
continue
|
||
if not entry.is_file(follow_symlinks=False):
|
||
continue
|
||
|
||
lower_name = entry.name.lower()
|
||
if lower_name.endswith(".hdr"):
|
||
base_name, _ = os.path.splitext(entry.name)
|
||
if not base_name.lower().endswith("_disp"):
|
||
continue
|
||
data_file_path = os.path.join(os.path.dirname(entry.path), base_name)
|
||
if not os.path.exists(data_file_path):
|
||
continue
|
||
source_type = "envi"
|
||
record_name = base_name
|
||
elif lower_name.endswith((".tif", ".tiff")):
|
||
if _DINSAR_ISCE2_SKIP_RE.match(entry.name):
|
||
continue
|
||
if not _DINSAR_ISCE2_DISP_RE.match(entry.name):
|
||
continue
|
||
data_file_path = entry.path
|
||
record_name, _ = os.path.splitext(entry.name)
|
||
source_type = "isce2"
|
||
else:
|
||
continue
|
||
|
||
try:
|
||
stat = entry.stat()
|
||
scan_ts = max(stat.st_mtime, stat.st_ctime)
|
||
except OSError:
|
||
scan_ts = 0.0
|
||
|
||
yield {
|
||
"name": record_name,
|
||
"file_path": data_file_path,
|
||
"scan_path": entry.path,
|
||
"scan_ts": scan_ts,
|
||
"source_type": source_type,
|
||
}
|
||
except OSError:
|
||
continue
|
||
except OSError:
|
||
continue
|
||
|
||
|
||
def _chunked(items: List[Any], size: int):
|
||
for i in range(0, len(items), size):
|
||
yield items[i:i + size]
|
||
|
||
|
||
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
|
||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview")
|
||
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
||
|
||
|
||
def _sanitize_cache_name(name: str) -> str:
|
||
cleaned = _RADAR_CACHE_NAME_RE.sub("_", name or "")
|
||
cleaned = cleaned.strip("._")
|
||
return cleaned[:48] or "scene"
|
||
|
||
|
||
def _build_cache_digest(unique_id: str, file_path: str) -> str:
|
||
return hashlib.sha1((unique_id or file_path).encode("utf-8", errors="ignore")).hexdigest()[:16]
|
||
|
||
|
||
def _build_radar_cache_filename(unique_id: str, file_path: str) -> str:
|
||
digest = _build_cache_digest(unique_id, file_path)
|
||
base = _sanitize_cache_name(os.path.basename(file_path))
|
||
return f"RID_{digest}_{base}.webp"
|
||
|
||
|
||
def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_version: str) -> str:
|
||
digest = _build_cache_digest(unique_id, file_path)
|
||
base = _sanitize_cache_name(os.path.basename(file_path))
|
||
safe_version = _sanitize_cache_name(cache_version or "v")
|
||
return f"RGID_{safe_version}_{digest}_{base}.webp"
|
||
|
||
|
||
def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]:
|
||
"""Extract coverage polygon from a GeoTIFF file using GDAL.
|
||
|
||
Returns a list of (lon, lat) tuples forming a closed polygon (5 points),
|
||
or None if the file cannot be read or has no valid geotransform.
|
||
"""
|
||
try:
|
||
from osgeo import gdal, osr
|
||
ds = gdal.Open(tiff_path, gdal.GA_ReadOnly)
|
||
if ds is None:
|
||
return None
|
||
gt = ds.GetGeoTransform()
|
||
if gt is None or gt == (0, 1, 0, 0, 0, 1):
|
||
ds = None
|
||
return None
|
||
w, h = ds.RasterXSize, ds.RasterYSize
|
||
|
||
# Compute 4 corners in the raster's CRS
|
||
corners_xy = [
|
||
(gt[0], gt[3]), # top-left
|
||
(gt[0] + w * gt[1], gt[3] + w * gt[4]), # top-right
|
||
(gt[0] + w * gt[1] + h * gt[2], gt[3] + w * gt[4] + h * gt[5]), # bottom-right
|
||
(gt[0] + h * gt[2], gt[3] + h * gt[5]), # bottom-left
|
||
]
|
||
|
||
# Transform to WGS84 if necessary
|
||
srs = osr.SpatialReference()
|
||
srs.ImportFromWkt(ds.GetProjection())
|
||
ds = None
|
||
|
||
wgs84 = osr.SpatialReference()
|
||
wgs84.ImportFromEPSG(4326)
|
||
|
||
if srs.IsSame(wgs84):
|
||
polygon = [(x, y) for x, y in corners_xy]
|
||
else:
|
||
srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
|
||
wgs84.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
|
||
transform = osr.CoordinateTransformation(srs, wgs84)
|
||
polygon = []
|
||
for x, y in corners_xy:
|
||
lon, lat, _ = transform.TransformPoint(x, y)
|
||
polygon.append((lon, lat))
|
||
|
||
polygon.append(polygon[0]) # close the ring
|
||
return polygon
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
class DataService:
|
||
@staticmethod
|
||
async def _get_scan_state(db: AsyncSession, data_type: str, root_path: str):
|
||
result = await db.execute(
|
||
select(ScanStateORM).where(
|
||
ScanStateORM.data_type == data_type,
|
||
ScanStateORM.root_path == root_path
|
||
)
|
||
)
|
||
return result.scalar_one_or_none()
|
||
|
||
@staticmethod
|
||
async def _upsert_scan_state(db: AsyncSession, data_type: str, root_path: str, last_scan_mtime: float):
|
||
stmt = pg_insert(ScanStateORM).values({
|
||
"data_type": data_type,
|
||
"root_path": root_path,
|
||
"last_scan_mtime": last_scan_mtime,
|
||
})
|
||
stmt = stmt.on_conflict_do_update(
|
||
index_elements=['data_type', 'root_path'],
|
||
set_={
|
||
"last_scan_mtime": last_scan_mtime,
|
||
"last_scan_at": datetime.now(),
|
||
}
|
||
)
|
||
await db.execute(stmt)
|
||
|
||
@staticmethod
|
||
def get_radar_raw_cache_path(unique_id: str, file_path: str) -> str:
|
||
filename = _build_radar_cache_filename(unique_id, file_path)
|
||
return os.path.join(settings.RADAR_RAW_CACHE_DIR, filename)
|
||
|
||
@staticmethod
|
||
def get_radar_geo_cache_path(unique_id: str, file_path: str) -> str:
|
||
filename = _build_radar_geo_cache_filename(
|
||
unique_id=unique_id,
|
||
file_path=file_path,
|
||
cache_version=settings.RADAR_GEO_CACHE_VERSION,
|
||
)
|
||
return os.path.join(settings.RADAR_GEO_CACHE_DIR, filename)
|
||
|
||
@staticmethod
|
||
def get_radar_cache_path(unique_id: str, file_path: str) -> str:
|
||
return DataService.get_radar_raw_cache_path(unique_id, file_path)
|
||
|
||
@staticmethod
|
||
def get_dinsar_cache_filename(record_id: int, record_name: str) -> str:
|
||
safe_name = _sanitize_cache_name(record_name)
|
||
return f"ID_{record_id}_{safe_name}.webp"
|
||
|
||
@staticmethod
|
||
def get_dinsar_cache_path(record_id: int, record_name: str) -> str:
|
||
filename = DataService.get_dinsar_cache_filename(record_id, record_name)
|
||
cache_dir = os.path.realpath(settings.DINSAR_CACHE_DIR)
|
||
target = os.path.realpath(os.path.join(cache_dir, filename))
|
||
if not target.startswith(cache_dir + os.sep) and target != cache_dir:
|
||
raise ValueError(f"Path traversal detected: {record_name!r}")
|
||
return target
|
||
|
||
@staticmethod
|
||
def get_dinsar_manifest_path() -> str:
|
||
return os.path.join(settings.DINSAR_CACHE_DIR, "cache_manifest.json")
|
||
|
||
@staticmethod
|
||
def find_radar_preview_source(scene_dir: str) -> Optional[str]:
|
||
if not scene_dir or not os.path.isdir(scene_dir):
|
||
return None
|
||
|
||
candidates: List[Tuple[int, int, int, int, str]] = []
|
||
for root, _, files in os.walk(scene_dir):
|
||
rel = os.path.relpath(root, scene_dir)
|
||
depth = 0 if rel == "." else rel.count(os.sep) + 1
|
||
if depth > 5:
|
||
continue
|
||
|
||
for name in files:
|
||
lower_name = name.lower()
|
||
if not lower_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
|
||
continue
|
||
|
||
path = os.path.join(root, name)
|
||
keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1
|
||
ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1
|
||
try:
|
||
size_score = -os.path.getsize(path)
|
||
except OSError:
|
||
size_score = 0
|
||
candidates.append((keyword_score, depth, ext_score, size_score, path))
|
||
|
||
if not candidates:
|
||
return None
|
||
|
||
candidates.sort()
|
||
return candidates[0][-1]
|
||
|
||
@staticmethod
|
||
def get_radar_source_corner_mapping(scene_dir: str) -> Optional[Dict[str, Any]]:
|
||
if not scene_dir or not os.path.isdir(scene_dir):
|
||
return None
|
||
xml_file_path = find_xml_file(scene_dir)
|
||
if not xml_file_path:
|
||
return None
|
||
_, xml_meta = parse_xml_metadata(xml_file_path)
|
||
if not xml_meta:
|
||
return None
|
||
corner_mapping = xml_meta.get("corner_pixel_mapping")
|
||
if isinstance(corner_mapping, dict):
|
||
return corner_mapping
|
||
return None
|
||
|
||
@staticmethod
|
||
def _normalize_coverage_polygon(coverage_polygon: Any) -> Optional[List[Tuple[float, float]]]:
|
||
if isinstance(coverage_polygon, list):
|
||
points: List[Tuple[float, float]] = []
|
||
for point in coverage_polygon:
|
||
if isinstance(point, (list, tuple)) and len(point) >= 2:
|
||
points.append((float(point[0]), float(point[1])))
|
||
return points or None
|
||
if isinstance(coverage_polygon, dict):
|
||
coordinates = coverage_polygon.get("coordinates")
|
||
if isinstance(coordinates, list) and coordinates:
|
||
ring = coordinates[0]
|
||
if isinstance(ring, list):
|
||
points = []
|
||
for point in ring:
|
||
if isinstance(point, (list, tuple)) and len(point) >= 2:
|
||
points.append((float(point[0]), float(point[1])))
|
||
return points or None
|
||
return None
|
||
|
||
|
||
"""
|
||
数据导入服务
|
||
|
||
负责扫描和导入各类数据到数据库。
|
||
"""
|
||
|
||
@staticmethod
|
||
async def scan_radar_data(
|
||
db: AsyncSession,
|
||
radar_dirs: List[str],
|
||
orbit_dir: Optional[str] = None,
|
||
task_id: Optional[str] = None,
|
||
progress_base: int = 0,
|
||
progress_span: int = 100
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
异步扫描雷达和轨道数据目录，解析文件，并将元数据存入数据库。
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
radar_dirs: 雷达数据目录列表
|
||
orbit_dir: 精轨数据目录
|
||
task_id: 任务 ID（用于进度更新）
|
||
|
||
Returns:
|
||
扫描结果字典
|
||
"""
|
||
def update_progress(msg: str, prog: int):
|
||
scaled = prog
|
||
if task_id:
|
||
scaled = progress_base + int((prog / 100) * progress_span)
|
||
scaled = min(progress_base + progress_span, max(progress_base, scaled))
|
||
asyncio.create_task(task_service.update_task(task_id, message=msg, progress=scaled))
|
||
print(f" [扫描进度] {scaled}% - {msg}")
|
||
|
||
os.makedirs(settings.RADAR_RAW_CACHE_DIR, exist_ok=True)
|
||
os.makedirs(settings.RADAR_GEO_CACHE_DIR, exist_ok=True)
|
||
|
||
update_progress("正在扫描精轨数据...", 5)
|
||
|
||
# 1. Pre-process all orbit data
|
||
orbit_files_map: Dict[Tuple[str, str], str] = {}
|
||
orbit_inventory = {"errors": [], "duplicate_count": 0, "files": {}}
|
||
invalid_orbit_stems = set()
|
||
if orbit_dir:
|
||
orbit_inventory = await asyncio.to_thread(
|
||
get_source_orbit_inventory,
|
||
orbit_dir,
|
||
True,
|
||
)
|
||
for item in orbit_inventory.get("files", {}).values():
|
||
orbit_files_map[(item["satellite"], item["date"])] = item["path"]
|
||
for err in orbit_inventory.get("errors", []):
|
||
print(f"警告: 精轨目录扫描异常: {err}")
|
||
if orbit_inventory.get("duplicate_count", 0):
|
||
print(f"警告: 精轨目录存在 {orbit_inventory['duplicate_count']} 个重复 stem，已按最新文件优先。")
|
||
else:
|
||
print("未配置精轨数据目录，跳过精轨扫描。")
|
||
|
||
# 1b. 同步精轨到本地引擎池（ENVI / ISCE2）
|
||
if orbit_dir and orbit_files_map:
|
||
update_progress("正在同步精轨到本地引擎池...", 8)
|
||
try:
|
||
sync_result = await asyncio.to_thread(
|
||
sync_orbit_pools,
|
||
orbit_dir,
|
||
settings.ORBIT_POOL_ENVI,
|
||
settings.ORBIT_POOL_ISCE2,
|
||
settings.ORBIT_POOL_LANDSAR,
|
||
)
|
||
envi_new = len(sync_result.get("envi", {}).get("copied", []))
|
||
envi_updated = len(sync_result.get("envi", {}).get("updated", []))
|
||
isce2_new = len(sync_result.get("isce2", {}).get("converted", []))
|
||
isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", []))
|
||
print(
|
||
f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated}，"
|
||
f"ISCE2 转换 {isce2_new}、重转 {isce2_updated}"
|
||
)
|
||
invalid_orbit_stems = {
|
||
item.get("name")
|
||
for item in sync_result.get("invalid_sources", [])
|
||
if item.get("name")
|
||
}
|
||
if invalid_orbit_stems:
|
||
print(f" [精轨同步] 发现 {len(invalid_orbit_stems)} 个坏源精轨，已禁止入池/入库。")
|
||
orbit_files_map = {
|
||
key: value
|
||
for key, value in orbit_files_map.items()
|
||
if f"{key[0]}_GpsData_GAS_C_{key[1]}" not in invalid_orbit_stems
|
||
}
|
||
sync_errors = (
|
||
sync_result.get("source", {}).get("errors", [])
|
||
+ [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])]
|
||
+ [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
|
||
)
|
||
if task_id and sync_errors:
|
||
await task_service.add_log(
|
||
task_id,
|
||
"WARN",
|
||
(
|
||
"精轨同步存在异常: "
|
||
f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, "
|
||
f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, "
|
||
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))} 项"
|
||
),
|
||
)
|
||
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
|
||
await task_service.add_log(
|
||
task_id,
|
||
"WARN",
|
||
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
|
||
)
|
||
if task_id and invalid_orbit_stems:
|
||
await task_service.add_log(
|
||
task_id,
|
||
"WARN",
|
||
f"发现 {len(invalid_orbit_stems)} 个坏源精轨，已跳过入池与轨道关联。",
|
||
)
|
||
for item in sync_result.get("invalid_sources", [])[:20]:
|
||
await task_service.add_log(
|
||
task_id,
|
||
"WARN",
|
||
(
|
||
f"坏源精轨: {item.get('name')} -> {item.get('error')}"
|
||
f" (NUL={item.get('nul_byte_count', 0)}, tail_nul={item.get('tail_has_nul_bytes', False)})"
|
||
),
|
||
)
|
||
except Exception as _sync_exc:
|
||
print(f" [精轨同步] 警告：同步失败（不影响入库）：{_sync_exc}")
|
||
elif task_id and orbit_inventory.get("errors"):
|
||
await task_service.add_log(
|
||
task_id,
|
||
"WARN",
|
||
f"精轨目录扫描异常: {orbit_inventory['errors'][0]}",
|
||
)
|
||
|
||
# 2. Process all radar data folders from multiple directories
|
||
update_progress("正在扫描雷达数据...", 10)
|
||
processed_scenes = 0
|
||
|
||
# 预先计算公共根目录
|
||
common_base = None
|
||
if radar_dirs:
|
||
try:
|
||
common_base = os.path.commonpath([os.path.abspath(d) for d in radar_dirs])
|
||
except ValueError:
|
||
common_base = None
|
||
|
||
total_progress = 80
|
||
scan_state_updates = []
|
||
radar_cache_candidates: Dict[str, Dict[str, str]] = {}
|
||
for idx, radar_dir in enumerate(radar_dirs):
|
||
radar_dir_abs = os.path.abspath(radar_dir)
|
||
state = await DataService._get_scan_state(db, "radar", radar_dir_abs)
|
||
last_mtime = state.last_scan_mtime if state else 0.0
|
||
max_mtime = last_mtime
|
||
if not os.path.exists(radar_dir):
|
||
print(f"警告: 雷达数据目录不存在，已跳过: {radar_dir}")
|
||
continue
|
||
root_mtime = _safe_mtime(radar_dir_abs)
|
||
if last_mtime and root_mtime <= last_mtime:
|
||
scan_state_updates.append(("radar", radar_dir_abs, last_mtime))
|
||
continue
|
||
|
||
for entry in _iter_dirs(radar_dir, last_mtime):
|
||
folder_name = entry.name
|
||
parsed_radar = get_parser(folder_name, RADAR_PARSERS)
|
||
if not parsed_radar:
|
||
continue
|
||
|
||
radar_folder_path = entry.path
|
||
try:
|
||
stat = entry.stat()
|
||
scan_ts = max(stat.st_mtime, stat.st_ctime)
|
||
except OSError:
|
||
scan_ts = 0
|
||
if scan_ts <= last_mtime:
|
||
continue
|
||
if scan_ts > max_mtime:
|
||
max_mtime = scan_ts
|
||
print(f" -> 正在处理: {folder_name}")
|
||
|
||
xml_file_path = find_xml_file(radar_folder_path)
|
||
if not xml_file_path:
|
||
continue
|
||
|
||
coverage_polygon, xml_meta = parse_xml_metadata(xml_file_path)
|
||
|
||
if not coverage_polygon or len(coverage_polygon) < 3:
|
||
continue
|
||
|
||
name_meta = parsed_radar or {}
|
||
merged_meta = dict(name_meta)
|
||
if xml_meta:
|
||
prefer_name_keys = {"product_unique_id"}
|
||
for key, value in xml_meta.items():
|
||
if value in (None, ""):
|
||
continue
|
||
if key in prefer_name_keys and merged_meta.get(key):
|
||
continue
|
||
merged_meta[key] = value
|
||
|
||
satellite = _text_or_none(merged_meta.get("satellite"))
|
||
imaging_date = _text_or_none(merged_meta.get("imaging_date"))
|
||
imaging_mode = _text_or_none(merged_meta.get("imaging_mode"))
|
||
polarization = _text_or_none(merged_meta.get("polarization"))
|
||
orbit_direction = _text_or_none(merged_meta.get("orbit_direction"))
|
||
satellite_mode = _text_or_none(merged_meta.get("satellite_mode"))
|
||
receiving_station = _text_or_none(merged_meta.get("receiving_station"))
|
||
orbit_circle = _text_or_none(merged_meta.get("orbit_circle"))
|
||
scene_center_lon = merged_meta.get("scene_center_lon")
|
||
scene_center_lat = merged_meta.get("scene_center_lat")
|
||
acquisition_time_utc = _text_or_none(merged_meta.get("acquisition_time_utc"))
|
||
product_type = _text_or_none(merged_meta.get("product_type"))
|
||
source_product_token = _text_or_none(merged_meta.get("source_product_token"))
|
||
image_data_type = _text_or_none(merged_meta.get("image_data_type"))
|
||
image_data_format = _text_or_none(merged_meta.get("image_data_format"))
|
||
product_variant = _text_or_none(merged_meta.get("product_variant"))
|
||
product_level = _text_or_none(merged_meta.get("product_level"))
|
||
product_unique_id = _text_or_none(merged_meta.get("product_unique_id"))
|
||
satellite_family = normalize_satellite_family(
|
||
_text_or_none(merged_meta.get("satellite_family")) or satellite
|
||
)
|
||
look_direction = _text_or_none(merged_meta.get("look_direction"))
|
||
if look_direction:
|
||
look_direction = look_direction.upper()
|
||
geocoded_flag = _bool_or_none(merged_meta.get("geocoded_flag"))
|
||
|
||
if not satellite:
|
||
continue
|
||
if not imaging_date and acquisition_time_utc:
|
||
imaging_date = _extract_date_from_text(acquisition_time_utc)
|
||
|
||
if not _valid_imaging_date(imaging_date):
|
||
print(f"警告: 影像日期格式无效，已跳过: {imaging_date} ({radar_folder_path})")
|
||
continue
|
||
has_orbit_data = (satellite, imaging_date) in orbit_files_map
|
||
orbit_file_path = orbit_files_map.get((satellite, imaging_date))
|
||
|
||
envi_import_path = os.path.join(radar_folder_path, 'envi_import')
|
||
is_envi_processed = os.path.isdir(envi_import_path) and bool(os.listdir(envi_import_path))
|
||
|
||
lons = [p[0] for p in coverage_polygon]
|
||
lats = [p[1] for p in coverage_polygon]
|
||
|
||
# 生成唯一 ID
|
||
if common_base:
|
||
try:
|
||
unique_id = os.path.relpath(radar_folder_path, start=common_base)
|
||
except ValueError:
|
||
unique_id = radar_folder_path
|
||
else:
|
||
unique_id = radar_folder_path
|
||
|
||
# Data to be inserted or updated
|
||
poly = Polygon(coverage_polygon)
|
||
if not poly.is_valid:
|
||
poly = poly.buffer(0)
|
||
if (scene_center_lon is None or scene_center_lat is None) and poly.is_valid:
|
||
scene_center_lon = scene_center_lon if scene_center_lon is not None else poly.centroid.x
|
||
scene_center_lat = scene_center_lat if scene_center_lat is not None else poly.centroid.y
|
||
|
||
readiness_meta = {
|
||
"satellite_family": satellite_family,
|
||
"imaging_date": imaging_date,
|
||
"imaging_mode": imaging_mode,
|
||
"orbit_direction": orbit_direction,
|
||
"polarization": polarization,
|
||
"product_type": product_type,
|
||
"source_product_token": source_product_token,
|
||
"image_data_type": image_data_type,
|
||
"product_variant": product_variant,
|
||
"geocoded_flag": geocoded_flag,
|
||
}
|
||
insar_source_ready, insar_source_reason = _build_insar_source_readiness(
|
||
readiness_meta,
|
||
coverage_polygon,
|
||
)
|
||
|
||
data_to_upsert = {
|
||
"unique_id": unique_id,
|
||
"satellite": satellite,
|
||
"satellite_family": satellite_family,
|
||
"imaging_date": imaging_date,
|
||
"imaging_mode": imaging_mode,
|
||
"orbit_direction": orbit_direction,
|
||
"polarization": polarization,
|
||
"satellite_mode": satellite_mode,
|
||
"receiving_station": receiving_station,
|
||
"orbit_circle": orbit_circle,
|
||
"scene_center_lon": scene_center_lon,
|
||
"scene_center_lat": scene_center_lat,
|
||
"acquisition_time_utc": acquisition_time_utc,
|
||
"product_type": product_type,
|
||
"source_product_token": source_product_token,
|
||
"image_data_type": image_data_type,
|
||
"image_data_format": image_data_format,
|
||
"product_variant": product_variant,
|
||
"product_level": product_level,
|
||
"product_unique_id": product_unique_id,
|
||
"look_direction": look_direction,
|
||
"geocoded_flag": geocoded_flag,
|
||
"insar_source_ready": insar_source_ready,
|
||
"insar_source_reason": insar_source_reason,
|
||
"file_path": radar_folder_path,
|
||
"has_orbit_data": has_orbit_data,
|
||
"orbit_file_path": orbit_file_path,
|
||
"is_envi_processed": is_envi_processed,
|
||
"coverage_polygon": coverage_polygon,
|
||
"geom": from_shape(poly, srid=4326),
|
||
"min_lon": min(lons),
|
||
"min_lat": min(lats),
|
||
"max_lon": max(lons),
|
||
"max_lat": max(lats),
|
||
}
|
||
|
||
stmt = pg_insert(RadarDataORM).values(data_to_upsert)
|
||
update_dict = {c.name: c for c in stmt.excluded if c.name != "unique_id"}
|
||
stmt = stmt.on_conflict_do_update(
|
||
index_elements=['unique_id'],
|
||
set_=update_dict,
|
||
)
|
||
await db.execute(stmt)
|
||
radar_cache_candidates[unique_id] = {
|
||
"unique_id": unique_id,
|
||
"file_path": radar_folder_path,
|
||
}
|
||
processed_scenes += 1
|
||
|
||
scan_state_updates.append(("radar", radar_dir_abs, max_mtime))
|
||
|
||
# 3. 更新缺失精轨的现有记录
|
||
update_progress("正在关联精轨数据...", 85)
|
||
updated_orbits = 0
|
||
updated_orbit_scene_ids = set()
|
||
if orbit_files_map:
|
||
stmt_select = select(RadarDataORM).where(RadarDataORM.has_orbit_data == False)
|
||
result = await db.execute(stmt_select)
|
||
records_to_update = result.scalars().all()
|
||
|
||
for record in records_to_update:
|
||
key = (record.satellite, record.imaging_date)
|
||
if key in orbit_files_map:
|
||
record.has_orbit_data = True
|
||
record.orbit_file_path = orbit_files_map[key]
|
||
db.add(record)
|
||
updated_orbits += 1
|
||
if record.id is not None:
|
||
updated_orbit_scene_ids.add(int(record.id))
|
||
|
||
for data_type, root_path, mtime in scan_state_updates:
|
||
await DataService._upsert_scan_state(db, data_type, root_path, mtime)
|
||
|
||
await db.commit()
|
||
|
||
pairing_dirty_summary: Dict[str, Any] = {}
|
||
dirty_scene_ids = set(updated_orbit_scene_ids)
|
||
processed_unique_ids = [key for key in radar_cache_candidates.keys() if key]
|
||
for chunk in _chunked(processed_unique_ids, 500):
|
||
id_result = await db.execute(
|
||
select(RadarDataORM.id).where(RadarDataORM.unique_id.in_(chunk))
|
||
)
|
||
dirty_scene_ids.update(int(value) for value in id_result.scalars().all() if value is not None)
|
||
|
||
if dirty_scene_ids:
|
||
pairing_dirty_summary = await pairing_state_service.mark_scenes_dirty(
|
||
db,
|
||
scene_ids=sorted(dirty_scene_ids),
|
||
reason="radar_scan",
|
||
commit=True,
|
||
)
|
||
elif processed_scenes > 0 or updated_orbits > 0:
|
||
pairing_dirty_summary = await pairing_state_service.mark_global_dirty(
|
||
db,
|
||
reason="radar_scan",
|
||
commit=True,
|
||
)
|
||
|
||
cached_previews = 0
|
||
skipped_previews = 0
|
||
missing_preview_sources = 0
|
||
failed_preview_cache = 0
|
||
cached_raw_previews = 0
|
||
skipped_raw_previews = 0
|
||
failed_raw_preview_cache = 0
|
||
|
||
radar_roots = [os.path.abspath(path) for path in radar_dirs if path]
|
||
new_candidate_keys = set(radar_cache_candidates.keys())
|
||
candidates: List[Dict[str, Any]] = []
|
||
if radar_roots:
|
||
existing_res = await db.execute(select(RadarDataORM))
|
||
for record in existing_res.scalars().all():
|
||
if not record.file_path:
|
||
continue
|
||
|
||
file_path_abs = os.path.abspath(record.file_path)
|
||
if not any(file_path_abs.startswith(root) for root in radar_roots):
|
||
continue
|
||
|
||
unique_id = record.unique_id or record.file_path
|
||
raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path)
|
||
geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path)
|
||
should_include = unique_id in new_candidate_keys
|
||
if not should_include:
|
||
if not os.path.exists(geo_cache_path):
|
||
should_include = True
|
||
elif (record.preview_cache_version or "") != settings.RADAR_GEO_CACHE_VERSION:
|
||
should_include = True
|
||
elif (record.preview_cache_status or "NONE") != "READY":
|
||
should_include = True
|
||
elif not os.path.exists(raw_cache_path):
|
||
should_include = True
|
||
|
||
if not should_include:
|
||
continue
|
||
|
||
candidates.append(
|
||
{
|
||
"id": record.id,
|
||
"unique_id": unique_id,
|
||
"file_path": record.file_path,
|
||
"coverage_polygon": record.coverage_polygon,
|
||
"min_lon": record.min_lon,
|
||
"min_lat": record.min_lat,
|
||
"max_lon": record.max_lon,
|
||
"max_lat": record.max_lat,
|
||
"preview_cache_version": record.preview_cache_version,
|
||
}
|
||
)
|
||
|
||
total_candidates = len(candidates)
|
||
if total_candidates > 0:
|
||
update_progress("正在生成源影像缓存...", 90)
|
||
workers = max(1, min(8, settings.RADAR_GEO_CACHE_WORKERS))
|
||
thumb_size = (settings.RADAR_THUMBNAIL_MAX_SIZE, settings.RADAR_THUMBNAIL_MAX_SIZE)
|
||
semaphore = asyncio.Semaphore(workers)
|
||
|
||
async def _cache_one(item: Dict[str, Any]) -> Dict[str, Any]:
|
||
async with semaphore:
|
||
item_id = int(item["id"])
|
||
unique_id = str(item.get("unique_id") or "")
|
||
file_path = str(item.get("file_path") or "")
|
||
raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, file_path)
|
||
geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, file_path)
|
||
|
||
preview_source = await asyncio.to_thread(DataService.find_radar_preview_source, file_path)
|
||
geo_exists = os.path.exists(geo_cache_path)
|
||
raw_exists = os.path.exists(raw_cache_path)
|
||
|
||
if not preview_source:
|
||
if geo_exists:
|
||
return {
|
||
"id": item_id,
|
||
"geo_status": "skipped",
|
||
"raw_status": "skipped" if raw_exists else "missing",
|
||
"db_values": {
|
||
"preview_cache_status": "READY",
|
||
"preview_cache_version": settings.RADAR_GEO_CACHE_VERSION,
|
||
"preview_cache_path": geo_cache_path,
|
||
"preview_cache_error": None,
|
||
},
|
||
}
|
||
return {
|
||
"id": item_id,
|
||
"geo_status": "missing_source",
|
||
"raw_status": "skipped" if raw_exists else "missing",
|
||
"db_values": {
|
||
"preview_cache_status": "NONE",
|
||
"preview_cache_version": settings.RADAR_GEO_CACHE_VERSION,
|
||
"preview_cache_path": None,
|
||
"preview_cache_error": "preview_source_not_found",
|
||
},
|
||
}
|
||
|
||
source_mtime = _safe_mtime(preview_source)
|
||
raw_cache_mtime = _safe_mtime(raw_cache_path)
|
||
geo_cache_mtime = _safe_mtime(geo_cache_path)
|
||
|
||
coverage_polygon = DataService._normalize_coverage_polygon(item.get("coverage_polygon"))
|
||
try:
|
||
bbox = (
|
||
float(item.get("min_lon")),
|
||
float(item.get("min_lat")),
|
||
float(item.get("max_lon")),
|
||
float(item.get("max_lat")),
|
||
)
|
||
except (TypeError, ValueError):
|
||
bbox = None
|
||
|
||
source_corner_mapping = await asyncio.to_thread(
|
||
DataService.get_radar_source_corner_mapping,
|
||
file_path,
|
||
)
|
||
|
||
need_geo_rebuild = (
|
||
(not geo_cache_mtime)
|
||
or (geo_cache_mtime < source_mtime)
|
||
or ((item.get("preview_cache_version") or "") != settings.RADAR_GEO_CACHE_VERSION)
|
||
)
|
||
if not need_geo_rebuild and os.path.exists(geo_cache_path):
|
||
geo_status = "skipped"
|
||
geo_error: Optional[str] = None
|
||
else:
|
||
if not coverage_polygon:
|
||
geo_status = "failed"
|
||
geo_error = "invalid_coverage_polygon"
|
||
elif not bbox:
|
||
geo_status = "failed"
|
||
geo_error = "invalid_bbox"
|
||
else:
|
||
ok_geo, geo_error = await asyncio.to_thread(
|
||
image_service.create_geocorrected_radar_cached_image,
|
||
preview_source,
|
||
geo_cache_path,
|
||
coverage_polygon,
|
||
bbox,
|
||
source_corner_mapping,
|
||
thumb_size,
|
||
settings.RADAR_GEO_CACHE_QUALITY,
|
||
)
|
||
geo_status = "cached" if ok_geo else "failed"
|
||
|
||
need_raw_rebuild = (not raw_cache_mtime) or (raw_cache_mtime < source_mtime)
|
||
if not need_raw_rebuild:
|
||
raw_status = "skipped"
|
||
else:
|
||
ok_raw = await asyncio.to_thread(
|
||
image_service.create_radar_cached_image,
|
||
preview_source,
|
||
raw_cache_path,
|
||
thumb_size,
|
||
)
|
||
raw_status = "cached" if ok_raw else "failed"
|
||
|
||
if geo_status in {"cached", "skipped"} and os.path.exists(geo_cache_path):
|
||
db_values = {
|
||
"preview_cache_status": "READY",
|
||
"preview_cache_version": settings.RADAR_GEO_CACHE_VERSION,
|
||
"preview_cache_path": geo_cache_path,
|
||
"preview_cache_updated_at": datetime.utcnow(),
|
||
"preview_cache_error": None,
|
||
}
|
||
elif geo_status == "failed":
|
||
db_values = {
|
||
"preview_cache_status": "FAILED",
|
||
"preview_cache_version": settings.RADAR_GEO_CACHE_VERSION,
|
||
"preview_cache_path": None,
|
||
"preview_cache_updated_at": datetime.utcnow(),
|
||
"preview_cache_error": geo_error or "geo_cache_build_failed",
|
||
}
|
||
else:
|
||
db_values = {
|
||
"preview_cache_status": "NONE",
|
||
"preview_cache_version": settings.RADAR_GEO_CACHE_VERSION,
|
||
"preview_cache_path": None,
|
||
"preview_cache_updated_at": datetime.utcnow(),
|
||
"preview_cache_error": "preview_source_not_found",
|
||
}
|
||
|
||
return {
|
||
"id": item_id,
|
||
"geo_status": geo_status,
|
||
"raw_status": raw_status,
|
||
"db_values": db_values,
|
||
}
|
||
|
||
processed = 0
|
||
for chunk in _chunked(candidates, workers):
|
||
results = await asyncio.gather(*[_cache_one(item) for item in chunk])
|
||
for result_item in results:
|
||
processed += 1
|
||
geo_status = result_item.get("geo_status")
|
||
raw_status = result_item.get("raw_status")
|
||
if geo_status == "cached":
|
||
cached_previews += 1
|
||
elif geo_status == "skipped":
|
||
skipped_previews += 1
|
||
elif geo_status == "missing_source":
|
||
missing_preview_sources += 1
|
||
else:
|
||
failed_preview_cache += 1
|
||
|
||
if raw_status == "cached":
|
||
cached_raw_previews += 1
|
||
elif raw_status == "skipped":
|
||
skipped_raw_previews += 1
|
||
elif raw_status == "failed":
|
||
failed_raw_preview_cache += 1
|
||
|
||
db_values = result_item.get("db_values")
|
||
if db_values:
|
||
await db.execute(
|
||
update(RadarDataORM)
|
||
.where(RadarDataORM.id == result_item["id"])
|
||
.values(**db_values)
|
||
)
|
||
|
||
prog = 90 + int((processed / total_candidates) * 10)
|
||
update_progress(
|
||
f"正在生成源影像缓存 ({processed}/{total_candidates})...",
|
||
min(100, prog),
|
||
)
|
||
|
||
await db.commit()
|
||
|
||
return {
|
||
"message": "数据扫描完成",
|
||
"processed_scenes": processed_scenes,
|
||
"total_orbit_files": len(orbit_files_map),
|
||
"updated_orbits": updated_orbits,
|
||
"pairing_dirty_scene_count": int(pairing_dirty_summary.get("dirty_scene_count") or 0),
|
||
"pairing_cache_status": pairing_dirty_summary.get("status"),
|
||
"cached_previews": cached_previews,
|
||
"skipped_previews": skipped_previews,
|
||
"missing_preview_sources": missing_preview_sources,
|
||
"failed_preview_cache": failed_preview_cache,
|
||
"cached_raw_previews": cached_raw_previews,
|
||
"skipped_raw_previews": skipped_raw_previews,
|
||
"failed_raw_preview_cache": failed_raw_preview_cache,
|
||
}
|
||
|
||
@staticmethod
|
||
async def scan_dinsar_results(
|
||
db: AsyncSession,
|
||
results_dirs: List[str],
|
||
task_id: Optional[str] = None,
|
||
progress_base: int = 0,
|
||
progress_span: int = 100
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
增量扫描 D-InSAR 结果目录，并为所有未缓存的结果创建图像缓存。
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
results_dirs: D-InSAR 结果目录列表
|
||
task_id: 任务 ID（用于进度更新）
|
||
|
||
Returns:
|
||
扫描结果字典
|
||
"""
|
||
def update_progress(msg: str, prog: int):
|
||
scaled = prog
|
||
if task_id:
|
||
scaled = progress_base + int((prog / 100) * progress_span)
|
||
scaled = min(progress_base + progress_span, max(progress_base, scaled))
|
||
asyncio.create_task(task_service.update_task(task_id, message=msg, progress=scaled))
|
||
print(f" [扫描进度] {scaled}% - {msg}")
|
||
|
||
os.makedirs(settings.DINSAR_CACHE_DIR, exist_ok=True)
|
||
MANIFEST_PATH = DataService.get_dinsar_manifest_path()
|
||
|
||
update_progress("正在检查数据库现有记录...", 5)
|
||
|
||
# 1. 从数据库获取现有记录（按 file_path 去重）
|
||
existing_rows_result = await db.execute(
|
||
select(DinsarResultORM.id, DinsarResultORM.file_path)
|
||
)
|
||
existing_rows = existing_rows_result.all()
|
||
|
||
path_map: Dict[str, int] = {}
|
||
duplicate_ids: List[int] = []
|
||
|
||
for record_id, file_path in existing_rows:
|
||
if not file_path:
|
||
continue
|
||
|
||
existing_id = path_map.get(file_path)
|
||
if existing_id is None:
|
||
path_map[file_path] = record_id
|
||
continue
|
||
if record_id > existing_id:
|
||
duplicate_ids.append(existing_id)
|
||
path_map[file_path] = record_id
|
||
else:
|
||
duplicate_ids.append(record_id)
|
||
|
||
if duplicate_ids:
|
||
await db.execute(
|
||
delete(DinsarResultORM).where(DinsarResultORM.id.in_(duplicate_ids))
|
||
)
|
||
await db.commit()
|
||
print(f"检测到重复记录，已清理 {len(duplicate_ids)} 条 (按 file_path 去重)。")
|
||
|
||
existing_paths = set(path_map.keys())
|
||
print(f"数据库中已存在 {len(existing_paths)} 条记录。开始增量文件扫描...")
|
||
|
||
new_files_metadata = []
|
||
updated_existing = 0
|
||
dinsar_state_updates = []
|
||
|
||
# 2. 统计总量并遍历文件系统收集新文件 (全量遍历，仅新增入库)
|
||
update_progress("正在统计 D-InSAR 结果总量...", 8)
|
||
total_candidates = 0
|
||
for results_dir in results_dirs:
|
||
if not os.path.exists(results_dir):
|
||
continue
|
||
total_candidates += sum(1 for _ in _iter_dinsar_result_candidates(results_dir))
|
||
|
||
if total_candidates == 0:
|
||
update_progress("未找到任何可识别的 D-InSAR 结果。", 10)
|
||
|
||
update_progress("正在遍历文件系统查找新结果...", 10)
|
||
processed_candidate_total = 0
|
||
scan_progress_base = 10
|
||
scan_progress_span = 15
|
||
for results_dir in results_dirs:
|
||
results_dir_abs = os.path.abspath(results_dir)
|
||
max_mtime = 0.0
|
||
if not os.path.exists(results_dir):
|
||
print(f"警告: Dinsar结果目录不存在，已跳过: {results_dir}")
|
||
continue
|
||
|
||
for candidate in _iter_dinsar_result_candidates(results_dir):
|
||
processed_candidate_total += 1
|
||
if processed_candidate_total % 50 == 0 or processed_candidate_total == total_candidates:
|
||
prog = scan_progress_base + int((processed_candidate_total / max(total_candidates, 1)) * scan_progress_span)
|
||
update_progress(
|
||
f"正在遍历文件系统查找新结果 ({processed_candidate_total}/{max(total_candidates, 1)})...",
|
||
min(scan_progress_base + scan_progress_span, prog),
|
||
)
|
||
|
||
scan_ts = candidate["scan_ts"]
|
||
if scan_ts > max_mtime:
|
||
max_mtime = scan_ts
|
||
base_name = candidate["name"]
|
||
data_file_path = candidate["file_path"]
|
||
|
||
if not os.path.exists(data_file_path):
|
||
continue
|
||
if data_file_path in existing_paths:
|
||
continue
|
||
|
||
update_progress(f"正在解析新文件元数据: {base_name}", 25)
|
||
|
||
try:
|
||
meta = image_service.extract_footprint(data_file_path)
|
||
poly = shape(meta["coverage_polygon"])
|
||
if not poly.is_valid:
|
||
poly = poly.buffer(0)
|
||
|
||
data_payload = {
|
||
"name": base_name,
|
||
"file_path": data_file_path,
|
||
"min_lon": meta["min_lon"],
|
||
"min_lat": meta["min_lat"],
|
||
"max_lon": meta["max_lon"],
|
||
"max_lat": meta["max_lat"],
|
||
"coverage_polygon": meta["coverage_polygon"],
|
||
"geom": from_shape(poly, srid=4326),
|
||
"is_cached": False,
|
||
}
|
||
|
||
new_files_metadata.append(data_payload)
|
||
existing_paths.add(data_file_path)
|
||
except Exception as e:
|
||
print(f"读取新文件元数据失败: {data_file_path}, 错误: {e}")
|
||
dinsar_state_updates.append(("dinsar", results_dir_abs, max_mtime))
|
||
|
||
# 3. 批量插入新文件
|
||
|
||
if new_files_metadata or updated_existing:
|
||
print(f"发现 {len(new_files_metadata)} 个新文件，更新 {updated_existing} 条已存在记录。")
|
||
try:
|
||
if new_files_metadata:
|
||
stmt = pg_insert(DinsarResultORM).values(new_files_metadata)
|
||
stmt = stmt.on_conflict_do_nothing(index_elements=['file_path'])
|
||
await db.execute(stmt)
|
||
await db.commit()
|
||
except Exception as e:
|
||
await db.rollback()
|
||
print(f"数据库批量写入新文件失败: {e}")
|
||
else:
|
||
print("未发现新的Dinsar结果文件。")
|
||
|
||
# 4. 检查需要处理的记录
|
||
update_progress("正在检查数据库一致性与缓存状态...", 25)
|
||
to_update_result = await db.execute(select(DinsarResultORM))
|
||
all_records = to_update_result.scalars().all()
|
||
|
||
# 5. 加载现有清单
|
||
manifest = {}
|
||
if os.path.exists(MANIFEST_PATH):
|
||
with open(MANIFEST_PATH, 'r', encoding='utf-8') as f:
|
||
try:
|
||
manifest = json.load(f)
|
||
except json.JSONDecodeError:
|
||
print("警告: 缓存清单文件损坏，将重新创建。")
|
||
|
||
records_to_process = []
|
||
cached_count = 0
|
||
updated_meta_count = 0
|
||
|
||
for record in all_records:
|
||
if not record.name:
|
||
continue
|
||
|
||
is_old_format = (
|
||
isinstance(record.coverage_polygon, list) or
|
||
(record.min_lon is None) or
|
||
(record.min_lon > 180) or
|
||
(record.coverage_polygon is None)
|
||
)
|
||
|
||
thumb_cache_filename = DataService.get_dinsar_cache_filename(record.id, record.name)
|
||
thumb_cache_path = DataService.get_dinsar_cache_path(record.id, record.name)
|
||
file_missing = not os.path.exists(thumb_cache_path)
|
||
|
||
force_recache = is_old_format or file_missing
|
||
needs_meta_update = force_recache or (record.coverage_polygon is None) or (record.min_lon is None) or (record.min_lon > 180)
|
||
needs_cache_gen = force_recache or (not record.is_cached and file_missing)
|
||
|
||
if not needs_meta_update and not needs_cache_gen:
|
||
if not record.is_cached and not file_missing:
|
||
await db.execute(
|
||
update(DinsarResultORM)
|
||
.where(DinsarResultORM.id == record.id)
|
||
.values(is_cached=True)
|
||
)
|
||
manifest[thumb_cache_filename] = record.file_path
|
||
cached_count += 1
|
||
continue
|
||
|
||
records_to_process.append({
|
||
"id": record.id,
|
||
"name": record.name,
|
||
"file_path": record.file_path,
|
||
"force_recache": force_recache,
|
||
"needs_meta_update": needs_meta_update,
|
||
"needs_cache_gen": needs_cache_gen,
|
||
"cache_filename": thumb_cache_filename,
|
||
"cache_path": thumb_cache_path,
|
||
})
|
||
|
||
if not records_to_process and not new_files_metadata and cached_count == 0:
|
||
for data_type, root_path, mtime in dinsar_state_updates:
|
||
await DataService._upsert_scan_state(db, data_type, root_path, mtime)
|
||
await db.commit()
|
||
return {
|
||
"message": "扫描完成，所有结果均已同步且已缓存。",
|
||
"new_files_found": 0,
|
||
"updated_existing": updated_existing,
|
||
"cached_now": cached_count
|
||
}
|
||
|
||
print(f"找到 {len(records_to_process)} 个需要处理（缓存或补全元数据）的记录...")
|
||
|
||
# 6. 处理所有待办记录 (CPU/IO in threads, DB updates sequential)
|
||
total_to_process = len(records_to_process)
|
||
processed_count = 0
|
||
worker_cap = max(1, min(settings.DINSAR_CACHE_WORKERS, os.cpu_count() or 2))
|
||
batch_size = max(1, worker_cap * 2)
|
||
thumb_size = (settings.DINSAR_THUMBNAIL_MAX_SIZE, settings.DINSAR_THUMBNAIL_MAX_SIZE)
|
||
|
||
def _cleanup_cache(cache_path: str, record_name: str):
|
||
if os.path.exists(cache_path):
|
||
try:
|
||
os.remove(cache_path)
|
||
except OSError:
|
||
pass
|
||
old_png = os.path.join(settings.DINSAR_CACHE_DIR, f"{record_name}.png")
|
||
if os.path.exists(old_png):
|
||
try:
|
||
os.remove(old_png)
|
||
except OSError:
|
||
pass
|
||
|
||
async def _process_record(info: Dict[str, Any]) -> Dict[str, Any]:
|
||
result = dict(info)
|
||
if info.get("needs_meta_update"):
|
||
try:
|
||
meta = await asyncio.to_thread(image_service.extract_footprint, info["file_path"])
|
||
result["meta"] = meta
|
||
except Exception as e:
|
||
result["meta_error"] = str(e)
|
||
|
||
if info.get("needs_cache_gen"):
|
||
if info.get("force_recache"):
|
||
await asyncio.to_thread(_cleanup_cache, info["cache_path"], info["name"])
|
||
try:
|
||
ok = await asyncio.to_thread(
|
||
image_service.create_cached_image,
|
||
info["file_path"],
|
||
info["cache_path"],
|
||
thumb_size
|
||
)
|
||
result["cache_ok"] = bool(ok)
|
||
except Exception as e:
|
||
result["cache_error"] = str(e)
|
||
return result
|
||
|
||
for chunk in _chunked(records_to_process, batch_size):
|
||
results = await asyncio.gather(*[_process_record(info) for info in chunk])
|
||
for result in results:
|
||
processed_count += 1
|
||
prog = 30 + int((processed_count / max(total_to_process, 1)) * 65)
|
||
update_progress(
|
||
f"正在处理缓存与自愈 ({processed_count}/{total_to_process}): {result.get('name', '')}",
|
||
prog
|
||
)
|
||
|
||
updates = {}
|
||
meta = result.get("meta")
|
||
if meta:
|
||
try:
|
||
poly = shape(meta["coverage_polygon"])
|
||
if not poly.is_valid:
|
||
poly = poly.buffer(0)
|
||
updates.update({
|
||
"coverage_polygon": meta["coverage_polygon"],
|
||
"geom": from_shape(poly, srid=4326),
|
||
"min_lon": meta["min_lon"],
|
||
"min_lat": meta["min_lat"],
|
||
"max_lon": meta["max_lon"],
|
||
"max_lat": meta["max_lat"],
|
||
})
|
||
updated_meta_count += 1
|
||
except Exception as e:
|
||
print(f"补全旧记录多边形失败 (ID: {result.get('id')}): {e}")
|
||
elif result.get("meta_error"):
|
||
print(f"读取元数据失败 (ID: {result.get('id')}): {result.get('meta_error')}")
|
||
|
||
if result.get("cache_ok"):
|
||
updates["is_cached"] = True
|
||
manifest[result["cache_filename"]] = result["file_path"]
|
||
cached_count += 1
|
||
elif result.get("needs_cache_gen") and result.get("cache_error"):
|
||
print(f"缓存生成失败 (ID: {result.get('id')}): {result.get('cache_error')}")
|
||
|
||
if updates:
|
||
await db.execute(
|
||
update(DinsarResultORM)
|
||
.where(DinsarResultORM.id == result["id"])
|
||
.values(**updates)
|
||
)
|
||
|
||
# 7. 提交并保存
|
||
for data_type, root_path, mtime in dinsar_state_updates:
|
||
await DataService._upsert_scan_state(db, data_type, root_path, mtime)
|
||
|
||
if cached_count > 0 or updated_meta_count > 0:
|
||
try:
|
||
await db.commit()
|
||
if cached_count > 0:
|
||
with open(MANIFEST_PATH, 'w', encoding='utf-8') as f:
|
||
json.dump(manifest, f, indent=4, ensure_ascii=False)
|
||
print(f"成功更新了 {cached_count} 条缓存记录和 {updated_meta_count} 条元数据。")
|
||
except Exception as e:
|
||
await db.rollback()
|
||
print(f"提交数据库或保存清单时出错: {e}")
|
||
|
||
else:
|
||
await db.commit()
|
||
return {
|
||
"message": "Dinsar结果扫描与自愈完成",
|
||
"new_files_found": len(new_files_metadata),
|
||
"updated_existing": updated_existing,
|
||
"cached_now": cached_count,
|
||
"meta_updated": updated_meta_count
|
||
}
|
||
|
||
@staticmethod
|
||
async def scan_hazard_points(
|
||
db: AsyncSession,
|
||
shp_path: str
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
扫描地质灾害点 Shapefile 并存入数据库。
|
||
|
||
Args:
|
||
db: 数据库会话
|
||
shp_path: Shapefile 路径
|
||
|
||
Returns:
|
||
扫描结果字典
|
||
"""
|
||
if not os.path.exists(shp_path):
|
||
raise FileNotFoundError(f"未找到地质灾害点文件: {shp_path}")
|
||
|
||
try:
|
||
gdf = gpd.read_file(shp_path, engine='pyogrio')
|
||
if gdf.crs and gdf.crs.to_epsg() != 4326:
|
||
gdf = gdf.to_crs(epsg=4326)
|
||
|
||
points_to_upsert = []
|
||
def pick_value(row_obj, candidates):
|
||
for key in candidates:
|
||
if key in row_obj and row_obj[key] not in (None, ""):
|
||
return row_obj[key]
|
||
return None
|
||
|
||
def to_float(value):
|
||
try:
|
||
if value is None or value == "":
|
||
return None
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
for _, row in gdf.iterrows():
|
||
tybh_value = pick_value(row, ["TYBH", "tybh"])
|
||
if tybh_value is None:
|
||
tybh_value = pick_value(row, ["统一编", "统一编号", "UNIFIED_ID"])
|
||
|
||
if tybh_value is None:
|
||
continue
|
||
|
||
hazard_type = pick_value(row, ["灾害类型", "灾害类", "ZHLX", "hazard_type", "TYPE"])
|
||
hazard_name = pick_value(row, ["灾害名", "ZHMC", "hazard_name", "NAME"])
|
||
city = pick_value(row, ["市", "CITY", "city"])
|
||
county = pick_value(row, ["县", "COUNTY", "county"])
|
||
township = pick_value(row, ["乡", "TOWNSHIP", "township", "乡镇"])
|
||
|
||
geom_x = getattr(row.geometry, "x", None)
|
||
geom_y = getattr(row.geometry, "y", None)
|
||
|
||
lon_value = to_float(geom_x)
|
||
lat_value = to_float(geom_y)
|
||
if lon_value is None:
|
||
lon_value = to_float(pick_value(row, ["经度", "LON", "longitude"]))
|
||
if lat_value is None:
|
||
lat_value = to_float(pick_value(row, ["纬度", "维度", "LAT", "latitude"]))
|
||
|
||
if lon_value is None or lat_value is None:
|
||
continue
|
||
|
||
point_data = {
|
||
"tybh": str(tybh_value).strip(),
|
||
"hazard_type": hazard_type,
|
||
"hazard_name": hazard_name,
|
||
"city": city,
|
||
"county": county,
|
||
"township": township,
|
||
"longitude": lon_value,
|
||
"latitude": lat_value,
|
||
}
|
||
points_to_upsert.append(point_data)
|
||
|
||
if points_to_upsert:
|
||
for p in points_to_upsert:
|
||
p['geom'] = from_shape(Point(p['longitude'], p['latitude']), srid=4326)
|
||
|
||
stmt = pg_insert(HazardPointORM).values(p)
|
||
update_dict = {c.name: c for c in stmt.excluded if c.name != "tybh"}
|
||
stmt = stmt.on_conflict_do_update(
|
||
index_elements=['tybh'],
|
||
set_=update_dict,
|
||
)
|
||
await db.execute(stmt)
|
||
|
||
await db.commit()
|
||
return {"message": "灾害点同步完成", "count": len(points_to_upsert)}
|
||
|
||
return {"message": "未发现有效的灾害点数据", "count": 0}
|
||
|
||
except Exception as e:
|
||
await db.rollback()
|
||
raise ValueError(f"处理灾害点 Shapefile 失败: {e}")
|
||
|
||
|
||
# 全局服务实例
|
||
data_service = DataService()
|