Refactor local InSAR asset and production workflows

This commit is contained in:
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
+77 -24
View File
@@ -13,6 +13,7 @@ from sklearn.preprocessing import StandardScaler
from scipy.stats import entropy
from concurrent.futures import ProcessPoolExecutor
import functools
from typing import List, Optional
from .config import settings
@@ -224,25 +225,74 @@ def get_model_info() -> dict:
OLLAMA_BASE_URL = settings.OLLAMA_BASE_URL
OLLAMA_API_URL = settings.OLLAMA_API_URL
DEFAULT_VLM_MODEL = settings.DEFAULT_VLM_MODEL
VLM_MODEL_MARKERS = (
"qwen3-vl",
"qwen2-vl",
"minicpm-v",
"llama3.2-vision",
"llava",
"vision",
"-vl",
"_vl",
)
async def _get_available_vlm_model() -> str:
def _normalize_model_name(model_name: Optional[str]) -> Optional[str]:
normalized = str(model_name or "").strip()
return normalized or None
def is_likely_vlm_model(model_name: Optional[str]) -> bool:
lower = str(model_name or "").strip().lower()
return bool(lower and any(marker in lower for marker in VLM_MODEL_MARKERS))
async def get_ollama_models(timeout: float = 2.0) -> List[str]:
"""Return model names reported by the local Ollama service."""
async with httpx.AsyncClient(timeout=timeout) as client:
resp = await client.get(f"{OLLAMA_BASE_URL.rstrip('/')}/api/tags")
resp.raise_for_status()
return [
str(model.get("name") or "").strip()
for model in resp.json().get("models", [])
if str(model.get("name") or "").strip()
]
async def get_ollama_vlm_models(timeout: float = 2.0) -> List[str]:
"""Return installed Ollama models whose names indicate image-input support."""
return [
model_name
for model_name in await get_ollama_models(timeout=timeout)
if is_likely_vlm_model(model_name)
]
async def _get_available_vlm_model(preferred_model: Optional[str] = None) -> str:
"""自动检测本地可用的 VLM 模型"""
preferred = _normalize_model_name(preferred_model)
try:
async with httpx.AsyncClient(timeout=2.0) as client:
resp = await client.get(f"{OLLAMA_BASE_URL}/api/tags")
if resp.status_code == 200:
models = [m['name'] for m in resp.json().get('models', [])]
# 优先级:qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl 的模型
for target in ["qwen3-vl:8b", "qwen2-vl", "minicpm-v"]:
for m in models:
if target in m: return m
for m in models:
if "vl" in m.lower(): return m
except:
models = await get_ollama_models(timeout=2.0)
if preferred and is_likely_vlm_model(preferred):
if preferred in models:
return preferred
for model in models:
if is_likely_vlm_model(model) and (model.startswith(preferred) or preferred in model):
return model
# 优先级:qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl/vision/llava 的模型
for target in ["qwen3-vl", "qwen2-vl", "minicpm-v", "llama3.2-vision", "llava"]:
for model in models:
if target in model.lower():
return model
for model in models:
if is_likely_vlm_model(model):
return model
except Exception:
pass
return DEFAULT_VLM_MODEL
async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callback=None) -> str:
async def analyze_map_with_vlm(
images_base64: list,
prompt: str,
progress_callback=None,
model_name: Optional[str] = None,
raise_on_error: bool = False,
) -> str:
"""
使用本地 Ollama 部署的多模态大模型分析地图截图。
已改为一次性返回模式,以提高连接稳定性。
@@ -250,10 +300,10 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba
if not images_base64:
return "未接收到有效的地图截图。"
model_name = await _get_available_vlm_model()
resolved_model_name = await _get_available_vlm_model(model_name)
payload = {
"model": model_name,
"model": resolved_model_name,
"prompt": prompt,
"images": [img.split(",")[1] if "," in img else img for img in images_base64],
"stream": False, # 关闭流式传输,改为一次性返回
@@ -282,9 +332,11 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba
final_output += f"> [!NOTE] 思考过程\n> {full_thinking}\n\n"
final_output += full_response
return final_output.strip() if final_output else f"模型 ({model_name}) 未返回任何内容。"
return final_output.strip() if final_output else f"模型 ({resolved_model_name}) 未返回任何内容。"
except Exception as e:
if raise_on_error:
raise RuntimeError(f"Ollama VLM request failed: {str(e)}") from e
return f"AI 分析过程中发生错误: {str(e)}"
async def generate_dinsar_diagnosis(
@@ -293,12 +345,13 @@ async def generate_dinsar_diagnosis(
date_str: str,
quality_context: str,
hazard_info: str,
progress_callback=None
progress_callback=None,
model_name: Optional[str] = None,
) -> str:
"""
针对 VLM 优化的 D-InSAR 专家诊断逻辑。
"""
model_name = await _get_available_vlm_model()
resolved_model_name = await _get_available_vlm_model(model_name)
prompt = (
f"你是一位拥有 20 年经验的资深 InSAR 地质灾害解译专家。请根据提供的 D-InSAR 形变图及背景信息,撰写一份专业的诊断报告。\n\n"
@@ -320,22 +373,22 @@ async def generate_dinsar_diagnosis(
f"- 使用 Markdown 格式,语言严谨、专业,严禁幻觉。\n"
f"- 报告末尾必须包含以下加粗文字:\n"
f"**--- 免责声明 ---**\n"
f"**本报告由 AI 自动生成(模型:{model_name}),仅供科研参考,不具备法律效力。**"
f"**本报告由 AI 自动生成(模型:{resolved_model_name}),仅供科研参考,不具备法律效力。**"
)
return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback)
return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback, model_name=resolved_model_name)
async def warm_up_vlm() -> bool:
async def warm_up_vlm(model_name: Optional[str] = None) -> bool:
"""
预热 VLM 模型,将其加载至显存。
发送一个轻量级请求以触发模型冷启动。
返回 True 表示成功,False 表示失败。
"""
# 预热时直接使用探测到的模型
model_name = await _get_available_vlm_model()
# 预热时直接使用指定模型或探测到的模型
resolved_model_name = await _get_available_vlm_model(model_name)
payload = {
"model": model_name,
"model": resolved_model_name,
"prompt": "hi",
"stream": False,
"keep_alive": "30m"
+97 -20
View File
@@ -101,6 +101,18 @@ def _default_task_pool_root(project_root: str) -> str:
return os.path.join(normalized_root, "Task_Pool")
def _default_gf3_data_root(project_root: str) -> str:
normalized_root = os.path.normpath(project_root)
drive, _tail = os.path.splitdrive(normalized_root)
if drive:
return os.path.join(drive + os.sep, "GaoFen3_Pool")
return os.path.join(normalized_root, "GaoFen3_Pool")
def _default_gf3_task_pool_root(project_root: str) -> str:
return os.path.join(_default_gf3_data_root(project_root), "task_pool")
def _default_runtime_dir(project_root: str, *parts: str) -> str:
return os.path.join(_default_runtime_root(project_root), *parts)
@@ -199,6 +211,7 @@ class Settings(BaseSettings):
DINSAR_TASK_POOL_ROOT: str = ""
SBAS_TASK_POOL_ROOT: str = ""
GF3_TASK_POOL_ROOT: str = ""
DATA_DISTRIBUTION_ROOT: str = ""
SOURCE_PRODUCT_DIRS: str = ""
SENTINEL1_STORAGE_DIRS: str = ""
ORBIT_SOURCE_DIRS: str = ""
@@ -239,7 +252,7 @@ class Settings(BaseSettings):
GF3_GEO_DEM_PATH: str = ""
GF3_ARCHIVE_SOURCE_DIRS: str = ""
GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz"
GF3_UNPACK_DELETE_ARCHIVE: bool = True
GF3_UNPACK_DELETE_ARCHIVE: bool = False
GF3_LEGACY_GDAL_ENABLED: bool = False
GF3_SOURCE_DIRS: str = ""
GF3_SARSCAPE_NATIVE_DIRS: str = ""
@@ -250,7 +263,7 @@ class Settings(BaseSettings):
GF3_SARSCAPE_DEM_PATH: str = ""
GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV"
GF3_SARSCAPE_KEEP_EXTRACTED: bool = True
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = False
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0
@@ -460,7 +473,9 @@ class Settings(BaseSettings):
if not self.SBAS_TASK_POOL_ROOT:
object.__setattr__(self, "SBAS_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "SBAS"))
if not self.GF3_TASK_POOL_ROOT:
object.__setattr__(self, "GF3_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "GF3"))
object.__setattr__(self, "GF3_TASK_POOL_ROOT", _default_gf3_task_pool_root(project_root))
if not self.DATA_DISTRIBUTION_ROOT:
object.__setattr__(self, "DATA_DISTRIBUTION_ROOT", os.path.join(self.TASK_POOL_ROOT, "Data_Distribution"))
if not self.SAR_ANALYSIS_READY_ROOT:
object.__setattr__(
self,
@@ -484,25 +499,25 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"GF3_ARCHIVE_SOURCE_DIRS",
os.path.join(_default_input_root(project_root), "gf3", "archives"),
os.path.join(_default_gf3_data_root(project_root), "archives"),
)
if not self.GF3_SARSCAPE_NATIVE_DIRS:
object.__setattr__(
self,
"GF3_SARSCAPE_NATIVE_DIRS",
os.path.join(_default_result_publish_root(project_root), "gf3", "sarscape_native"),
os.path.join(_default_gf3_data_root(project_root), "native_geo"),
)
if not self.GF3_STORAGE_DIRS:
object.__setattr__(
self,
"GF3_STORAGE_DIRS",
os.path.join(_default_result_publish_root(project_root), "gf3", "standard_l2"),
os.path.join(_default_gf3_data_root(project_root), "catalog"),
)
if not self.GF3_SARSCAPE_RUNTIME_DIR:
object.__setattr__(
self,
"GF3_SARSCAPE_RUNTIME_DIR",
_default_runtime_dir(project_root, "gf3", "sarscape_runtime"),
os.path.join(self.GF3_TASK_POOL_ROOT, "sarscape_runtime"),
)
if not self.ORBIT_QUARANTINE_DIR and self.MONITOR_ORBIT_DIR:
object.__setattr__(
@@ -835,7 +850,8 @@ class Settings(BaseSettings):
if "lutan" in item.lower() or "lt1" in item.lower()
]
lt1_roots = list(dict.fromkeys(lt1_roots))
object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", ";".join(lt1_roots) or r"D:\LuTan1_Image_Pool")
materialized_lt1_root = os.path.join(self.TASK_POOL_ROOT, "source_materialized", "lutan1")
object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", materialized_lt1_root)
if not self.GAMMA_SBAS_ORBIT_ROOTS:
object.__setattr__(self, "GAMMA_SBAS_ORBIT_ROOTS", self.PYINT_ORBIT_POOL_TXT or self.ORBIT_POOL_ENVI)
object.__setattr__(self, "GAMMA_SBAS_DEFAULT_RLKS", max(1, int(self.GAMMA_SBAS_DEFAULT_RLKS or 8)))
@@ -1014,6 +1030,7 @@ class Settings(BaseSettings):
os.makedirs(settings.DINSAR_TASK_POOL_ROOT, exist_ok=True)
os.makedirs(settings.SBAS_TASK_POOL_ROOT, exist_ok=True)
os.makedirs(settings.GF3_TASK_POOL_ROOT, exist_ok=True)
os.makedirs(settings.DATA_DISTRIBUTION_ROOT, exist_ok=True)
for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS):
os.makedirs(path, exist_ok=True)
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
@@ -1100,6 +1117,24 @@ def _is_wsl_posix_path(value: str) -> bool:
return text.startswith("/home/") or text.startswith("/mnt/")
def _is_unc_path(value: str) -> bool:
return str(value or "").strip().strip('"').strip("'").startswith("\\\\")
def _enforce_local_runtime_paths(
*,
pairs: list[tuple[str, str]],
errors: list[str],
) -> None:
for label, raw_value in pairs:
raw_text = str(raw_value or "")
values = split_env_paths(raw_text) if (";" in raw_text or "," in raw_text) else [raw_text]
for item in values:
text = str(item or "").strip().strip('"').strip("'")
if text and _is_unc_path(text):
errors.append(f"{label} points to UNC path {text}; active production paths must be local.")
def _check_path(
*,
label: str,
@@ -1171,21 +1206,66 @@ def validate_runtime_config() -> dict[str, Any]:
else:
info.append("DATABASE_URL 已配置。")
_enforce_local_runtime_paths(
pairs=[
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS),
("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS),
("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS),
("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS),
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
("MONITOR_ORBIT_DIR", settings.MONITOR_ORBIT_DIR),
("TASK_POOL_ROOT", settings.TASK_POOL_ROOT),
("DINSAR_TASK_POOL_ROOT", settings.DINSAR_TASK_POOL_ROOT),
("SBAS_TASK_POOL_ROOT", settings.SBAS_TASK_POOL_ROOT),
("GF3_TASK_POOL_ROOT", settings.GF3_TASK_POOL_ROOT),
("DATA_DISTRIBUTION_ROOT", settings.DATA_DISTRIBUTION_ROOT),
("ORBIT_POOL_ENVI", settings.ORBIT_POOL_ENVI),
("ORBIT_POOL_ISCE2", settings.ORBIT_POOL_ISCE2),
("ORBIT_POOL_LANDSAR", settings.ORBIT_POOL_LANDSAR),
("PYINT_ORBIT_POOL_TXT", settings.PYINT_ORBIT_POOL_TXT),
("PYINT_WORK_ROOT", settings.PYINT_WORK_ROOT),
("PYINT_OUTPUT_ROOT", settings.PYINT_OUTPUT_ROOT),
("LANDSAR_WORK_ROOT", settings.LANDSAR_WORK_ROOT),
("RESULT_PUBLISH_ROOT", settings.RESULT_PUBLISH_ROOT),
("DINSAR_PRODUCT_DIR", settings.DINSAR_PRODUCT_DIR),
("TIMESERIES_PRODUCT_DIR", settings.TIMESERIES_PRODUCT_DIR),
("SAR_ANALYSIS_READY_ROOT", settings.SAR_ANALYSIS_READY_ROOT),
("SAR_ANALYSIS_WORK_ROOT", settings.SAR_ANALYSIS_WORK_ROOT),
("GAMMA_SBAS_WORK_ROOT", settings.GAMMA_SBAS_WORK_ROOT),
("GAMMA_SBAS_PRODUCT_ROOT", settings.GAMMA_SBAS_PRODUCT_ROOT),
("GAMMA_SBAS_SOURCE_ROOTS", settings.GAMMA_SBAS_SOURCE_ROOTS),
("GAMMA_SBAS_ORBIT_ROOTS", settings.GAMMA_SBAS_ORBIT_ROOTS),
("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS),
("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS),
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
("GF3_SARSCAPE_RUNTIME_DIR", settings.GF3_SARSCAPE_RUNTIME_DIR),
],
errors=errors,
)
_check_path(label="PYTHON_PATH", value=settings.PYTHON_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="NGINX_PATH", value=settings.NGINX_PATH, errors=errors, warnings=warnings, required=True, expect_file=True)
_check_path(label="LICENSE_PATH", value=settings.LICENSE_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(
label="GF3_SARSCAPE_DEM_PATH",
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
errors=errors,
warnings=warnings,
expect_file=True,
gf3_local_sarscape_enabled = bool(
settings.GF3_LEGACY_GDAL_ENABLED
or settings.GF3_SARSCAPE_AUTO_STANDARDIZE
)
if gf3_local_sarscape_enabled:
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(
label="GF3_SARSCAPE_DEM_PATH",
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
errors=errors,
warnings=warnings,
expect_file=True,
)
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
@@ -1213,9 +1293,6 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="RESULT_QUARANTINE_ROOT", value=settings.RESULT_QUARANTINE_ROOT, errors=errors, warnings=warnings, expect_file=False)
for label, raw_value in (
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS),
("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS),
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
@@ -1230,7 +1307,7 @@ def validate_runtime_config() -> dict[str, Any]:
gf3_archive_dirs = split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
if not gf3_archive_dirs:
warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置;无法触发 GF3 SARscape 生产,只能扫描已有原生结果")
warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置;GF3 本机生产已停用,原始归档仅用于本地资产追踪")
for item in gf3_archive_dirs:
_check_path(label="GF3_ARCHIVE_SOURCE_DIRS", value=item, errors=errors, warnings=warnings, expect_file=False)
+292 -13
View File
@@ -6,11 +6,12 @@ import tarfile
import zipfile
import json
import hashlib
import re
from datetime import datetime
from typing import List, Tuple, Optional, Dict, Any
from .services.task_service import task_service
from .services.dinsar_naming import write_pair_metadata
from .services.dinsar_naming import build_task_alias, write_pair_metadata
# --- Core Logic ---
@@ -44,7 +45,20 @@ def find_dinsar_source_to_copy(path: str) -> str:
return path
def _resolve_dinsar_task_names(item: Dict[str, Any]) -> Tuple[str, str]:
task_name = str(item.get("task_name") or item.get("task_alias") or "").strip()
task_alias = str(item.get("task_alias") or "").strip()
if not task_alias:
task_alias = build_task_alias(item.get("master_imaging_date"), item.get("slave_imaging_date"))
if task_alias == "Task_unknown_unknown" and task_name:
task_alias = task_name
if not task_name:
task_name = task_alias or "task"
return task_name or "task", task_alias or task_name or "task"
_DINSAR_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".zip", ".tar")
_LT1_RASTER_SUFFIXES = (".tif", ".tiff")
def _is_supported_archive(path: str) -> bool:
@@ -52,6 +66,243 @@ def _is_supported_archive(path: str) -> bool:
return any(lower.endswith(suffix) for suffix in _DINSAR_ARCHIVE_SUFFIXES)
def _is_lt1_scene_file_name(filename: str) -> bool:
return str(filename or "").lower().startswith("lt1")
def _is_lt1_meta_name(filename: str) -> bool:
lower = str(filename or "").lower()
return _is_lt1_scene_file_name(lower) and lower.endswith(".meta.xml")
def _is_lt1_raster_name(filename: str) -> bool:
lower = str(filename or "").lower()
return _is_lt1_scene_file_name(lower) and lower.endswith(_LT1_RASTER_SUFFIXES)
def _extract_yyyymmdd_from_name(filename: str) -> str:
match = re.search(r"(20\d{6})", str(filename or ""))
return match.group(1) if match else ""
def _iter_lt1_scene_files(root_dir: str) -> List[str]:
normalized = os.path.normpath(os.path.abspath(str(root_dir or "")))
if not os.path.isdir(normalized):
return []
files: List[str] = []
for current_root, _, filenames in os.walk(normalized):
for filename in filenames:
if _is_lt1_scene_file_name(filename):
files.append(os.path.join(current_root, filename))
return sorted(files, key=lambda item: os.path.normcase(os.path.relpath(item, normalized)))
def _link_or_copy_file(source_path: str, dest_path: str) -> str:
source = os.path.normpath(os.path.abspath(str(source_path or "")))
dest = os.path.normpath(os.path.abspath(str(dest_path or "")))
if not os.path.isfile(source):
raise FileNotFoundError(source)
os.makedirs(os.path.dirname(dest), exist_ok=True)
if os.path.exists(dest):
try:
if os.path.samefile(source, dest):
return "exists"
except OSError:
pass
if os.path.isdir(dest):
raise IsADirectoryError(dest)
os.remove(dest)
try:
os.link(source, dest)
return "linked"
except OSError:
shutil.copy2(source, dest)
return "copied"
def _list_direct_files(directory: str) -> List[str]:
if not os.path.isdir(directory):
return []
return sorted(
os.path.join(directory, entry.name)
for entry in os.scandir(directory)
if entry.is_file()
)
def _flatten_lt1_side_inputs(side_dir: str) -> Dict[str, Any]:
normalized = os.path.normpath(os.path.abspath(str(side_dir or "")))
summary: Dict[str, Any] = {
"side_dir": normalized,
"lt1_source_file_count": 0,
"linked": 0,
"copied": 0,
"exists": 0,
"direct_meta_count": 0,
"direct_raster_count": 0,
}
if not os.path.isdir(normalized):
summary["status"] = "missing_side_dir"
return summary
source_files = _iter_lt1_scene_files(normalized)
summary["lt1_source_file_count"] = len(source_files)
for source in source_files:
dest = os.path.join(normalized, os.path.basename(source))
action = _link_or_copy_file(source, dest)
if action in {"linked", "copied", "exists"}:
summary[action] = int(summary.get(action, 0)) + 1
direct_files = _list_direct_files(normalized)
summary["direct_meta_count"] = sum(1 for path in direct_files if _is_lt1_meta_name(os.path.basename(path)))
summary["direct_raster_count"] = sum(1 for path in direct_files if _is_lt1_raster_name(os.path.basename(path)))
summary["status"] = "ready" if summary["direct_meta_count"] and summary["direct_raster_count"] else "no_lt1_direct_pair"
return summary
def _select_lt1_meta_raster(side_dir: str, expected_date: Any = None) -> Dict[str, str]:
expected = re.sub(r"\D", "", str(expected_date or ""))[:8]
direct_files = _list_direct_files(side_dir)
metas = [path for path in direct_files if _is_lt1_meta_name(os.path.basename(path))]
rasters = [path for path in direct_files if _is_lt1_raster_name(os.path.basename(path))]
def sort_key(path: str) -> tuple[int, str]:
name = os.path.basename(path)
date_mismatch = 0 if expected and expected in name else 1 if expected else 0
return date_mismatch, name.lower()
metas.sort(key=sort_key)
rasters.sort(key=sort_key)
return {
"meta": metas[0] if metas else "",
"raster": rasters[0] if rasters else "",
}
def _prepare_landsar_input_data(
task_dir: str,
master_selection: Dict[str, str],
slave_selection: Dict[str, str],
) -> Dict[str, Any]:
input_dir = os.path.join(task_dir, "Input_Data")
copied: List[Dict[str, Any]] = []
for role, selection in (("master", master_selection), ("slave", slave_selection)):
for kind, source in (("meta", selection.get("meta")), ("raster", selection.get("raster"))):
if not source:
return {
"status": "missing_selected_lt1_file",
"input_data_dir": input_dir,
"role": role,
"kind": kind,
"copied": copied,
}
dest = os.path.join(input_dir, os.path.basename(source))
action = _link_or_copy_file(source, dest)
copied.append(
{
"role": role,
"kind": kind,
"source_path": source,
"relative_path": os.path.relpath(dest, start=task_dir),
"action": action,
}
)
return {
"status": "ready",
"input_data_dir": input_dir,
"copied": copied,
}
def _landsar_input_data_ready(input_data_dir: str) -> bool:
if not os.path.isdir(input_data_dir):
return False
by_date: Dict[str, Dict[str, bool]] = {}
for path in _list_direct_files(input_data_dir):
name = os.path.basename(path)
if not _is_lt1_scene_file_name(name):
continue
lower_name = name.lower()
if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"):
continue
if lower_name.endswith(".xml") and "_slc" not in lower_name:
continue
date_text = _extract_yyyymmdd_from_name(name)
if not date_text:
continue
entry = by_date.setdefault(date_text, {"meta": False, "raster": False})
if lower_name.endswith(".xml"):
entry["xml"] = True
elif _is_lt1_raster_name(name):
entry["raster"] = True
return sum(1 for item in by_date.values() if item.get("xml") and item.get("raster")) >= 2
def _lt1_side_direct_raw_ready(side_dir: str) -> bool:
direct_files = _list_direct_files(side_dir)
return (
any(_is_lt1_meta_name(os.path.basename(path)) for path in direct_files)
and any(_is_lt1_raster_name(os.path.basename(path)) for path in direct_files)
)
def _prepare_dinsar_engine_inputs(task_dir: str, item: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
normalized = os.path.normpath(os.path.abspath(str(task_dir or "")))
payload = dict(item or {})
master_dir = os.path.join(normalized, "master")
slave_dir = os.path.join(normalized, "slave")
master_summary = _flatten_lt1_side_inputs(master_dir)
slave_summary = _flatten_lt1_side_inputs(slave_dir)
summary: Dict[str, Any] = {
"master": master_summary,
"slave": slave_summary,
"landsar": {"status": "skipped_non_lt1"},
}
has_lt1 = bool(master_summary.get("lt1_source_file_count")) or bool(slave_summary.get("lt1_source_file_count"))
if not has_lt1:
return summary
master_selection = _select_lt1_meta_raster(master_dir, payload.get("master_imaging_date"))
slave_selection = _select_lt1_meta_raster(slave_dir, payload.get("slave_imaging_date"))
summary["selected"] = {
"master": {
"meta": os.path.relpath(master_selection["meta"], start=normalized) if master_selection.get("meta") else "",
"raster": os.path.relpath(master_selection["raster"], start=normalized) if master_selection.get("raster") else "",
},
"slave": {
"meta": os.path.relpath(slave_selection["meta"], start=normalized) if slave_selection.get("meta") else "",
"raster": os.path.relpath(slave_selection["raster"], start=normalized) if slave_selection.get("raster") else "",
},
}
if not all(master_selection.values()) or not all(slave_selection.values()):
summary["landsar"] = {"status": "missing_lt1_meta_or_raster"}
return summary
summary["landsar"] = {
"status": "raw_ready_for_import",
"input_data_policy": "LandSAR 100016 import creates native landsar_input at run time.",
}
return summary
def _normalize_source_bundle_archive_path(source_path: str) -> str:
normalized = os.path.normpath(os.path.abspath(str(source_path or "")))
if not os.path.exists(normalized):
raise FileNotFoundError(normalized)
if os.path.isdir(normalized):
raise ValueError(
"D-InSAR source bundle distribution requires source archive files; "
f"unpacked directories are retired: {normalized}"
)
if not os.path.isfile(normalized) or not _is_supported_archive(normalized):
raise ValueError(
"D-InSAR source bundle distribution requires .zip/.tar.gz/.tgz/.tar source archives: "
f"{normalized}"
)
return normalized
def _safe_archive_member_name(member_name: str, archive_path: str) -> str:
name = str(member_name or "").replace("\\", "/").strip("/")
if not name or name.startswith("../") or "/../" in f"/{name}/":
@@ -104,10 +355,20 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int:
raise ValueError(f"Unsupported archive format: {archive_path}")
def _materialize_dinsar_source(source_path: str, dest_dir: str) -> Dict[str, Any]:
def _materialize_dinsar_source(
source_path: str,
dest_dir: str,
*,
require_archive: bool = False,
) -> Dict[str, Any]:
normalized = os.path.normpath(os.path.abspath(str(source_path or "")))
if not os.path.exists(normalized):
raise FileNotFoundError(normalized)
if require_archive and (not os.path.isfile(normalized) or not _is_supported_archive(normalized)):
raise ValueError(
"D-InSAR production preparation requires source archive files; "
f"rebuild the batch from LT-1/Sentinel-1 archive assets: {normalized}"
)
if os.path.isdir(normalized):
shutil.copytree(normalized, dest_dir, dirs_exist_ok=True)
@@ -266,9 +527,16 @@ def _directory_has_entries(path: str) -> bool:
def _is_existing_dinsar_folder_complete(task_dir: str) -> bool:
master_dir = os.path.join(task_dir, "master")
slave_dir = os.path.join(task_dir, "slave")
input_data_dir = os.path.join(task_dir, "Input_Data")
has_lt1_inputs = bool(_iter_lt1_scene_files(master_dir) or _iter_lt1_scene_files(slave_dir))
lt1_raw_ready = _lt1_side_direct_raw_ready(master_dir) and _lt1_side_direct_raw_ready(slave_dir)
input_data_ready = (not has_lt1_inputs) or lt1_raw_ready or _landsar_input_data_ready(input_data_dir)
return (
_directory_has_entries(os.path.join(task_dir, "master"))
and _directory_has_entries(os.path.join(task_dir, "slave"))
_directory_has_entries(master_dir)
and _directory_has_entries(slave_dir)
and input_data_ready
)
@@ -532,8 +800,7 @@ async def run_dinsar_source_bundle_items(
candidate_tasks: List[Dict[str, Any]] = []
for item in items:
task_name = item.get("task_name") or item.get("task_alias") or "task"
task_alias = item.get("task_alias") or task_name
task_name, task_alias = _resolve_dinsar_task_names(item)
master_path = item.get("master_path")
slave_path = item.get("slave_path")
if not master_path or not slave_path:
@@ -749,7 +1016,7 @@ async def run_dinsar_source_bundle_items(
)
def ensure_scene_entry(scene_path: str, role_item: Dict[str, Any], prefix: str) -> Dict[str, Any]:
source_path = os.path.normpath(os.path.abspath(str(scene_path)))
source_path = _normalize_source_bundle_archive_path(scene_path)
source_key = os.path.normcase(source_path)
relative_path = _bundle_relative_scene_path(source_path)
existing = scene_entries_by_source.get(source_key) or scene_entries_by_relative.get(relative_path)
@@ -1100,8 +1367,7 @@ async def run_dinsar_copy_items(
tasks: List[Dict[str, Any]] = []
for item in items:
task_name = item.get("task_name") or item.get("task_alias") or "task"
task_alias = item.get("task_alias") or task_name
task_name, task_alias = _resolve_dinsar_task_names(item)
master_path = item.get("master_path")
slave_path = item.get("slave_path")
if not master_path or not slave_path:
@@ -1195,11 +1461,13 @@ async def run_dinsar_copy_items(
_materialize_dinsar_source,
master_src_path,
master_dir,
require_archive=True,
)
slave_materialization = await asyncio.to_thread(
_materialize_dinsar_source,
slave_src_path,
slave_dir,
require_archive=True,
)
source_materialization = {
"master": {
@@ -1211,6 +1479,19 @@ async def run_dinsar_copy_items(
"target_relative_path": "slave",
},
}
engine_inputs = await asyncio.to_thread(
_prepare_dinsar_engine_inputs,
task_dir,
item,
)
source_materialization["engine_inputs"] = engine_inputs
landsar_status = str(engine_inputs.get("landsar", {}).get("status") or "")
if landsar_status == "raw_ready_for_import":
await _log_and_update(task_id, " -> LT-1 raw inputs ready; LandSAR will run 100016 import at production time")
elif landsar_status == "skipped_non_lt1":
await _log_and_update(task_id, " -> Skipped LandSAR input preparation (non-LT1 source)")
else:
await _log_and_update(task_id, f" -> LandSAR raw inputs not ready: {landsar_status or 'unknown'}")
orbit_entries = await _copy_dinsar_orbit_files(
task_id,
item,
@@ -1235,10 +1516,8 @@ async def run_dinsar_copy_items(
await _log_and_update(task_id, f" -> ZIP: {zip_path}")
elif not export_zip:
if os.path.exists(final_task_dir):
raise CopyTaskExecutionError(
"Destination folder already exists and cannot be atomically replaced: "
f"{final_task_dir}"
)
await asyncio.to_thread(shutil.rmtree, final_task_dir)
await _log_and_update(task_id, f" -> Replaced incomplete existing Task: {final_task_dir}")
await asyncio.to_thread(os.replace, task_dir, final_task_dir)
await _log_and_update(task_id, f" -> Folder: {final_task_dir}")
+2
View File
@@ -40,6 +40,8 @@ MIGRATION_FILES = [
"008_timeseries_stack_plan_edges.sql",
"009_raw_source_pairing_fields.sql",
"010_source_orbit_asset_inventory.sql",
"011_source_metadata_documents.sql",
"012_source_archive_integrity.sql",
]
+498 -13
View File
@@ -9,18 +9,22 @@ can run in the service process without desktop dependencies.
"""
from __future__ import annotations
import hashlib
import json
import math
import os
import queue
import re
import shutil
import socket
import subprocess
import tempfile
import threading
import time
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
import xml.etree.ElementTree as ET
from ..config import get_env_text, read_bool_env, settings
from ..services.dinsar_naming import (
@@ -42,8 +46,10 @@ SUPPORTED_PROFILES = {"lt1_dinsar", "standard"}
_PROJECT_ROOT = Path(__file__).resolve().parents[3]
_DATE_RE = re.compile(r"(?:^|[_-])((?:19|20)\d{6})(?:[_-]|$)")
_CENTER_RE = re.compile(r"(?:^|_)E([+-]?\d+(?:\.\d+)?)_N([+-]?\d+(?:\.\d+)?)(?:_|$)", re.IGNORECASE)
_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+")
_SUCCESS_RE = re.compile(r"(success|成功)", re.IGNORECASE)
_DEFAULT_DEM_CROP_MARGIN_DEGREES = 0.35
_DEFAULT_PARAM_VALUES: Dict[str, Any] = {
"dem_file_type": 0,
@@ -271,6 +277,33 @@ def _extract_date(name: str) -> str:
return fallback.group(1) if fallback else ""
def _safe_path_name(value: Any, fallback: str = "item") -> str:
text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._-")
return text or fallback
def _extract_center_from_name(name: str) -> Optional[tuple[float, float]]:
match = _CENTER_RE.search(name or "")
if not match:
return None
try:
return float(match.group(1)), float(match.group(2))
except (TypeError, ValueError):
return None
def _extract_center_from_path(path: str) -> Optional[tuple[float, float]]:
source = _norm_path(path)
if not source:
return None
candidates = [os.path.basename(source), os.path.basename(os.path.dirname(source))]
for candidate in candidates:
center = _extract_center_from_name(candidate)
if center:
return center
return None
def _collect_tail(text: str, max_chars: int = 4000) -> str:
content = str(text or "")
if len(content) <= max_chars:
@@ -296,6 +329,13 @@ def _summarize_landsar_failure(stdout_text: str, stage: str, return_code: int) -
return f"{prefix}: LandSAR license dongle login failed{status_text}."
if "cann't find 'config.csv'" in lowered or "can't find 'config.csv'" in lowered:
return f"{prefix}: LandSAR config.csv is missing. Run versionControl.exe once from LANDSAR_HOME."
if "cannot operate regis module" in lowered or "invalid parameters" in lowered:
return (
f"{prefix}: registration module failed. "
"LandSAR could not obtain a valid coregistration solution for this pair."
)
if "invalid data type" in lowered or "access window out of range" in lowered or "subterrain phase failed" in lowered:
return f"{prefix}: DEM/sub-terrain processing failed. Check DEM coverage, data type, and LandSAR-readable format."
lines = [line.strip() for line in content.splitlines() if line.strip()]
error_lines = [
line
@@ -418,6 +458,348 @@ def _looks_like_raw_task_dir(task_dir: str) -> bool:
)
def _find_lt1_scene_token(directory: str) -> str:
source_dir = _norm_path(directory)
if not os.path.isdir(source_dir):
return ""
for entry in os.scandir(source_dir):
if entry.is_file():
match = re.match(r"^(LT1[AB])_", entry.name, re.IGNORECASE)
if match:
return match.group(1).upper()
return ""
def _infer_lt1_import_sat_mode(master_dir: str, slave_dir: str) -> str:
master_sat = _find_lt1_scene_token(master_dir)
slave_sat = _find_lt1_scene_token(slave_dir)
if master_sat and slave_sat and master_sat != slave_sat:
return "BIST"
return "MONO"
def _read_dem_bounds(dem_path: str) -> Optional[tuple[float, float, float, float]]:
path = _norm_path(dem_path)
if not path or not os.path.isfile(path):
return None
try:
import rasterio # type: ignore
with rasterio.open(path) as dataset:
bounds = dataset.bounds
return float(bounds.left), float(bounds.bottom), float(bounds.right), float(bounds.top)
except Exception:
return None
def _scene_centers_outside_dem(
parsed_pair: Dict[str, Any],
dem_path: str,
*,
margin_degrees: float = 0.25,
) -> List[str]:
bounds = _read_dem_bounds(dem_path)
if not bounds:
return []
left, bottom, right, top = bounds
blockers: List[str] = []
for role, key in (("master", "master_xml"), ("slave", "slave_xml")):
source_path = str(parsed_pair.get(key) or "")
center = _extract_center_from_path(source_path)
if not center:
continue
lon, lat = center
if lon < left + margin_degrees or lon > right - margin_degrees or lat < bottom + margin_degrees or lat > top - margin_degrees:
blockers.append(
f"{role} center E{lon:.3f}/N{lat:.3f} is outside or too close to DEM bounds "
f"E{left:.3f}-{right:.3f}, N{bottom:.3f}-{top:.3f}"
)
return blockers
def _xml_local_name(tag: str) -> str:
return str(tag or "").rsplit("}", 1)[-1].lower()
def _xml_float_text(value: Any) -> Optional[float]:
try:
text = str(value or "").strip()
if not text:
return None
return float(text)
except (TypeError, ValueError):
return None
def _xml_child_float_by_local_names(element: ET.Element, names: Iterable[str]) -> Optional[float]:
wanted = {str(name).lower() for name in names}
for child in element.iter():
if child is element:
continue
if _xml_local_name(child.tag) in wanted:
value = _xml_float_text(child.text)
if value is not None:
return value
return None
def _extract_scene_lonlat_points_from_xml(xml_path: str) -> List[tuple[float, float]]:
source = _norm_path(xml_path)
if not source or not os.path.isfile(source):
return []
try:
root = ET.parse(source).getroot()
except Exception:
return []
points: List[tuple[float, float]] = []
for element in root.iter():
if _xml_local_name(element.tag) != "scenecornercoord":
continue
lon = _xml_child_float_by_local_names(element, ("lon", "longitude"))
lat = _xml_child_float_by_local_names(element, ("lat", "latitude"))
if lon is not None and lat is not None:
points.append((lon, lat))
if points:
return points
for element in root.iter():
if _xml_local_name(element.tag) != "scenecentercoord":
continue
lon = _xml_child_float_by_local_names(element, ("lon", "longitude"))
lat = _xml_child_float_by_local_names(element, ("lat", "latitude"))
if lon is not None and lat is not None:
return [(lon, lat)]
return []
def _bbox_from_points(points: Iterable[tuple[float, float]]) -> Optional[tuple[float, float, float, float]]:
cleaned = [
(float(lon), float(lat))
for lon, lat in points
if math.isfinite(float(lon)) and math.isfinite(float(lat))
]
if not cleaned:
return None
lons = [item[0] for item in cleaned]
lats = [item[1] for item in cleaned]
return min(lons), min(lats), max(lons), max(lats)
def _expand_lonlat_bbox(
bbox: tuple[float, float, float, float],
margin_degrees: float,
) -> tuple[float, float, float, float]:
margin = max(0.0, float(margin_degrees or 0.0))
left, bottom, right, top = bbox
return (
max(-180.0, float(left) - margin),
max(-90.0, float(bottom) - margin),
min(180.0, float(right) + margin),
min(90.0, float(top) + margin),
)
def derive_landsar_dem_bbox_from_xml_paths(
xml_paths: Iterable[str],
*,
fallback_paths: Iterable[str] = (),
) -> Optional[tuple[float, float, float, float]]:
points: List[tuple[float, float]] = []
for xml_path in xml_paths:
points.extend(_extract_scene_lonlat_points_from_xml(str(xml_path or "")))
bbox = _bbox_from_points(points)
if bbox:
return bbox
fallback_points: List[tuple[float, float]] = []
for path in fallback_paths:
center = _extract_center_from_path(str(path or ""))
if center:
fallback_points.append(center)
return _bbox_from_points(fallback_points)
def derive_landsar_pair_dem_bbox(parsed_pair: Dict[str, Any]) -> Optional[tuple[float, float, float, float]]:
return derive_landsar_dem_bbox_from_xml_paths(
[
str(parsed_pair.get("master_xml") or ""),
str(parsed_pair.get("slave_xml") or ""),
],
fallback_paths=[
str(parsed_pair.get("master_xml") or parsed_pair.get("master_tif") or ""),
str(parsed_pair.get("slave_xml") or parsed_pair.get("slave_tif") or ""),
],
)
def _align_raster_window(window: Any, width: int, height: int) -> Any:
from rasterio.windows import Window
col_off = max(0, int(math.floor(window.col_off)))
row_off = max(0, int(math.floor(window.row_off)))
col_stop = min(width, int(math.ceil(window.col_off + window.width)))
row_stop = min(height, int(math.ceil(window.row_off + window.height)))
if col_stop <= col_off or row_stop <= row_off:
raise ValueError("DEM crop bbox does not overlap source DEM")
return Window(col_off, row_off, col_stop - col_off, row_stop - row_off)
def _iter_raster_windows(width: int, height: int, block_size: int = 2048) -> Iterable[Any]:
from rasterio.windows import Window
step = max(256, int(block_size or 2048))
for row in range(0, int(height), step):
h = min(step, int(height) - row)
for col in range(0, int(width), step):
w = min(step, int(width) - col)
yield Window(col, row, w, h)
def _format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
return ",".join(f"{value:.8f}" for value in bbox)
def prepare_landsar_dem_crop(
source_dem_path: str,
crop_root: str,
bbox: tuple[float, float, float, float],
*,
label: str = "task",
margin_degrees: float = _DEFAULT_DEM_CROP_MARGIN_DEGREES,
block_size: int = 2048,
) -> Dict[str, Any]:
source = _norm_path(source_dem_path)
if not source or not os.path.isfile(source):
raise FileNotFoundError(f"LandSAR DEM source file is missing: {source or '<empty>'}")
if not bbox:
raise ValueError("LandSAR DEM crop bbox is empty")
expanded_bbox = _expand_lonlat_bbox(bbox, margin_degrees)
crop_dir = Path(_norm_path(crop_root))
crop_dir.mkdir(parents=True, exist_ok=True)
safe_label = _safe_path_name(label, "task")
key = hashlib.sha1(f"{source}|{_format_bbox_key(expanded_bbox)}".encode("utf-8", errors="ignore")).hexdigest()[:16]
target = crop_dir / f"{safe_label}_{key}_dem.tif"
manifest = target.with_suffix(target.suffix + ".json")
if target.is_file() and manifest.is_file():
try:
payload = json.loads(manifest.read_text(encoding="utf-8"))
except Exception:
payload = {}
return {
"source_dem_path": source,
"dem_path": str(target),
"bbox": list(bbox),
"expanded_bbox": list(expanded_bbox),
"reused": True,
"manifest_path": str(manifest),
**({"bounds": payload.get("bounds")} if payload.get("bounds") else {}),
}
try:
import rasterio # type: ignore
from rasterio.transform import array_bounds
from rasterio.windows import Window, from_bounds
except Exception as exc:
raise RuntimeError("rasterio is required to crop LandSAR DEMs") from exc
temp_path: Optional[Path] = None
with rasterio.open(source) as src:
source_dtype = str(src.dtypes[0]).lower()
if source_dtype != "int16":
raise ValueError(f"LandSAR DEM source must be a prepared Int16 GeoTIFF, got dtype={src.dtypes[0]}: {source}")
bounds = src.bounds
left, bottom, right, top = expanded_bbox
if left < bounds.left or right > bounds.right or bottom < bounds.bottom or top > bounds.top:
raise ValueError(
"LandSAR DEM crop bbox is outside source DEM bounds: "
f"bbox=E{left:.6f}-{right:.6f},N{bottom:.6f}-{top:.6f}; "
f"source=E{bounds.left:.6f}-{bounds.right:.6f},N{bounds.bottom:.6f}-{bounds.top:.6f}"
)
source_window = _align_raster_window(from_bounds(*expanded_bbox, transform=src.transform), src.width, src.height)
source_window = Window(
int(source_window.col_off),
int(source_window.row_off),
int(source_window.width),
int(source_window.height),
)
transform = src.window_transform(source_window)
crop_bounds = array_bounds(int(source_window.height), int(source_window.width), transform)
profile = src.profile.copy()
profile.update(
driver="GTiff",
height=int(source_window.height),
width=int(source_window.width),
count=1,
dtype=src.dtypes[0],
crs=src.crs,
transform=transform,
nodata=src.nodata,
compress="NONE",
BIGTIFF="YES",
interleave="band",
)
profile.pop("photometric", None)
profile.pop("predictor", None)
if int(source_window.width) >= 512 and int(source_window.height) >= 512:
profile.update(tiled=True, blockxsize=512, blockysize=512)
else:
profile.pop("blockxsize", None)
profile.pop("blockysize", None)
profile["tiled"] = False
try:
with tempfile.NamedTemporaryFile(
prefix=f"{target.stem}.",
suffix=".tmp.tif",
dir=str(crop_dir),
delete=False,
) as tmp:
temp_path = Path(tmp.name)
with rasterio.open(temp_path, "w", **profile) as dst:
for rel_window in _iter_raster_windows(int(source_window.width), int(source_window.height), block_size):
src_window = Window(
source_window.col_off + rel_window.col_off,
source_window.row_off + rel_window.row_off,
rel_window.width,
rel_window.height,
)
dst.write(src.read(1, window=src_window, masked=False), 1, window=rel_window)
os.replace(temp_path, target)
temp_path = None
payload = {
"source_dem_path": source,
"dem_path": str(target),
"bbox": list(bbox),
"expanded_bbox": list(expanded_bbox),
"bounds": [float(value) for value in crop_bounds],
"width": int(source_window.width),
"height": int(source_window.height),
"dtype": src.dtypes[0],
"nodata": src.nodata,
"margin_degrees": float(margin_degrees),
}
manifest.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
finally:
if temp_path and temp_path.exists():
temp_path.unlink(missing_ok=True)
return {
"source_dem_path": source,
"dem_path": str(target),
"bbox": list(bbox),
"expanded_bbox": list(expanded_bbox),
"bounds": [float(value) for value in crop_bounds],
"reused": False,
"manifest_path": str(manifest),
}
def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]:
"""Return the first chronological LT-1 SLC pair from Input_Data."""
@@ -432,6 +814,10 @@ def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]:
lower_name = entry.name.lower()
if not lower_name.endswith((".xml", ".tif", ".tiff")):
continue
if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"):
continue
if lower_name.endswith(".xml") and "_slc" not in lower_name:
continue
date_text = _extract_date(entry.name)
if not date_text:
continue
@@ -830,6 +1216,9 @@ class LandsarEngine(DinsarEngine):
"type": "string",
"default": self._default_dem_path,
"section": "输入数据",
"readonly": True,
"readonly_label": "服务器固定",
"include_in_payload": False,
"description": "传给 LandSAR 200014 D-InSAR 模块的外部参考 DEM,建议使用已验证可用的 GeoTIFF。",
},
"az_looks": {
@@ -1178,14 +1567,14 @@ class LandsarEngine(DinsarEngine):
)
extra = self.normalize_extra(request.extra)
dem_path = _norm_path(extra.get("dem_path") or self._default_dem_path)
if not dem_path or not os.path.isfile(dem_path):
dem_source_path = _norm_path(extra.get("dem_path") or self._default_dem_path)
if not dem_source_path or not os.path.isfile(dem_source_path):
return RunResult(
success=False,
engine_code=self.engine_code,
profile=request.profile,
job_id=request.job_id,
error=f"LandSAR DEM file is missing: {dem_path or '<empty>'}",
error=f"LandSAR DEM source file is missing: {dem_source_path or '<empty>'}",
)
if _coerce_bool(extra.get("do_atmosphere")):
gacos_path = _norm_path(extra.get("gacos_file"))
@@ -1341,6 +1730,85 @@ class LandsarEngine(DinsarEngine):
continue
task_alias, pair_key, pair_meta = self._resolve_task_identity(task_dir, task_name, parsed_pair)
dem_blockers = _scene_centers_outside_dem(parsed_pair, dem_source_path)
if dem_blockers:
pairs_failed += 1
error_text = "LandSAR DEM coverage preflight failed: " + "; ".join(dem_blockers)
emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text)
task_results.append(
self._build_task_result(
task_name=task_name,
task_alias=task_alias,
pair_key=pair_key,
run_key=run_key,
task_dir=task_dir,
run_dir=run_dir,
native_output_dir=native_output_dir,
landsar_output_dir=landsar_output_dir,
command=command,
success=False,
returncode=-2,
error=error_text,
stdout_tail="",
param_file="",
dem_source_path=dem_source_path,
dem_path=effective_dem_path,
dem_crop=dem_crop_info,
)
)
continue
effective_dem_path = dem_source_path
dem_crop_info: Dict[str, Any] = {}
try:
dem_bbox = derive_landsar_pair_dem_bbox(parsed_pair)
if not dem_bbox:
raise ValueError("cannot derive DEM crop bbox from LT-1 XML corner coordinates")
dem_crop_info = prepare_landsar_dem_crop(
dem_source_path,
os.path.join(native_output_dir, "dem_crop"),
dem_bbox,
label=task_alias,
)
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
emit_progress(
"log",
pair_index=pair_index,
pair_total=len(task_dirs),
task_name=task_name,
task_alias=task_alias,
pair_key=pair_key,
level="INFO",
source="dem",
message=(
f"LandSAR DEM crop ready: {effective_dem_path} "
f"from {dem_source_path}"
),
)
except Exception as exc:
pairs_failed += 1
error_text = f"LandSAR DEM crop failed: {exc}"
emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text)
task_results.append(
self._build_task_result(
task_name=task_name,
task_alias=task_alias,
pair_key=pair_key,
run_key=run_key,
task_dir=task_dir,
run_dir=run_dir,
native_output_dir=native_output_dir,
landsar_output_dir=landsar_output_dir,
command=command,
success=False,
returncode=-2,
error=error_text,
stdout_tail="",
param_file="",
)
)
continue
param_values = {**_DEFAULT_PARAM_VALUES}
param_values.update({key: value for key, value in extra.items() if key in _DEFAULT_PARAM_VALUES})
param_file = _generate_dinsar_param_file(
@@ -1349,7 +1817,7 @@ class LandsarEngine(DinsarEngine):
master_tif=parsed_pair["master_tif"],
slave_xml=parsed_pair["slave_xml"],
slave_tif=parsed_pair["slave_tif"],
dem_path=dem_path,
dem_path=effective_dem_path,
output_dir=landsar_output_dir,
params=param_values,
)
@@ -1375,7 +1843,7 @@ class LandsarEngine(DinsarEngine):
pair_key=pair_key,
level="INFO",
source="dem",
message=dem_path,
message=effective_dem_path,
)
rc, stdout_text, timed_out = self._run_console(
@@ -1461,7 +1929,9 @@ class LandsarEngine(DinsarEngine):
request=request,
pair_meta=pair_meta_payload,
params=param_values,
dem_path=dem_path,
dem_path=effective_dem_path,
dem_source_path=dem_source_path,
dem_crop=dem_crop_info,
landsar_output_dir=landsar_output_dir,
primary_file=primary_file,
source_files=source_files,
@@ -1510,18 +1980,20 @@ class LandsarEngine(DinsarEngine):
param_file=param_file,
raw_primary_file=primary_raw,
raw_coherence_file=coherence_raw,
dem_source_path=dem_source_path,
dem_path=effective_dem_path,
dem_crop=dem_crop_info,
)
)
invalid_candidates = validation.get("invalid_candidates", [])
pairs_failed += len(invalid_candidates)
overall_success = pairs_processed > 0 and pairs_failed == 0
if pairs_processed > 0 and pairs_failed > 0:
overall_success = False
overall_success = pairs_processed > 0 or (pairs_processed == 0 and pairs_failed == 0)
run_status = "COMPLETED" if pairs_failed == 0 else ("PARTIAL" if pairs_processed > 0 else "FAILED")
error = None
if not overall_success:
failed_names = [item.get("task_alias") or item.get("task_name") for item in task_results if not item.get("success")]
error = f"LandSAR run failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed."
error = f"All LandSAR tasks failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed."
return RunResult(
success=overall_success,
@@ -1534,6 +2006,7 @@ class LandsarEngine(DinsarEngine):
error=error,
detail={
"mode": validation["mode"],
"run_status": run_status,
"task_count": len(task_dirs),
"selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results],
"invalid_candidates": invalid_candidates,
@@ -1541,7 +2014,8 @@ class LandsarEngine(DinsarEngine):
"run_key": run_key,
"started_at": run_started_at_text,
"timeout_seconds": timeout,
"dem_path": dem_path,
"dem_path": dem_source_path,
"dem_role": "global_prepared_source",
"console_path": console_path,
},
)
@@ -1674,13 +2148,14 @@ class LandsarEngine(DinsarEngine):
"error": "Task directory has neither valid Input_Data nor valid master/slave raw LT-1 folders.",
}
sat_mode = _infer_lt1_import_sat_mode(master_dir, slave_dir)
param_file = _generate_import_param_file(
os.path.join(export_dir, f"{IMPORT_PROID}.txt"),
master_dir=master_dir,
slave_dir=slave_dir,
export_dir=export_dir,
import_method="dir",
sat_mode="BIST",
sat_mode=sat_mode,
read_xml=True,
read_slc=True,
export_to_new=True,
@@ -1695,7 +2170,7 @@ class LandsarEngine(DinsarEngine):
pair_key=pair_key,
level="INFO",
source="import",
message=f"LandSAR import 100016 -> {export_dir}",
message=f"LandSAR import 100016 ({sat_mode}) -> {export_dir}",
)
rc, stdout_text, timed_out = self._run_console(
command,
@@ -1832,6 +2307,8 @@ class LandsarEngine(DinsarEngine):
source_files: List[str],
returncode: int,
started_at: str,
dem_source_path: str = "",
dem_crop: Optional[Dict[str, Any]] = None,
) -> None:
payload = {
"run_key": run_key,
@@ -1850,6 +2327,8 @@ class LandsarEngine(DinsarEngine):
"params": {
**dict(params or {}),
"dem_path": dem_path,
"dem_source_path": dem_source_path or dem_path,
"dem_crop": dem_crop or {},
},
"metrics": {
"returncode": returncode,
@@ -1902,6 +2381,9 @@ class LandsarEngine(DinsarEngine):
param_file: str = "",
raw_primary_file: str = "",
raw_coherence_file: str = "",
dem_source_path: str = "",
dem_path: str = "",
dem_crop: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
command_text = command if isinstance(command, str) else " ".join(str(part) for part in command)
return {
@@ -1919,6 +2401,9 @@ class LandsarEngine(DinsarEngine):
"raw_primary_file": _norm_path(raw_primary_file) if raw_primary_file else "",
"raw_coherence_file": _norm_path(raw_coherence_file) if raw_coherence_file else "",
"param_file": _norm_path(param_file) if param_file else "",
"dem_source_path": _norm_path(dem_source_path) if dem_source_path else "",
"dem_path": _norm_path(dem_path) if dem_path else "",
"dem_crop": dem_crop or {},
"validation": validation or {},
"layout": layout or {},
"command": command_text,
+4 -1
View File
@@ -27,6 +27,8 @@ from .orm import (
ScanCursorORM,
PathInventoryORM,
SourceProductAssetORM,
SourceMetadataDocumentORM,
SARSceneGeometryProfileORM,
OrbitAssetORM,
SceneOrbitBindingORM,
OrbitAssetDerivativeORM,
@@ -103,7 +105,8 @@ __all__ = [
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
"SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM",
"SourceProductAssetORM", "SourceMetadataDocumentORM", "SARSceneGeometryProfileORM",
"OrbitAssetORM", "SceneOrbitBindingORM",
"OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM",
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
"SystemWorkerHeartbeatORM",
+93 -1
View File
@@ -3,7 +3,7 @@ SQLAlchemy ORM 模型定义。
所有数据库表对应的 ORM 类均在此文件中定义。
"""
from sqlalchemy import (
Column, Integer, BigInteger, String, Boolean, Float, JSON, Text,
Column, Integer, BigInteger, String, Boolean, Float, JSON, Text, LargeBinary,
DateTime, func, ForeignKey, UniqueConstraint, Index,
)
from sqlalchemy.orm import relationship
@@ -384,7 +384,15 @@ class PairingMetricCacheORM(Base):
spatial_baseline_meters = Column(Float, index=True, nullable=True)
scene_center_distance_meters = Column(Float, index=True, nullable=True)
scene_overlap_ratio = Column(Float, index=True, nullable=True)
pair_aoi_overlap_ratio = Column(Float, nullable=True)
orbit_direction = Column(String, index=True, nullable=True)
same_relative_orbit = Column(Boolean, nullable=False, default=False, server_default="false", index=True)
master_relative_orbit = Column(String(64), nullable=True)
slave_relative_orbit = Column(String(64), nullable=True)
dinsar_quality_tier = Column(String(16), nullable=False, default="C", server_default="C", index=True)
dinsar_quality_score = Column(Float, nullable=True)
dinsar_readiness = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
dinsar_reasons_json = Column(JSON, nullable=True)
same_satellite = Column(Boolean, nullable=False, default=True)
same_satellite_family = Column(Boolean, nullable=False, default=True, server_default="true")
same_look_direction = Column(Boolean, nullable=False, default=True, server_default="true")
@@ -875,6 +883,12 @@ class SourceProductAssetORM(Base):
mtime_epoch = Column(Float, nullable=True)
checksum_sha256 = Column(String(64), nullable=True)
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
archive_integrity_status = Column(String(32), nullable=False, default="NOT_CHECKED", server_default="NOT_CHECKED", index=True)
archive_integrity_method = Column(String(64), nullable=True)
archive_integrity_checked_at = Column(DateTime, nullable=True)
archive_integrity_error = Column(Text, nullable=True)
archive_integrity_version = Column(String(32), nullable=True)
archive_integrity_member_count = Column(Integer, nullable=True)
parser_name = Column(String(64), nullable=True)
parser_version = Column(String(32), nullable=True)
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
@@ -895,6 +909,84 @@ class SourceProductAssetORM(Base):
)
class SourceMetadataDocumentORM(Base):
__tablename__ = "source_metadata_documents"
id = Column(Integer, primary_key=True, autoincrement=True)
source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), index=True, nullable=False)
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True)
satellite_family = Column(String(32), index=True, nullable=True)
source_format = Column(String(32), index=True, nullable=True)
document_type = Column(String(32), index=True, nullable=False)
member_path = Column(String, nullable=False)
content_sha256 = Column(String(64), index=True, nullable=False)
content_encoding = Column(String(16), nullable=False, default="gzip", server_default="gzip")
content_bytes = Column(LargeBinary, nullable=False)
content_size_bytes = Column(BigInteger, nullable=True)
archive_path = Column(String, nullable=True)
archive_mtime = Column(Float, nullable=True)
parser_version = Column(String(32), nullable=True)
parse_status = Column(String(32), nullable=False, default="OK", server_default="OK", index=True)
parse_error = Column(Text, nullable=True)
extracted_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
source_asset = relationship("SourceProductAssetORM")
radar_data = relationship("RadarDataORM")
__table_args__ = (
UniqueConstraint("source_asset_id", "document_type", "member_path", name="uq_source_metadata_document_member"),
Index("idx_source_metadata_documents_asset_type", "source_asset_id", "document_type"),
Index("idx_source_metadata_documents_radar_type", "radar_data_id", "document_type"),
)
class SARSceneGeometryProfileORM(Base):
__tablename__ = "sar_scene_geometry_profiles"
id = Column(Integer, primary_key=True, autoincrement=True)
source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), unique=True, index=True, nullable=False)
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), unique=True, index=True, nullable=True)
satellite_family = Column(String(32), index=True, nullable=True)
satellite = Column(String(32), index=True, nullable=True)
source_format = Column(String(32), index=True, nullable=True)
imaging_mode = Column(String(64), index=True, nullable=True)
polarization = Column(String(64), index=True, nullable=True)
orbit_direction = Column(String(32), index=True, nullable=True)
look_direction = Column(String(32), index=True, nullable=True)
absolute_orbit = Column(String(64), index=True, nullable=True)
relative_orbit = Column(String(64), index=True, nullable=True)
acquisition_start_time_utc = Column(DateTime, index=True, nullable=True)
acquisition_stop_time_utc = Column(DateTime, nullable=True)
scene_center_lon = Column(Float, nullable=True)
scene_center_lat = Column(Float, nullable=True)
footprint_geom = Column(Geometry("POLYGON", srid=4326), index=True, nullable=True)
footprint_polygon = Column(JSON, nullable=True)
swath_summary_json = Column(JSON, nullable=True)
burst_summary_json = Column(JSON, nullable=True)
incidence_angle_min = Column(Float, nullable=True)
incidence_angle_max = Column(Float, nullable=True)
doppler_summary_json = Column(JSON, nullable=True)
state_vector_summary_json = Column(JSON, nullable=True)
metadata_quality = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True)
production_readiness = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True)
readiness_reasons_json = Column(JSON, nullable=True)
parser_version = Column(String(32), nullable=True)
parsed_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
source_asset = relationship("SourceProductAssetORM")
radar_data = relationship("RadarDataORM")
__table_args__ = (
Index("idx_sar_scene_geometry_profiles_family_date", "satellite_family", "acquisition_start_time_utc"),
Index("idx_sar_scene_geometry_profiles_track", "satellite_family", "relative_orbit", "orbit_direction"),
Index("idx_sar_scene_geometry_profiles_readiness", "production_readiness", "metadata_quality"),
)
class OrbitAssetORM(Base):
__tablename__ = "orbit_assets"
+24 -6
View File
@@ -261,10 +261,10 @@ class PairingRequest(BaseModel):
"""D-InSAR 配对请求的参数模型(增强版 v2.0)"""
# === 时空约束(保留) ===
time_baseline_min: int = Field(default=1, ge=0, le=3650)
time_baseline_max: int = Field(default=90, ge=1, le=3650)
time_baseline_max: int = Field(default=30, ge=1, le=3650)
overlap_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
spatial_baseline_max_meters: int = Field(default=3000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS)
limit_footprint_center_distance: bool = False
spatial_baseline_max_meters: int = Field(default=5000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS)
limit_footprint_center_distance: bool = True
coverage_diversity_penalty: float = Field(default=0.3, ge=0.0, le=1.0)
require_same_imaging_mode: bool = True
require_same_polarization: bool = True
@@ -280,7 +280,7 @@ class PairingRequest(BaseModel):
slave_date_to: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$')
# === 配对策略(新增) ===
strategy: str = Field(default="all", pattern=r'^(all|sbas|sequential|star)$')
strategy: str = Field(default="dinsar_production", pattern=r'^dinsar_production$')
num_connections: int = Field(default=1, ge=1, le=10)
reference_image_id: Optional[int] = None
@@ -303,6 +303,7 @@ class PairingRequest(BaseModel):
normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio']
if normalized.get('footprint_center_distance_max_meters') not in (None, ''):
normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters']
normalized['strategy'] = 'dinsar_production'
return normalized
@field_validator(
@@ -343,8 +344,15 @@ class PairingRequest(BaseModel):
return None
if not isinstance(value, list):
return value
normalized = [str(item).strip() for item in value if str(item).strip()]
return normalized or None
normalized = []
for item in value:
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
normalized.append("LT1")
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
normalized.append("S1")
deduped = list(dict.fromkeys(normalized))
return deduped or ["__UNSUPPORTED_DINSAR_FAMILY__"]
@field_validator('master_date_to')
@classmethod
@@ -381,6 +389,16 @@ class RadarPair(BaseModel):
time_baseline_days: int
spatial_baseline_meters: float
scene_center_distance_meters: Optional[float] = None
scene_overlap_ratio: Optional[float] = None
pair_aoi_overlap_ratio: Optional[float] = None
dinsar_quality_tier: Optional[str] = None
dinsar_quality_score: Optional[float] = None
dinsar_readiness: Optional[str] = None
dinsar_reasons: Optional[List[str]] = None
same_relative_orbit: Optional[bool] = None
master_relative_orbit: Optional[str] = None
slave_relative_orbit: Optional[str] = None
production_summary: Optional[Dict[str, Any]] = None
class PairingResponse(BaseModel):
+12 -6
View File
@@ -15,7 +15,9 @@ from typing import List, Optional
from ..ai_service import (
analyze_map_with_vlm,
get_ollama_models,
get_model_info,
is_likely_vlm_model,
is_model_trained,
predict_quality,
train_quality_model,
@@ -146,12 +148,12 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
counts = await dinsar_read_service.get_ai_status_counts(db)
ollama_online = False
ollama_models: List[str] = []
ollama_vlm_models: List[str] = []
try:
import httpx
async with httpx.AsyncClient(timeout=1.0) as client:
ollama_base = settings.OLLAMA_BASE_URL
response = await client.get(f"{ollama_base.rstrip('/')}/api/tags")
ollama_online = response.status_code == 200
ollama_models = await get_ollama_models(timeout=1.0)
ollama_vlm_models = [model for model in ollama_models if is_likely_vlm_model(model)]
ollama_online = True
except Exception:
ollama_online = False
@@ -161,7 +163,11 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
"labeled_count": counts["labeled_count"],
"good_count": counts["good_count"],
"bad_count": counts["bad_count"],
"ollama_online": ollama_online
"ollama_online": ollama_online,
"ollama_models": ollama_models,
"ollama_vlm_models": ollama_vlm_models,
"ollama_base_url": settings.OLLAMA_BASE_URL,
"default_vlm_model": settings.DEFAULT_VLM_MODEL,
}
+53 -4
View File
@@ -22,21 +22,29 @@ router = APIRouter(prefix="/assets", tags=["assets"])
class AssetScanRequest(BaseModel):
inventory_types: List[str] = Field(default_factory=list)
root_ids: List[int] = Field(default_factory=list)
families: List[str] = Field(default_factory=list)
bind_orbits: bool = True
build_previews: bool = True
class ArchiveIntegrityAuditRequest(BaseModel):
families: List[str] = Field(default_factory=list)
source_formats: List[str] = Field(default_factory=list)
asset_ids: List[int] = Field(default_factory=list)
force: bool = False
limit: Optional[int] = Field(default=None, ge=0)
class S1UnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
class S1BatchUnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
scan_before_unpack: bool = True
@@ -89,7 +97,50 @@ async def run_asset_inventory_scan_now(
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
families=payload.get("families") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
build_previews=bool(payload.get("build_previews", True)),
)
@router.post("/inventory/archive-integrity-audit", status_code=202)
async def run_archive_integrity_audit(
request: Optional[ArchiveIntegrityAuditRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
try:
task_id = await task_service.create_task(
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"Source archive integrity audit",
params=payload,
)
job_id = await job_queue_service.create_job(
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
payload=payload,
task_id=task_id,
)
return {"message": "Source archive integrity audit queued", "task_id": task_id, "job_id": job_id}
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/inventory/archive-integrity-audit-now")
async def run_archive_integrity_audit_now(
request: Optional[ArchiveIntegrityAuditRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = admin_user
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
return await asset_inventory_service.audit_source_archive_integrity(
db,
families=payload.get("families") or None,
source_formats=payload.get("source_formats") or None,
asset_ids=payload.get("asset_ids") or None,
force=bool(payload.get("force", False)),
limit=payload.get("limit"),
)
@@ -186,8 +237,6 @@ async def unpack_sentinel1_source_asset(
}
if request_data.min_disk_space_gb is not None:
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
if request_data.delete_archive is not None:
payload["delete_archive"] = request_data.delete_archive
try:
task_id = await task_service.create_task(
"UNPACK_SENTINEL1",
+89
View File
@@ -2,7 +2,9 @@
from __future__ import annotations
import asyncio
import os
import re
from datetime import datetime
from typing import Any, Dict, Literal, Optional
from fastapi import APIRouter, Depends, HTTPException
@@ -65,6 +67,43 @@ class PreviewInputAssetsRequest(BaseModel):
num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all")
def _is_dinsar_task_dir(path: str) -> bool:
return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave"))
def _is_landsar_task_dir(path: str) -> bool:
return os.path.isdir(os.path.join(path, "Input_Data")) or _is_dinsar_task_dir(path)
def _count_candidate_tasks(root_dir: str) -> Dict[str, Any]:
try:
if _is_dinsar_task_dir(root_dir) or _is_landsar_task_dir(root_dir):
return {"task_count": 1, "invalid_child_count": 0, "mode": "single_task"}
task_count = 0
invalid_child_count = 0
with os.scandir(root_dir) as entries:
for entry in entries:
if not entry.is_dir() or entry.name.startswith("._"):
continue
child_path = entry.path
if _is_dinsar_task_dir(child_path) or _is_landsar_task_dir(child_path):
task_count += 1
elif entry.name.lower().startswith("task_"):
invalid_child_count += 1
return {
"task_count": task_count,
"invalid_child_count": invalid_child_count,
"mode": "task_root",
}
except OSError as exc:
return {
"task_count": 0,
"invalid_child_count": 0,
"mode": "unreadable",
"error": str(exc),
}
def _get_registry():
from ..dinsar_engines import registry
@@ -136,6 +175,56 @@ async def run_wsl_check(
return report.to_dict()
@router.get("/task-roots")
async def list_task_roots(
current_user: AuthUserORM = Depends(_get_current_user),
):
_ = current_user
root_dir = os.path.normpath(os.path.abspath(str(settings.DINSAR_TASK_POOL_ROOT or "").strip()))
root_exists = bool(root_dir and os.path.isdir(root_dir))
items = []
child_dir_count = 0
if root_exists:
try:
with os.scandir(root_dir) as entries:
child_dirs = sorted(
[entry for entry in entries if entry.is_dir() and not entry.name.startswith("._")],
key=lambda entry: entry.name.lower(),
)
except OSError as exc:
raise HTTPException(status_code=500, detail=f"无法读取 D-InSAR Task_Pool: {exc}") from exc
child_dir_count = len(child_dirs)
for entry in child_dirs:
path = os.path.normpath(entry.path)
stat = entry.stat()
summary = _count_candidate_tasks(path)
task_count = int(summary.get("task_count") or 0)
invalid_child_count = int(summary.get("invalid_child_count") or 0)
items.append(
{
"name": entry.name,
"path": path,
"task_count": task_count,
"invalid_child_count": invalid_child_count,
"valid": task_count > 0,
"mode": summary.get("mode"),
"error": summary.get("error"),
"mtime": stat.st_mtime,
"updated_at": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"),
}
)
items.sort(key=lambda item: str(item.get("name") or "").lower())
return {
"root": root_dir,
"root_exists": root_exists,
"items": items,
"count": len(items),
"child_dir_count": child_dir_count,
}
@router.post("/engines/pyint/preview-input-assets")
async def preview_pyint_input_assets(
req: PreviewInputAssetsRequest,
+409 -120
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
from typing import List, Optional
import os
import shutil
from typing import Any, Iterable, List, Optional
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import delete, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
@@ -17,6 +20,15 @@ from .dependencies import _require_admin
router = APIRouter()
MONITOR_ALLOWED_TARGETS = {"radar", "orbit", "dinsar", "gf3"}
MONITOR_SCAN_TASK_TYPES = {
"SCAN_ASSET_INVENTORY",
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"SCAN_DATA",
"SCAN_DINSAR",
"GF3_SARSCAPE_SYNC",
"GF3_QUICKLOOK_WEBP",
}
MONITOR_TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
MONITOR_LOG_DEFAULT_LIMIT = read_int_env(
"MONITOR_LOG_DEFAULT_LIMIT",
50,
@@ -37,10 +49,272 @@ MONITOR_LOG_MAX_OFFSET = read_int_env(
)
def _path_kind(path: str) -> str:
text = str(path or "").strip()
if text.startswith("\\\\"):
return "unc"
drive, _tail = os.path.splitdrive(os.path.normpath(text))
if drive:
return "windows"
if text.startswith("/mnt/"):
return "wsl_mount"
if text.startswith("/"):
return "posix"
return "relative"
def _nearest_existing_path(path: str) -> str:
candidate = os.path.normpath(str(path or "").strip())
while candidate and not os.path.exists(candidate):
parent = os.path.dirname(candidate)
if not parent or parent == candidate:
break
candidate = parent
return candidate
def _storage_status(label: str, path: str, role: str) -> dict[str, Any]:
text = str(path or "").strip()
item: dict[str, Any] = {
"label": label,
"role": role,
"path": text,
"path_kind": _path_kind(text),
"exists": bool(text and os.path.exists(text)),
"probe_path": "",
"total_gb": None,
"used_gb": None,
"free_gb": None,
"free_ratio": None,
"status": "missing" if text else "empty",
"message": "",
}
if not text:
item["message"] = "Path is not configured."
return item
if item["path_kind"] == "unc":
item["status"] = "blocked"
item["message"] = "UNC paths are not allowed for active local production."
return item
probe = _nearest_existing_path(text)
if not probe or not os.path.exists(probe):
item["message"] = "No existing parent path found."
return item
item["probe_path"] = probe
try:
total, used, free = shutil.disk_usage(probe)
except OSError as exc:
item["status"] = "error"
item["message"] = str(exc)
return item
gb = 1024 ** 3
free_ratio = float(free) / float(total or 1)
item.update(
{
"total_gb": round(total / gb, 2),
"used_gb": round(used / gb, 2),
"free_gb": round(free / gb, 2),
"free_ratio": round(free_ratio, 4),
"status": "ok",
"message": "",
}
)
if not item["exists"]:
item["status"] = "missing"
item["message"] = f"Path does not exist; disk usage probed from {probe}."
elif free_ratio < 0.10:
item["status"] = "critical"
item["message"] = "Free space is below 10%."
elif free_ratio < 0.20:
item["status"] = "warning"
item["message"] = "Free space is below 20%."
return item
def _add_storage_roots(rows: list[dict[str, str]], label: str, paths: Iterable[str], role: str) -> None:
for path in paths:
text = str(path or "").strip()
if text:
rows.append({"label": label, "path": text, "role": role})
def _storage_group_identity(item: dict[str, Any]) -> tuple[str, str, str]:
path = str(item.get("path") or "").strip()
path_kind = str(item.get("path_kind") or "")
if path_kind == "windows":
base = str(item.get("probe_path") or path).strip()
drive, _tail = os.path.splitdrive(os.path.normpath(base))
if drive:
drive = drive.upper()
volume_path = f"{drive}\\"
return f"windows:{drive}", volume_path, f"本机磁盘 {volume_path}"
if path_kind == "unc":
key = os.path.normcase(os.path.normpath(path))
return f"unc:{key}", path, "UNC path"
if path_kind in {"posix", "wsl_mount"}:
base = str(item.get("probe_path") or path or "/").strip()
root = "/mnt/" + base.split("/")[2] if base.startswith("/mnt/") and len(base.split("/")) > 2 else "/"
return f"{path_kind}:{root}", root, root
key = os.path.normcase(os.path.normpath(path))
return f"path:{key}", path, str(item.get("label") or item.get("role") or path or "Storage")
def _capacity_status(free_ratio: Any) -> str:
if free_ratio is None:
return "missing"
try:
ratio = float(free_ratio)
except (TypeError, ValueError):
return "missing"
if ratio < 0.10:
return "critical"
if ratio < 0.20:
return "warning"
return "ok"
def _group_storage_statuses(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
groups: dict[str, dict[str, Any]] = {}
order: list[str] = []
for item in items:
group_key, volume_path, volume_label = _storage_group_identity(item)
group = groups.get(group_key)
if group is None:
group = {
"label": volume_label,
"role": "storage_volume",
"path": volume_path,
"path_kind": item.get("path_kind"),
"exists": False,
"probe_path": item.get("probe_path") or "",
"total_gb": None,
"used_gb": None,
"free_gb": None,
"free_ratio": None,
"status": "missing",
"message": "",
"paths": [],
"configured_path_count": 0,
"existing_path_count": 0,
"missing_path_count": 0,
"blocked_path_count": 0,
"error_path_count": 0,
}
groups[group_key] = group
order.append(group_key)
group["configured_path_count"] += 1
if item.get("exists"):
group["existing_path_count"] += 1
if item.get("status") == "missing":
group["missing_path_count"] += 1
elif item.get("status") == "blocked":
group["blocked_path_count"] += 1
elif item.get("status") == "error":
group["error_path_count"] += 1
group["paths"].append(
{
"label": item.get("label"),
"role": item.get("role"),
"path": item.get("path"),
"exists": item.get("exists"),
"status": item.get("status"),
"message": item.get("message"),
}
)
if item.get("total_gb") is not None:
group["exists"] = True
group["probe_path"] = item.get("probe_path") or group["probe_path"]
group["total_gb"] = item.get("total_gb")
group["used_gb"] = item.get("used_gb")
group["free_gb"] = item.get("free_gb")
group["free_ratio"] = item.get("free_ratio")
result: list[dict[str, Any]] = []
for key in order:
group = groups[key]
status = _capacity_status(group.get("free_ratio"))
if group["blocked_path_count"]:
status = "blocked"
elif group["error_path_count"]:
status = "error"
elif group["missing_path_count"] and status == "ok":
status = "partial"
group["status"] = status
messages = [f"{group['configured_path_count']} 个配置路径"]
if group["missing_path_count"]:
messages.append(f"{group['missing_path_count']} 个路径缺失")
if group["blocked_path_count"]:
messages.append(f"{group['blocked_path_count']} 个 UNC 被禁用")
if group["error_path_count"]:
messages.append(f"{group['error_path_count']} 个探测失败")
if status == "critical":
messages.append("剩余空间低于 10%")
elif status == "warning" and not group["missing_path_count"]:
messages.append("剩余空间低于 20%")
group["message"] = "".join(messages) + ""
result.append(group)
return result
def _collect_storage_roots() -> list[dict[str, Any]]:
rows: list[dict[str, str]] = []
_add_storage_roots(rows, "Task_Pool", [settings.TASK_POOL_ROOT], "task_pool")
_add_storage_roots(rows, "D-InSAR Task_Pool", [settings.DINSAR_TASK_POOL_ROOT], "dinsar_task_pool")
_add_storage_roots(rows, "SBAS Task_Pool", [settings.SBAS_TASK_POOL_ROOT], "sbas_task_pool")
_add_storage_roots(rows, "Data distribution root", [settings.DATA_DISTRIBUTION_ROOT], "data_distribution")
source_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
source_path_keys = {os.path.normcase(os.path.normpath(path)) for path in source_paths}
s1_storage_paths = [
path
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
if os.path.normcase(os.path.normpath(path)) not in source_path_keys
]
_add_storage_roots(rows, "LT/S1 local source pools", source_paths, "source_local")
_add_storage_roots(rows, "Sentinel-1 local source pool", s1_storage_paths, "source_local")
_add_storage_roots(rows, "Orbit source pool", split_env_paths(settings.ORBIT_SOURCE_DIRS), "orbit_source")
_add_storage_roots(rows, "LT-1 radar scan pool", split_env_paths(settings.MONITOR_RADAR_DIRS), "lt1_storage")
_add_storage_roots(rows, "D-InSAR product root", [settings.DINSAR_PRODUCT_DIR], "dinsar_product")
_add_storage_roots(rows, "GF3 native _geo", split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), "gf3_native")
_add_storage_roots(rows, "GF3 standard/index", split_env_paths(settings.GF3_STORAGE_DIRS), "gf3_storage")
_add_storage_roots(rows, "GF3 task/runtime pool", [settings.GF3_TASK_POOL_ROOT], "gf3_task_pool")
_add_storage_roots(rows, "GF3 SARscape runtime", [settings.GF3_SARSCAPE_RUNTIME_DIR], "gf3_runtime")
_add_storage_roots(rows, "Result publish root", [settings.RESULT_PUBLISH_ROOT], "result_publish")
_add_storage_roots(rows, "SBAS work root", [settings.GAMMA_SBAS_WORK_ROOT], "sbas_work")
_add_storage_roots(rows, "SBAS product root", [settings.GAMMA_SBAS_PRODUCT_ROOT], "sbas_product")
_add_storage_roots(rows, "SAR analysis ready", [settings.SAR_ANALYSIS_READY_ROOT], "analysis_ready")
_add_storage_roots(rows, "SAR analysis work", [settings.SAR_ANALYSIS_WORK_ROOT], "analysis_work")
seen: set[str] = set()
result: list[dict[str, Any]] = []
for row in rows:
key = os.path.normcase(os.path.normpath(row["path"]))
if key in seen:
continue
seen.add(key)
result.append(_storage_status(row["label"], row["path"], row["role"]))
return _group_storage_statuses(result)
class MonitorConfig(BaseModel):
radar_dirs: List[str] = []
orbit_dir: Optional[str] = None
orbit_source_dirs: List[str] = []
orbit_production_txt_pool: Optional[str] = None
dinsar_dirs: List[str] = []
dinsar_product_dir: Optional[str] = None
sbas_product_root: Optional[str] = None
task_pool_root: Optional[str] = None
dinsar_task_pool_root: Optional[str] = None
sbas_task_pool_root: Optional[str] = None
gf3_task_pool_root: Optional[str] = None
data_distribution_root: Optional[str] = None
gf3_archive_source_dirs: List[str] = []
gf3_source_dirs: List[str] = []
gf3_legacy_gdal_enabled: bool = False
@@ -53,6 +327,7 @@ class MonitorConfig(BaseModel):
gf3_sarscape_polarizations: Optional[str] = None
gf3_sarscape_auto_standardize: bool = True
gf3_sarscape_clean_after_success: bool = True
storage_roots: List[dict[str, Any]] = []
# Manual-only: config is read from .env
@@ -70,6 +345,13 @@ class GF3UnpackRunRequest(BaseModel):
class GF3SarscapeSyncRequest(BaseModel):
force: bool = False
register: bool = True
quicklook_only: bool = False
native_dirs: List[str] = []
class GF3QuicklookWebpRequest(BaseModel):
force: bool = False
max_records: Optional[int] = Field(default=None, ge=0)
class GF3SarscapeProduceRequest(BaseModel):
@@ -91,7 +373,7 @@ class GF3SarscapeCleanRequest(BaseModel):
@router.post("/monitor/config")
async def update_monitor_config(config: MonitorConfig):
"""
更新数据监控配置(雷达、精轨和Dinsar结果)。
鏇存柊鏁版嵁鐩戞帶閰嶇疆锛堥浄杈俱€佺簿杞ㄥ拰Dinsar缁撴灉锛夈€?
"""
raise HTTPException(
status_code=403,
@@ -102,8 +384,8 @@ async def update_monitor_config(config: MonitorConfig):
@router.post("/monitor/run-now")
async def run_monitor_now(target: Optional[str] = None, background_tasks: BackgroundTasks = None, admin_user: AuthUserORM = Depends(_require_admin)):
"""
手动触发一次监控任务(扫描所有配置的目录)。
target: 'radar', 'orbit', 'dinsar' None (全部)
鎵嬪姩瑙﹀彂涓€娆$洃鎺т换鍔★紙鎵弿鎵€鏈夐厤缃殑鐩綍锛夈€?
target: 'radar', 'orbit', 'dinsar' 鎴?None (鍏ㄩ儴)
"""
normalized_target = (target or "").strip().lower() or None
if normalized_target and normalized_target not in MONITOR_ALLOWED_TARGETS:
@@ -127,7 +409,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
raise HTTPException(status_code=400, detail="Monitor paths are not configured in .env.")
task_type = "SCAN_DATA" if normalized_target in ["radar", "orbit", "gf3", None] else "SCAN_DINSAR"
task_name = f"手动触发扫描 ({normalized_target or '全部'})"
task_name = f"Manual scan ({normalized_target or 'all'})"
try:
task_id = await task_service.create_task(task_type, task_name)
@@ -135,7 +417,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
payload = {"dirs": dinsar_dirs}
job_type = "SCAN_DINSAR"
elif normalized_target == "gf3":
# GF3 扫描:用 gf3_storage_dirs 作为 radar_dirs
# GF3 scan uses gf3_storage_dirs as radar_dirs.
payload = {
"radar_dirs": gf3_storage_dirs,
"target": "radar",
@@ -150,7 +432,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
await job_queue_service.create_job(job_type, payload=payload, task_id=task_id)
return {
"message": f"已触发{normalized_target or '全部'}手动扫描任务(已进入队列)",
"message": f"Manual scan queued for {normalized_target or 'all'}.",
"task_id": task_id
}
except ValueError as e:
@@ -160,14 +442,14 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
@router.post("/monitor/gf3-process")
async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin)):
"""
批量 GF3 L1A→L2 处理:扫描 GF3_SOURCE_DIRS,过滤已处理的,逐个处理并自动入库。
Batch GF3 legacy L1A to L2 processing.
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or []
@@ -175,7 +457,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.")
task_type = "GF3_BATCH_PROCESS"
task_name = "GF3 批量 L1A→L2 处理"
task_name = "GF3 legacy batch process"
try:
task_id = await task_service.create_task(task_type, task_name)
@@ -185,7 +467,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
task_id=task_id,
)
return {
"message": "GF3 批量处理任务已提交",
"message": "GF3 batch process task submitted",
"task_id": task_id,
}
except ValueError as e:
@@ -198,147 +480,100 @@ async def run_gf3_sarscape_sync(
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
扫描 GF3 SARscape 原生 _geo 二进制池,转换为标准 GeoTIFF,并登记入库。
Scan GF3 SARscape native _geo outputs.
"""
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
options = request_data or GF3SarscapeSyncRequest()
gf3_native_dirs = options.native_dirs or MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
options = request_data or GF3SarscapeSyncRequest()
task_type = "GF3_SARSCAPE_SYNC"
task_name = "GF3 SARscape 原生结果标准化"
task_name = "GF3 SARscape native result inventory" if options.quicklook_only else "GF3 SARscape native standardize"
payload = {
"native_dirs": gf3_native_dirs,
"storage_root": gf3_storage_dirs[0],
"force": bool(options.force),
"register": bool(options.register),
"quicklook_only": bool(options.quicklook_only),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape 原生结果标准化任务已提交",
"message": (
"GF3 SARscape native result inventory task submitted"
if options.quicklook_only
else "GF3 SARscape native standardize task submitted"
),
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-quicklook-webp", status_code=202)
async def run_gf3_quicklook_webp(
request_data: GF3QuicklookWebpRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Generate local WebP cache files from registered GF3 SARscape native _geo records.
"""
options = request_data or GF3QuicklookWebpRequest()
task_type = "GF3_QUICKLOOK_WEBP"
task_name = "GF3 native _geo WebP cache"
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
payload = {
"force": bool(options.force),
"max_records": int(options.max_records or 0),
"native_dirs": gf3_native_dirs,
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 native _geo WebP cache task submitted",
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-sarscape-produce", status_code=202)
async def run_gf3_sarscape_produce(
request_data: GF3SarscapeProduceRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates.
GF3 SARscape production is disabled on this management machine.
"""
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
if not settings.GF3_SARSCAPE_WRAPPER_EXE:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.")
if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH):
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
options = request_data or GF3SarscapeProduceRequest()
selected_dates = []
for raw_date in options.selected_dates or []:
text = str(raw_date or "").strip()
if not text:
continue
normalized = text.replace("-", "").replace("_", "")
if len(normalized) != 8 or not normalized.isdigit():
raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}")
if normalized not in selected_dates:
selected_dates.append(normalized)
task_type = "GF3_SARSCAPE_PRODUCE"
task_name = "GF3 SARscape production"
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success)
payload = {
"source_dirs": gf3_archive_source_dirs,
"native_dirs": gf3_native_dirs,
"native_root": gf3_native_dirs[0],
"storage_root": gf3_storage_dirs[0],
"wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
"idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
"dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
"max_archives_per_run": int(options.max_archives_per_run or 0),
"selected_dates": selected_dates,
"local_staging_root": settings.GF3_TASK_POOL_ROOT,
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
"auto_standardize": bool(auto_standardize),
"clean_after_success": bool(clean_after_success),
"force_standardize": bool(options.force_standardize),
"register": bool(options.register),
"cleanup_require_standardized": True,
"cleanup_dry_run": bool(options.cleanup_dry_run),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": (
f"GF3 SARscape production task submitted for {', '.join(selected_dates)}"
if selected_dates
else "GF3 SARscape production task submitted"
),
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
raise HTTPException(
status_code=409,
detail=(
"GF3 SARscape production is disabled on this management machine. "
"Run GF3 production on the SARscape host, copy completed _geo results to local "
"GF3_SARSCAPE_NATIVE_DIRS, then use GF3 native result registration and GF3 _geo WebP generation."
),
)
@router.get("/monitor/gf3-sarscape-dates")
async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)):
"""
List available GF3 SARscape source dates from configured raw archive roots.
GF3 SARscape production date selection is disabled on this management machine.
"""
from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
discovery = discover_gf3_sarscape_inputs(
gf3_archive_source_dirs,
archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS),
raise HTTPException(
status_code=409,
detail="GF3 SARscape production date selection is disabled. Register local _geo native results instead.",
)
by_date: dict[str, dict[str, object]] = {}
undated = 0
for item in discovery.get("inputs") or []:
scene_name = str(item.get("scene_name") or "")
date_text = str(item.get("scene_date") or "")
if not date_text:
undated += 1
continue
bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []})
bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1
scenes = bucket.get("scenes")
if isinstance(scenes, list) and len(scenes) < 20:
scenes.append(scene_name)
dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True)
return {
"dates": dates,
"input_count": discovery.get("input_count") or 0,
"undated_count": undated,
"missing_roots": discovery.get("missing_roots") or [],
}
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
@@ -394,14 +629,14 @@ async def run_gf3_unpack(
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
将 GF3 压缩包池解包到 GF3_SOURCE_DIRS,作为后续 L1A→L2 预处理输入。
Unpack GF3 archives into GF3_SOURCE_DIRS for the legacy pipeline.
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
@@ -416,7 +651,7 @@ async def run_gf3_unpack(
max_files = max(0, int(request_data.max_files_per_run))
task_type = "GF3_UNPACK"
task_name = "GF3 压缩包解包"
task_name = "GF3 archive unpack"
payload = {
"source_dirs": gf3_archive_source_dirs,
"target_dirs": gf3_source_dirs,
@@ -429,7 +664,7 @@ async def run_gf3_unpack(
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 解包任务已提交",
"message": "GF3 unpack task submitted",
"task_id": task_id,
}
except ValueError as e:
@@ -439,9 +674,24 @@ async def run_gf3_unpack(
@router.get("/monitor/status")
async def get_monitor_status():
"""
获取当前监控状态。
鑾峰彇褰撳墠鐩戞帶鐘舵€併€?
"""
return MONITOR_CONFIG
config = dict(MONITOR_CONFIG)
config.update(
{
"task_pool_root": settings.TASK_POOL_ROOT,
"dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT,
"sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT,
"gf3_task_pool_root": settings.GF3_TASK_POOL_ROOT,
"data_distribution_root": settings.DATA_DISTRIBUTION_ROOT,
"dinsar_product_dir": settings.DINSAR_PRODUCT_DIR,
"sbas_product_root": settings.GAMMA_SBAS_PRODUCT_ROOT,
"orbit_source_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS) or split_env_paths(settings.MONITOR_ORBIT_DIR),
"orbit_production_txt_pool": settings.ORBIT_POOL_ENVI,
"storage_roots": _collect_storage_roots(),
}
)
return config
@router.get("/monitor/logs")
@@ -451,7 +701,7 @@ async def get_monitor_logs(
db: AsyncSession = Depends(get_db),
):
"""
获取最新的监控日志 (来自任务日志表)。
鑾峰彇鏈€鏂扮殑鐩戞帶鏃ュ織 (鏉ヨ嚜浠诲姟鏃ュ織琛?銆?
"""
safe_limit = min(MONITOR_LOG_MAX_LIMIT, max(1, int(limit or MONITOR_LOG_DEFAULT_LIMIT)))
safe_offset = min(MONITOR_LOG_MAX_OFFSET, max(0, int(offset or 0)))
@@ -478,3 +728,42 @@ async def get_monitor_logs(
"count": len(logs),
"logs": logs,
}
@router.delete("/monitor/scan-task-history")
async def clear_monitor_scan_task_history(
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
"""
Clear finished scan task records and their logs from the monitor panel.
Running and pending tasks are intentionally preserved.
"""
task_rows = await db.execute(
select(SystemTaskORM.task_id)
.where(SystemTaskORM.task_type.in_(MONITOR_SCAN_TASK_TYPES))
.where(SystemTaskORM.status.in_(MONITOR_TERMINAL_TASK_STATUSES))
)
task_ids = [str(task_id) for task_id in task_rows.scalars().all() if task_id]
if not task_ids:
return {
"deleted_task_count": 0,
"deleted_log_count": 0,
"preserved_active": True,
}
log_count_result = await db.execute(
select(func.count(TaskLogORM.id)).where(TaskLogORM.task_id.in_(task_ids))
)
deleted_log_count = int(log_count_result.scalar_one() or 0)
await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids)))
task_delete_result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids)))
await db.commit()
return {
"deleted_task_count": int(task_delete_result.rowcount or len(task_ids)),
"deleted_log_count": deleted_log_count,
"preserved_active": True,
}
+18 -10
View File
@@ -35,6 +35,8 @@ class OrbitPoolActionRequest(BaseModel):
async def _build_orbit_database_stats(
db: AsyncSession,
pool_inventory: Dict[str, Any],
*,
isce2_enabled: bool = False,
) -> Dict[str, Any]:
total_radar_count = (
await db.execute(select(func.count(RadarDataORM.id)))
@@ -94,9 +96,9 @@ async def _build_orbit_database_stats(
}
envi_stems = set(pool_inventory["envi"]["files"].keys())
isce2_stems = set(pool_inventory["isce2"]["files"].keys())
isce2_stems = set(pool_inventory["isce2"]["files"].keys()) if isce2_enabled else set()
missing_in_envi = sorted(db_expected_stems - envi_stems)
missing_in_isce2 = sorted(db_expected_stems - isce2_stems)
missing_in_isce2 = sorted(db_expected_stems - isce2_stems) if isce2_enabled else []
return {
"total_radar_count": int(total_radar_count or 0),
@@ -110,6 +112,7 @@ async def _build_orbit_database_stats(
"db_expected_stem_count": len(db_expected_stems),
"stems_missing_in_envi_count": len(missing_in_envi),
"stems_missing_in_isce2_count": len(missing_in_isce2),
"isce2_enabled": bool(isce2_enabled),
"sample_missing_in_envi": missing_in_envi[:20],
"sample_missing_in_isce2": missing_in_isce2[:20],
"path_errors": db_path_errors,
@@ -123,26 +126,27 @@ async def get_orbit_status(
):
"""Return source, pool, consistency, and database orbit status."""
source_dir = settings.MONITOR_ORBIT_DIR
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
source_stats = await asyncio.to_thread(scan_orbit_dir, source_dir)
pool_inventory = await asyncio.to_thread(
get_orbit_pool_inventory,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
True,
)
consistency = await asyncio.to_thread(
check_orbit_consistency,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
)
source_gap_summary = await asyncio.to_thread(
summarize_source_orbit_gaps,
source_dir,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_QUARANTINE_DIR,
)
database_stats = await _build_orbit_database_stats(db, pool_inventory)
database_stats = await _build_orbit_database_stats(db, pool_inventory, isce2_enabled=bool(settings.ISCE2_ENABLED))
return {
"orbit_root": source_stats.orbit_root,
@@ -184,7 +188,8 @@ async def get_orbit_status(
"errors": pool_inventory["envi"]["errors"],
},
"isce2": {
"path": settings.ORBIT_POOL_ISCE2,
"path": isce2_pool,
"enabled": bool(settings.ISCE2_ENABLED),
"total": pool_inventory["isce2"]["total"],
"duplicate_count": pool_inventory["isce2"]["duplicate_count"],
"errors": pool_inventory["isce2"]["errors"],
@@ -207,25 +212,28 @@ async def sync_orbit_pool_action(
Check pool consistency, or repair missing entries when repair=true.
"""
if payload and payload.quarantine_bad:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
quarantine_bad_orbits,
settings.MONITOR_ORBIT_DIR,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_QUARANTINE_DIR,
)
if payload and payload.repair:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
repair_orbit_pools,
settings.MONITOR_ORBIT_DIR,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_POOL_LANDSAR,
)
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
check_orbit_consistency,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
)
+5 -5
View File
@@ -40,10 +40,10 @@ router = APIRouter()
def get_pairing_request_from_form(
time_baseline_min: int = Form(1),
time_baseline_max: int = Form(90),
time_baseline_max: int = Form(30),
overlap_threshold: float = Form(0.5),
spatial_baseline_max_meters: int = Form(3000),
limit_footprint_center_distance: bool = Form(False),
spatial_baseline_max_meters: int = Form(5000),
limit_footprint_center_distance: bool = Form(True),
max_temporal_baseline_days: Optional[int] = Form(None),
pair_footprint_overlap_min_ratio: Optional[float] = Form(None),
footprint_center_distance_max_meters: Optional[int] = Form(None),
@@ -57,7 +57,7 @@ def get_pairing_request_from_form(
master_date_to: Optional[str] = Form(None),
slave_date_from: Optional[str] = Form(None),
slave_date_to: Optional[str] = Form(None),
strategy: str = Form("all"),
strategy: str = Form("dinsar_production"),
num_connections: int = Form(1),
reference_image_id: Optional[int] = Form(None),
allowed_satellites: Optional[str] = Form(None), # JSON string
@@ -91,7 +91,7 @@ def get_pairing_request_from_form(
master_date_to=master_date_to,
slave_date_from=slave_date_from,
slave_date_to=slave_date_to,
strategy=strategy,
strategy="dinsar_production",
num_connections=num_connections,
reference_image_id=reference_image_id,
allowed_satellites=satellites_list,
+65
View File
@@ -41,6 +41,7 @@ from .dependencies import (
router = APIRouter()
logger = logging.getLogger(__name__)
GF3_NATIVE_PREVIEW_SOURCE_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
LIST_QUERY_MAX_LIMIT = read_int_env(
"LIST_QUERY_MAX_LIMIT",
2000,
@@ -188,6 +189,38 @@ def _radar_preview_paths(record: RadarDataORM) -> Tuple[str, str]:
return raw_cache_path, geo_cache_path
def _is_gf3_native_preview_record(record: RadarDataORM) -> bool:
return str(record.source_format or "") == GF3_NATIVE_PREVIEW_SOURCE_FORMAT
def _build_gf3_native_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo:
preview_path = str(record.preview_cache_path or "")
has_native_cache = (
(record.preview_cache_status or "NONE") == "READY"
and preview_path.lower().endswith(".webp")
and os.path.exists(preview_path)
)
metadata = record.metadata_json or {}
source_path = str(metadata.get("default_native_path") or "")
source_found = bool(source_path and os.path.exists(source_path))
return RadarPreviewStatusInfo(
radar_id=record.id,
status="READY" if has_native_cache else (record.preview_cache_status or "NONE"),
cache_version=record.preview_cache_version,
cache_updated_at=record.preview_cache_updated_at,
has_geo_cache=has_native_cache,
has_raw_cache=has_native_cache,
source_found=source_found,
fallback_in_use=False,
message=(
"GF3 native _geo WebP cache is available."
if has_native_cache
else "GF3 native _geo WebP cache has not been generated."
),
error=None if has_native_cache else record.preview_cache_error,
)
def _build_radar_preview_status(
record: RadarDataORM,
source_found: bool,
@@ -228,6 +261,9 @@ async def _build_radar_preview_cache(
db: AsyncSession,
force: bool = False,
) -> RadarPreviewStatusInfo:
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
@@ -355,7 +391,33 @@ async def _get_cached_radar_preview(data_id: int, db: AsyncSession):
if not record:
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
if _is_gf3_native_preview_record(record):
preview_path = str(record.preview_cache_path or "")
if (
(record.preview_cache_status or "NONE") == "READY"
and preview_path.lower().endswith(".webp")
and os.path.exists(preview_path)
):
return FileResponse(
preview_path,
media_type="image/webp",
headers={"Cache-Control": "public, max-age=31536000"},
)
raise HTTPException(status_code=404, detail="GF3 native _geo WebP cache has not been generated.")
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
if (
(record.preview_cache_status or "NONE") == "READY"
and record.preview_cache_path
and str(record.preview_cache_path).lower().endswith(".webp")
and os.path.exists(record.preview_cache_path)
):
return FileResponse(
record.preview_cache_path,
media_type="image/webp",
headers={"Cache-Control": "public, max-age=31536000"},
)
if os.path.exists(geo_cache_path):
return FileResponse(
geo_cache_path,
@@ -664,6 +726,9 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep
if not record:
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
+77 -19
View File
@@ -1,12 +1,14 @@
from __future__ import annotations
import os
import re
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field, field_validator
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import read_int_env
from ..config import read_int_env, settings
from ..database import get_db
from ..services.job_queue_service import job_queue_service
from ..services.task_service import TASK_LOG_DEFAULT_LIMIT, TASK_LOG_MAX_LIMIT, TASK_QUERY_MAX_OFFSET, task_service
@@ -33,12 +35,14 @@ COPY_BATCH_MAX_COPY_ITEMS = read_int_env(
minimum=1,
maximum=200000,
)
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "task_zip", "source_bundle"}
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "source_bundle"}
_COPY_TARGET_NAME_RE = re.compile(r"[^0-9A-Za-z\u4e00-\u9fff._-]+")
class CopyBatchRequest(BaseModel):
batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
dest_dir: str = Field(default="", max_length=COPY_BATCH_TEXT_MAX_LENGTH)
target_name: Optional[str] = Field(default=None, max_length=255)
copy_statuses: Optional[List[str]] = None
include_orbit_files: bool = False
export_zip: bool = False
@@ -46,7 +50,7 @@ class CopyBatchRequest(BaseModel):
skip_existing: bool = True
max_items: Optional[int] = None
@field_validator("batch_id", "dest_dir", mode="before")
@field_validator("batch_id", mode="before")
@classmethod
def _normalize_required_text(cls, value):
normalized = str(value or "").strip()
@@ -54,6 +58,17 @@ class CopyBatchRequest(BaseModel):
raise ValueError("Field must not be empty.")
return normalized
@field_validator("dest_dir", mode="before")
@classmethod
def _normalize_optional_dest_dir(cls, value):
return str(value or "").strip()
@field_validator("target_name", mode="before")
@classmethod
def _normalize_optional_target_name(cls, value):
normalized = str(value or "").strip()
return normalized or None
@field_validator("copy_statuses", mode="before")
@classmethod
def _validate_copy_statuses_length(cls, value):
@@ -88,6 +103,10 @@ class CopyBatchRequest(BaseModel):
@classmethod
def _normalize_package_mode(cls, value):
normalized = str(value or "task_folder").strip().lower()
if normalized == "task_zip":
normalized = "task_folder"
if normalized in {"bundle", "dedupe_source"}:
normalized = "source_bundle"
if normalized not in COPY_DINSAR_PACKAGE_MODES:
raise ValueError(
f"package_mode must be one of: {sorted(COPY_DINSAR_PACKAGE_MODES)}."
@@ -115,6 +134,28 @@ def _normalize_copy_batch_statuses(copy_statuses: Optional[List[str]]) -> List[s
return normalized or ["COMPLETED"]
def _safe_copy_target_name(value: Optional[str]) -> str:
raw = str(value or "").strip()
if not raw:
raise HTTPException(status_code=400, detail="target_name 不能为空")
if os.path.isabs(raw) or os.path.splitdrive(raw)[0] or "\\" in raw or "/" in raw:
raise HTTPException(status_code=400, detail="target_name 只能是任务名,不能包含路径")
normalized = _COPY_TARGET_NAME_RE.sub("_", raw).strip("._ ")
if not normalized:
raise HTTPException(status_code=400, detail="target_name 不合法")
if normalized.upper() in {"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "LPT1", "LPT2", "LPT3"}:
raise HTTPException(status_code=400, detail="target_name 是 Windows 保留名称")
return normalized[:120]
def _server_copy_destination(package_mode: str, target_name: Optional[str]) -> str:
safe_name = _safe_copy_target_name(target_name)
root = settings.DINSAR_TASK_POOL_ROOT if package_mode == "task_folder" else settings.DATA_DISTRIBUTION_ROOT
if not root:
raise HTTPException(status_code=500, detail="服务器目标根目录未配置")
return os.path.normpath(os.path.join(root, safe_name))
@router.post("/tools/copy-ps-stack")
async def copy_ps_stack_endpoint(
request: CopyBatchRequest,
@@ -124,20 +165,26 @@ async def copy_ps_stack_endpoint(
"""
Start PS-InSAR copy task from a batch.
"""
_validate_export_path(request.dest_dir, "dest_dir")
try:
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
if request.target_name:
dest_dir = _server_copy_destination("source_bundle", request.target_name)
else:
dest_dir = request.dest_dir
_validate_export_path(dest_dir, "dest_dir")
params = {
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"file_type": "PS_STACK",
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
}
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.dest_dir}", params=params)
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.target_name or dest_dir}", params=params)
payload = {
"file_type": "PS_STACK",
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
}
@@ -150,7 +197,8 @@ async def copy_ps_stack_endpoint(
detail={
"task_id": task_id,
"batch_id": request.batch_id,
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"copy_statuses": copy_statuses,
},
)
@@ -169,32 +217,41 @@ async def copy_dinsar_pairs_endpoint(
"""
Start D-InSAR copy task from a batch.
"""
_validate_export_path(request.dest_dir, "dest_dir")
try:
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
package_mode = request.package_mode
if bool(request.export_zip) and package_mode == "task_folder":
package_mode = "task_zip"
if request.target_name:
dest_dir = _server_copy_destination(package_mode, request.target_name)
else:
dest_dir = request.dest_dir
_validate_export_path(dest_dir, "dest_dir")
params = {
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"file_type": "DINSAR_PAIRS",
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,
}
task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params)
task_name = (
f"D-InSAR 生产数据准备: {request.target_name or dest_dir}"
if package_mode == "task_folder"
else f"D-InSAR 数据分发: {request.target_name or dest_dir}"
)
task_id = await task_service.create_task("COPY_DATA", task_name, params=params)
payload = {
"file_type": "DINSAR_PAIRS",
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,
@@ -208,10 +265,11 @@ async def copy_dinsar_pairs_endpoint(
detail={
"task_id": task_id,
"batch_id": request.batch_id,
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,
+5 -2
View File
@@ -13,8 +13,11 @@ MONITOR_CONFIG = {
# LT-1 链路
"radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS),
"orbit_dir": settings.MONITOR_ORBIT_DIR,
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS),
# Sentinel-1 链路
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS) or [settings.DINSAR_PRODUCT_DIR],
"task_pool_root": settings.TASK_POOL_ROOT,
"dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT,
"sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT,
# LT-1 / Sentinel-1 source inventory
"s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS),
"s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS),
"s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS),
File diff suppressed because it is too large Load Diff
+231 -13
View File
@@ -16,6 +16,8 @@ import json
import asyncio
import hashlib
import re
import tarfile
import zipfile
from typing import List, Dict, Optional, Tuple, Any
from datetime import datetime
from shapely.geometry import Point, Polygon, shape, mapping
@@ -240,6 +242,8 @@ def _chunked(items: List[Any], size: int):
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
_RADAR_ARCHIVE_SUFFIXES = (".zip", ".tar", ".tgz", ".tar.gz")
_RADAR_ARCHIVE_PREVIEW_MAX_BYTES = 256 * 1024 * 1024
def _sanitize_cache_name(name: str) -> str:
@@ -265,6 +269,208 @@ def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_versio
return f"RGID_{safe_version}_{digest}_{base}.webp"
def _has_radar_archive_suffix(path: str) -> bool:
return str(path or "").lower().endswith(_RADAR_ARCHIVE_SUFFIXES)
def _archive_product_stem(path: str) -> str:
base = os.path.basename(str(path or ""))
lower_base = base.lower()
for suffix in (".tar.gz", ".tgz", ".tar", ".zip"):
if lower_base.endswith(suffix):
return base[: -len(suffix)]
return os.path.splitext(base)[0]
def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> Optional[int]:
product_stem = _archive_product_stem(archive_path)
if not product_stem:
return None
member = str(member_name or "").replace("\\", "/").strip("/")
member_lower = member.lower()
product_lower = product_stem.lower()
expected_names = [
f"{product_lower}/{product_lower}.browse.jpg",
f"{product_lower}/{product_lower}.browse.jpeg",
f"{product_lower}/{product_lower}.browse.png",
f"{product_lower}/{product_lower}.quicklook.jpg",
f"{product_lower}/{product_lower}.quicklook.png",
f"{product_lower}/{product_lower}.quick-look.png",
f"{product_lower}/{product_lower}.thumb.jpg",
f"{product_lower}/{product_lower}.thumb.jpeg",
f"{product_lower}/{product_lower}.thumb.png",
f"{product_lower}/preview/quick-look.png",
f"{product_lower}.browse.jpg",
f"{product_lower}.browse.jpeg",
f"{product_lower}.browse.png",
f"{product_lower}.thumb.jpg",
f"{product_lower}.thumb.jpeg",
f"{product_lower}.thumb.png",
]
try:
return expected_names.index(member_lower)
except ValueError:
return None
def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]:
lower_name = str(member_name or "").replace("\\", "/").lower()
base_name = os.path.basename(lower_name)
if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
return None
if size_bytes and size_bytes > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
return None
has_keyword = any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS)
if base_name.endswith((".tif", ".tiff")) and not has_keyword:
return None
keyword_score = 0 if has_keyword else 2
if base_name == "quick-look.png":
keyword_score = -4
elif base_name == "quicklook.png":
keyword_score = -3
elif base_name.startswith("quick-look.") or base_name.startswith("quicklook."):
keyword_score = min(keyword_score, -2)
if "/preview/" in lower_name:
keyword_score -= 1
if "/icons/" in lower_name:
keyword_score += 2
if base_name.endswith((".jpg", ".jpeg", ".png", ".webp")):
ext_score = 0
elif base_name.endswith(".bmp"):
ext_score = 1
else:
ext_score = 2
depth = lower_name.count("/")
size_score = -int(size_bytes or 0)
return (keyword_score, depth, ext_score, size_score, member_name)
def _radar_archive_preview_cache_path(archive_path: str, member_name: str) -> str:
digest = hashlib.sha1(
f"{archive_path}|{member_name}".encode("utf-8", errors="ignore")
).hexdigest()[:16]
raw_base = os.path.basename(str(member_name or "preview"))
base = _sanitize_cache_name(raw_base)
_, ext = os.path.splitext(raw_base)
ext = _RADAR_CACHE_NAME_RE.sub("", ext.lower())[:12]
if ext and not base.lower().endswith(ext):
base = f"{base[:max(1, 48 - len(ext))]}{ext}"
return os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources", f"APS_{digest}_{base}")
def _write_archive_preview_cache(target_path: str, source_obj: Any, archive_mtime: float) -> Optional[str]:
try:
os.makedirs(os.path.dirname(target_path), exist_ok=True)
if os.path.exists(target_path) and _safe_mtime(target_path) >= archive_mtime:
return target_path
tmp_path = f"{target_path}.tmp"
with open(tmp_path, "wb") as target:
while True:
chunk = source_obj.read(1024 * 1024)
if not chunk:
break
target.write(chunk)
os.replace(tmp_path, target_path)
if archive_mtime:
os.utime(target_path, (archive_mtime, archive_mtime))
return target_path
except Exception:
try:
if "tmp_path" in locals() and os.path.exists(tmp_path):
os.remove(tmp_path)
except OSError:
pass
return None
def _find_radar_archive_preview_source(archive_path: str) -> Optional[str]:
if not archive_path or not os.path.isfile(archive_path) or not _has_radar_archive_suffix(archive_path):
return None
archive_mtime = _safe_mtime(archive_path)
lower_path = archive_path.lower()
try:
if lower_path.endswith(".zip"):
with zipfile.ZipFile(archive_path) as archive:
candidates = []
info_by_name = {}
for info in archive.infolist():
if info.is_dir():
continue
score = _radar_archive_preview_score(info.filename, int(info.file_size or 0))
if score is None:
continue
candidates.append(score)
info_by_name[info.filename] = info
if not candidates:
return None
candidates.sort()
member_name = candidates[0][-1]
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
with archive.open(info_by_name[member_name], "r") as source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
with tarfile.open(archive_path, "r:*") as archive:
candidates = []
member_by_name = {}
best_rank: Optional[int] = None
best_member: Optional[tarfile.TarInfo] = None
for member in archive:
if not member.isfile():
continue
expected_rank = _radar_archive_expected_preview_rank(archive_path, member.name)
if expected_rank is not None:
if best_rank is None or expected_rank < best_rank:
best_rank = expected_rank
best_member = member
if expected_rank == 0:
source_obj = archive.extractfile(member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
continue
score = _radar_archive_preview_score(member.name, int(member.size or 0))
if score is None:
if best_member is not None and int(member.size or 0) > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
break
continue
candidates.append(score)
member_by_name[member.name] = member
if os.path.basename(str(member.name or "").lower()) in {"quick-look.png", "quicklook.png"}:
source_obj = archive.extractfile(member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
if best_member is not None:
source_obj = archive.extractfile(best_member)
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, best_member.name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
if not candidates:
return None
candidates.sort()
member_name = candidates[0][-1]
source_obj = archive.extractfile(member_by_name[member_name])
if source_obj is None:
return None
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
with source_obj:
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
except Exception:
return None
return None
def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]:
"""Extract coverage polygon from a GeoTIFF file using GDAL.
@@ -380,6 +586,8 @@ class DataService:
@staticmethod
def find_radar_preview_source(scene_dir: str) -> Optional[str]:
if scene_dir and os.path.isfile(scene_dir) and _has_radar_archive_suffix(scene_dir):
return _find_radar_archive_preview_source(scene_dir)
if not scene_dir or not os.path.isdir(scene_dir):
return None
@@ -520,21 +728,26 @@ class DataService:
if orbit_dir and orbit_files_map:
update_progress("正在同步精轨到本地引擎池...", 8)
try:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
sync_result = await asyncio.to_thread(
sync_orbit_pools,
orbit_dir,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_POOL_LANDSAR,
bool(settings.ISCE2_ENABLED),
)
envi_new = len(sync_result.get("envi", {}).get("copied", []))
envi_updated = len(sync_result.get("envi", {}).get("updated", []))
isce2_new = len(sync_result.get("isce2", {}).get("converted", []))
isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", []))
print(
f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated}"
f"ISCE2 转换 {isce2_new}重转 {isce2_updated}"
)
if settings.ISCE2_ENABLED:
print(
f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}刷新 {envi_updated}"
f"ISCE2 XML 转换 {isce2_new}、重转 {isce2_updated}"
)
else:
print(f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated}ISCE2 已停用")
invalid_orbit_stems = {
item.get("name")
for item in sync_result.get("invalid_sources", [])
@@ -550,7 +763,11 @@ class DataService:
sync_errors = (
sync_result.get("source", {}).get("errors", [])
+ [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])]
+ [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
+ (
[item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
if settings.ISCE2_ENABLED
else []
)
)
if task_id and sync_errors:
await task_service.add_log(
@@ -560,15 +777,16 @@ class DataService:
"精轨同步存在异常: "
f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, "
f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, "
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))}"
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', [])) if settings.ISCE2_ENABLED else 0}"
),
)
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
await task_service.add_log(
task_id,
"WARN",
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
)
if settings.ISCE2_ENABLED:
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
await task_service.add_log(
task_id,
"WARN",
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
)
if task_id and invalid_orbit_stems:
await task_service.add_log(
task_id,
@@ -17,10 +17,13 @@ from ..utils import parse_gf3_l2_dirname
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes"
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
SKIP_DIR_NAMES = {
".git",
FLAT_SCENE_CATALOG_DIR_NAME,
".gf3_extract",
".gf3_runtime",
".sarmap",
"__pycache__",
"temp",
@@ -87,6 +90,16 @@ def _polarization_from_geo_name(name: str) -> str | None:
return None
def _scene_name_from_geo_name(name: str) -> str:
text = str(name or "").strip()
match = re.match(r"^(?P<scene>.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE)
if match:
return match.group("scene")
if text.lower().endswith("_geo"):
return text[:-4]
return Path(text).stem
def _is_geo_native_data_file(path: Path) -> bool:
return path.is_file() and path.name.lower().endswith("_geo")
@@ -128,6 +141,62 @@ def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict
return metadata
def _native_asset_from_base(base: Path) -> dict[str, Any]:
hdr = Path(str(base) + ".hdr")
sml = Path(str(base) + ".sml")
aux_xml = Path(str(base) + ".aux.xml")
ovr = Path(str(base) + ".ovr")
kml = Path(str(base) + ".kml")
quicklook = base.with_name(base.name + "_ql.tif")
ql_kml = base.with_name(base.name + "_ql.kml")
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
return {
"polarization": polarization,
"role": "geo_native",
"path": str(base),
"hdr": str(hdr) if hdr.exists() else None,
"sml": str(sml) if sml.exists() else None,
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
"ovr": str(ovr) if ovr.exists() else None,
"quicklook": str(quicklook) if quicklook.exists() else None,
"kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None),
"complete": bool(complete),
"source": _safe_stat(base),
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
"sml_info": _safe_stat(sml) if sml.exists() else None,
}
def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]:
return {
"polarization": str(asset.get("polarization") or "").upper(),
"path": asset.get("path"),
"complete": bool(asset.get("complete")),
"source": asset.get("source"),
"hdr": asset.get("hdr_info"),
"sml": asset.get("sml_info"),
}
def _native_fingerprint(assets: list[dict[str, Any]]) -> str:
payload = [
_asset_fingerprint_part(asset)
for asset in sorted(
assets,
key=lambda item: (
str(item.get("polarization") or ""),
str(item.get("path") or ""),
),
)
]
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
import hashlib
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
assets: list[dict[str, Any]] = []
try:
@@ -139,43 +208,25 @@ def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
if not _is_geo_native_data_file(path):
continue
base = path
hdr = Path(str(base) + ".hdr")
sml = Path(str(base) + ".sml")
aux_xml = Path(str(base) + ".aux.xml")
ovr = Path(str(base) + ".ovr")
kml = Path(str(base) + ".kml")
quicklook = base.with_name(base.name + "_ql.tif")
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
asset: dict[str, Any] = {
"polarization": polarization,
"role": "geo_native",
"path": str(base),
"hdr": str(hdr) if hdr.exists() else None,
"sml": str(sml) if sml.exists() else None,
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
"ovr": str(ovr) if ovr.exists() else None,
"quicklook": str(quicklook) if quicklook.exists() else None,
"kml": str(kml) if kml.exists() else None,
"complete": bool(complete),
"source": _safe_stat(base),
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
"sml_info": _safe_stat(sml) if sml.exists() else None,
}
assets.append(asset)
assets.append(_native_asset_from_base(path))
return assets
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
assets = _collect_native_assets(scene_dir)
def _build_scene_manifest(
root: Path,
*,
scene_name: str,
native_dir: Path,
manifest_dir: Path,
assets: list[dict[str, Any]],
source_dir: Path,
storage_layout: str,
) -> dict[str, Any] | None:
if not assets:
return None
scene_name = scene_dir.name
batch_name = _scene_batch_name(root, scene_dir, scene_name)
batch_name = _scene_batch_name(root, source_dir, scene_name)
complete_assets = [asset for asset in assets if asset.get("complete")]
complete_pols = [
pol
@@ -200,25 +251,30 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
logs = []
for name in ("gf3_sarscape_cli.log",):
log_path = scene_dir / name
log_path = source_dir / name
if log_path.is_file():
logs.append(str(log_path))
try:
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
except OSError:
pass
metadata = _parse_scene_metadata(scene_name, assets)
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
native_fingerprint = _native_fingerprint(assets)
metadata["native_fingerprint"] = native_fingerprint
manifest_path = manifest_dir / NATIVE_MANIFEST_NAME
return {
"schema": NATIVE_MANIFEST_SCHEMA,
"generated_at": _utc_now(),
"scene_name": scene_name,
"batch_name": batch_name,
"native_root": str(root),
"native_dir": str(scene_dir),
"native_dir": str(native_dir),
"source_dir": str(source_dir),
"manifest_path": str(manifest_path),
"source_archive": None,
"storage_layout": storage_layout,
"native_fingerprint": native_fingerprint,
"status": status,
"polarizations": complete_pols,
"metadata": metadata,
@@ -227,6 +283,49 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
}
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
assets = _collect_native_assets(scene_dir)
return _build_scene_manifest(
root,
scene_name=scene_dir.name,
native_dir=scene_dir,
manifest_dir=scene_dir,
assets=assets,
source_dir=scene_dir,
storage_layout="scene_dir",
)
def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]:
try:
entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower())
except OSError:
return []
grouped_assets: dict[str, list[dict[str, Any]]] = {}
for path in entries:
if not _is_geo_native_data_file(path):
continue
scene_name = _scene_name_from_geo_name(path.name)
grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path))
manifests: list[dict[str, Any]] = []
for scene_name in sorted(grouped_assets):
scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name
manifest = _build_scene_manifest(
root,
scene_name=scene_name,
native_dir=scene_catalog_dir,
manifest_dir=scene_catalog_dir,
assets=grouped_assets[scene_name],
source_dir=source_dir,
storage_layout="flat_files",
)
if manifest:
manifests.append(manifest)
return manifests
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
roots: list[Path] = []
missing: list[str] = []
@@ -251,6 +350,7 @@ def scan_gf3_sarscape_native_roots(
native_dirs: list[str] | tuple[str, ...] | None,
*,
write_manifest: bool = True,
scene_dirs_only: bool = False,
) -> dict[str, Any]:
"""Scan configured native roots and return discovered scene manifests."""
roots, missing_roots = _normalize_roots(native_dirs)
@@ -258,12 +358,56 @@ def scan_gf3_sarscape_native_roots(
seen_scene_dirs: set[str] = set()
write_errors: list[dict[str, str]] = []
def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None:
seen_scene_dirs.add(scene_key)
if write_manifest:
try:
_write_json(Path(manifest["manifest_path"]), manifest)
except OSError as exc:
write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)})
scenes.append(manifest)
for root in roots:
if scene_dirs_only:
try:
candidate_dirs = [root] + [
child
for child in sorted(root.iterdir(), key=lambda item: item.name.lower())
if child.is_dir()
and child.name not in SKIP_DIR_NAMES
and not child.name.startswith(".SARscape")
and not child.name.startswith(".gf3_")
]
except OSError:
candidate_dirs = [root]
for scene_dir in candidate_dirs:
scene_key = str(scene_dir).lower()
if scene_key in seen_scene_dirs:
continue
if not scene_dir.name.upper().startswith("GF3_"):
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
if flat_manifests:
for manifest in flat_manifests:
flat_key = str(manifest.get("native_dir") or "").lower()
if flat_key and flat_key not in seen_scene_dirs:
append_manifest(manifest, flat_key, scene_dir)
continue
manifest = _collect_scene_manifest(root, scene_dir)
if not manifest:
continue
append_manifest(manifest, scene_key, scene_dir)
continue
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
and not name.startswith(".gf3_")
]
scene_dir = Path(current_dir)
scene_key = str(scene_dir).lower()
@@ -271,17 +415,20 @@ def scan_gf3_sarscape_native_roots(
dir_names[:] = []
continue
if not scene_dir.name.upper().startswith("GF3_"):
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
if flat_manifests:
for manifest in flat_manifests:
flat_key = str(manifest.get("native_dir") or "").lower()
if flat_key and flat_key not in seen_scene_dirs:
append_manifest(manifest, flat_key, scene_dir)
continue
manifest = _collect_scene_manifest(root, scene_dir)
if not manifest:
continue
seen_scene_dirs.add(scene_key)
if write_manifest:
try:
_write_json(Path(manifest["manifest_path"]), manifest)
except OSError as exc:
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
scenes.append(manifest)
append_manifest(manifest, scene_key, scene_dir)
dir_names[:] = []
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
+737 -9
View File
@@ -15,12 +15,13 @@ from shapely.geometry import Polygon
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..config import settings, split_env_paths
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
from .data_service import extract_geotiff_bounds
from .data_service import DataService, extract_geotiff_bounds
from .gf3_native_inventory_service import (
NATIVE_MANIFEST_NAME,
POLARIZATION_PRIORITY,
SKIP_DIR_NAMES,
scan_gf3_sarscape_native_roots,
)
from .image_service import image_service
@@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
CONVERTER_VERSION = "v1"
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK"
SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2"
def _utc_now() -> str:
@@ -72,6 +76,112 @@ def _path_kind(path: str) -> str:
return "relative"
def _path_key(path: str | Path) -> str:
return os.path.normcase(os.path.normpath(str(path))).lower()
def _has_native_geo_assets(path: Path) -> bool:
try:
return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir())
except OSError:
return False
def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]:
"""Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning.
Operators usually configure a durable GF3 root rather than a date folder.
Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer
direct ``*_geo`` children and fall back to recursive discovery.
"""
roots: list[str] = []
seen: set[str] = set()
def add(path: str | Path) -> None:
text = os.path.normpath(str(path))
key = _path_key(text)
if key and key not in seen:
seen.add(key)
roots.append(text)
for raw in native_dirs or []:
text = str(raw or "").strip()
if not text:
continue
if any(ch in text for ch in "*?"):
glob_root = Path(os.path.normpath(text))
try:
matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower())
except OSError:
matches = []
if matches:
for match in matches:
if match.is_dir():
add(match)
continue
add(glob_root)
continue
root = Path(os.path.normpath(text))
try:
is_dir = root.is_dir()
except OSError:
is_dir = False
if not is_dir:
add(root)
continue
if root.name.lower().endswith("_geo"):
add(root)
try:
immediate_geo_roots = sorted(
(
child
for child in root.iterdir()
if child.is_dir() and child.name.lower().endswith("_geo")
),
key=lambda item: item.name.lower(),
)
except OSError:
immediate_geo_roots = []
if immediate_geo_roots:
for child in immediate_geo_roots:
add(child)
continue
if _has_native_geo_assets(root):
add(root)
continue
found_recursive = 0
for current_dir, dir_names, _file_names in os.walk(root):
dir_names[:] = [
name
for name in dir_names
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
and not name.startswith(".gf3_")
]
selected: list[str] = []
remaining: list[str] = []
for name in dir_names:
if name.lower().endswith("_geo"):
selected.append(name)
else:
remaining.append(name)
for name in selected:
add(Path(current_dir) / name)
found_recursive += len(selected)
dir_names[:] = remaining
if found_recursive == 0:
add(root)
return roots
def _source_asset_uid(path: str) -> str:
normalized = os.path.normpath(str(path or "").strip())
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
@@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool:
return False
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None:
polygon = extract_geotiff_bounds(str(path))
if polygon and len(polygon) >= 4:
return polygon
@@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
return None
def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None:
if not asset:
return None
for key in ("path", "quicklook"):
text = _path_text(asset.get(key))
if not text:
continue
path = Path(text)
if not path.is_file():
continue
polygon = _polygon_from_raster(path)
if polygon and len(polygon) >= 4:
return polygon
return None
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
if not polygon:
return None, None
@@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None
return assets[0] if assets else None
def _is_quicklook_raster_path(path: Any) -> bool:
text = _path_text(path).lower()
return text.endswith("_ql.tif") or text.endswith("_ql.tiff")
def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
candidates = [
asset
for asset in assets
if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path"))
]
complete = [asset for asset in candidates if asset.get("complete") is not False]
return _select_default_asset(complete or candidates)
def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]:
output_assets: list[dict[str, Any]] = []
for asset in scene_manifest.get("assets") or []:
native_path = _path_text(asset.get("path"))
if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True:
continue
output_assets.append(
{
"polarization": str(asset.get("polarization") or "UNKNOWN").upper(),
"role": "native_geo",
"path": native_path,
"hdr": _path_text(asset.get("hdr")) or None,
"sml": _path_text(asset.get("sml")) or None,
"quicklook": _path_text(asset.get("quicklook")) or None,
"complete": bool(asset.get("complete")),
"status": "registered_native_geo",
}
)
return output_assets
def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]:
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
out_dir = _standard_scene_dir(scene_manifest, root)
manifest_path = out_dir / "gf3_native_preview_manifest.json"
assets = _quicklook_assets_from_scene_manifest(scene_manifest)
manifest = {
"schema": "gf3_sarscape_native_preview.v1",
"generated_at": _utc_now(),
"scene_name": scene_manifest.get("scene_name"),
"batch_name": scene_manifest.get("batch_name"),
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
"native_dir": scene_manifest.get("native_dir"),
"standard_dir": str(out_dir),
"manifest_path": str(manifest_path),
"status": "NATIVE_READY" if assets else "FAILED",
"assets": assets,
"summary": {
"native_assets": len(assets),
"quicklook_assets": 0,
"full_raster_materialized": True,
"webp_source": "native_geo",
},
"errors": [] if assets else [{"error": "no native _geo assets found"}],
}
_write_json(manifest_path, manifest)
return manifest
def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool:
"""Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``."""
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except Exception:
return False
try:
target = Path(target_path)
target.parent.mkdir(parents=True, exist_ok=True)
limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600)
with rasterio.open(source_path) as src:
scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0)
out_width = max(1, int(src.width * scale))
out_height = max(1, int(src.height * scale))
band = src.read(
1,
out_shape=(out_height, out_width),
masked=True,
resampling=Resampling.bilinear,
)
data = band.filled(0)
if data.dtype != np.uint8:
valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)]
if valid.size:
p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98])
scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1)
data = (scaled * 255).astype("uint8")
else:
data = np.zeros(data.shape, dtype="uint8")
else:
data = data.astype("uint8", copy=False)
mask = getattr(band, "mask", None)
if mask is None or np.ndim(mask) == 0:
alpha = np.full(data.shape, 255, dtype="uint8")
else:
alpha = np.where(mask, 0, 255).astype("uint8")
rgba = np.stack([data, data, data, alpha], axis=-1)
image = Image.fromarray(rgba, "RGBA")
image = image_service.make_edge_dark_transparent(image)
image_service.save_image_as_webp(image, target_path, quality=82)
return target.exists() and target.stat().st_size > 0
except Exception:
return False
def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str:
candidates: list[Any] = [metadata.get("default_native_path")]
for key in ("native_assets", "quicklook_assets"):
for asset in metadata.get(key) or []:
if isinstance(asset, dict):
candidates.append(asset.get("path"))
for candidate in candidates:
text = _path_text(candidate)
if text and not _is_quicklook_raster_path(text):
return text
return ""
def _path_is_under_any(path: str, roots: list[str]) -> bool:
target = _path_key(path)
if not target:
return False
for root in roots:
root_key = _path_key(root)
if root_key and (target == root_key or target.startswith(root_key + os.sep)):
return True
return False
def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]:
roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if not roots:
roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
return [os.path.normpath(root) for root in roots if str(root or "").strip()]
def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str:
return str(
scene_manifest.get("native_fingerprint")
or (scene_manifest.get("metadata") or {}).get("native_fingerprint")
or ""
).strip()
def _record_has_bounds(record: RadarDataORM) -> bool:
return (
record.coverage_polygon is not None
and record.min_lon is not None
and record.min_lat is not None
and record.max_lon is not None
and record.max_lat is not None
)
async def _find_existing_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> RadarDataORM | None:
native_dir = str(scene_manifest.get("native_dir") or "")
scene_name = scene_manifest.get("scene_name") or Path(native_dir).name
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
return result.scalars().first()
async def _can_skip_unchanged_quicklook_radar(
db: AsyncSession,
*,
scene_manifest: dict[str, Any],
) -> tuple[bool, int | None, str]:
native_fingerprint = _scene_native_fingerprint(scene_manifest)
if not native_fingerprint:
return False, None, "missing_native_fingerprint"
record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest)
if record is None:
return False, None, "missing_record"
metadata = record.metadata_json or {}
if str(metadata.get("native_fingerprint") or "") != native_fingerprint:
return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed"
if str(metadata.get("registration_mode") or "") != "native_preview":
return False, int(record.id) if record.id is not None else None, "registration_mode_changed"
if not _record_has_bounds(record):
return False, int(record.id) if record.id is not None else None, "missing_bounds"
return True, int(record.id) if record.id is not None else None, "unchanged"
async def generate_gf3_quicklook_webp_cache(
db: AsyncSession,
*,
force: bool = False,
max_records: int | None = None,
native_dirs: list[str] | tuple[str, ...] | None = None,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters."""
stmt = (
select(RadarDataORM)
.where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT)
.order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc())
)
result = await db.execute(stmt)
configured_roots = _configured_gf3_native_roots(native_dirs)
records = [
record
for record in result.scalars().all()
if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots)
]
if max_records and max_records > 0:
records = records[: int(max_records)]
total = len(records)
generated = 0
skipped = 0
failed = 0
record_results: list[dict[str, Any]] = []
for idx, record in enumerate(records):
metadata = record.metadata_json or {}
source_path = _native_preview_source_from_metadata(metadata)
cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path)
if progress_callback:
pct = 5 + int((idx / max(total, 1)) * 90)
progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}")
if (
not force
and (record.preview_cache_status or "NONE") == "READY"
and record.preview_cache_path
and os.path.exists(record.preview_cache_path)
):
skipped += 1
record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path})
continue
if not source_path:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "default_native_path_missing"
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
continue
ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path)
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
record.preview_cache_updated_at = _db_now()
if ok and os.path.exists(cache_path):
generated += 1
record.preview_cache_status = "READY"
record.preview_cache_path = cache_path
record.preview_cache_error = None
record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path})
else:
failed += 1
record.preview_cache_status = "FAILED"
record.preview_cache_path = None
record.preview_cache_error = "native_webp_build_failed"
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
await db.commit()
return {
"ok": failed == 0,
"mode": "gf3_native_webp",
"total": total,
"generated": generated,
"skipped": skipped,
"failed": failed,
"records": record_results,
}
async def _upsert_quicklook_source_product_asset(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
) -> int | None:
native_dir_text = _path_text(scene_manifest.get("native_dir"))
if not native_dir_text:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name
now = _db_now()
root = await _find_managed_root_for_path(db, native_dir_text)
asset_metadata = _json_safe(
{
"source": "GF3 SARscape native _geo",
"native_dir": native_dir_text,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": quicklook_manifest.get("assets") or [],
"full_raster_materialized": True,
"analysis_engine": "gf3_sarscape",
}
)
data = {
"asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"),
"logical_product_uid": scene_name,
"satellite_family": "GF3",
"satellite": "GF3",
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"product_type": metadata.get("product_type") or "SARSCAPE_NATIVE",
"product_level": "L2_NATIVE",
"imaging_mode": metadata.get("imaging_mode"),
"polarization": metadata.get("polarization"),
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"orbit_direction": metadata.get("orbit_direction"),
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"imaging_date": imaging_date,
"root_ref_id": root.id if root else None,
"root_path": root.path if root else str(Path(native_dir_text).parent),
"file_path": native_dir_text,
"archive_path": scene_manifest.get("source_archive"),
"path_kind": _path_kind(native_dir_text),
"file_name": Path(native_dir_text).name,
"file_stem": Path(native_dir_text).name,
"file_ext": "",
"size_bytes": None,
"mtime_epoch": None,
"checksum_status": "NOT_COMPUTED",
"parser_name": "gf3_sarscape_native_preview_manifest",
"parser_version": CONVERTER_VERSION,
"parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED",
"parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None,
"parsed_at": now,
"metadata_json": asset_metadata,
"is_active": True,
"missing_since": None,
"updated_at": now,
}
result = await db.execute(
select(SourceProductAssetORM).where(
or_(
SourceProductAssetORM.asset_uid == data["asset_uid"],
SourceProductAssetORM.file_path == native_dir_text,
)
)
)
asset = result.scalars().first()
if asset is None:
asset = SourceProductAssetORM(**data)
db.add(asset)
else:
for key, value in data.items():
setattr(asset, key, value)
await db.flush()
return int(asset.id) if asset.id is not None else None
async def _upsert_quicklook_radar_data(
db: AsyncSession,
scene_manifest: dict[str, Any],
quicklook_manifest: dict[str, Any],
source_product_ref_id: int | None = None,
) -> int | None:
assets = quicklook_manifest.get("assets") or []
default_asset = _select_default_native_asset(assets)
if not default_asset:
return None
metadata = scene_manifest.get("metadata") or {}
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
acquisition_start = _date_to_naive_utc(imaging_date)
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
native_dir = str(scene_manifest.get("native_dir") or "")
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
default_native_path = str(default_asset.get("path") or "")
default_quicklook_path = str(default_asset.get("quicklook") or "")
polygon = _polygon_from_native_asset(default_asset)
min_lon = min_lat = max_lon = max_lat = None
geom = None
if polygon:
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
else:
center_lon = metadata.get("scene_center_lon")
center_lat = metadata.get("scene_center_lat")
preview_cache_path = None
preview_cache_kind = "deferred"
preview_cache_error = "native_webp_not_generated"
radar_metadata = _json_safe(
{
**metadata,
"native_dir": native_dir,
"native_manifest": scene_manifest.get("manifest_path"),
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"native_assets": assets,
"quicklook_assets": [],
"default_native_path": default_native_path,
"default_quicklook_path": default_quicklook_path,
"coverage_source": "native_geo_or_quicklook" if polygon else "scene_name",
"preview_cache_kind": preview_cache_kind,
"analysis_engine": "gf3_sarscape",
"registration_mode": "native_preview",
"full_raster_materialized": True,
}
)
data_to_upsert = {
"unique_id": unique_id,
"satellite": "GF3",
"satellite_family": "GF3",
"imaging_date": imaging_date,
"imaging_mode": metadata.get("imaging_mode"),
"polarization": ",".join(
pol
for pol in POLARIZATION_PRIORITY
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
)
or metadata.get("polarization"),
"scene_center_lon": center_lon,
"scene_center_lat": center_lat,
"acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None,
"product_level": "L2_NATIVE",
"product_unique_id": metadata.get("product_unique_id") or scene_name,
"source_product_token": scene_name,
"acquisition_start_time_utc": acquisition_start,
"acquisition_stop_time_utc": None,
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
"relative_orbit": metadata.get("relative_orbit"),
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
"source_product_ref_id": source_product_ref_id,
"image_data_format": "ENVI_NATIVE",
"geocoded_flag": True,
"metadata_json": radar_metadata,
"file_path": native_dir,
"has_orbit_data": False,
"orbit_file_path": None,
"is_envi_processed": True,
"coverage_polygon": polygon,
"geom": geom,
"min_lon": min_lon,
"min_lat": min_lat,
"max_lon": max_lon,
"max_lat": max_lat,
"preview_cache_status": "NONE",
"preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION,
"preview_cache_path": preview_cache_path,
"preview_cache_updated_at": _db_now(),
"preview_cache_error": preview_cache_error,
}
result = await db.execute(
select(RadarDataORM).where(
or_(
RadarDataORM.unique_id == unique_id,
RadarDataORM.file_path == native_dir,
)
)
)
radar = result.scalars().first()
if (
radar is not None
and (radar.preview_cache_status or "").upper() == "READY"
and radar.preview_cache_path
and os.path.exists(radar.preview_cache_path)
and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path
):
data_to_upsert["preview_cache_status"] = radar.preview_cache_status
data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION
data_to_upsert["preview_cache_path"] = radar.preview_cache_path
data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at
data_to_upsert["preview_cache_error"] = radar.preview_cache_error
if radar is None:
radar = RadarDataORM(**data_to_upsert)
db.add(radar)
else:
for key, value in data_to_upsert.items():
setattr(radar, key, value)
await db.flush()
return int(radar.id) if radar.id is not None else None
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
metadata = dict(scene_manifest.get("metadata") or {})
metadata.update(
@@ -635,7 +1260,7 @@ async def _upsert_radar_data(
if not default_asset:
return None
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path"))))
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
center_lon, center_lat = _scene_center_from_polygon(polygon)
geom = _geom_from_polygon(polygon)
@@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots(
storage_root: str | None = None,
force: bool = False,
register: bool = True,
quicklook_only: bool = False,
progress_callback: Any | None = None,
) -> dict[str, Any]:
"""Scan native roots, convert complete assets, and register standard scenes."""
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
native_dirs,
write_manifest=True,
)
requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
if quicklook_only:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
else:
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
inventory = await asyncio.to_thread(
scan_gf3_sarscape_native_roots,
scan_roots,
write_manifest=True,
)
scenes = inventory.get("scenes") or []
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
if quicklook_only:
registered = 0
quicklook_assets = 0
failed_scenes = 0
skipped_unchanged = 0
scene_results: list[dict[str, Any]] = []
total = len(ready_scenes)
for idx, scene_manifest in enumerate(ready_scenes):
if progress_callback:
pct = 10 + int((idx / max(total, 1)) * 80)
progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
quicklook_manifest = await asyncio.to_thread(
_quicklook_manifest_for_scene,
scene_manifest,
storage_root,
)
assets = quicklook_manifest.get("assets") or []
quicklook_assets += len(assets)
source_asset_id = None
radar_id = None
skipped_reason = None
if register and assets:
can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar(
db,
scene_manifest=scene_manifest,
)
if can_skip:
skipped_unchanged += 1
radar_id = existing_radar_id
skipped_reason = skip_reason
if progress_callback:
pct = 10 + int(((idx + 1) / max(total, 1)) * 80)
progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
else:
skipped_reason = skip_reason
source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest)
radar_id = await _upsert_quicklook_radar_data(
db,
scene_manifest,
quicklook_manifest,
source_product_ref_id=source_asset_id,
)
if radar_id:
registered += 1
await db.commit()
if not assets:
failed_scenes += 1
scene_results.append(
{
"scene_name": scene_manifest.get("scene_name"),
"native_status": scene_manifest.get("status"),
"standard_status": quicklook_manifest.get("status"),
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
"source_asset_id": source_asset_id,
"radar_id": radar_id,
"skipped_unchanged": bool(skipped_reason == "unchanged"),
"skip_reason": skipped_reason,
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
"summary": quicklook_manifest.get("summary") or {},
"errors": quicklook_manifest.get("errors") or [],
}
)
return {
"ok": failed_scenes == 0,
"mode": "native_preview",
"inventory": {
key: value
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": 0,
"partial_scenes": 0,
"failed_scenes": failed_scenes,
"converted_assets": 0,
"skipped_assets": skipped_unchanged,
"skipped_unchanged": skipped_unchanged,
"failed_assets": failed_scenes,
"quicklook_assets": quicklook_assets,
"native_assets": quicklook_assets,
"registered": registered,
"analysis_ready": 0,
"scenes": scene_results,
}
converted_scenes = 0
partial_scenes = 0
failed_scenes = 0
@@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots(
for key, value in inventory.items()
if key != "scenes"
},
"requested_native_dirs": requested_native_dirs,
"scan_roots": scan_roots,
"scene_count": len(scenes),
"ready_scene_count": len(ready_scenes),
"converted_scenes": converted_scenes,
+130 -161
View File
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
from sqlalchemy import select
from .. import database
from ..config import settings
from ..config import settings, split_env_paths
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job
from .data_service import data_service
@@ -88,6 +88,7 @@ JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
JOB_TYPE_GF3_QUICKLOOK_WEBP = "GF3_QUICKLOOK_WEBP"
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
@@ -97,6 +98,7 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY"
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM"
JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS"
@@ -253,9 +255,12 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
families=payload.get("families") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
build_previews=bool(payload.get("build_previews", True)),
task_id=job.task_id,
)
preview = result.get("preview_cache") or {}
await task_service.update_task(
job.task_id,
status="COMPLETED",
@@ -265,7 +270,42 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
f"sources={result.get('source_assets', 0)}, "
f"orbits={result.get('orbit_assets', 0)}, "
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
f"missing={((result.get('binding') or {}).get('missing_count', 0))}, "
f"previews_ready={preview.get('ready', 0)}, "
f"previews_skipped={preview.get('skipped_ready', 0)}, "
f"previews_failed={preview.get('failed', 0)}, "
f"previews_missing={preview.get('missing_source', 0)}"
),
db=db,
)
async def _handle_archive_integrity_audit(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("AUDIT_SOURCE_ARCHIVE_INTEGRITY requires task_id for progress tracking.")
payload = job.payload or {}
await task_service.start_task(job.task_id, message="Source archive integrity audit started")
async with AsyncSessionLocal() as db:
result = await asset_inventory_service.audit_source_archive_integrity(
db,
families=payload.get("families") or None,
source_formats=payload.get("source_formats") or None,
asset_ids=payload.get("asset_ids") or None,
force=bool(payload.get("force", False)),
limit=payload.get("limit"),
task_id=job.task_id,
)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"Source archive integrity audit completed: "
f"checked={result.get('checked', 0)}, "
f"skipped={result.get('skipped', 0)}, "
f"ok={result.get('ok', 0)}, "
f"failed={result.get('failed', 0)}, "
f"unsupported={result.get('unsupported', 0)}"
),
db=db,
)
@@ -345,7 +385,7 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses"))
include_orbit_files = bool(payload.get("include_orbit_files"))
export_zip = bool(payload.get("export_zip"))
package_mode = str(payload.get("package_mode") or ("task_zip" if export_zip else "task_folder")).strip().lower()
package_mode = str(payload.get("package_mode") or ("source_bundle" if export_zip else "task_folder")).strip().lower()
skip_existing = payload.get("skip_existing") is not False
max_items = _normalize_positive_int(payload.get("max_items"))
@@ -445,17 +485,27 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
skip_existing=skip_existing,
max_items=max_items,
)
else:
return
if package_mode in {"task_folder", "task"}:
await run_dinsar_copy_items(
job.task_id,
items,
dest_dir,
include_orbit_files=include_orbit_files,
export_zip=(package_mode == "task_zip" or export_zip),
export_zip=False,
skip_existing=skip_existing,
max_items=max_items,
)
return
return
if package_mode == "task_zip":
raise ValueError(
"D-InSAR Task ZIP export is not a production-preparation format. "
"Use package_mode=task_folder for Task_Pool preparation or source_bundle for archive distribution."
)
raise ValueError(
"Unsupported D-InSAR package_mode. "
"Use task_folder for production preparation or source_bundle for archive distribution."
)
raise ValueError(f"Unknown COPY_DATA file_type: {file_type}")
@@ -884,7 +934,9 @@ async def _handle_ai_diagnosis(job: SystemJobORM) -> None:
from ..ai_service import analyze_map_with_vlm
diagnosis_markdown = await analyze_map_with_vlm(
images_base64=[img_base64],
prompt=full_prompt
prompt=full_prompt,
model_name=model_name,
raise_on_error=True,
)
# 8. 解析风险等级和置信度(简单正则匹配)
@@ -3646,7 +3698,7 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
@@ -3725,7 +3777,7 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
if not job.task_id:
@@ -3792,7 +3844,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
@@ -3901,7 +3953,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
"""Scan SARscape native GF3 _geo outputs and register requested assets."""
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
if not job.task_id:
@@ -3915,7 +3967,15 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
if not storage_root:
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
quicklook_only = bool(payload.get("quicklook_only", False))
await task_service.start_task(
job.task_id,
message=(
"按 GF3 日期/场景命名规则登记本机 SARscape _geo 原生结果..."
if quicklook_only
else "递归扫描 GF3 SARscape 原生 _geo 结果池..."
),
)
loop = asyncio.get_running_loop()
@@ -3943,6 +4003,7 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
storage_root=storage_root,
force=bool(payload.get("force", False)),
register=bool(payload.get("register", True)),
quicklook_only=quicklook_only,
progress_callback=_progress_cb,
)
except Exception as exc:
@@ -3954,44 +4015,44 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
)
raise
message = (
"GF3 SARscape 标准化完成: "
f"发现 {int(result.get('scene_count') or 0)} 景, "
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
f"部分 {int(result.get('partial_scenes') or 0)} , "
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} , "
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)}"
)
if quicklook_only:
message = (
"GF3 _geo 原生结果登记完成: "
f"规则匹配 {int(result.get('scene_count') or 0)} 景, "
f"可登记 {int(result.get('ready_scene_count') or 0)} 景, "
f"原生资产 {int(result.get('native_assets') or result.get('quicklook_assets') or 0)} , "
f"入库 {int(result.get('registered') or 0)} 景, "
f"跳过未变化 {int(result.get('skipped_unchanged') or 0)} , "
f"失败 {int(result.get('failed_scenes') or 0)} "
)
else:
message = (
"GF3 SARscape 标准化完成: "
f"发现 {int(result.get('scene_count') or 0)} 景, "
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
f"入库 {int(result.get('registered') or 0)}"
)
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
from .gf3_sarscape_production_service import (
cleanup_gf3_sarscape_native_pool,
run_gf3_sarscape_production,
)
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
async def _handle_gf3_quicklook_webp(job: SystemJobORM) -> None:
"""Generate local WebP previews from registered GF3 SARscape native _geo records."""
from .gf3_standardize_service import generate_gf3_quicklook_webp_cache
if not job.task_id:
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
raise ValueError("GF3_QUICKLOOK_WEBP requires task_id for progress tracking.")
payload = job.payload or {}
source_dirs = payload.get("source_dirs") or []
native_dirs = payload.get("native_dirs") or []
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
if not source_dirs:
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
if not native_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
if not storage_root:
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
force = bool(payload.get("force", False))
max_records = _normalize_positive_int(payload.get("max_records"))
native_dirs = payload.get("native_dirs") or split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
await task_service.start_task(job.task_id, message="开始从 GF3 _geo 原生数据生成 WebP 缓存...")
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
loop = asyncio.get_running_loop()
def _progress_cb(progress: int, message: str) -> None:
@@ -4005,143 +4066,49 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
logger.warning("[GF3 native WebP] progress callback failed: %s", exc)
future.add_done_callback(_swallow_progress_error)
except RuntimeError:
return
def _log_cb(level: str, message: str) -> None:
try:
future = asyncio.run_coroutine_threadsafe(
task_service.add_log(job.task_id, level, message),
loop,
)
def _swallow_log_error(fut):
try:
fut.result()
except Exception as exc:
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
future.add_done_callback(_swallow_log_error)
except RuntimeError:
return
async def _production_keepalive() -> None:
progress = 8
while True:
await asyncio.sleep(60)
progress = min(68, progress + 1)
await task_service.update_task(
job.task_id,
progress=progress,
message="GF3 SARscape production is still running...",
)
try:
production_task = asyncio.create_task(
asyncio.to_thread(
run_gf3_sarscape_production,
source_dirs=source_dirs,
native_root=native_root,
wrapper_exe=payload.get("wrapper_exe"),
dem_path=payload.get("dem_path"),
idlrt_path=payload.get("idlrt_path"),
polarizations=payload.get("polarizations"),
archive_exts=payload.get("archive_exts") or [],
max_archives_per_run=payload.get("max_archives_per_run"),
selected_dates=payload.get("selected_dates") or [],
task_id=job.task_id,
local_staging_root=payload.get("local_staging_root") or settings.GF3_TASK_POOL_ROOT,
timeout_seconds=payload.get("timeout_seconds"),
keep_extracted=payload.get("keep_extracted"),
log_callback=_log_cb,
async with AsyncSessionLocal() as db:
result = await generate_gf3_quicklook_webp_cache(
db,
force=force,
max_records=max_records,
native_dirs=native_dirs,
progress_callback=_progress_cb,
)
)
keepalive_task = asyncio.create_task(_production_keepalive())
try:
production_result = await production_task
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
standardize_result: Dict[str, Any] = {}
if bool(payload.get("auto_standardize", True)):
await task_service.update_task(
job.task_id,
progress=72,
message="GF3 SARscape production finished; standardizing native _geo outputs...",
)
async with AsyncSessionLocal() as db:
standardize_result = await standardize_gf3_sarscape_native_roots(
db,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
force=bool(payload.get("force_standardize", False)),
register=bool(payload.get("register", True)),
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
)
cleanup_result: Dict[str, Any] = {}
production_ok = int(production_result.get("failed_count") or 0) == 0
standardize_ok = (
not standardize_result
or (
int(standardize_result.get("failed_assets") or 0) == 0
and int(standardize_result.get("failed_scenes") or 0) == 0
)
)
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
await task_service.update_task(
job.task_id,
progress=90,
message="Cleaning GF3 SARscape intermediate files...",
)
cleanup_result = await asyncio.to_thread(
cleanup_gf3_sarscape_native_pool,
native_dirs=native_dirs or [native_root],
storage_root=storage_root,
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
dry_run=bool(payload.get("cleanup_dry_run", False)),
max_scenes=payload.get("cleanup_max_scenes"),
log_callback=_log_cb,
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
)
elif bool(payload.get("clean_after_success", True)):
_log_cb(
"WARNING",
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
)
except Exception as exc:
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=f"GF3 SARscape production failed: {exc}",
message=f"GF3 _geo WebP 生成失败: {exc}",
)
raise
failed_count = int(production_result.get("failed_count") or 0)
failed_assets = int(standardize_result.get("failed_assets") or 0)
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
message = (
"GF3 SARscape production chain finished: "
f"found={int(production_result.get('found_count') or 0)}, "
f"produced={int(production_result.get('processed_count') or 0)}, "
f"skipped={int(production_result.get('skipped_count') or 0)}, "
f"failed={failed_count}, "
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
f"registered={int(standardize_result.get('registered') or 0)}, "
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
"GF3 _geo WebP 生成完成: "
f"总数 {int(result.get('total') or 0)}, "
f"生成 {int(result.get('generated') or 0)}, "
f"跳过 {int(result.get('skipped') or 0)}, "
f"失败 {int(result.get('failed') or 0)} "
)
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
"""Reject GF3 production jobs on this management machine."""
message = (
"GF3 SARscape production is disabled on this management machine. "
"Run production on the SARscape host and register local _geo native results here."
)
if job.task_id:
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=message)
raise ValueError(message)
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
@@ -5263,6 +5230,7 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None:
_HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit,
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
@@ -5301,6 +5269,7 @@ _HANDLERS = {
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
JOB_TYPE_GF3_QUICKLOOK_WEBP: _handle_gf3_quicklook_webp,
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
+71 -6
View File
@@ -24,6 +24,8 @@ from ..dinsar_engines.landsar_engine import (
_norm_path,
_path_search_dirs,
_summarize_landsar_failure,
derive_landsar_dem_bbox_from_xml_paths,
prepare_landsar_dem_crop,
)
@@ -633,6 +635,16 @@ class LandsarSbasService:
"errors": errors,
}
def _derive_task_dem_bbox(self, task: dict[str, Any], input_dir: str) -> tuple[float, float, float, float] | None:
xml_paths = [
str(scene.get("xml") or "")
for scene in (task.get("scenes") or [])
if str(scene.get("xml") or "").strip()
]
if not xml_paths and input_dir and os.path.isdir(input_dir):
xml_paths = [str(path) for path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower())]
return derive_landsar_dem_bbox_from_xml_paths(xml_paths, fallback_paths=xml_paths)
def materialize_stack(
self,
*,
@@ -949,7 +961,7 @@ class LandsarSbasService:
params_payload = self.normalize_params(params or {})
normalized_dem = _norm_path(dem_path or self.default_dem_path)
if not normalized_dem or not os.path.isfile(normalized_dem):
raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or '<empty>'}")
raise ValueError(f"LandSAR SBAS DEM source file is missing: {normalized_dem or '<empty>'}")
from .sbas_insar_production_service import sbas_insar_production_service
@@ -1035,6 +1047,8 @@ class LandsarSbasService:
"input_task_root": import_dest_root,
"publish_root": str(publish_root),
"dem_path": normalized_dem,
"dem_source_path": normalized_dem,
"dem_role": "global_prepared_source",
"params": params_payload,
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
"import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
@@ -1541,9 +1555,9 @@ class LandsarSbasService:
if not bool(settings.LANDSAR_SBAS_ENABLED):
raise ValueError("LandSAR SBAS is disabled.")
params = self.normalize_params(extra or {})
dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
if not dem_path or not os.path.isfile(dem_path):
raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or '<empty>'}")
dem_source_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
if not dem_source_path or not os.path.isfile(dem_source_path):
raise ValueError(f"LandSAR SBAS DEM source file is missing: {dem_source_path or '<empty>'}")
validation = self.validate_root_dir(
root_dir,
min_scenes=min_scenes,
@@ -1585,7 +1599,9 @@ class LandsarSbasService:
"work_root_strategy": "short_landsar_execution_path",
"native_root": str(native_root),
"publish_root": str(publish_root),
"dem_path": dem_path,
"dem_path": dem_source_path,
"dem_source_path": dem_source_path,
"dem_role": "global_prepared_source",
"params": params,
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
"min_scenes": validation.get("min_scenes"),
@@ -1716,10 +1732,56 @@ class LandsarSbasService:
native_output_dir = native_root / task_alias / "Output_Data"
native_output_dir.mkdir(parents=True, exist_ok=True)
project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias
dem_source_path = _norm_path(str(manifest.get("dem_path") or ""))
effective_dem_path = dem_source_path
dem_crop_info: dict[str, Any] = {}
try:
dem_bbox = self._derive_task_dem_bbox(task, input_dir)
if not dem_bbox:
raise ValueError("cannot derive DEM crop bbox from LandSAR SBAS Input_Data XML corner coordinates")
dem_crop_info = prepare_landsar_dem_crop(
dem_source_path,
str(native_root / task_alias / "dem_crop"),
dem_bbox,
label=task_alias,
)
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
self._emit(
progress_callback,
"INFO",
f"[{index}/{len(tasks)}] LandSAR SBAS DEM crop ready: {effective_dem_path}",
)
except Exception as exc:
failed_count += 1
error = f"LandSAR SBAS DEM crop failed: {exc}"
self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] {task_name} failed: {error}")
task_results.append(
{
"task_name": task_name,
"task_alias": task_alias,
"input_data_dir": input_dir,
"native_output_dir": str(native_output_dir),
"param_file": "",
"process_name": SBAS_PROCESS_NAME,
"command": "",
"returncode": -2,
"success": False,
"timed_out": False,
"failure_kind": "dem_crop_failed",
"error": error,
"stdout_tail": "",
"native_logs": {},
"publish": {},
"dem_source_path": dem_source_path,
"dem_path": effective_dem_path,
"dem_crop": dem_crop_info,
}
)
continue
param_file = _generate_sbas_param_file(
str(native_output_dir / f"{SBAS_PROID}.txt"),
slc_folder=input_dir,
dem_path=str(manifest.get("dem_path") or ""),
dem_path=effective_dem_path,
output_dir=str(native_output_dir),
project_name=project_name,
params=dict(manifest.get("params") or {}),
@@ -1784,6 +1846,9 @@ class LandsarSbasService:
"stdout_tail": _collect_tail(stdout_text, 4000),
"native_logs": log_publish_result,
"publish": publish_result,
"dem_source_path": dem_source_path,
"dem_path": effective_dem_path,
"dem_crop": dem_crop_info,
}
)
+40 -14
View File
@@ -1,7 +1,7 @@
"""轨道文件管理与格式转换服务。
目录约定
MONITOR_ORBIT_DIR/ 源精轨目录可为 UNC平铺 .txt
MONITOR_ORBIT_DIR/ 本机源精轨目录平铺 .txt
ORBIT_POOL_ENVI/ ENVI 本地精轨池
LT1A/ LT-1A 精轨 .txt
LT1B/ LT-1B 精轨 .txt
@@ -320,16 +320,17 @@ def summarize_source_orbit_gaps(
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
isce2_enabled = bool(str(isce2_pool or "").strip())
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
source_stems = set(source_files)
envi_stems = set(envi_files)
isce2_stems = set(isce2_files)
source_without_isce2 = sorted(source_stems - isce2_stems)
source_without_isce2 = sorted(source_stems - isce2_stems) if isce2_enabled else []
source_without_envi = sorted(source_stems - envi_stems)
envi_without_source = sorted(envi_stems - source_stems)
isce2_without_source = sorted(isce2_stems - source_stems)
isce2_without_source = sorted(isce2_stems - source_stems) if isce2_enabled else []
sample_limit = 20
def _make_samples(
@@ -349,7 +350,7 @@ def summarize_source_orbit_gaps(
)
return samples
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) if isce2_enabled else []
bad_source_samples = [
item for item in suspect_bad_samples
if item.get("has_corruption_signal")
@@ -358,6 +359,7 @@ def summarize_source_orbit_gaps(
return {
"sample_limit": sample_limit,
"quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
"isce2_enabled": isce2_enabled,
"source_without_isce2_count": len(source_without_isce2),
"source_without_envi_count": len(source_without_envi),
"envi_without_source_count": len(envi_without_source),
@@ -809,6 +811,7 @@ def sync_orbit_pools(
envi_pool: str = "",
isce2_pool: str = "",
landsar_pool: str = "",
validate_without_isce2: bool = True,
) -> Dict:
"""从 source_dir 扫描平铺 .txt 精轨,同步到各引擎本地池。
@@ -863,11 +866,16 @@ def sync_orbit_pools(
xml_needs_refresh = bool(isce2_pool) and (
not os.path.exists(xml_path) or _needs_generated_refresh(fpath, xml_path)
)
requires_validation = xml_needs_refresh or (not isce2_pool and envi_needs_refresh)
requires_validation = xml_needs_refresh or (
bool(validate_without_isce2) and not isce2_pool and envi_needs_refresh
)
staged_dir = ""
staged_xml = ""
try:
health = _inspect_orbit_txt_health(fpath)
if _has_orbit_corruption_signal(health):
raise RuntimeError("Source TXT contains NUL bytes or cannot be read as a healthy LT-1 orbit text file.")
if requires_validation:
staged_dir, staged_xml = _stage_isce2_xml(
fpath,
@@ -924,8 +932,15 @@ def repair_orbit_pools(
isce2_pool: str = "",
landsar_pool: str = "",
) -> Dict[str, Any]:
isce2_enabled = bool(str(isce2_pool or "").strip())
before = check_orbit_consistency(envi_pool, isce2_pool)
sync_result = sync_orbit_pools(source_dir, envi_pool, isce2_pool, landsar_pool)
sync_result = sync_orbit_pools(
source_dir,
envi_pool,
isce2_pool,
landsar_pool,
validate_without_isce2=isce2_enabled,
)
repaired_from_envi: List[str] = []
repair_errors: List[Dict[str, str]] = []
@@ -934,7 +949,7 @@ def repair_orbit_pools(
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
if isce2_pool:
if isce2_enabled:
os.makedirs(isce2_pool, exist_ok=True)
for stem in sorted(set(envi_files) - set(isce2_files)):
txt_path = envi_files[stem]["path"]
@@ -952,6 +967,7 @@ def repair_orbit_pools(
"before": before,
"after": after,
"sync_result": sync_result,
"isce2_enabled": isce2_enabled,
"repaired_from_envi": repaired_from_envi,
"repair_error_count": len(repair_errors),
"repair_errors": repair_errors,
@@ -968,17 +984,19 @@ def quarantine_bad_orbits(
isce2_pool: str = "",
quarantine_root: str = "",
) -> Dict[str, Any]:
isce2_enabled = bool(str(isce2_pool or "").strip())
source_inventory = get_source_orbit_inventory(source_dir, recursive=True)
pool_inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
quarantine_dir = _default_quarantine_root(source_dir, quarantine_root)
candidate_stems = sorted(set(source_files) - set(isce2_files))
candidate_stems = sorted(set(source_files) - set(isce2_files)) if isce2_enabled else []
result: Dict[str, Any] = {
"isce2_enabled": isce2_enabled,
"quarantine_root": quarantine_dir,
"candidate_count": len(candidate_stems),
"validated_count": 0,
@@ -1083,14 +1101,18 @@ def check_orbit_consistency(
"healthy": bool,
}
"""
isce2_enabled = bool(str(isce2_pool or "").strip())
inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
envi_files: Dict[str, Dict[str, str]] = inventory["envi"]["files"]
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"]
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] if isce2_enabled else {}
by_satellite: Dict[str, int] = inventory["envi"]["by_satellite"]
# 对比
envi_stems = set(envi_files)
isce2_stems = set(isce2_files)
if not isce2_enabled:
envi_stems = set()
isce2_stems = set()
mismatches = []
for stem in envi_stems - isce2_stems:
mismatches.append(
@@ -1112,6 +1134,9 @@ def check_orbit_consistency(
)
mismatches.sort(key=lambda item: item["name"])
errors = list(inventory["envi"]["errors"])
if isce2_enabled:
errors.extend(inventory["isce2"]["errors"])
return {
"envi": {
@@ -1121,11 +1146,12 @@ def check_orbit_consistency(
},
"isce2": {
"path": isce2_pool,
"enabled": isce2_enabled,
"total": len(isce2_files),
},
"error_count": len(inventory["envi"]["errors"]) + len(inventory["isce2"]["errors"]),
"errors": list(inventory["envi"]["errors"]) + list(inventory["isce2"]["errors"]),
"error_count": len(errors),
"errors": errors,
"mismatch_count": len(mismatches),
"mismatches": mismatches,
"healthy": len(mismatches) == 0,
"healthy": len(mismatches) == 0 and len(errors) == 0,
}
@@ -52,6 +52,15 @@ def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str:
)
def _supported_dinsar_family_pair_expr(left_alias: str, right_alias: str) -> str:
left_family = _satellite_family_expr(left_alias)
right_family = _satellite_family_expr(right_alias)
return (
f"(({left_family} = 'LT1' AND {right_family} = 'LT1') "
f"OR ({left_family} = 'S1' AND {right_family} = 'S1'))"
)
def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
return (
f"(NULLIF({left_alias}.look_direction, '') IS NULL "
@@ -60,6 +69,60 @@ def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
)
def _relative_orbit_value_expr(alias: str) -> str:
return f"NULLIF(upper(trim(COALESCE({alias}.relative_orbit, ''))), '')"
def _same_relative_orbit_expr(left_alias: str, right_alias: str) -> str:
left_rel = _relative_orbit_value_expr(left_alias)
right_rel = _relative_orbit_value_expr(right_alias)
return f"({left_rel} IS NOT NULL AND {right_rel} IS NOT NULL AND {left_rel} = {right_rel})"
def _dinsar_quality_tier_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"CASE "
f"WHEN {same_rel} AND {center_distance_expr} <= 5000 THEN 'A' "
f"WHEN {center_distance_expr} <= 5000 THEN 'B' "
f"WHEN {center_distance_expr} <= 12000 THEN 'C' "
"ELSE 'REJECT' END"
)
def _dinsar_readiness_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
tier = _dinsar_quality_tier_expr(left_alias, right_alias, center_distance_expr)
return (
"CASE "
f"WHEN {tier} IN ('A', 'B') THEN 'RECOMMENDED' "
f"WHEN {tier} = 'C' THEN 'CANDIDATE' "
"ELSE 'NOT_RECOMMENDED' END"
)
def _dinsar_quality_score_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"GREATEST(0.0, LEAST(1.0, "
f"0.45 * COALESCE((ST_Area(ST_Intersection({left_alias}.geom, {right_alias}.geom)::geography) / "
f"NULLIF(GREATEST(ST_Area({left_alias}.geom::geography), ST_Area({right_alias}.geom::geography)), 0)), 0) + "
f"0.30 * CASE WHEN {same_rel} THEN 1 ELSE 0 END + "
f"0.25 * GREATEST(0.0, 1.0 - ({center_distance_expr} / 12000.0))"
"))"
)
def _dinsar_reasons_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
return (
"COALESCE(jsonb_path_query_array(jsonb_build_array("
f"CASE WHEN NOT ({same_rel}) THEN 'relative_orbit_missing_or_mismatch' END, "
f"CASE WHEN {center_distance_expr} > 5000 THEN 'center_distance_over_5km' END, "
f"CASE WHEN {center_distance_expr} > 12000 THEN 'center_distance_over_12km' END"
"), '$[*] ? (@ != null)')::json, '[]'::json)"
)
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
left_uid = _scene_uid_expr(left_alias)
right_uid = _scene_uid_expr(right_alias)
@@ -82,6 +145,7 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
f"AND COALESCE({left_alias}.insar_source_ready, false) "
f"AND COALESCE({right_alias}.insar_source_ready, false) "
f"AND { _supported_dinsar_family_pair_expr(left_alias, right_alias) } "
f"AND { _same_look_direction_expr(left_alias, right_alias) } "
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
)
@@ -106,7 +170,15 @@ def _full_rebuild_insert_sql() -> str:
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
pair_aoi_overlap_ratio,
orbit_direction,
same_relative_orbit,
master_relative_orbit,
slave_relative_orbit,
dinsar_quality_tier,
dinsar_quality_score,
dinsar_readiness,
dinsar_reasons_json,
same_satellite,
same_satellite_family,
same_look_direction,
@@ -144,7 +216,15 @@ def _full_rebuild_insert_sql() -> str:
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
NULL::double precision AS pair_aoi_overlap_ratio,
m.orbit_direction,
{ _same_relative_orbit_expr('m', 's') } AS same_relative_orbit,
{_relative_orbit_value_expr('m')} AS master_relative_orbit,
{_relative_orbit_value_expr('s')} AS slave_relative_orbit,
{ _dinsar_quality_tier_expr('m', 's', center_distance) } AS dinsar_quality_tier,
{ _dinsar_quality_score_expr('m', 's', center_distance) } AS dinsar_quality_score,
{ _dinsar_readiness_expr('m', 's', center_distance) } AS dinsar_readiness,
{ _dinsar_reasons_expr('m', 's', center_distance) } AS dinsar_reasons_json,
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
{ _same_satellite_family_expr('m', 's') } AS same_satellite_family,
{ _same_look_direction_expr('m', 's') } AS same_look_direction,
@@ -201,7 +281,15 @@ def _incremental_insert_sql() -> str:
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
pair_aoi_overlap_ratio,
orbit_direction,
same_relative_orbit,
master_relative_orbit,
slave_relative_orbit,
dinsar_quality_tier,
dinsar_quality_score,
dinsar_readiness,
dinsar_reasons_json,
same_satellite,
same_satellite_family,
same_look_direction,
@@ -244,7 +332,15 @@ def _incremental_insert_sql() -> str:
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
NULL::double precision AS pair_aoi_overlap_ratio,
d.orbit_direction,
{ _same_relative_orbit_expr('d', 'o') } AS same_relative_orbit,
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('d')} ELSE {_relative_orbit_value_expr('o')} END AS master_relative_orbit,
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('o')} ELSE {_relative_orbit_value_expr('d')} END AS slave_relative_orbit,
{ _dinsar_quality_tier_expr('d', 'o', center_distance) } AS dinsar_quality_tier,
{ _dinsar_quality_score_expr('d', 'o', center_distance) } AS dinsar_quality_score,
{ _dinsar_readiness_expr('d', 'o', center_distance) } AS dinsar_readiness,
{ _dinsar_reasons_expr('d', 'o', center_distance) } AS dinsar_reasons_json,
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
{ _same_satellite_family_expr('d', 'o') } AS same_satellite_family,
{ _same_look_direction_expr('d', 'o') } AS same_look_direction,
@@ -18,7 +18,7 @@ from ..models import (
PAIRING_CACHE_SCOPE_GLOBAL = "global"
DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1"
DEFAULT_PAIRING_METRIC_VERSION = "2026.06.dinsar.family.v1"
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
@@ -235,15 +235,6 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="directory_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_ARCHIVE_SOURCE_DIRS",
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
root_role="source_pool_gf3_archive",
display_prefix="GF3 Archive Pool",
scan_mode="archive_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_SOURCE_DIRS",
+603 -46
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import and_, case, cast, func, or_
from sqlalchemy import and_, case, cast, func, or_, text
from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
@@ -24,6 +24,8 @@ from shapely.geometry import Polygon
from shapely.ops import unary_union
from ..models import (
DinsarProductionRunItemORM,
DinsarProductionRunORM,
HazardPoint,
HazardPointORM,
PairingNetworkEdgeORM,
@@ -43,7 +45,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
PAIRING_POLICY_VERSION = "2026.06.dinsar-production.v1"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000
logger = logging.getLogger(__name__)
@@ -145,10 +147,10 @@ class SpatialService:
require_orbit_data=require_orbit_data,
)
if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
if len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
raise RuntimeError(
f"全部配对命中 {len(candidate_pool)} 条候选边,超过系统一次性返回上限 "
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略,或收紧 AOI、日期范围、重叠率。"
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请收紧 AOI、日期范围、重叠率或中心距离阈值"
)
if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD:
@@ -156,15 +158,21 @@ class SpatialService:
f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}(当前: {len(candidate_pool)}),建议收紧参数或缩小 AOI。"
)
selected_candidates, strategy_warnings = self._apply_strategy(
candidate_pool,
effective_params,
aoi_wkt=aoi_wkt,
)
selected_candidates, strategy_warnings = self._apply_dinsar_production_strategy(candidate_pool, effective_params)
warnings.extend(strategy_warnings)
if not selected_candidates:
warnings.extend(
await self._build_empty_pairing_diagnostics(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
)
for candidate in selected_candidates:
candidate.setdefault("selection_strategy", effective_params.strategy)
candidate["selection_strategy"] = "dinsar_production"
self._ensure_candidate_identity(candidate)
network_run_id = await self._persist_network_run(
db,
@@ -176,6 +184,7 @@ class SpatialService:
selected_candidates=selected_candidates,
)
await self._attach_dinsar_production_summaries(db, selected_candidates)
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
metadata = {
"fallback_used": False,
@@ -227,13 +236,20 @@ class SpatialService:
PairingMetricCacheORM.status == "READY",
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.master_imaging_date < PairingMetricCacheORM.slave_imaging_date,
PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold,
PairingMetricCacheORM.same_look_direction.is_(True),
PairingMetricCacheORM.dinsar_readiness.in_(["RECOMMENDED", "CANDIDATE"]),
)
)
if params.limit_footprint_center_distance:
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
stmt = stmt.where(
or_(
and_(master_family_expr == "LT1", slave_family_expr == "LT1"),
and_(master_family_expr == "S1", slave_family_expr == "S1"),
)
)
if require_orbit_data:
stmt = stmt.where(
@@ -241,8 +257,7 @@ class SpatialService:
slave_alias.has_orbit_data.is_(True),
)
if not params.cross_satellite_pairing:
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
if params.require_same_imaging_mode:
stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True))
@@ -259,20 +274,19 @@ class SpatialService:
)
if params.allowed_satellites:
allowed_satellites = [
str(item).strip().upper()
for item in params.allowed_satellites
if str(item).strip()
]
allowed_satellites = []
for item in params.allowed_satellites:
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
allowed_satellites.append("LT1")
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
allowed_satellites.append("S1")
allowed_satellites = list(dict.fromkeys(allowed_satellites))
if not allowed_satellites:
return []
stmt = stmt.where(
or_(
func.upper(master_alias.satellite).in_(allowed_satellites),
func.upper(master_alias.satellite_family).in_(allowed_satellites),
),
or_(
func.upper(slave_alias.satellite).in_(allowed_satellites),
func.upper(slave_alias.satellite_family).in_(allowed_satellites),
),
master_family_expr.in_(allowed_satellites),
slave_family_expr.in_(allowed_satellites),
)
if params.master_date_from:
@@ -286,23 +300,23 @@ class SpatialService:
if aoi_wkt:
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
aoi_geog = cast(aoi_geom, Geography)
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
pair_overlap_geom = ST_Intersection(master_alias.geom, slave_alias.geom)
pair_aoi_geom = ST_Intersection(pair_overlap_geom, aoi_geom)
pair_aoi_overlap_expr = (ST_Area(cast(pair_aoi_geom, Geography)) / aoi_area).label("pair_aoi_overlap_ratio")
stmt = stmt.add_columns(pair_aoi_overlap_expr)
stmt = stmt.where(
ST_Intersects(master_alias.geom, aoi_geom),
ST_Intersects(slave_alias.geom, aoi_geom),
ST_Intersects(pair_overlap_geom, aoi_geom),
)
if params.aoi_overlap_threshold is not None:
aoi_geog = cast(aoi_geom, Geography)
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
master_inter_geog = cast(ST_Intersection(master_alias.geom, aoi_geom), Geography)
slave_inter_geog = cast(ST_Intersection(slave_alias.geom, aoi_geom), Geography)
stmt = stmt.where(
ST_Area(master_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
ST_Area(slave_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
)
stmt = stmt.where(pair_aoi_overlap_expr >= params.aoi_overlap_threshold)
stmt = stmt.order_by(
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
PairingMetricCacheORM.dinsar_quality_tier.asc(),
func.coalesce(PairingMetricCacheORM.dinsar_quality_score, 0).desc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
center_distance_expr.asc(),
PairingMetricCacheORM.pair_uid.asc(),
@@ -310,7 +324,9 @@ class SpatialService:
result = await db.execute(stmt)
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
for row in result.all():
metric_row, master_row, slave_row = row[0], row[1], row[2]
pair_aoi_overlap_ratio = row[3] if len(row) > 3 else None
center_distance = float(
metric_row.scene_center_distance_meters
if metric_row.scene_center_distance_meters is not None
@@ -328,16 +344,54 @@ class SpatialService:
"dist": center_distance,
"scene_center_distance_meters": center_distance,
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0),
"dinsar_quality_tier": metric_row.dinsar_quality_tier,
"dinsar_quality_score": (
float(metric_row.dinsar_quality_score)
if metric_row.dinsar_quality_score is not None
else None
),
"dinsar_readiness": metric_row.dinsar_readiness,
"dinsar_reasons": [
str(item)
for item in (metric_row.dinsar_reasons_json or [])
if isinstance(item, str) and item.strip()
],
"same_relative_orbit": bool(metric_row.same_relative_orbit),
"master_relative_orbit": metric_row.master_relative_orbit,
"slave_relative_orbit": metric_row.slave_relative_orbit,
"pair_aoi_overlap_ratio": (
float(pair_aoi_overlap_ratio)
if pair_aoi_overlap_ratio is not None
else None
),
}
)
return candidate_pool
def _ensure_candidate_identity(self, candidate: dict) -> Tuple[str, str]:
master = candidate["master"]
slave = candidate["slave"]
task_alias = str(candidate.get("task_alias") or "").strip() or build_task_alias(
master.imaging_date,
slave.imaging_date,
)
pair_key = str(candidate.get("pair_key") or "").strip() or build_pair_key(
master.file_path,
slave.file_path,
master.imaging_date,
slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
)
candidate["task_alias"] = task_alias
candidate["pair_key"] = pair_key
return task_alias, pair_key
def _build_radar_pairs(self, selected_candidates: List[dict]) -> List[RadarPair]:
result_pairs: List[RadarPair] = []
for candidate in selected_candidates:
master = candidate["master"]
slave = candidate["slave"]
task_alias = build_task_alias(master.imaging_date, slave.imaging_date)
task_alias, pair_key = self._ensure_candidate_identity(candidate)
selection_score = candidate.get("selection_score")
result_pairs.append(
RadarPair(
@@ -345,13 +399,7 @@ class SpatialService:
slave=slave,
task_name=task_alias,
task_alias=task_alias,
pair_key=build_pair_key(
master.file_path,
slave.file_path,
master.imaging_date,
slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
),
pair_key=pair_key,
pair_uid=candidate.get("pair_uid"),
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
network_run_id=candidate.get("network_run_id"),
@@ -367,10 +415,292 @@ class SpatialService:
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0
),
scene_overlap_ratio=float(candidate.get("overlap_ratio") or 0.0),
pair_aoi_overlap_ratio=(
float(candidate["pair_aoi_overlap_ratio"])
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
dinsar_quality_tier=candidate.get("dinsar_quality_tier"),
dinsar_quality_score=(
float(candidate["dinsar_quality_score"])
if candidate.get("dinsar_quality_score") is not None
else None
),
dinsar_readiness=candidate.get("dinsar_readiness"),
dinsar_reasons=candidate.get("dinsar_reasons") or [],
same_relative_orbit=bool(candidate.get("same_relative_orbit")),
master_relative_orbit=candidate.get("master_relative_orbit"),
slave_relative_orbit=candidate.get("slave_relative_orbit"),
production_summary=candidate.get("production_summary"),
)
)
return result_pairs
async def _attach_dinsar_production_summaries(
self,
db: AsyncSession,
selected_candidates: List[dict],
) -> None:
if not selected_candidates:
return
pair_uids: set[str] = set()
pair_keys: set[str] = set()
aliases: set[str] = set()
for candidate in selected_candidates:
task_alias, pair_key = self._ensure_candidate_identity(candidate)
pair_uid = str(candidate.get("pair_uid") or "").strip()
if pair_uid:
pair_uids.add(pair_uid)
if pair_key:
pair_keys.add(pair_key)
if task_alias:
aliases.add(task_alias)
run_conditions = []
if pair_uids:
run_conditions.append(DinsarProductionRunItemORM.pair_uid.in_(pair_uids))
if pair_keys:
run_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys))
if aliases:
run_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases))
run_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases))
product_conditions = []
if pair_uids:
product_conditions.append(ResultProductORM.pair_uid.in_(pair_uids))
if pair_keys:
product_conditions.append(ResultProductORM.pair_key.in_(pair_keys))
if aliases:
product_conditions.append(ResultProductORM.task_alias.in_(aliases))
product_conditions.append(ResultProductORM.task_name.in_(aliases))
run_rows = []
if run_conditions:
result = await db.execute(
select(DinsarProductionRunItemORM, DinsarProductionRunORM)
.join(DinsarProductionRunORM, DinsarProductionRunItemORM.run_id == DinsarProductionRunORM.run_id)
.where(or_(*run_conditions))
.order_by(
DinsarProductionRunItemORM.updated_at.desc().nullslast(),
DinsarProductionRunItemORM.id.desc(),
)
)
run_rows = result.all()
products = []
if product_conditions:
result = await db.execute(
select(ResultProductORM)
.where(ResultProductORM.catalog_name == "dinsar")
.where(or_(*product_conditions))
.order_by(
ResultProductORM.published_at.desc().nullslast(),
ResultProductORM.id.desc(),
)
)
products = result.scalars().all()
run_by_uid: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
run_by_key: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
run_by_alias: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
for item, run in run_rows:
pair_uid = str(item.pair_uid or "").strip()
pair_key = str(item.pair_key or "").strip()
if pair_uid:
run_by_uid[pair_uid].append((item, run))
if pair_key:
run_by_key[pair_key].append((item, run))
for alias in {str(item.task_alias or "").strip(), str(item.task_name or "").strip()}:
if alias:
run_by_alias[alias].append((item, run))
products_by_uid: Dict[str, List[ResultProductORM]] = defaultdict(list)
products_by_key: Dict[str, List[ResultProductORM]] = defaultdict(list)
products_by_alias: Dict[str, List[ResultProductORM]] = defaultdict(list)
for product in products:
pair_uid = str(product.pair_uid or "").strip()
pair_key = str(product.pair_key or "").strip()
if pair_uid:
products_by_uid[pair_uid].append(product)
if pair_key:
products_by_key[pair_key].append(product)
for alias in {str(product.task_alias or "").strip(), str(product.task_name or "").strip()}:
if alias:
products_by_alias[alias].append(product)
for candidate in selected_candidates:
task_alias, pair_key = self._ensure_candidate_identity(candidate)
pair_uid = str(candidate.get("pair_uid") or "").strip()
exact_runs = self._dedupe_by_object_id(
[*run_by_uid.get(pair_uid, []), *run_by_key.get(pair_key, [])],
key=lambda row: getattr(row[0], "id", None),
)
alias_runs = self._dedupe_by_object_id(
run_by_alias.get(task_alias, []),
key=lambda row: getattr(row[0], "id", None),
)
exact_products = self._dedupe_by_object_id(
[*products_by_uid.get(pair_uid, []), *products_by_key.get(pair_key, [])],
key=lambda product: getattr(product, "id", None),
)
alias_products = self._dedupe_by_object_id(
products_by_alias.get(task_alias, []),
key=lambda product: getattr(product, "id", None),
)
matched_runs = exact_runs or alias_runs
matched_products = exact_products or alias_products
candidate["production_summary"] = self._summarize_dinsar_production(
matched_runs,
matched_products,
match_level="identity" if (exact_runs or exact_products) else ("task_alias" if (alias_runs or alias_products) else "none"),
)
def _summarize_dinsar_production(
self,
run_rows: List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]],
products: List[ResultProductORM],
*,
match_level: str = "none",
) -> Dict[str, Any]:
latest_run_row = max(
run_rows,
key=lambda row: self._datetime_sort_key(
row[0].updated_at,
row[0].ended_at,
row[0].started_at,
row[0].created_at,
),
default=None,
)
latest_product = max(
products,
key=lambda product: self._datetime_sort_key(
product.published_at,
product.produced_at,
product.updated_at,
product.registered_at,
),
default=None,
)
ready_products = [product for product in products if self._is_ready_result_product(product)]
completed_statuses = {"COMPLETED", "READY", "SUCCESS", "PUBLISHED"}
failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
completed_run_count = sum(
1
for item, run in run_rows
if str(item.status or "").strip().upper() in completed_statuses
or str(run.status or "").strip().upper() in completed_statuses
)
failed_run_count = sum(
1
for item, run in run_rows
if str(item.status or "").strip().upper() in failed_statuses
or str(run.status or "").strip().upper() in failed_statuses
)
latest_item = latest_run_row[0] if latest_run_row else None
latest_run = latest_run_row[1] if latest_run_row else None
if ready_products and latest_product is not None:
latest_status = str(latest_product.status or "").strip().upper()
elif latest_item is not None or latest_run is not None:
latest_status = str(
(latest_item.status if latest_item is not None else None)
or (latest_run.status if latest_run is not None else None)
or ""
).strip().upper()
elif latest_product is not None:
latest_status = str(latest_product.status or "").strip().upper()
else:
latest_status = ""
if ready_products:
status = "READY"
elif completed_run_count > 0:
status = "COMPLETED"
elif latest_status:
status = latest_status
else:
status = "MISSING"
engine_codes = sorted(
{
str(value or "").strip().lower()
for value in [
*(product.engine_code for product in products),
*(run.engine_code for _, run in run_rows),
]
if str(value or "").strip()
}
)
return {
"has_record": bool(run_rows or products),
"is_produced": bool(ready_products or completed_run_count > 0),
"status": status,
"match_level": match_level,
"run_item_count": len(run_rows),
"completed_run_count": completed_run_count,
"failed_run_count": failed_run_count,
"product_count": len(products),
"ready_product_count": len(ready_products),
"engine_codes": engine_codes,
"latest_engine_code": (
str(latest_product.engine_code or "").strip().lower()
if latest_product is not None and latest_product.engine_code
else (
str(latest_run.engine_code or "").strip().lower()
if latest_run is not None and latest_run.engine_code
else None
)
),
"latest_run_id": latest_run.run_id if latest_run is not None else None,
"latest_run_status": latest_run.status if latest_run is not None else None,
"latest_item_status": latest_item.status if latest_item is not None else None,
"latest_output_dir": latest_item.latest_output_dir if latest_item is not None else None,
"latest_product_id": latest_product.id if latest_product is not None else None,
"latest_product_identifier": latest_product.product_id if latest_product is not None else None,
"latest_product_status": latest_product.status if latest_product is not None else None,
"latest_product_health": latest_product.health_status if latest_product is not None else None,
"latest_product_published_at": latest_product.published_at if latest_product is not None else None,
"updated_at": (
latest_item.updated_at
if latest_item is not None
else (
latest_product.updated_at
if latest_product is not None
else None
)
),
}
def _is_ready_result_product(self, product: ResultProductORM) -> bool:
status = str(product.status or "").strip().upper()
health = str(product.health_status or "").strip().upper()
return status in {"READY", "COMPLETED", "SUCCESS"} and health not in {"ERROR", "FAILED"}
def _datetime_sort_key(self, *values: Any) -> float:
for value in values:
if value is None:
continue
try:
return float(value.timestamp())
except Exception:
continue
return 0.0
def _dedupe_by_object_id(self, items: List[Any], *, key) -> List[Any]:
seen: set[Any] = set()
output: List[Any] = []
for item in items:
item_key = key(item)
if item_key is None:
item_key = id(item)
if item_key in seen:
continue
seen.add(item_key)
output.append(item)
return output
async def _persist_network_run(
self,
db: AsyncSession,
@@ -451,6 +781,8 @@ class SpatialService:
"master_scene_uid": candidate.get("master_scene_uid"),
"slave_scene_uid": candidate.get("slave_scene_uid"),
"pair_uid": candidate.get("pair_uid"),
"pair_key": candidate.get("pair_key"),
"task_alias": candidate.get("task_alias"),
"time_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_center_distance_meters": float(
@@ -460,6 +792,11 @@ class SpatialService:
),
"legacy_spatial_baseline_field": "scene_center_distance_meters",
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
"pair_aoi_overlap_ratio": (
float(candidate["pair_aoi_overlap_ratio"])
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
}
def _stable_sha1(self, value: Any) -> str:
@@ -814,6 +1151,226 @@ class SpatialService:
"scenes": scene_payloads,
}
def _apply_dinsar_production_strategy(
self,
candidate_pool: List[dict],
params: PairingRequest,
) -> Tuple[List[dict], List[str]]:
if not candidate_pool:
return [], []
warnings: List[str] = []
rejected_count = sum(
1
for candidate in candidate_pool
if str(candidate.get("dinsar_readiness") or "").upper() == "NOT_RECOMMENDED"
)
if rejected_count:
warnings.append(f"{rejected_count}条候选因D-InSAR生产前置条件不足被过滤。")
selected = []
for candidate in candidate_pool:
readiness = str(candidate.get("dinsar_readiness") or "CANDIDATE").upper()
if readiness == "NOT_RECOMMENDED":
continue
tier = str(candidate.get("dinsar_quality_tier") or "C").upper()
quality_score = candidate.get("dinsar_quality_score")
selected.append(
{
**candidate,
"selection_reason": f"dinsar_{readiness.lower()}_{tier.lower()}",
"selection_score": (
float(quality_score)
if quality_score is not None
else self._score_pair_candidate(candidate, params)
),
}
)
selected.sort(
key=lambda item: (
{"A": 0, "B": 1, "C": 2}.get(str(item.get("dinsar_quality_tier") or "C").upper(), 9),
-float(item.get("selection_score") or 0),
int(item.get("days") or 0),
float(item.get("dist") or 0),
str(getattr(item.get("master"), "imaging_date", "") or ""),
str(getattr(item.get("slave"), "imaging_date", "") or ""),
)
)
return selected, warnings
async def _build_empty_pairing_diagnostics(
self,
db: AsyncSession,
params: PairingRequest,
*,
aoi_wkt: Optional[str],
require_orbit_data: bool,
) -> List[str]:
allowed_families: List[str] = []
for item in params.allowed_satellites or []:
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
allowed_families.append("LT1")
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
allowed_families.append("S1")
allowed_families = list(dict.fromkeys(allowed_families))
sql = text(
"""
WITH base AS (
SELECT
pmc.*,
m.satellite AS master_satellite_actual,
s.satellite AS slave_satellite_actual,
m.has_orbit_data AS master_has_orbit,
s.has_orbit_data AS slave_has_orbit,
COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters) AS center_m
FROM pairing_metric_cache pmc
JOIN radar_data m ON m.id = pmc.master_scene_ref_id
JOIN radar_data s ON s.id = pmc.slave_scene_ref_id
WHERE pmc.metric_version = :metric_version
AND pmc.status = 'READY'
AND pmc.master_imaging_date < pmc.slave_imaging_date
AND pmc.same_look_direction IS TRUE
AND pmc.same_satellite_family IS TRUE
AND pmc.dinsar_readiness IN ('RECOMMENDED', 'CANDIDATE')
AND (:require_orbit_data IS FALSE OR (m.has_orbit_data IS TRUE AND s.has_orbit_data IS TRUE))
AND (:require_same_imaging_mode IS FALSE OR pmc.same_imaging_mode IS TRUE)
AND (:require_same_polarization IS FALSE OR pmc.same_polarization IS TRUE)
AND (
:allowed_families_is_empty IS TRUE
OR pmc.master_satellite_family = ANY(:allowed_families)
OR pmc.master_satellite = ANY(:allowed_families)
)
AND (CAST(:master_date_from AS text) IS NULL OR pmc.master_imaging_date >= CAST(:master_date_from AS text))
AND (CAST(:master_date_to AS text) IS NULL OR pmc.master_imaging_date <= CAST(:master_date_to AS text))
AND (CAST(:slave_date_from AS text) IS NULL OR pmc.slave_imaging_date >= CAST(:slave_date_from AS text))
AND (CAST(:slave_date_to AS text) IS NULL OR pmc.slave_imaging_date <= CAST(:slave_date_to AS text))
AND (
CAST(:aoi_wkt AS text) IS NULL
OR ST_Intersects(
ST_Intersection(m.geom, s.geom),
ST_GeomFromText(CAST(:aoi_wkt AS text), 4326)
)
)
),
time_ok AS (
SELECT * FROM base
WHERE time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
),
overlap_ok AS (
SELECT * FROM time_ok
WHERE scene_overlap_ratio >= :overlap_threshold
),
center_ok AS (
SELECT * FROM overlap_ok
WHERE center_m <= :center_distance_max
)
SELECT
(SELECT count(*) FROM base) AS base_count,
(SELECT count(*) FROM time_ok) AS time_ok_count,
(SELECT count(*) FROM overlap_ok) AS overlap_ok_count,
(SELECT count(*) FROM center_ok) AS center_ok_count,
(
SELECT json_build_object(
'master_date', master_imaging_date,
'slave_date', slave_imaging_date,
'master_satellite', master_satellite_actual,
'slave_satellite', slave_satellite_actual,
'time_baseline_days', time_baseline_days,
'center_meters', center_m,
'overlap_ratio', scene_overlap_ratio,
'quality_tier', dinsar_quality_tier,
'readiness', dinsar_readiness
)
FROM base
ORDER BY
CASE
WHEN time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
THEN 0 ELSE 1
END,
CASE WHEN scene_overlap_ratio >= :overlap_threshold THEN 0 ELSE 1 END,
abs(time_baseline_days - :time_baseline_max),
center_m ASC NULLS LAST
LIMIT 1
) AS nearest_candidate;
"""
)
result = await db.execute(
sql,
{
"metric_version": pairing_state_service.metric_version,
"require_orbit_data": require_orbit_data,
"require_same_imaging_mode": bool(params.require_same_imaging_mode),
"require_same_polarization": bool(params.require_same_polarization),
"allowed_families": allowed_families or ["__NONE__"],
"allowed_families_is_empty": not allowed_families,
"master_date_from": params.master_date_from or None,
"master_date_to": params.master_date_to or None,
"slave_date_from": params.slave_date_from or None,
"slave_date_to": params.slave_date_to or None,
"aoi_wkt": aoi_wkt,
"time_baseline_min": int(params.time_baseline_min),
"time_baseline_max": int(params.time_baseline_max),
"overlap_threshold": float(params.overlap_threshold),
"center_distance_max": float(params.spatial_baseline_max_meters),
},
)
row = result.mappings().first()
if not row:
return ["未找到满足条件的 D-InSAR 配对;诊断查询未返回统计结果。"]
base_count = int(row.get("base_count") or 0)
time_ok_count = int(row.get("time_ok_count") or 0)
overlap_ok_count = int(row.get("overlap_ok_count") or 0)
center_ok_count = int(row.get("center_ok_count") or 0)
if base_count <= 0:
family_text = "".join(allowed_families) if allowed_families else "LT-1/Sentinel-1"
return [
f"未找到 {family_text} 的可生产基础候选边。请检查数据体系、主从日期范围、AOI、精轨绑定和配对缓存状态。"
]
messages = [
(
"当前筛选下基础候选 {base} 条;时间基线 {min_days}-{max_days} 天后剩 {time_ok} 条;"
"重叠率 >= {overlap:.2f} 后剩 {overlap_ok} 条;footprint 中心距离 <= {center:.0f} 米后剩 {center_ok} 条。"
).format(
base=base_count,
min_days=int(params.time_baseline_min),
max_days=int(params.time_baseline_max),
time_ok=time_ok_count,
overlap=float(params.overlap_threshold),
overlap_ok=overlap_ok_count,
center=float(params.spatial_baseline_max_meters),
center_ok=center_ok_count,
)
]
nearest = row.get("nearest_candidate")
if isinstance(nearest, str):
try:
nearest = json.loads(nearest)
except Exception:
nearest = None
if isinstance(nearest, dict):
messages.append(
(
"最接近的一对是 {master_satellite} {master_date} -> {slave_satellite} {slave_date}"
"时间基线 {days} 天,中心距离 {center:.1f} 米,重叠率 {overlap:.3f},质量 {tier}/{readiness}"
).format(
master_satellite=nearest.get("master_satellite") or "?",
master_date=nearest.get("master_date") or "?",
slave_satellite=nearest.get("slave_satellite") or "?",
slave_date=nearest.get("slave_date") or "?",
days=int(nearest.get("time_baseline_days") or 0),
center=float(nearest.get("center_meters") or 0.0),
overlap=float(nearest.get("overlap_ratio") or 0.0),
tier=nearest.get("quality_tier") or "?",
readiness=nearest.get("readiness") or "?",
)
)
return messages
def _apply_strategy(
self,
candidate_pool: List[dict],
+3
View File
@@ -259,12 +259,15 @@ class TaskService:
task = result.scalar_one_or_none()
if task:
previous_message = task.message
if status:
task.status = status
if progress is not None:
task.progress = progress
if message:
task.message = message
if message != previous_message:
await self._add_log(task_id, "INFO", message, db=db)
# 如果任务结束,更新结束时间
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
+1 -1
View File
@@ -39,7 +39,7 @@ def get_unpack_config() -> Dict[str, Any]:
"source_dirs": source_dirs,
"insar_storage_dirs": insar_storage_dirs,
"min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")),
"delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")),
"delete_archive": False,
"tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"),
"archive_exts": archive_exts,
"scan_workers": module.parse_int(
+79 -1
View File
@@ -1,5 +1,6 @@
import os
import re
import math
from datetime import datetime
from typing import Optional, Tuple, List, Callable, Dict, Any
from lxml import etree
@@ -20,6 +21,80 @@ def _create_secure_xml_parser() -> etree.XMLParser:
recover=False,
)
def _ordered_closed_polygon(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]:
unique: List[Tuple[float, float]] = []
for point in points or []:
try:
lon = float(point[0])
lat = float(point[1])
except (TypeError, ValueError, IndexError):
continue
current = (lon, lat)
if unique and abs(unique[-1][0] - lon) < 1e-12 and abs(unique[-1][1] - lat) < 1e-12:
continue
if unique and abs(unique[0][0] - lon) < 1e-12 and abs(unique[0][1] - lat) < 1e-12:
continue
if current not in unique:
unique.append(current)
if len(unique) < 3:
return None
if len(unique) == 4:
center_lon = sum(item[0] for item in unique) / len(unique)
center_lat = sum(item[1] for item in unique) / len(unique)
ordered = sorted(
unique,
key=lambda item: math.atan2(item[1] - center_lat, item[0] - center_lon),
)
else:
ordered = unique
if ordered[0] != ordered[-1]:
ordered.append(ordered[0])
return ordered
def _closed_polygon_if_valid(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]:
ring = [(float(point[0]), float(point[1])) for point in points or []]
if len(ring) < 3:
return None
if ring[0] != ring[-1]:
ring.append(ring[0])
return ring
def _ordered_closed_polygon_from_corner_details(corner_details: Dict[str, Dict[str, Any]]) -> Optional[List[Tuple[float, float]]]:
by_name = {str(key or "").strip().lower(): value for key, value in (corner_details or {}).items()}
name_order = ["bottomleft", "bottomright", "topright", "topleft"]
if all(name in by_name for name in name_order):
return _closed_polygon_if_valid([(by_name[name]["lon"], by_name[name]["lat"]) for name in name_order])
entries = [
value
for value in (corner_details or {}).values()
if value.get("lon") is not None
and value.get("lat") is not None
and value.get("ref_row") is not None
and value.get("ref_col") is not None
]
if len(entries) >= 4:
min_row = min(float(item["ref_row"]) for item in entries)
max_row = max(float(item["ref_row"]) for item in entries)
min_col = min(float(item["ref_col"]) for item in entries)
max_col = max(float(item["ref_col"]) for item in entries)
targets = [(min_row, min_col), (min_row, max_col), (max_row, max_col), (max_row, min_col)]
remaining = list(entries)
ordered_entries: List[Dict[str, Any]] = []
for target_row, target_col in targets:
chosen = min(
remaining,
key=lambda item: abs(float(item["ref_row"]) - target_row) + abs(float(item["ref_col"]) - target_col),
)
ordered_entries.append(chosen)
remaining.remove(chosen)
return _closed_polygon_if_valid([(item["lon"], item["lat"]) for item in ordered_entries])
return _ordered_closed_polygon([(value["lon"], value["lat"]) for value in (corner_details or {}).values()])
# --- Sentinel-1 (S1A/S1B/S1C) Parsers ---
def _radar_meta_base() -> Dict[str, Any]:
@@ -573,7 +648,10 @@ def parse_xml_metadata(
return None, None
if len(polygon) == 4:
polygon.append(polygon[0])
ordered_polygon = _ordered_closed_polygon_from_corner_details(corner_details)
if not ordered_polygon:
return None, None
polygon = ordered_polygon
corner_pixel_mapping = _build_corner_pixel_mapping(corner_details)
meta = {
"orbit_direction": orbit_direction,
@@ -0,0 +1,138 @@
-- Persist source XML/manifest documents and normalized scene geometry profiles.
CREATE TABLE IF NOT EXISTS source_metadata_documents (
id SERIAL PRIMARY KEY,
source_asset_id INTEGER NOT NULL REFERENCES source_product_assets(id) ON DELETE CASCADE,
radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
satellite_family VARCHAR(32) NULL,
source_format VARCHAR(32) NULL,
document_type VARCHAR(32) NOT NULL,
member_path TEXT NOT NULL,
content_sha256 VARCHAR(64) NOT NULL,
content_encoding VARCHAR(16) NOT NULL DEFAULT 'gzip',
content_bytes BYTEA NOT NULL,
content_size_bytes BIGINT NULL,
archive_path TEXT NULL,
archive_mtime DOUBLE PRECISION NULL,
parser_version VARCHAR(32) NULL,
parse_status VARCHAR(32) NOT NULL DEFAULT 'OK',
parse_error TEXT NULL,
extracted_at TIMESTAMP NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NULL DEFAULT NOW(),
CONSTRAINT uq_source_metadata_document_member UNIQUE (source_asset_id, document_type, member_path)
);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_asset_id
ON source_metadata_documents (source_asset_id);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_data_id
ON source_metadata_documents (radar_data_id);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_satellite_family
ON source_metadata_documents (satellite_family);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_format
ON source_metadata_documents (source_format);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_document_type
ON source_metadata_documents (document_type);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_content_sha256
ON source_metadata_documents (content_sha256);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_parse_status
ON source_metadata_documents (parse_status);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_asset_type
ON source_metadata_documents (source_asset_id, document_type);
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_type
ON source_metadata_documents (radar_data_id, document_type);
CREATE TABLE IF NOT EXISTS sar_scene_geometry_profiles (
id SERIAL PRIMARY KEY,
source_asset_id INTEGER NOT NULL UNIQUE REFERENCES source_product_assets(id) ON DELETE CASCADE,
radar_data_id INTEGER NULL UNIQUE REFERENCES radar_data(id) ON DELETE CASCADE,
satellite_family VARCHAR(32) NULL,
satellite VARCHAR(32) NULL,
source_format VARCHAR(32) NULL,
imaging_mode VARCHAR(64) NULL,
polarization VARCHAR(64) NULL,
orbit_direction VARCHAR(32) NULL,
look_direction VARCHAR(32) NULL,
absolute_orbit VARCHAR(64) NULL,
relative_orbit VARCHAR(64) NULL,
acquisition_start_time_utc TIMESTAMP NULL,
acquisition_stop_time_utc TIMESTAMP NULL,
scene_center_lon DOUBLE PRECISION NULL,
scene_center_lat DOUBLE PRECISION NULL,
footprint_geom GEOMETRY(POLYGON, 4326) NULL,
footprint_polygon JSON NULL,
swath_summary_json JSON NULL,
burst_summary_json JSON NULL,
incidence_angle_min DOUBLE PRECISION NULL,
incidence_angle_max DOUBLE PRECISION NULL,
doppler_summary_json JSON NULL,
state_vector_summary_json JSON NULL,
metadata_quality VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN',
production_readiness VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN',
readiness_reasons_json JSON NULL,
parser_version VARCHAR(32) NULL,
parsed_at TIMESTAMP NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_asset_id
ON sar_scene_geometry_profiles (source_asset_id);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_radar_data_id
ON sar_scene_geometry_profiles (radar_data_id);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite_family
ON sar_scene_geometry_profiles (satellite_family);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite
ON sar_scene_geometry_profiles (satellite);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_format
ON sar_scene_geometry_profiles (source_format);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_imaging_mode
ON sar_scene_geometry_profiles (imaging_mode);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_polarization
ON sar_scene_geometry_profiles (polarization);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_orbit_direction
ON sar_scene_geometry_profiles (orbit_direction);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_look_direction
ON sar_scene_geometry_profiles (look_direction);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_absolute_orbit
ON sar_scene_geometry_profiles (absolute_orbit);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_relative_orbit
ON sar_scene_geometry_profiles (relative_orbit);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_acquisition_start_time_utc
ON sar_scene_geometry_profiles (acquisition_start_time_utc);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_footprint_geom
ON sar_scene_geometry_profiles USING GIST (footprint_geom);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_metadata_quality
ON sar_scene_geometry_profiles (metadata_quality);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_production_readiness
ON sar_scene_geometry_profiles (production_readiness);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_family_date
ON sar_scene_geometry_profiles (satellite_family, acquisition_start_time_utc);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_track
ON sar_scene_geometry_profiles (satellite_family, relative_orbit, orbit_direction);
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_readiness
ON sar_scene_geometry_profiles (production_readiness, metadata_quality);
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS pair_aoi_overlap_ratio DOUBLE PRECISION NULL;
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS same_relative_orbit BOOLEAN NOT NULL DEFAULT FALSE;
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS master_relative_orbit VARCHAR(64) NULL;
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS slave_relative_orbit VARCHAR(64) NULL;
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS dinsar_quality_tier VARCHAR(16) NOT NULL DEFAULT 'C';
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS dinsar_quality_score DOUBLE PRECISION NULL;
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS dinsar_readiness VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE';
ALTER TABLE IF EXISTS pairing_metric_cache
ADD COLUMN IF NOT EXISTS dinsar_reasons_json JSON NULL;
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_same_relative_orbit
ON pairing_metric_cache (same_relative_orbit);
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_quality_tier
ON pairing_metric_cache (dinsar_quality_tier);
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_readiness
ON pairing_metric_cache (dinsar_readiness);
@@ -0,0 +1,20 @@
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_status VARCHAR(32) NOT NULL DEFAULT 'NOT_CHECKED';
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_method VARCHAR(64) NULL;
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_checked_at TIMESTAMP NULL;
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_error TEXT NULL;
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_version VARCHAR(32) NULL;
ALTER TABLE source_product_assets
ADD COLUMN IF NOT EXISTS archive_integrity_member_count INTEGER NULL;
CREATE INDEX IF NOT EXISTS idx_source_product_assets_archive_integrity_status
ON source_product_assets (archive_integrity_status);