Refactor local InSAR asset and production workflows
This commit is contained in:
+77
-24
@@ -13,6 +13,7 @@ from sklearn.preprocessing import StandardScaler
|
||||
from scipy.stats import entropy
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
import functools
|
||||
from typing import List, Optional
|
||||
|
||||
from .config import settings
|
||||
|
||||
@@ -224,25 +225,74 @@ def get_model_info() -> dict:
|
||||
OLLAMA_BASE_URL = settings.OLLAMA_BASE_URL
|
||||
OLLAMA_API_URL = settings.OLLAMA_API_URL
|
||||
DEFAULT_VLM_MODEL = settings.DEFAULT_VLM_MODEL
|
||||
VLM_MODEL_MARKERS = (
|
||||
"qwen3-vl",
|
||||
"qwen2-vl",
|
||||
"minicpm-v",
|
||||
"llama3.2-vision",
|
||||
"llava",
|
||||
"vision",
|
||||
"-vl",
|
||||
"_vl",
|
||||
)
|
||||
|
||||
async def _get_available_vlm_model() -> str:
|
||||
def _normalize_model_name(model_name: Optional[str]) -> Optional[str]:
|
||||
normalized = str(model_name or "").strip()
|
||||
return normalized or None
|
||||
|
||||
def is_likely_vlm_model(model_name: Optional[str]) -> bool:
|
||||
lower = str(model_name or "").strip().lower()
|
||||
return bool(lower and any(marker in lower for marker in VLM_MODEL_MARKERS))
|
||||
|
||||
async def get_ollama_models(timeout: float = 2.0) -> List[str]:
|
||||
"""Return model names reported by the local Ollama service."""
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
resp = await client.get(f"{OLLAMA_BASE_URL.rstrip('/')}/api/tags")
|
||||
resp.raise_for_status()
|
||||
return [
|
||||
str(model.get("name") or "").strip()
|
||||
for model in resp.json().get("models", [])
|
||||
if str(model.get("name") or "").strip()
|
||||
]
|
||||
|
||||
async def get_ollama_vlm_models(timeout: float = 2.0) -> List[str]:
|
||||
"""Return installed Ollama models whose names indicate image-input support."""
|
||||
return [
|
||||
model_name
|
||||
for model_name in await get_ollama_models(timeout=timeout)
|
||||
if is_likely_vlm_model(model_name)
|
||||
]
|
||||
|
||||
async def _get_available_vlm_model(preferred_model: Optional[str] = None) -> str:
|
||||
"""自动检测本地可用的 VLM 模型"""
|
||||
preferred = _normalize_model_name(preferred_model)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=2.0) as client:
|
||||
resp = await client.get(f"{OLLAMA_BASE_URL}/api/tags")
|
||||
if resp.status_code == 200:
|
||||
models = [m['name'] for m in resp.json().get('models', [])]
|
||||
# 优先级:qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl 的模型
|
||||
for target in ["qwen3-vl:8b", "qwen2-vl", "minicpm-v"]:
|
||||
for m in models:
|
||||
if target in m: return m
|
||||
for m in models:
|
||||
if "vl" in m.lower(): return m
|
||||
except:
|
||||
models = await get_ollama_models(timeout=2.0)
|
||||
if preferred and is_likely_vlm_model(preferred):
|
||||
if preferred in models:
|
||||
return preferred
|
||||
for model in models:
|
||||
if is_likely_vlm_model(model) and (model.startswith(preferred) or preferred in model):
|
||||
return model
|
||||
# 优先级:qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl/vision/llava 的模型
|
||||
for target in ["qwen3-vl", "qwen2-vl", "minicpm-v", "llama3.2-vision", "llava"]:
|
||||
for model in models:
|
||||
if target in model.lower():
|
||||
return model
|
||||
for model in models:
|
||||
if is_likely_vlm_model(model):
|
||||
return model
|
||||
except Exception:
|
||||
pass
|
||||
return DEFAULT_VLM_MODEL
|
||||
|
||||
async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callback=None) -> str:
|
||||
async def analyze_map_with_vlm(
|
||||
images_base64: list,
|
||||
prompt: str,
|
||||
progress_callback=None,
|
||||
model_name: Optional[str] = None,
|
||||
raise_on_error: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
使用本地 Ollama 部署的多模态大模型分析地图截图。
|
||||
已改为一次性返回模式,以提高连接稳定性。
|
||||
@@ -250,10 +300,10 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba
|
||||
if not images_base64:
|
||||
return "未接收到有效的地图截图。"
|
||||
|
||||
model_name = await _get_available_vlm_model()
|
||||
resolved_model_name = await _get_available_vlm_model(model_name)
|
||||
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"model": resolved_model_name,
|
||||
"prompt": prompt,
|
||||
"images": [img.split(",")[1] if "," in img else img for img in images_base64],
|
||||
"stream": False, # 关闭流式传输,改为一次性返回
|
||||
@@ -282,9 +332,11 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba
|
||||
final_output += f"> [!NOTE] 思考过程\n> {full_thinking}\n\n"
|
||||
final_output += full_response
|
||||
|
||||
return final_output.strip() if final_output else f"模型 ({model_name}) 未返回任何内容。"
|
||||
return final_output.strip() if final_output else f"模型 ({resolved_model_name}) 未返回任何内容。"
|
||||
|
||||
except Exception as e:
|
||||
if raise_on_error:
|
||||
raise RuntimeError(f"Ollama VLM request failed: {str(e)}") from e
|
||||
return f"AI 分析过程中发生错误: {str(e)}"
|
||||
|
||||
async def generate_dinsar_diagnosis(
|
||||
@@ -293,12 +345,13 @@ async def generate_dinsar_diagnosis(
|
||||
date_str: str,
|
||||
quality_context: str,
|
||||
hazard_info: str,
|
||||
progress_callback=None
|
||||
progress_callback=None,
|
||||
model_name: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
针对 VLM 优化的 D-InSAR 专家诊断逻辑。
|
||||
"""
|
||||
model_name = await _get_available_vlm_model()
|
||||
resolved_model_name = await _get_available_vlm_model(model_name)
|
||||
|
||||
prompt = (
|
||||
f"你是一位拥有 20 年经验的资深 InSAR 地质灾害解译专家。请根据提供的 D-InSAR 形变图及背景信息,撰写一份专业的诊断报告。\n\n"
|
||||
@@ -320,22 +373,22 @@ async def generate_dinsar_diagnosis(
|
||||
f"- 使用 Markdown 格式,语言严谨、专业,严禁幻觉。\n"
|
||||
f"- 报告末尾必须包含以下加粗文字:\n"
|
||||
f"**--- 免责声明 ---**\n"
|
||||
f"**本报告由 AI 自动生成(模型:{model_name}),仅供科研参考,不具备法律效力。**"
|
||||
f"**本报告由 AI 自动生成(模型:{resolved_model_name}),仅供科研参考,不具备法律效力。**"
|
||||
)
|
||||
|
||||
return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback)
|
||||
return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback, model_name=resolved_model_name)
|
||||
|
||||
async def warm_up_vlm() -> bool:
|
||||
async def warm_up_vlm(model_name: Optional[str] = None) -> bool:
|
||||
"""
|
||||
预热 VLM 模型,将其加载至显存。
|
||||
发送一个轻量级请求以触发模型冷启动。
|
||||
返回 True 表示成功,False 表示失败。
|
||||
"""
|
||||
# 预热时直接使用探测到的模型
|
||||
model_name = await _get_available_vlm_model()
|
||||
# 预热时直接使用指定模型或探测到的模型
|
||||
resolved_model_name = await _get_available_vlm_model(model_name)
|
||||
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"model": resolved_model_name,
|
||||
"prompt": "hi",
|
||||
"stream": False,
|
||||
"keep_alive": "30m"
|
||||
|
||||
+97
-20
@@ -101,6 +101,18 @@ def _default_task_pool_root(project_root: str) -> str:
|
||||
return os.path.join(normalized_root, "Task_Pool")
|
||||
|
||||
|
||||
def _default_gf3_data_root(project_root: str) -> str:
|
||||
normalized_root = os.path.normpath(project_root)
|
||||
drive, _tail = os.path.splitdrive(normalized_root)
|
||||
if drive:
|
||||
return os.path.join(drive + os.sep, "GaoFen3_Pool")
|
||||
return os.path.join(normalized_root, "GaoFen3_Pool")
|
||||
|
||||
|
||||
def _default_gf3_task_pool_root(project_root: str) -> str:
|
||||
return os.path.join(_default_gf3_data_root(project_root), "task_pool")
|
||||
|
||||
|
||||
def _default_runtime_dir(project_root: str, *parts: str) -> str:
|
||||
return os.path.join(_default_runtime_root(project_root), *parts)
|
||||
|
||||
@@ -199,6 +211,7 @@ class Settings(BaseSettings):
|
||||
DINSAR_TASK_POOL_ROOT: str = ""
|
||||
SBAS_TASK_POOL_ROOT: str = ""
|
||||
GF3_TASK_POOL_ROOT: str = ""
|
||||
DATA_DISTRIBUTION_ROOT: str = ""
|
||||
SOURCE_PRODUCT_DIRS: str = ""
|
||||
SENTINEL1_STORAGE_DIRS: str = ""
|
||||
ORBIT_SOURCE_DIRS: str = ""
|
||||
@@ -239,7 +252,7 @@ class Settings(BaseSettings):
|
||||
GF3_GEO_DEM_PATH: str = ""
|
||||
GF3_ARCHIVE_SOURCE_DIRS: str = ""
|
||||
GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz"
|
||||
GF3_UNPACK_DELETE_ARCHIVE: bool = True
|
||||
GF3_UNPACK_DELETE_ARCHIVE: bool = False
|
||||
GF3_LEGACY_GDAL_ENABLED: bool = False
|
||||
GF3_SOURCE_DIRS: str = ""
|
||||
GF3_SARSCAPE_NATIVE_DIRS: str = ""
|
||||
@@ -250,7 +263,7 @@ class Settings(BaseSettings):
|
||||
GF3_SARSCAPE_DEM_PATH: str = ""
|
||||
GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV"
|
||||
GF3_SARSCAPE_KEEP_EXTRACTED: bool = True
|
||||
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True
|
||||
GF3_SARSCAPE_AUTO_STANDARDIZE: bool = False
|
||||
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True
|
||||
GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0
|
||||
|
||||
@@ -460,7 +473,9 @@ class Settings(BaseSettings):
|
||||
if not self.SBAS_TASK_POOL_ROOT:
|
||||
object.__setattr__(self, "SBAS_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "SBAS"))
|
||||
if not self.GF3_TASK_POOL_ROOT:
|
||||
object.__setattr__(self, "GF3_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "GF3"))
|
||||
object.__setattr__(self, "GF3_TASK_POOL_ROOT", _default_gf3_task_pool_root(project_root))
|
||||
if not self.DATA_DISTRIBUTION_ROOT:
|
||||
object.__setattr__(self, "DATA_DISTRIBUTION_ROOT", os.path.join(self.TASK_POOL_ROOT, "Data_Distribution"))
|
||||
if not self.SAR_ANALYSIS_READY_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
@@ -484,25 +499,25 @@ class Settings(BaseSettings):
|
||||
object.__setattr__(
|
||||
self,
|
||||
"GF3_ARCHIVE_SOURCE_DIRS",
|
||||
os.path.join(_default_input_root(project_root), "gf3", "archives"),
|
||||
os.path.join(_default_gf3_data_root(project_root), "archives"),
|
||||
)
|
||||
if not self.GF3_SARSCAPE_NATIVE_DIRS:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"GF3_SARSCAPE_NATIVE_DIRS",
|
||||
os.path.join(_default_result_publish_root(project_root), "gf3", "sarscape_native"),
|
||||
os.path.join(_default_gf3_data_root(project_root), "native_geo"),
|
||||
)
|
||||
if not self.GF3_STORAGE_DIRS:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"GF3_STORAGE_DIRS",
|
||||
os.path.join(_default_result_publish_root(project_root), "gf3", "standard_l2"),
|
||||
os.path.join(_default_gf3_data_root(project_root), "catalog"),
|
||||
)
|
||||
if not self.GF3_SARSCAPE_RUNTIME_DIR:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"GF3_SARSCAPE_RUNTIME_DIR",
|
||||
_default_runtime_dir(project_root, "gf3", "sarscape_runtime"),
|
||||
os.path.join(self.GF3_TASK_POOL_ROOT, "sarscape_runtime"),
|
||||
)
|
||||
if not self.ORBIT_QUARANTINE_DIR and self.MONITOR_ORBIT_DIR:
|
||||
object.__setattr__(
|
||||
@@ -835,7 +850,8 @@ class Settings(BaseSettings):
|
||||
if "lutan" in item.lower() or "lt1" in item.lower()
|
||||
]
|
||||
lt1_roots = list(dict.fromkeys(lt1_roots))
|
||||
object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", ";".join(lt1_roots) or r"D:\LuTan1_Image_Pool")
|
||||
materialized_lt1_root = os.path.join(self.TASK_POOL_ROOT, "source_materialized", "lutan1")
|
||||
object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", materialized_lt1_root)
|
||||
if not self.GAMMA_SBAS_ORBIT_ROOTS:
|
||||
object.__setattr__(self, "GAMMA_SBAS_ORBIT_ROOTS", self.PYINT_ORBIT_POOL_TXT or self.ORBIT_POOL_ENVI)
|
||||
object.__setattr__(self, "GAMMA_SBAS_DEFAULT_RLKS", max(1, int(self.GAMMA_SBAS_DEFAULT_RLKS or 8)))
|
||||
@@ -1014,6 +1030,7 @@ class Settings(BaseSettings):
|
||||
os.makedirs(settings.DINSAR_TASK_POOL_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.SBAS_TASK_POOL_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.GF3_TASK_POOL_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.DATA_DISTRIBUTION_ROOT, exist_ok=True)
|
||||
for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS):
|
||||
os.makedirs(path, exist_ok=True)
|
||||
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
|
||||
@@ -1100,6 +1117,24 @@ def _is_wsl_posix_path(value: str) -> bool:
|
||||
return text.startswith("/home/") or text.startswith("/mnt/")
|
||||
|
||||
|
||||
def _is_unc_path(value: str) -> bool:
|
||||
return str(value or "").strip().strip('"').strip("'").startswith("\\\\")
|
||||
|
||||
|
||||
def _enforce_local_runtime_paths(
|
||||
*,
|
||||
pairs: list[tuple[str, str]],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
for label, raw_value in pairs:
|
||||
raw_text = str(raw_value or "")
|
||||
values = split_env_paths(raw_text) if (";" in raw_text or "," in raw_text) else [raw_text]
|
||||
for item in values:
|
||||
text = str(item or "").strip().strip('"').strip("'")
|
||||
if text and _is_unc_path(text):
|
||||
errors.append(f"{label} points to UNC path {text}; active production paths must be local.")
|
||||
|
||||
|
||||
def _check_path(
|
||||
*,
|
||||
label: str,
|
||||
@@ -1171,21 +1206,66 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
else:
|
||||
info.append("DATABASE_URL 已配置。")
|
||||
|
||||
_enforce_local_runtime_paths(
|
||||
pairs=[
|
||||
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
|
||||
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
|
||||
("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS),
|
||||
("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS),
|
||||
("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS),
|
||||
("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS),
|
||||
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
|
||||
("MONITOR_ORBIT_DIR", settings.MONITOR_ORBIT_DIR),
|
||||
("TASK_POOL_ROOT", settings.TASK_POOL_ROOT),
|
||||
("DINSAR_TASK_POOL_ROOT", settings.DINSAR_TASK_POOL_ROOT),
|
||||
("SBAS_TASK_POOL_ROOT", settings.SBAS_TASK_POOL_ROOT),
|
||||
("GF3_TASK_POOL_ROOT", settings.GF3_TASK_POOL_ROOT),
|
||||
("DATA_DISTRIBUTION_ROOT", settings.DATA_DISTRIBUTION_ROOT),
|
||||
("ORBIT_POOL_ENVI", settings.ORBIT_POOL_ENVI),
|
||||
("ORBIT_POOL_ISCE2", settings.ORBIT_POOL_ISCE2),
|
||||
("ORBIT_POOL_LANDSAR", settings.ORBIT_POOL_LANDSAR),
|
||||
("PYINT_ORBIT_POOL_TXT", settings.PYINT_ORBIT_POOL_TXT),
|
||||
("PYINT_WORK_ROOT", settings.PYINT_WORK_ROOT),
|
||||
("PYINT_OUTPUT_ROOT", settings.PYINT_OUTPUT_ROOT),
|
||||
("LANDSAR_WORK_ROOT", settings.LANDSAR_WORK_ROOT),
|
||||
("RESULT_PUBLISH_ROOT", settings.RESULT_PUBLISH_ROOT),
|
||||
("DINSAR_PRODUCT_DIR", settings.DINSAR_PRODUCT_DIR),
|
||||
("TIMESERIES_PRODUCT_DIR", settings.TIMESERIES_PRODUCT_DIR),
|
||||
("SAR_ANALYSIS_READY_ROOT", settings.SAR_ANALYSIS_READY_ROOT),
|
||||
("SAR_ANALYSIS_WORK_ROOT", settings.SAR_ANALYSIS_WORK_ROOT),
|
||||
("GAMMA_SBAS_WORK_ROOT", settings.GAMMA_SBAS_WORK_ROOT),
|
||||
("GAMMA_SBAS_PRODUCT_ROOT", settings.GAMMA_SBAS_PRODUCT_ROOT),
|
||||
("GAMMA_SBAS_SOURCE_ROOTS", settings.GAMMA_SBAS_SOURCE_ROOTS),
|
||||
("GAMMA_SBAS_ORBIT_ROOTS", settings.GAMMA_SBAS_ORBIT_ROOTS),
|
||||
("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS),
|
||||
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
|
||||
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
|
||||
("GF3_SARSCAPE_RUNTIME_DIR", settings.GF3_SARSCAPE_RUNTIME_DIR),
|
||||
],
|
||||
errors=errors,
|
||||
)
|
||||
|
||||
_check_path(label="PYTHON_PATH", value=settings.PYTHON_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="NGINX_PATH", value=settings.NGINX_PATH, errors=errors, warnings=warnings, required=True, expect_file=True)
|
||||
_check_path(label="LICENSE_PATH", value=settings.LICENSE_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(
|
||||
label="GF3_SARSCAPE_DEM_PATH",
|
||||
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
gf3_local_sarscape_enabled = bool(
|
||||
settings.GF3_LEGACY_GDAL_ENABLED
|
||||
or settings.GF3_SARSCAPE_AUTO_STANDARDIZE
|
||||
)
|
||||
if gf3_local_sarscape_enabled:
|
||||
_check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True)
|
||||
_check_path(
|
||||
label="GF3_SARSCAPE_DEM_PATH",
|
||||
value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH),
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=True,
|
||||
)
|
||||
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
@@ -1213,9 +1293,6 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
_check_path(label="RESULT_QUARANTINE_ROOT", value=settings.RESULT_QUARANTINE_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
|
||||
for label, raw_value in (
|
||||
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
|
||||
("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS),
|
||||
("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS),
|
||||
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
|
||||
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
|
||||
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
|
||||
@@ -1230,7 +1307,7 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
|
||||
gf3_archive_dirs = split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
|
||||
if not gf3_archive_dirs:
|
||||
warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置;无法触发 GF3 SARscape 生产,只能扫描已有原生结果。")
|
||||
warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置;GF3 本机生产已停用,原始归档仅用于本地资产追踪。")
|
||||
for item in gf3_archive_dirs:
|
||||
_check_path(label="GF3_ARCHIVE_SOURCE_DIRS", value=item, errors=errors, warnings=warnings, expect_file=False)
|
||||
|
||||
|
||||
+292
-13
@@ -6,11 +6,12 @@ import tarfile
|
||||
import zipfile
|
||||
import json
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import List, Tuple, Optional, Dict, Any
|
||||
|
||||
from .services.task_service import task_service
|
||||
from .services.dinsar_naming import write_pair_metadata
|
||||
from .services.dinsar_naming import build_task_alias, write_pair_metadata
|
||||
|
||||
# --- Core Logic ---
|
||||
|
||||
@@ -44,7 +45,20 @@ def find_dinsar_source_to_copy(path: str) -> str:
|
||||
return path
|
||||
|
||||
|
||||
def _resolve_dinsar_task_names(item: Dict[str, Any]) -> Tuple[str, str]:
|
||||
task_name = str(item.get("task_name") or item.get("task_alias") or "").strip()
|
||||
task_alias = str(item.get("task_alias") or "").strip()
|
||||
if not task_alias:
|
||||
task_alias = build_task_alias(item.get("master_imaging_date"), item.get("slave_imaging_date"))
|
||||
if task_alias == "Task_unknown_unknown" and task_name:
|
||||
task_alias = task_name
|
||||
if not task_name:
|
||||
task_name = task_alias or "task"
|
||||
return task_name or "task", task_alias or task_name or "task"
|
||||
|
||||
|
||||
_DINSAR_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".zip", ".tar")
|
||||
_LT1_RASTER_SUFFIXES = (".tif", ".tiff")
|
||||
|
||||
|
||||
def _is_supported_archive(path: str) -> bool:
|
||||
@@ -52,6 +66,243 @@ def _is_supported_archive(path: str) -> bool:
|
||||
return any(lower.endswith(suffix) for suffix in _DINSAR_ARCHIVE_SUFFIXES)
|
||||
|
||||
|
||||
def _is_lt1_scene_file_name(filename: str) -> bool:
|
||||
return str(filename or "").lower().startswith("lt1")
|
||||
|
||||
|
||||
def _is_lt1_meta_name(filename: str) -> bool:
|
||||
lower = str(filename or "").lower()
|
||||
return _is_lt1_scene_file_name(lower) and lower.endswith(".meta.xml")
|
||||
|
||||
|
||||
def _is_lt1_raster_name(filename: str) -> bool:
|
||||
lower = str(filename or "").lower()
|
||||
return _is_lt1_scene_file_name(lower) and lower.endswith(_LT1_RASTER_SUFFIXES)
|
||||
|
||||
|
||||
def _extract_yyyymmdd_from_name(filename: str) -> str:
|
||||
match = re.search(r"(20\d{6})", str(filename or ""))
|
||||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _iter_lt1_scene_files(root_dir: str) -> List[str]:
|
||||
normalized = os.path.normpath(os.path.abspath(str(root_dir or "")))
|
||||
if not os.path.isdir(normalized):
|
||||
return []
|
||||
files: List[str] = []
|
||||
for current_root, _, filenames in os.walk(normalized):
|
||||
for filename in filenames:
|
||||
if _is_lt1_scene_file_name(filename):
|
||||
files.append(os.path.join(current_root, filename))
|
||||
return sorted(files, key=lambda item: os.path.normcase(os.path.relpath(item, normalized)))
|
||||
|
||||
|
||||
def _link_or_copy_file(source_path: str, dest_path: str) -> str:
|
||||
source = os.path.normpath(os.path.abspath(str(source_path or "")))
|
||||
dest = os.path.normpath(os.path.abspath(str(dest_path or "")))
|
||||
if not os.path.isfile(source):
|
||||
raise FileNotFoundError(source)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
if os.path.exists(dest):
|
||||
try:
|
||||
if os.path.samefile(source, dest):
|
||||
return "exists"
|
||||
except OSError:
|
||||
pass
|
||||
if os.path.isdir(dest):
|
||||
raise IsADirectoryError(dest)
|
||||
os.remove(dest)
|
||||
try:
|
||||
os.link(source, dest)
|
||||
return "linked"
|
||||
except OSError:
|
||||
shutil.copy2(source, dest)
|
||||
return "copied"
|
||||
|
||||
|
||||
def _list_direct_files(directory: str) -> List[str]:
|
||||
if not os.path.isdir(directory):
|
||||
return []
|
||||
return sorted(
|
||||
os.path.join(directory, entry.name)
|
||||
for entry in os.scandir(directory)
|
||||
if entry.is_file()
|
||||
)
|
||||
|
||||
|
||||
def _flatten_lt1_side_inputs(side_dir: str) -> Dict[str, Any]:
|
||||
normalized = os.path.normpath(os.path.abspath(str(side_dir or "")))
|
||||
summary: Dict[str, Any] = {
|
||||
"side_dir": normalized,
|
||||
"lt1_source_file_count": 0,
|
||||
"linked": 0,
|
||||
"copied": 0,
|
||||
"exists": 0,
|
||||
"direct_meta_count": 0,
|
||||
"direct_raster_count": 0,
|
||||
}
|
||||
if not os.path.isdir(normalized):
|
||||
summary["status"] = "missing_side_dir"
|
||||
return summary
|
||||
|
||||
source_files = _iter_lt1_scene_files(normalized)
|
||||
summary["lt1_source_file_count"] = len(source_files)
|
||||
for source in source_files:
|
||||
dest = os.path.join(normalized, os.path.basename(source))
|
||||
action = _link_or_copy_file(source, dest)
|
||||
if action in {"linked", "copied", "exists"}:
|
||||
summary[action] = int(summary.get(action, 0)) + 1
|
||||
|
||||
direct_files = _list_direct_files(normalized)
|
||||
summary["direct_meta_count"] = sum(1 for path in direct_files if _is_lt1_meta_name(os.path.basename(path)))
|
||||
summary["direct_raster_count"] = sum(1 for path in direct_files if _is_lt1_raster_name(os.path.basename(path)))
|
||||
summary["status"] = "ready" if summary["direct_meta_count"] and summary["direct_raster_count"] else "no_lt1_direct_pair"
|
||||
return summary
|
||||
|
||||
|
||||
def _select_lt1_meta_raster(side_dir: str, expected_date: Any = None) -> Dict[str, str]:
|
||||
expected = re.sub(r"\D", "", str(expected_date or ""))[:8]
|
||||
direct_files = _list_direct_files(side_dir)
|
||||
metas = [path for path in direct_files if _is_lt1_meta_name(os.path.basename(path))]
|
||||
rasters = [path for path in direct_files if _is_lt1_raster_name(os.path.basename(path))]
|
||||
|
||||
def sort_key(path: str) -> tuple[int, str]:
|
||||
name = os.path.basename(path)
|
||||
date_mismatch = 0 if expected and expected in name else 1 if expected else 0
|
||||
return date_mismatch, name.lower()
|
||||
|
||||
metas.sort(key=sort_key)
|
||||
rasters.sort(key=sort_key)
|
||||
return {
|
||||
"meta": metas[0] if metas else "",
|
||||
"raster": rasters[0] if rasters else "",
|
||||
}
|
||||
|
||||
|
||||
def _prepare_landsar_input_data(
|
||||
task_dir: str,
|
||||
master_selection: Dict[str, str],
|
||||
slave_selection: Dict[str, str],
|
||||
) -> Dict[str, Any]:
|
||||
input_dir = os.path.join(task_dir, "Input_Data")
|
||||
copied: List[Dict[str, Any]] = []
|
||||
for role, selection in (("master", master_selection), ("slave", slave_selection)):
|
||||
for kind, source in (("meta", selection.get("meta")), ("raster", selection.get("raster"))):
|
||||
if not source:
|
||||
return {
|
||||
"status": "missing_selected_lt1_file",
|
||||
"input_data_dir": input_dir,
|
||||
"role": role,
|
||||
"kind": kind,
|
||||
"copied": copied,
|
||||
}
|
||||
dest = os.path.join(input_dir, os.path.basename(source))
|
||||
action = _link_or_copy_file(source, dest)
|
||||
copied.append(
|
||||
{
|
||||
"role": role,
|
||||
"kind": kind,
|
||||
"source_path": source,
|
||||
"relative_path": os.path.relpath(dest, start=task_dir),
|
||||
"action": action,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"status": "ready",
|
||||
"input_data_dir": input_dir,
|
||||
"copied": copied,
|
||||
}
|
||||
|
||||
|
||||
def _landsar_input_data_ready(input_data_dir: str) -> bool:
|
||||
if not os.path.isdir(input_data_dir):
|
||||
return False
|
||||
by_date: Dict[str, Dict[str, bool]] = {}
|
||||
for path in _list_direct_files(input_data_dir):
|
||||
name = os.path.basename(path)
|
||||
if not _is_lt1_scene_file_name(name):
|
||||
continue
|
||||
lower_name = name.lower()
|
||||
if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"):
|
||||
continue
|
||||
if lower_name.endswith(".xml") and "_slc" not in lower_name:
|
||||
continue
|
||||
date_text = _extract_yyyymmdd_from_name(name)
|
||||
if not date_text:
|
||||
continue
|
||||
entry = by_date.setdefault(date_text, {"meta": False, "raster": False})
|
||||
if lower_name.endswith(".xml"):
|
||||
entry["xml"] = True
|
||||
elif _is_lt1_raster_name(name):
|
||||
entry["raster"] = True
|
||||
return sum(1 for item in by_date.values() if item.get("xml") and item.get("raster")) >= 2
|
||||
|
||||
|
||||
def _lt1_side_direct_raw_ready(side_dir: str) -> bool:
|
||||
direct_files = _list_direct_files(side_dir)
|
||||
return (
|
||||
any(_is_lt1_meta_name(os.path.basename(path)) for path in direct_files)
|
||||
and any(_is_lt1_raster_name(os.path.basename(path)) for path in direct_files)
|
||||
)
|
||||
|
||||
|
||||
def _prepare_dinsar_engine_inputs(task_dir: str, item: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
normalized = os.path.normpath(os.path.abspath(str(task_dir or "")))
|
||||
payload = dict(item or {})
|
||||
master_dir = os.path.join(normalized, "master")
|
||||
slave_dir = os.path.join(normalized, "slave")
|
||||
master_summary = _flatten_lt1_side_inputs(master_dir)
|
||||
slave_summary = _flatten_lt1_side_inputs(slave_dir)
|
||||
summary: Dict[str, Any] = {
|
||||
"master": master_summary,
|
||||
"slave": slave_summary,
|
||||
"landsar": {"status": "skipped_non_lt1"},
|
||||
}
|
||||
|
||||
has_lt1 = bool(master_summary.get("lt1_source_file_count")) or bool(slave_summary.get("lt1_source_file_count"))
|
||||
if not has_lt1:
|
||||
return summary
|
||||
|
||||
master_selection = _select_lt1_meta_raster(master_dir, payload.get("master_imaging_date"))
|
||||
slave_selection = _select_lt1_meta_raster(slave_dir, payload.get("slave_imaging_date"))
|
||||
summary["selected"] = {
|
||||
"master": {
|
||||
"meta": os.path.relpath(master_selection["meta"], start=normalized) if master_selection.get("meta") else "",
|
||||
"raster": os.path.relpath(master_selection["raster"], start=normalized) if master_selection.get("raster") else "",
|
||||
},
|
||||
"slave": {
|
||||
"meta": os.path.relpath(slave_selection["meta"], start=normalized) if slave_selection.get("meta") else "",
|
||||
"raster": os.path.relpath(slave_selection["raster"], start=normalized) if slave_selection.get("raster") else "",
|
||||
},
|
||||
}
|
||||
if not all(master_selection.values()) or not all(slave_selection.values()):
|
||||
summary["landsar"] = {"status": "missing_lt1_meta_or_raster"}
|
||||
return summary
|
||||
|
||||
summary["landsar"] = {
|
||||
"status": "raw_ready_for_import",
|
||||
"input_data_policy": "LandSAR 100016 import creates native landsar_input at run time.",
|
||||
}
|
||||
return summary
|
||||
|
||||
|
||||
def _normalize_source_bundle_archive_path(source_path: str) -> str:
|
||||
normalized = os.path.normpath(os.path.abspath(str(source_path or "")))
|
||||
if not os.path.exists(normalized):
|
||||
raise FileNotFoundError(normalized)
|
||||
if os.path.isdir(normalized):
|
||||
raise ValueError(
|
||||
"D-InSAR source bundle distribution requires source archive files; "
|
||||
f"unpacked directories are retired: {normalized}"
|
||||
)
|
||||
if not os.path.isfile(normalized) or not _is_supported_archive(normalized):
|
||||
raise ValueError(
|
||||
"D-InSAR source bundle distribution requires .zip/.tar.gz/.tgz/.tar source archives: "
|
||||
f"{normalized}"
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _safe_archive_member_name(member_name: str, archive_path: str) -> str:
|
||||
name = str(member_name or "").replace("\\", "/").strip("/")
|
||||
if not name or name.startswith("../") or "/../" in f"/{name}/":
|
||||
@@ -104,10 +355,20 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int:
|
||||
raise ValueError(f"Unsupported archive format: {archive_path}")
|
||||
|
||||
|
||||
def _materialize_dinsar_source(source_path: str, dest_dir: str) -> Dict[str, Any]:
|
||||
def _materialize_dinsar_source(
|
||||
source_path: str,
|
||||
dest_dir: str,
|
||||
*,
|
||||
require_archive: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
normalized = os.path.normpath(os.path.abspath(str(source_path or "")))
|
||||
if not os.path.exists(normalized):
|
||||
raise FileNotFoundError(normalized)
|
||||
if require_archive and (not os.path.isfile(normalized) or not _is_supported_archive(normalized)):
|
||||
raise ValueError(
|
||||
"D-InSAR production preparation requires source archive files; "
|
||||
f"rebuild the batch from LT-1/Sentinel-1 archive assets: {normalized}"
|
||||
)
|
||||
|
||||
if os.path.isdir(normalized):
|
||||
shutil.copytree(normalized, dest_dir, dirs_exist_ok=True)
|
||||
@@ -266,9 +527,16 @@ def _directory_has_entries(path: str) -> bool:
|
||||
|
||||
|
||||
def _is_existing_dinsar_folder_complete(task_dir: str) -> bool:
|
||||
master_dir = os.path.join(task_dir, "master")
|
||||
slave_dir = os.path.join(task_dir, "slave")
|
||||
input_data_dir = os.path.join(task_dir, "Input_Data")
|
||||
has_lt1_inputs = bool(_iter_lt1_scene_files(master_dir) or _iter_lt1_scene_files(slave_dir))
|
||||
lt1_raw_ready = _lt1_side_direct_raw_ready(master_dir) and _lt1_side_direct_raw_ready(slave_dir)
|
||||
input_data_ready = (not has_lt1_inputs) or lt1_raw_ready or _landsar_input_data_ready(input_data_dir)
|
||||
return (
|
||||
_directory_has_entries(os.path.join(task_dir, "master"))
|
||||
and _directory_has_entries(os.path.join(task_dir, "slave"))
|
||||
_directory_has_entries(master_dir)
|
||||
and _directory_has_entries(slave_dir)
|
||||
and input_data_ready
|
||||
)
|
||||
|
||||
|
||||
@@ -532,8 +800,7 @@ async def run_dinsar_source_bundle_items(
|
||||
|
||||
candidate_tasks: List[Dict[str, Any]] = []
|
||||
for item in items:
|
||||
task_name = item.get("task_name") or item.get("task_alias") or "task"
|
||||
task_alias = item.get("task_alias") or task_name
|
||||
task_name, task_alias = _resolve_dinsar_task_names(item)
|
||||
master_path = item.get("master_path")
|
||||
slave_path = item.get("slave_path")
|
||||
if not master_path or not slave_path:
|
||||
@@ -749,7 +1016,7 @@ async def run_dinsar_source_bundle_items(
|
||||
)
|
||||
|
||||
def ensure_scene_entry(scene_path: str, role_item: Dict[str, Any], prefix: str) -> Dict[str, Any]:
|
||||
source_path = os.path.normpath(os.path.abspath(str(scene_path)))
|
||||
source_path = _normalize_source_bundle_archive_path(scene_path)
|
||||
source_key = os.path.normcase(source_path)
|
||||
relative_path = _bundle_relative_scene_path(source_path)
|
||||
existing = scene_entries_by_source.get(source_key) or scene_entries_by_relative.get(relative_path)
|
||||
@@ -1100,8 +1367,7 @@ async def run_dinsar_copy_items(
|
||||
|
||||
tasks: List[Dict[str, Any]] = []
|
||||
for item in items:
|
||||
task_name = item.get("task_name") or item.get("task_alias") or "task"
|
||||
task_alias = item.get("task_alias") or task_name
|
||||
task_name, task_alias = _resolve_dinsar_task_names(item)
|
||||
master_path = item.get("master_path")
|
||||
slave_path = item.get("slave_path")
|
||||
if not master_path or not slave_path:
|
||||
@@ -1195,11 +1461,13 @@ async def run_dinsar_copy_items(
|
||||
_materialize_dinsar_source,
|
||||
master_src_path,
|
||||
master_dir,
|
||||
require_archive=True,
|
||||
)
|
||||
slave_materialization = await asyncio.to_thread(
|
||||
_materialize_dinsar_source,
|
||||
slave_src_path,
|
||||
slave_dir,
|
||||
require_archive=True,
|
||||
)
|
||||
source_materialization = {
|
||||
"master": {
|
||||
@@ -1211,6 +1479,19 @@ async def run_dinsar_copy_items(
|
||||
"target_relative_path": "slave",
|
||||
},
|
||||
}
|
||||
engine_inputs = await asyncio.to_thread(
|
||||
_prepare_dinsar_engine_inputs,
|
||||
task_dir,
|
||||
item,
|
||||
)
|
||||
source_materialization["engine_inputs"] = engine_inputs
|
||||
landsar_status = str(engine_inputs.get("landsar", {}).get("status") or "")
|
||||
if landsar_status == "raw_ready_for_import":
|
||||
await _log_and_update(task_id, " -> LT-1 raw inputs ready; LandSAR will run 100016 import at production time")
|
||||
elif landsar_status == "skipped_non_lt1":
|
||||
await _log_and_update(task_id, " -> Skipped LandSAR input preparation (non-LT1 source)")
|
||||
else:
|
||||
await _log_and_update(task_id, f" -> LandSAR raw inputs not ready: {landsar_status or 'unknown'}")
|
||||
orbit_entries = await _copy_dinsar_orbit_files(
|
||||
task_id,
|
||||
item,
|
||||
@@ -1235,10 +1516,8 @@ async def run_dinsar_copy_items(
|
||||
await _log_and_update(task_id, f" -> ZIP: {zip_path}")
|
||||
elif not export_zip:
|
||||
if os.path.exists(final_task_dir):
|
||||
raise CopyTaskExecutionError(
|
||||
"Destination folder already exists and cannot be atomically replaced: "
|
||||
f"{final_task_dir}"
|
||||
)
|
||||
await asyncio.to_thread(shutil.rmtree, final_task_dir)
|
||||
await _log_and_update(task_id, f" -> Replaced incomplete existing Task: {final_task_dir}")
|
||||
await asyncio.to_thread(os.replace, task_dir, final_task_dir)
|
||||
await _log_and_update(task_id, f" -> Folder: {final_task_dir}")
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ MIGRATION_FILES = [
|
||||
"008_timeseries_stack_plan_edges.sql",
|
||||
"009_raw_source_pairing_fields.sql",
|
||||
"010_source_orbit_asset_inventory.sql",
|
||||
"011_source_metadata_documents.sql",
|
||||
"012_source_archive_integrity.sql",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -9,18 +9,22 @@ can run in the service process without desktop dependencies.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from ..services.dinsar_naming import (
|
||||
@@ -42,8 +46,10 @@ SUPPORTED_PROFILES = {"lt1_dinsar", "standard"}
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
_DATE_RE = re.compile(r"(?:^|[_-])((?:19|20)\d{6})(?:[_-]|$)")
|
||||
_CENTER_RE = re.compile(r"(?:^|_)E([+-]?\d+(?:\.\d+)?)_N([+-]?\d+(?:\.\d+)?)(?:_|$)", re.IGNORECASE)
|
||||
_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
_SUCCESS_RE = re.compile(r"(success|成功)", re.IGNORECASE)
|
||||
_DEFAULT_DEM_CROP_MARGIN_DEGREES = 0.35
|
||||
|
||||
_DEFAULT_PARAM_VALUES: Dict[str, Any] = {
|
||||
"dem_file_type": 0,
|
||||
@@ -271,6 +277,33 @@ def _extract_date(name: str) -> str:
|
||||
return fallback.group(1) if fallback else ""
|
||||
|
||||
|
||||
def _safe_path_name(value: Any, fallback: str = "item") -> str:
|
||||
text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._-")
|
||||
return text or fallback
|
||||
|
||||
|
||||
def _extract_center_from_name(name: str) -> Optional[tuple[float, float]]:
|
||||
match = _CENTER_RE.search(name or "")
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return float(match.group(1)), float(match.group(2))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _extract_center_from_path(path: str) -> Optional[tuple[float, float]]:
|
||||
source = _norm_path(path)
|
||||
if not source:
|
||||
return None
|
||||
candidates = [os.path.basename(source), os.path.basename(os.path.dirname(source))]
|
||||
for candidate in candidates:
|
||||
center = _extract_center_from_name(candidate)
|
||||
if center:
|
||||
return center
|
||||
return None
|
||||
|
||||
|
||||
def _collect_tail(text: str, max_chars: int = 4000) -> str:
|
||||
content = str(text or "")
|
||||
if len(content) <= max_chars:
|
||||
@@ -296,6 +329,13 @@ def _summarize_landsar_failure(stdout_text: str, stage: str, return_code: int) -
|
||||
return f"{prefix}: LandSAR license dongle login failed{status_text}."
|
||||
if "cann't find 'config.csv'" in lowered or "can't find 'config.csv'" in lowered:
|
||||
return f"{prefix}: LandSAR config.csv is missing. Run versionControl.exe once from LANDSAR_HOME."
|
||||
if "cannot operate regis module" in lowered or "invalid parameters" in lowered:
|
||||
return (
|
||||
f"{prefix}: registration module failed. "
|
||||
"LandSAR could not obtain a valid coregistration solution for this pair."
|
||||
)
|
||||
if "invalid data type" in lowered or "access window out of range" in lowered or "subterrain phase failed" in lowered:
|
||||
return f"{prefix}: DEM/sub-terrain processing failed. Check DEM coverage, data type, and LandSAR-readable format."
|
||||
lines = [line.strip() for line in content.splitlines() if line.strip()]
|
||||
error_lines = [
|
||||
line
|
||||
@@ -418,6 +458,348 @@ def _looks_like_raw_task_dir(task_dir: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _find_lt1_scene_token(directory: str) -> str:
|
||||
source_dir = _norm_path(directory)
|
||||
if not os.path.isdir(source_dir):
|
||||
return ""
|
||||
for entry in os.scandir(source_dir):
|
||||
if entry.is_file():
|
||||
match = re.match(r"^(LT1[AB])_", entry.name, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1).upper()
|
||||
return ""
|
||||
|
||||
|
||||
def _infer_lt1_import_sat_mode(master_dir: str, slave_dir: str) -> str:
|
||||
master_sat = _find_lt1_scene_token(master_dir)
|
||||
slave_sat = _find_lt1_scene_token(slave_dir)
|
||||
if master_sat and slave_sat and master_sat != slave_sat:
|
||||
return "BIST"
|
||||
return "MONO"
|
||||
|
||||
|
||||
def _read_dem_bounds(dem_path: str) -> Optional[tuple[float, float, float, float]]:
|
||||
path = _norm_path(dem_path)
|
||||
if not path or not os.path.isfile(path):
|
||||
return None
|
||||
try:
|
||||
import rasterio # type: ignore
|
||||
|
||||
with rasterio.open(path) as dataset:
|
||||
bounds = dataset.bounds
|
||||
return float(bounds.left), float(bounds.bottom), float(bounds.right), float(bounds.top)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _scene_centers_outside_dem(
|
||||
parsed_pair: Dict[str, Any],
|
||||
dem_path: str,
|
||||
*,
|
||||
margin_degrees: float = 0.25,
|
||||
) -> List[str]:
|
||||
bounds = _read_dem_bounds(dem_path)
|
||||
if not bounds:
|
||||
return []
|
||||
left, bottom, right, top = bounds
|
||||
blockers: List[str] = []
|
||||
for role, key in (("master", "master_xml"), ("slave", "slave_xml")):
|
||||
source_path = str(parsed_pair.get(key) or "")
|
||||
center = _extract_center_from_path(source_path)
|
||||
if not center:
|
||||
continue
|
||||
lon, lat = center
|
||||
if lon < left + margin_degrees or lon > right - margin_degrees or lat < bottom + margin_degrees or lat > top - margin_degrees:
|
||||
blockers.append(
|
||||
f"{role} center E{lon:.3f}/N{lat:.3f} is outside or too close to DEM bounds "
|
||||
f"E{left:.3f}-{right:.3f}, N{bottom:.3f}-{top:.3f}"
|
||||
)
|
||||
return blockers
|
||||
|
||||
|
||||
def _xml_local_name(tag: str) -> str:
|
||||
return str(tag or "").rsplit("}", 1)[-1].lower()
|
||||
|
||||
|
||||
def _xml_float_text(value: Any) -> Optional[float]:
|
||||
try:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
return float(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _xml_child_float_by_local_names(element: ET.Element, names: Iterable[str]) -> Optional[float]:
|
||||
wanted = {str(name).lower() for name in names}
|
||||
for child in element.iter():
|
||||
if child is element:
|
||||
continue
|
||||
if _xml_local_name(child.tag) in wanted:
|
||||
value = _xml_float_text(child.text)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _extract_scene_lonlat_points_from_xml(xml_path: str) -> List[tuple[float, float]]:
|
||||
source = _norm_path(xml_path)
|
||||
if not source or not os.path.isfile(source):
|
||||
return []
|
||||
try:
|
||||
root = ET.parse(source).getroot()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
points: List[tuple[float, float]] = []
|
||||
for element in root.iter():
|
||||
if _xml_local_name(element.tag) != "scenecornercoord":
|
||||
continue
|
||||
lon = _xml_child_float_by_local_names(element, ("lon", "longitude"))
|
||||
lat = _xml_child_float_by_local_names(element, ("lat", "latitude"))
|
||||
if lon is not None and lat is not None:
|
||||
points.append((lon, lat))
|
||||
if points:
|
||||
return points
|
||||
|
||||
for element in root.iter():
|
||||
if _xml_local_name(element.tag) != "scenecentercoord":
|
||||
continue
|
||||
lon = _xml_child_float_by_local_names(element, ("lon", "longitude"))
|
||||
lat = _xml_child_float_by_local_names(element, ("lat", "latitude"))
|
||||
if lon is not None and lat is not None:
|
||||
return [(lon, lat)]
|
||||
return []
|
||||
|
||||
|
||||
def _bbox_from_points(points: Iterable[tuple[float, float]]) -> Optional[tuple[float, float, float, float]]:
|
||||
cleaned = [
|
||||
(float(lon), float(lat))
|
||||
for lon, lat in points
|
||||
if math.isfinite(float(lon)) and math.isfinite(float(lat))
|
||||
]
|
||||
if not cleaned:
|
||||
return None
|
||||
lons = [item[0] for item in cleaned]
|
||||
lats = [item[1] for item in cleaned]
|
||||
return min(lons), min(lats), max(lons), max(lats)
|
||||
|
||||
|
||||
def _expand_lonlat_bbox(
|
||||
bbox: tuple[float, float, float, float],
|
||||
margin_degrees: float,
|
||||
) -> tuple[float, float, float, float]:
|
||||
margin = max(0.0, float(margin_degrees or 0.0))
|
||||
left, bottom, right, top = bbox
|
||||
return (
|
||||
max(-180.0, float(left) - margin),
|
||||
max(-90.0, float(bottom) - margin),
|
||||
min(180.0, float(right) + margin),
|
||||
min(90.0, float(top) + margin),
|
||||
)
|
||||
|
||||
|
||||
def derive_landsar_dem_bbox_from_xml_paths(
|
||||
xml_paths: Iterable[str],
|
||||
*,
|
||||
fallback_paths: Iterable[str] = (),
|
||||
) -> Optional[tuple[float, float, float, float]]:
|
||||
points: List[tuple[float, float]] = []
|
||||
for xml_path in xml_paths:
|
||||
points.extend(_extract_scene_lonlat_points_from_xml(str(xml_path or "")))
|
||||
bbox = _bbox_from_points(points)
|
||||
if bbox:
|
||||
return bbox
|
||||
|
||||
fallback_points: List[tuple[float, float]] = []
|
||||
for path in fallback_paths:
|
||||
center = _extract_center_from_path(str(path or ""))
|
||||
if center:
|
||||
fallback_points.append(center)
|
||||
return _bbox_from_points(fallback_points)
|
||||
|
||||
|
||||
def derive_landsar_pair_dem_bbox(parsed_pair: Dict[str, Any]) -> Optional[tuple[float, float, float, float]]:
|
||||
return derive_landsar_dem_bbox_from_xml_paths(
|
||||
[
|
||||
str(parsed_pair.get("master_xml") or ""),
|
||||
str(parsed_pair.get("slave_xml") or ""),
|
||||
],
|
||||
fallback_paths=[
|
||||
str(parsed_pair.get("master_xml") or parsed_pair.get("master_tif") or ""),
|
||||
str(parsed_pair.get("slave_xml") or parsed_pair.get("slave_tif") or ""),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _align_raster_window(window: Any, width: int, height: int) -> Any:
|
||||
from rasterio.windows import Window
|
||||
|
||||
col_off = max(0, int(math.floor(window.col_off)))
|
||||
row_off = max(0, int(math.floor(window.row_off)))
|
||||
col_stop = min(width, int(math.ceil(window.col_off + window.width)))
|
||||
row_stop = min(height, int(math.ceil(window.row_off + window.height)))
|
||||
if col_stop <= col_off or row_stop <= row_off:
|
||||
raise ValueError("DEM crop bbox does not overlap source DEM")
|
||||
return Window(col_off, row_off, col_stop - col_off, row_stop - row_off)
|
||||
|
||||
|
||||
def _iter_raster_windows(width: int, height: int, block_size: int = 2048) -> Iterable[Any]:
|
||||
from rasterio.windows import Window
|
||||
|
||||
step = max(256, int(block_size or 2048))
|
||||
for row in range(0, int(height), step):
|
||||
h = min(step, int(height) - row)
|
||||
for col in range(0, int(width), step):
|
||||
w = min(step, int(width) - col)
|
||||
yield Window(col, row, w, h)
|
||||
|
||||
|
||||
def _format_bbox_key(bbox: tuple[float, float, float, float]) -> str:
|
||||
return ",".join(f"{value:.8f}" for value in bbox)
|
||||
|
||||
|
||||
def prepare_landsar_dem_crop(
|
||||
source_dem_path: str,
|
||||
crop_root: str,
|
||||
bbox: tuple[float, float, float, float],
|
||||
*,
|
||||
label: str = "task",
|
||||
margin_degrees: float = _DEFAULT_DEM_CROP_MARGIN_DEGREES,
|
||||
block_size: int = 2048,
|
||||
) -> Dict[str, Any]:
|
||||
source = _norm_path(source_dem_path)
|
||||
if not source or not os.path.isfile(source):
|
||||
raise FileNotFoundError(f"LandSAR DEM source file is missing: {source or '<empty>'}")
|
||||
if not bbox:
|
||||
raise ValueError("LandSAR DEM crop bbox is empty")
|
||||
|
||||
expanded_bbox = _expand_lonlat_bbox(bbox, margin_degrees)
|
||||
crop_dir = Path(_norm_path(crop_root))
|
||||
crop_dir.mkdir(parents=True, exist_ok=True)
|
||||
safe_label = _safe_path_name(label, "task")
|
||||
key = hashlib.sha1(f"{source}|{_format_bbox_key(expanded_bbox)}".encode("utf-8", errors="ignore")).hexdigest()[:16]
|
||||
target = crop_dir / f"{safe_label}_{key}_dem.tif"
|
||||
manifest = target.with_suffix(target.suffix + ".json")
|
||||
|
||||
if target.is_file() and manifest.is_file():
|
||||
try:
|
||||
payload = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
payload = {}
|
||||
return {
|
||||
"source_dem_path": source,
|
||||
"dem_path": str(target),
|
||||
"bbox": list(bbox),
|
||||
"expanded_bbox": list(expanded_bbox),
|
||||
"reused": True,
|
||||
"manifest_path": str(manifest),
|
||||
**({"bounds": payload.get("bounds")} if payload.get("bounds") else {}),
|
||||
}
|
||||
|
||||
try:
|
||||
import rasterio # type: ignore
|
||||
from rasterio.transform import array_bounds
|
||||
from rasterio.windows import Window, from_bounds
|
||||
except Exception as exc:
|
||||
raise RuntimeError("rasterio is required to crop LandSAR DEMs") from exc
|
||||
|
||||
temp_path: Optional[Path] = None
|
||||
with rasterio.open(source) as src:
|
||||
source_dtype = str(src.dtypes[0]).lower()
|
||||
if source_dtype != "int16":
|
||||
raise ValueError(f"LandSAR DEM source must be a prepared Int16 GeoTIFF, got dtype={src.dtypes[0]}: {source}")
|
||||
bounds = src.bounds
|
||||
left, bottom, right, top = expanded_bbox
|
||||
if left < bounds.left or right > bounds.right or bottom < bounds.bottom or top > bounds.top:
|
||||
raise ValueError(
|
||||
"LandSAR DEM crop bbox is outside source DEM bounds: "
|
||||
f"bbox=E{left:.6f}-{right:.6f},N{bottom:.6f}-{top:.6f}; "
|
||||
f"source=E{bounds.left:.6f}-{bounds.right:.6f},N{bounds.bottom:.6f}-{bounds.top:.6f}"
|
||||
)
|
||||
|
||||
source_window = _align_raster_window(from_bounds(*expanded_bbox, transform=src.transform), src.width, src.height)
|
||||
source_window = Window(
|
||||
int(source_window.col_off),
|
||||
int(source_window.row_off),
|
||||
int(source_window.width),
|
||||
int(source_window.height),
|
||||
)
|
||||
transform = src.window_transform(source_window)
|
||||
crop_bounds = array_bounds(int(source_window.height), int(source_window.width), transform)
|
||||
|
||||
profile = src.profile.copy()
|
||||
profile.update(
|
||||
driver="GTiff",
|
||||
height=int(source_window.height),
|
||||
width=int(source_window.width),
|
||||
count=1,
|
||||
dtype=src.dtypes[0],
|
||||
crs=src.crs,
|
||||
transform=transform,
|
||||
nodata=src.nodata,
|
||||
compress="NONE",
|
||||
BIGTIFF="YES",
|
||||
interleave="band",
|
||||
)
|
||||
profile.pop("photometric", None)
|
||||
profile.pop("predictor", None)
|
||||
if int(source_window.width) >= 512 and int(source_window.height) >= 512:
|
||||
profile.update(tiled=True, blockxsize=512, blockysize=512)
|
||||
else:
|
||||
profile.pop("blockxsize", None)
|
||||
profile.pop("blockysize", None)
|
||||
profile["tiled"] = False
|
||||
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
prefix=f"{target.stem}.",
|
||||
suffix=".tmp.tif",
|
||||
dir=str(crop_dir),
|
||||
delete=False,
|
||||
) as tmp:
|
||||
temp_path = Path(tmp.name)
|
||||
|
||||
with rasterio.open(temp_path, "w", **profile) as dst:
|
||||
for rel_window in _iter_raster_windows(int(source_window.width), int(source_window.height), block_size):
|
||||
src_window = Window(
|
||||
source_window.col_off + rel_window.col_off,
|
||||
source_window.row_off + rel_window.row_off,
|
||||
rel_window.width,
|
||||
rel_window.height,
|
||||
)
|
||||
dst.write(src.read(1, window=src_window, masked=False), 1, window=rel_window)
|
||||
os.replace(temp_path, target)
|
||||
temp_path = None
|
||||
payload = {
|
||||
"source_dem_path": source,
|
||||
"dem_path": str(target),
|
||||
"bbox": list(bbox),
|
||||
"expanded_bbox": list(expanded_bbox),
|
||||
"bounds": [float(value) for value in crop_bounds],
|
||||
"width": int(source_window.width),
|
||||
"height": int(source_window.height),
|
||||
"dtype": src.dtypes[0],
|
||||
"nodata": src.nodata,
|
||||
"margin_degrees": float(margin_degrees),
|
||||
}
|
||||
manifest.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
finally:
|
||||
if temp_path and temp_path.exists():
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
return {
|
||||
"source_dem_path": source,
|
||||
"dem_path": str(target),
|
||||
"bbox": list(bbox),
|
||||
"expanded_bbox": list(expanded_bbox),
|
||||
"bounds": [float(value) for value in crop_bounds],
|
||||
"reused": False,
|
||||
"manifest_path": str(manifest),
|
||||
}
|
||||
|
||||
|
||||
def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return the first chronological LT-1 SLC pair from Input_Data."""
|
||||
|
||||
@@ -432,6 +814,10 @@ def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]:
|
||||
lower_name = entry.name.lower()
|
||||
if not lower_name.endswith((".xml", ".tif", ".tiff")):
|
||||
continue
|
||||
if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"):
|
||||
continue
|
||||
if lower_name.endswith(".xml") and "_slc" not in lower_name:
|
||||
continue
|
||||
date_text = _extract_date(entry.name)
|
||||
if not date_text:
|
||||
continue
|
||||
@@ -830,6 +1216,9 @@ class LandsarEngine(DinsarEngine):
|
||||
"type": "string",
|
||||
"default": self._default_dem_path,
|
||||
"section": "输入数据",
|
||||
"readonly": True,
|
||||
"readonly_label": "服务器固定",
|
||||
"include_in_payload": False,
|
||||
"description": "传给 LandSAR 200014 D-InSAR 模块的外部参考 DEM,建议使用已验证可用的 GeoTIFF。",
|
||||
},
|
||||
"az_looks": {
|
||||
@@ -1178,14 +1567,14 @@ class LandsarEngine(DinsarEngine):
|
||||
)
|
||||
|
||||
extra = self.normalize_extra(request.extra)
|
||||
dem_path = _norm_path(extra.get("dem_path") or self._default_dem_path)
|
||||
if not dem_path or not os.path.isfile(dem_path):
|
||||
dem_source_path = _norm_path(extra.get("dem_path") or self._default_dem_path)
|
||||
if not dem_source_path or not os.path.isfile(dem_source_path):
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
profile=request.profile,
|
||||
job_id=request.job_id,
|
||||
error=f"LandSAR DEM file is missing: {dem_path or '<empty>'}",
|
||||
error=f"LandSAR DEM source file is missing: {dem_source_path or '<empty>'}",
|
||||
)
|
||||
if _coerce_bool(extra.get("do_atmosphere")):
|
||||
gacos_path = _norm_path(extra.get("gacos_file"))
|
||||
@@ -1341,6 +1730,85 @@ class LandsarEngine(DinsarEngine):
|
||||
continue
|
||||
task_alias, pair_key, pair_meta = self._resolve_task_identity(task_dir, task_name, parsed_pair)
|
||||
|
||||
dem_blockers = _scene_centers_outside_dem(parsed_pair, dem_source_path)
|
||||
if dem_blockers:
|
||||
pairs_failed += 1
|
||||
error_text = "LandSAR DEM coverage preflight failed: " + "; ".join(dem_blockers)
|
||||
emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text)
|
||||
task_results.append(
|
||||
self._build_task_result(
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
run_key=run_key,
|
||||
task_dir=task_dir,
|
||||
run_dir=run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
landsar_output_dir=landsar_output_dir,
|
||||
command=command,
|
||||
success=False,
|
||||
returncode=-2,
|
||||
error=error_text,
|
||||
stdout_tail="",
|
||||
param_file="",
|
||||
dem_source_path=dem_source_path,
|
||||
dem_path=effective_dem_path,
|
||||
dem_crop=dem_crop_info,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
effective_dem_path = dem_source_path
|
||||
dem_crop_info: Dict[str, Any] = {}
|
||||
try:
|
||||
dem_bbox = derive_landsar_pair_dem_bbox(parsed_pair)
|
||||
if not dem_bbox:
|
||||
raise ValueError("cannot derive DEM crop bbox from LT-1 XML corner coordinates")
|
||||
dem_crop_info = prepare_landsar_dem_crop(
|
||||
dem_source_path,
|
||||
os.path.join(native_output_dir, "dem_crop"),
|
||||
dem_bbox,
|
||||
label=task_alias,
|
||||
)
|
||||
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
|
||||
emit_progress(
|
||||
"log",
|
||||
pair_index=pair_index,
|
||||
pair_total=len(task_dirs),
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
level="INFO",
|
||||
source="dem",
|
||||
message=(
|
||||
f"LandSAR DEM crop ready: {effective_dem_path} "
|
||||
f"from {dem_source_path}"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
pairs_failed += 1
|
||||
error_text = f"LandSAR DEM crop failed: {exc}"
|
||||
emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text)
|
||||
task_results.append(
|
||||
self._build_task_result(
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
run_key=run_key,
|
||||
task_dir=task_dir,
|
||||
run_dir=run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
landsar_output_dir=landsar_output_dir,
|
||||
command=command,
|
||||
success=False,
|
||||
returncode=-2,
|
||||
error=error_text,
|
||||
stdout_tail="",
|
||||
param_file="",
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
param_values = {**_DEFAULT_PARAM_VALUES}
|
||||
param_values.update({key: value for key, value in extra.items() if key in _DEFAULT_PARAM_VALUES})
|
||||
param_file = _generate_dinsar_param_file(
|
||||
@@ -1349,7 +1817,7 @@ class LandsarEngine(DinsarEngine):
|
||||
master_tif=parsed_pair["master_tif"],
|
||||
slave_xml=parsed_pair["slave_xml"],
|
||||
slave_tif=parsed_pair["slave_tif"],
|
||||
dem_path=dem_path,
|
||||
dem_path=effective_dem_path,
|
||||
output_dir=landsar_output_dir,
|
||||
params=param_values,
|
||||
)
|
||||
@@ -1375,7 +1843,7 @@ class LandsarEngine(DinsarEngine):
|
||||
pair_key=pair_key,
|
||||
level="INFO",
|
||||
source="dem",
|
||||
message=dem_path,
|
||||
message=effective_dem_path,
|
||||
)
|
||||
|
||||
rc, stdout_text, timed_out = self._run_console(
|
||||
@@ -1461,7 +1929,9 @@ class LandsarEngine(DinsarEngine):
|
||||
request=request,
|
||||
pair_meta=pair_meta_payload,
|
||||
params=param_values,
|
||||
dem_path=dem_path,
|
||||
dem_path=effective_dem_path,
|
||||
dem_source_path=dem_source_path,
|
||||
dem_crop=dem_crop_info,
|
||||
landsar_output_dir=landsar_output_dir,
|
||||
primary_file=primary_file,
|
||||
source_files=source_files,
|
||||
@@ -1510,18 +1980,20 @@ class LandsarEngine(DinsarEngine):
|
||||
param_file=param_file,
|
||||
raw_primary_file=primary_raw,
|
||||
raw_coherence_file=coherence_raw,
|
||||
dem_source_path=dem_source_path,
|
||||
dem_path=effective_dem_path,
|
||||
dem_crop=dem_crop_info,
|
||||
)
|
||||
)
|
||||
|
||||
invalid_candidates = validation.get("invalid_candidates", [])
|
||||
pairs_failed += len(invalid_candidates)
|
||||
overall_success = pairs_processed > 0 and pairs_failed == 0
|
||||
if pairs_processed > 0 and pairs_failed > 0:
|
||||
overall_success = False
|
||||
overall_success = pairs_processed > 0 or (pairs_processed == 0 and pairs_failed == 0)
|
||||
run_status = "COMPLETED" if pairs_failed == 0 else ("PARTIAL" if pairs_processed > 0 else "FAILED")
|
||||
error = None
|
||||
if not overall_success:
|
||||
failed_names = [item.get("task_alias") or item.get("task_name") for item in task_results if not item.get("success")]
|
||||
error = f"LandSAR run failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed."
|
||||
error = f"All LandSAR tasks failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed."
|
||||
|
||||
return RunResult(
|
||||
success=overall_success,
|
||||
@@ -1534,6 +2006,7 @@ class LandsarEngine(DinsarEngine):
|
||||
error=error,
|
||||
detail={
|
||||
"mode": validation["mode"],
|
||||
"run_status": run_status,
|
||||
"task_count": len(task_dirs),
|
||||
"selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results],
|
||||
"invalid_candidates": invalid_candidates,
|
||||
@@ -1541,7 +2014,8 @@ class LandsarEngine(DinsarEngine):
|
||||
"run_key": run_key,
|
||||
"started_at": run_started_at_text,
|
||||
"timeout_seconds": timeout,
|
||||
"dem_path": dem_path,
|
||||
"dem_path": dem_source_path,
|
||||
"dem_role": "global_prepared_source",
|
||||
"console_path": console_path,
|
||||
},
|
||||
)
|
||||
@@ -1674,13 +2148,14 @@ class LandsarEngine(DinsarEngine):
|
||||
"error": "Task directory has neither valid Input_Data nor valid master/slave raw LT-1 folders.",
|
||||
}
|
||||
|
||||
sat_mode = _infer_lt1_import_sat_mode(master_dir, slave_dir)
|
||||
param_file = _generate_import_param_file(
|
||||
os.path.join(export_dir, f"{IMPORT_PROID}.txt"),
|
||||
master_dir=master_dir,
|
||||
slave_dir=slave_dir,
|
||||
export_dir=export_dir,
|
||||
import_method="dir",
|
||||
sat_mode="BIST",
|
||||
sat_mode=sat_mode,
|
||||
read_xml=True,
|
||||
read_slc=True,
|
||||
export_to_new=True,
|
||||
@@ -1695,7 +2170,7 @@ class LandsarEngine(DinsarEngine):
|
||||
pair_key=pair_key,
|
||||
level="INFO",
|
||||
source="import",
|
||||
message=f"LandSAR import 100016 -> {export_dir}",
|
||||
message=f"LandSAR import 100016 ({sat_mode}) -> {export_dir}",
|
||||
)
|
||||
rc, stdout_text, timed_out = self._run_console(
|
||||
command,
|
||||
@@ -1832,6 +2307,8 @@ class LandsarEngine(DinsarEngine):
|
||||
source_files: List[str],
|
||||
returncode: int,
|
||||
started_at: str,
|
||||
dem_source_path: str = "",
|
||||
dem_crop: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
payload = {
|
||||
"run_key": run_key,
|
||||
@@ -1850,6 +2327,8 @@ class LandsarEngine(DinsarEngine):
|
||||
"params": {
|
||||
**dict(params or {}),
|
||||
"dem_path": dem_path,
|
||||
"dem_source_path": dem_source_path or dem_path,
|
||||
"dem_crop": dem_crop or {},
|
||||
},
|
||||
"metrics": {
|
||||
"returncode": returncode,
|
||||
@@ -1902,6 +2381,9 @@ class LandsarEngine(DinsarEngine):
|
||||
param_file: str = "",
|
||||
raw_primary_file: str = "",
|
||||
raw_coherence_file: str = "",
|
||||
dem_source_path: str = "",
|
||||
dem_path: str = "",
|
||||
dem_crop: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
command_text = command if isinstance(command, str) else " ".join(str(part) for part in command)
|
||||
return {
|
||||
@@ -1919,6 +2401,9 @@ class LandsarEngine(DinsarEngine):
|
||||
"raw_primary_file": _norm_path(raw_primary_file) if raw_primary_file else "",
|
||||
"raw_coherence_file": _norm_path(raw_coherence_file) if raw_coherence_file else "",
|
||||
"param_file": _norm_path(param_file) if param_file else "",
|
||||
"dem_source_path": _norm_path(dem_source_path) if dem_source_path else "",
|
||||
"dem_path": _norm_path(dem_path) if dem_path else "",
|
||||
"dem_crop": dem_crop or {},
|
||||
"validation": validation or {},
|
||||
"layout": layout or {},
|
||||
"command": command_text,
|
||||
|
||||
@@ -27,6 +27,8 @@ from .orm import (
|
||||
ScanCursorORM,
|
||||
PathInventoryORM,
|
||||
SourceProductAssetORM,
|
||||
SourceMetadataDocumentORM,
|
||||
SARSceneGeometryProfileORM,
|
||||
OrbitAssetORM,
|
||||
SceneOrbitBindingORM,
|
||||
OrbitAssetDerivativeORM,
|
||||
@@ -103,7 +105,8 @@ __all__ = [
|
||||
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
|
||||
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
|
||||
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
|
||||
"SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM",
|
||||
"SourceProductAssetORM", "SourceMetadataDocumentORM", "SARSceneGeometryProfileORM",
|
||||
"OrbitAssetORM", "SceneOrbitBindingORM",
|
||||
"OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM",
|
||||
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
|
||||
"SystemWorkerHeartbeatORM",
|
||||
|
||||
@@ -3,7 +3,7 @@ SQLAlchemy ORM 模型定义。
|
||||
所有数据库表对应的 ORM 类均在此文件中定义。
|
||||
"""
|
||||
from sqlalchemy import (
|
||||
Column, Integer, BigInteger, String, Boolean, Float, JSON, Text,
|
||||
Column, Integer, BigInteger, String, Boolean, Float, JSON, Text, LargeBinary,
|
||||
DateTime, func, ForeignKey, UniqueConstraint, Index,
|
||||
)
|
||||
from sqlalchemy.orm import relationship
|
||||
@@ -384,7 +384,15 @@ class PairingMetricCacheORM(Base):
|
||||
spatial_baseline_meters = Column(Float, index=True, nullable=True)
|
||||
scene_center_distance_meters = Column(Float, index=True, nullable=True)
|
||||
scene_overlap_ratio = Column(Float, index=True, nullable=True)
|
||||
pair_aoi_overlap_ratio = Column(Float, nullable=True)
|
||||
orbit_direction = Column(String, index=True, nullable=True)
|
||||
same_relative_orbit = Column(Boolean, nullable=False, default=False, server_default="false", index=True)
|
||||
master_relative_orbit = Column(String(64), nullable=True)
|
||||
slave_relative_orbit = Column(String(64), nullable=True)
|
||||
dinsar_quality_tier = Column(String(16), nullable=False, default="C", server_default="C", index=True)
|
||||
dinsar_quality_score = Column(Float, nullable=True)
|
||||
dinsar_readiness = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
|
||||
dinsar_reasons_json = Column(JSON, nullable=True)
|
||||
same_satellite = Column(Boolean, nullable=False, default=True)
|
||||
same_satellite_family = Column(Boolean, nullable=False, default=True, server_default="true")
|
||||
same_look_direction = Column(Boolean, nullable=False, default=True, server_default="true")
|
||||
@@ -875,6 +883,12 @@ class SourceProductAssetORM(Base):
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
|
||||
archive_integrity_status = Column(String(32), nullable=False, default="NOT_CHECKED", server_default="NOT_CHECKED", index=True)
|
||||
archive_integrity_method = Column(String(64), nullable=True)
|
||||
archive_integrity_checked_at = Column(DateTime, nullable=True)
|
||||
archive_integrity_error = Column(Text, nullable=True)
|
||||
archive_integrity_version = Column(String(32), nullable=True)
|
||||
archive_integrity_member_count = Column(Integer, nullable=True)
|
||||
parser_name = Column(String(64), nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
@@ -895,6 +909,84 @@ class SourceProductAssetORM(Base):
|
||||
)
|
||||
|
||||
|
||||
class SourceMetadataDocumentORM(Base):
|
||||
__tablename__ = "source_metadata_documents"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
source_format = Column(String(32), index=True, nullable=True)
|
||||
document_type = Column(String(32), index=True, nullable=False)
|
||||
member_path = Column(String, nullable=False)
|
||||
content_sha256 = Column(String(64), index=True, nullable=False)
|
||||
content_encoding = Column(String(16), nullable=False, default="gzip", server_default="gzip")
|
||||
content_bytes = Column(LargeBinary, nullable=False)
|
||||
content_size_bytes = Column(BigInteger, nullable=True)
|
||||
archive_path = Column(String, nullable=True)
|
||||
archive_mtime = Column(Float, nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="OK", server_default="OK", index=True)
|
||||
parse_error = Column(Text, nullable=True)
|
||||
extracted_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
source_asset = relationship("SourceProductAssetORM")
|
||||
radar_data = relationship("RadarDataORM")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_asset_id", "document_type", "member_path", name="uq_source_metadata_document_member"),
|
||||
Index("idx_source_metadata_documents_asset_type", "source_asset_id", "document_type"),
|
||||
Index("idx_source_metadata_documents_radar_type", "radar_data_id", "document_type"),
|
||||
)
|
||||
|
||||
|
||||
class SARSceneGeometryProfileORM(Base):
|
||||
__tablename__ = "sar_scene_geometry_profiles"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), unique=True, index=True, nullable=False)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), unique=True, index=True, nullable=True)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
satellite = Column(String(32), index=True, nullable=True)
|
||||
source_format = Column(String(32), index=True, nullable=True)
|
||||
imaging_mode = Column(String(64), index=True, nullable=True)
|
||||
polarization = Column(String(64), index=True, nullable=True)
|
||||
orbit_direction = Column(String(32), index=True, nullable=True)
|
||||
look_direction = Column(String(32), index=True, nullable=True)
|
||||
absolute_orbit = Column(String(64), index=True, nullable=True)
|
||||
relative_orbit = Column(String(64), index=True, nullable=True)
|
||||
acquisition_start_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
acquisition_stop_time_utc = Column(DateTime, nullable=True)
|
||||
scene_center_lon = Column(Float, nullable=True)
|
||||
scene_center_lat = Column(Float, nullable=True)
|
||||
footprint_geom = Column(Geometry("POLYGON", srid=4326), index=True, nullable=True)
|
||||
footprint_polygon = Column(JSON, nullable=True)
|
||||
swath_summary_json = Column(JSON, nullable=True)
|
||||
burst_summary_json = Column(JSON, nullable=True)
|
||||
incidence_angle_min = Column(Float, nullable=True)
|
||||
incidence_angle_max = Column(Float, nullable=True)
|
||||
doppler_summary_json = Column(JSON, nullable=True)
|
||||
state_vector_summary_json = Column(JSON, nullable=True)
|
||||
metadata_quality = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True)
|
||||
production_readiness = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True)
|
||||
readiness_reasons_json = Column(JSON, nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parsed_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
source_asset = relationship("SourceProductAssetORM")
|
||||
radar_data = relationship("RadarDataORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_sar_scene_geometry_profiles_family_date", "satellite_family", "acquisition_start_time_utc"),
|
||||
Index("idx_sar_scene_geometry_profiles_track", "satellite_family", "relative_orbit", "orbit_direction"),
|
||||
Index("idx_sar_scene_geometry_profiles_readiness", "production_readiness", "metadata_quality"),
|
||||
)
|
||||
|
||||
|
||||
class OrbitAssetORM(Base):
|
||||
__tablename__ = "orbit_assets"
|
||||
|
||||
|
||||
@@ -261,10 +261,10 @@ class PairingRequest(BaseModel):
|
||||
"""D-InSAR 配对请求的参数模型(增强版 v2.0)"""
|
||||
# === 时空约束(保留) ===
|
||||
time_baseline_min: int = Field(default=1, ge=0, le=3650)
|
||||
time_baseline_max: int = Field(default=90, ge=1, le=3650)
|
||||
time_baseline_max: int = Field(default=30, ge=1, le=3650)
|
||||
overlap_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
|
||||
spatial_baseline_max_meters: int = Field(default=3000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS)
|
||||
limit_footprint_center_distance: bool = False
|
||||
spatial_baseline_max_meters: int = Field(default=5000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS)
|
||||
limit_footprint_center_distance: bool = True
|
||||
coverage_diversity_penalty: float = Field(default=0.3, ge=0.0, le=1.0)
|
||||
require_same_imaging_mode: bool = True
|
||||
require_same_polarization: bool = True
|
||||
@@ -280,7 +280,7 @@ class PairingRequest(BaseModel):
|
||||
slave_date_to: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$')
|
||||
|
||||
# === 配对策略(新增) ===
|
||||
strategy: str = Field(default="all", pattern=r'^(all|sbas|sequential|star)$')
|
||||
strategy: str = Field(default="dinsar_production", pattern=r'^dinsar_production$')
|
||||
num_connections: int = Field(default=1, ge=1, le=10)
|
||||
reference_image_id: Optional[int] = None
|
||||
|
||||
@@ -303,6 +303,7 @@ class PairingRequest(BaseModel):
|
||||
normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio']
|
||||
if normalized.get('footprint_center_distance_max_meters') not in (None, ''):
|
||||
normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters']
|
||||
normalized['strategy'] = 'dinsar_production'
|
||||
return normalized
|
||||
|
||||
@field_validator(
|
||||
@@ -343,8 +344,15 @@ class PairingRequest(BaseModel):
|
||||
return None
|
||||
if not isinstance(value, list):
|
||||
return value
|
||||
normalized = [str(item).strip() for item in value if str(item).strip()]
|
||||
return normalized or None
|
||||
normalized = []
|
||||
for item in value:
|
||||
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
normalized.append("LT1")
|
||||
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
normalized.append("S1")
|
||||
deduped = list(dict.fromkeys(normalized))
|
||||
return deduped or ["__UNSUPPORTED_DINSAR_FAMILY__"]
|
||||
|
||||
@field_validator('master_date_to')
|
||||
@classmethod
|
||||
@@ -381,6 +389,16 @@ class RadarPair(BaseModel):
|
||||
time_baseline_days: int
|
||||
spatial_baseline_meters: float
|
||||
scene_center_distance_meters: Optional[float] = None
|
||||
scene_overlap_ratio: Optional[float] = None
|
||||
pair_aoi_overlap_ratio: Optional[float] = None
|
||||
dinsar_quality_tier: Optional[str] = None
|
||||
dinsar_quality_score: Optional[float] = None
|
||||
dinsar_readiness: Optional[str] = None
|
||||
dinsar_reasons: Optional[List[str]] = None
|
||||
same_relative_orbit: Optional[bool] = None
|
||||
master_relative_orbit: Optional[str] = None
|
||||
slave_relative_orbit: Optional[str] = None
|
||||
production_summary: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class PairingResponse(BaseModel):
|
||||
|
||||
@@ -15,7 +15,9 @@ from typing import List, Optional
|
||||
|
||||
from ..ai_service import (
|
||||
analyze_map_with_vlm,
|
||||
get_ollama_models,
|
||||
get_model_info,
|
||||
is_likely_vlm_model,
|
||||
is_model_trained,
|
||||
predict_quality,
|
||||
train_quality_model,
|
||||
@@ -146,12 +148,12 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
|
||||
counts = await dinsar_read_service.get_ai_status_counts(db)
|
||||
|
||||
ollama_online = False
|
||||
ollama_models: List[str] = []
|
||||
ollama_vlm_models: List[str] = []
|
||||
try:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=1.0) as client:
|
||||
ollama_base = settings.OLLAMA_BASE_URL
|
||||
response = await client.get(f"{ollama_base.rstrip('/')}/api/tags")
|
||||
ollama_online = response.status_code == 200
|
||||
ollama_models = await get_ollama_models(timeout=1.0)
|
||||
ollama_vlm_models = [model for model in ollama_models if is_likely_vlm_model(model)]
|
||||
ollama_online = True
|
||||
except Exception:
|
||||
ollama_online = False
|
||||
|
||||
@@ -161,7 +163,11 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
|
||||
"labeled_count": counts["labeled_count"],
|
||||
"good_count": counts["good_count"],
|
||||
"bad_count": counts["bad_count"],
|
||||
"ollama_online": ollama_online
|
||||
"ollama_online": ollama_online,
|
||||
"ollama_models": ollama_models,
|
||||
"ollama_vlm_models": ollama_vlm_models,
|
||||
"ollama_base_url": settings.OLLAMA_BASE_URL,
|
||||
"default_vlm_model": settings.DEFAULT_VLM_MODEL,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -22,21 +22,29 @@ router = APIRouter(prefix="/assets", tags=["assets"])
|
||||
class AssetScanRequest(BaseModel):
|
||||
inventory_types: List[str] = Field(default_factory=list)
|
||||
root_ids: List[int] = Field(default_factory=list)
|
||||
families: List[str] = Field(default_factory=list)
|
||||
bind_orbits: bool = True
|
||||
build_previews: bool = True
|
||||
|
||||
|
||||
class ArchiveIntegrityAuditRequest(BaseModel):
|
||||
families: List[str] = Field(default_factory=list)
|
||||
source_formats: List[str] = Field(default_factory=list)
|
||||
asset_ids: List[int] = Field(default_factory=list)
|
||||
force: bool = False
|
||||
limit: Optional[int] = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class S1UnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
|
||||
|
||||
class S1BatchUnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
scan_before_unpack: bool = True
|
||||
|
||||
|
||||
@@ -89,7 +97,50 @@ async def run_asset_inventory_scan_now(
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
families=payload.get("families") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
build_previews=bool(payload.get("build_previews", True)),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/inventory/archive-integrity-audit", status_code=202)
|
||||
async def run_archive_integrity_audit(
|
||||
request: Optional[ArchiveIntegrityAuditRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
|
||||
"Source archive integrity audit",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Source archive integrity audit queued", "task_id": task_id, "job_id": job_id}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/inventory/archive-integrity-audit-now")
|
||||
async def run_archive_integrity_audit_now(
|
||||
request: Optional[ArchiveIntegrityAuditRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
|
||||
return await asset_inventory_service.audit_source_archive_integrity(
|
||||
db,
|
||||
families=payload.get("families") or None,
|
||||
source_formats=payload.get("source_formats") or None,
|
||||
asset_ids=payload.get("asset_ids") or None,
|
||||
force=bool(payload.get("force", False)),
|
||||
limit=payload.get("limit"),
|
||||
)
|
||||
|
||||
|
||||
@@ -186,8 +237,6 @@ async def unpack_sentinel1_source_asset(
|
||||
}
|
||||
if request_data.min_disk_space_gb is not None:
|
||||
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
|
||||
if request_data.delete_archive is not None:
|
||||
payload["delete_archive"] = request_data.delete_archive
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"UNPACK_SENTINEL1",
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
@@ -65,6 +67,43 @@ class PreviewInputAssetsRequest(BaseModel):
|
||||
num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all")
|
||||
|
||||
|
||||
def _is_dinsar_task_dir(path: str) -> bool:
|
||||
return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave"))
|
||||
|
||||
|
||||
def _is_landsar_task_dir(path: str) -> bool:
|
||||
return os.path.isdir(os.path.join(path, "Input_Data")) or _is_dinsar_task_dir(path)
|
||||
|
||||
|
||||
def _count_candidate_tasks(root_dir: str) -> Dict[str, Any]:
|
||||
try:
|
||||
if _is_dinsar_task_dir(root_dir) or _is_landsar_task_dir(root_dir):
|
||||
return {"task_count": 1, "invalid_child_count": 0, "mode": "single_task"}
|
||||
task_count = 0
|
||||
invalid_child_count = 0
|
||||
with os.scandir(root_dir) as entries:
|
||||
for entry in entries:
|
||||
if not entry.is_dir() or entry.name.startswith("._"):
|
||||
continue
|
||||
child_path = entry.path
|
||||
if _is_dinsar_task_dir(child_path) or _is_landsar_task_dir(child_path):
|
||||
task_count += 1
|
||||
elif entry.name.lower().startswith("task_"):
|
||||
invalid_child_count += 1
|
||||
return {
|
||||
"task_count": task_count,
|
||||
"invalid_child_count": invalid_child_count,
|
||||
"mode": "task_root",
|
||||
}
|
||||
except OSError as exc:
|
||||
return {
|
||||
"task_count": 0,
|
||||
"invalid_child_count": 0,
|
||||
"mode": "unreadable",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
|
||||
def _get_registry():
|
||||
from ..dinsar_engines import registry
|
||||
|
||||
@@ -136,6 +175,56 @@ async def run_wsl_check(
|
||||
return report.to_dict()
|
||||
|
||||
|
||||
@router.get("/task-roots")
|
||||
async def list_task_roots(
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
):
|
||||
_ = current_user
|
||||
root_dir = os.path.normpath(os.path.abspath(str(settings.DINSAR_TASK_POOL_ROOT or "").strip()))
|
||||
root_exists = bool(root_dir and os.path.isdir(root_dir))
|
||||
items = []
|
||||
child_dir_count = 0
|
||||
if root_exists:
|
||||
try:
|
||||
with os.scandir(root_dir) as entries:
|
||||
child_dirs = sorted(
|
||||
[entry for entry in entries if entry.is_dir() and not entry.name.startswith("._")],
|
||||
key=lambda entry: entry.name.lower(),
|
||||
)
|
||||
except OSError as exc:
|
||||
raise HTTPException(status_code=500, detail=f"无法读取 D-InSAR Task_Pool: {exc}") from exc
|
||||
child_dir_count = len(child_dirs)
|
||||
|
||||
for entry in child_dirs:
|
||||
path = os.path.normpath(entry.path)
|
||||
stat = entry.stat()
|
||||
summary = _count_candidate_tasks(path)
|
||||
task_count = int(summary.get("task_count") or 0)
|
||||
invalid_child_count = int(summary.get("invalid_child_count") or 0)
|
||||
items.append(
|
||||
{
|
||||
"name": entry.name,
|
||||
"path": path,
|
||||
"task_count": task_count,
|
||||
"invalid_child_count": invalid_child_count,
|
||||
"valid": task_count > 0,
|
||||
"mode": summary.get("mode"),
|
||||
"error": summary.get("error"),
|
||||
"mtime": stat.st_mtime,
|
||||
"updated_at": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"),
|
||||
}
|
||||
)
|
||||
|
||||
items.sort(key=lambda item: str(item.get("name") or "").lower())
|
||||
return {
|
||||
"root": root_dir,
|
||||
"root_exists": root_exists,
|
||||
"items": items,
|
||||
"count": len(items),
|
||||
"child_dir_count": child_dir_count,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/engines/pyint/preview-input-assets")
|
||||
async def preview_pyint_input_assets(
|
||||
req: PreviewInputAssetsRequest,
|
||||
|
||||
+409
-120
@@ -1,9 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
import os
|
||||
import shutil
|
||||
from typing import Any, Iterable, List, Optional
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
|
||||
@@ -17,6 +20,15 @@ from .dependencies import _require_admin
|
||||
|
||||
router = APIRouter()
|
||||
MONITOR_ALLOWED_TARGETS = {"radar", "orbit", "dinsar", "gf3"}
|
||||
MONITOR_SCAN_TASK_TYPES = {
|
||||
"SCAN_ASSET_INVENTORY",
|
||||
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
|
||||
"SCAN_DATA",
|
||||
"SCAN_DINSAR",
|
||||
"GF3_SARSCAPE_SYNC",
|
||||
"GF3_QUICKLOOK_WEBP",
|
||||
}
|
||||
MONITOR_TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
|
||||
MONITOR_LOG_DEFAULT_LIMIT = read_int_env(
|
||||
"MONITOR_LOG_DEFAULT_LIMIT",
|
||||
50,
|
||||
@@ -37,10 +49,272 @@ MONITOR_LOG_MAX_OFFSET = read_int_env(
|
||||
)
|
||||
|
||||
|
||||
def _path_kind(path: str) -> str:
|
||||
text = str(path or "").strip()
|
||||
if text.startswith("\\\\"):
|
||||
return "unc"
|
||||
drive, _tail = os.path.splitdrive(os.path.normpath(text))
|
||||
if drive:
|
||||
return "windows"
|
||||
if text.startswith("/mnt/"):
|
||||
return "wsl_mount"
|
||||
if text.startswith("/"):
|
||||
return "posix"
|
||||
return "relative"
|
||||
|
||||
|
||||
def _nearest_existing_path(path: str) -> str:
|
||||
candidate = os.path.normpath(str(path or "").strip())
|
||||
while candidate and not os.path.exists(candidate):
|
||||
parent = os.path.dirname(candidate)
|
||||
if not parent or parent == candidate:
|
||||
break
|
||||
candidate = parent
|
||||
return candidate
|
||||
|
||||
|
||||
def _storage_status(label: str, path: str, role: str) -> dict[str, Any]:
|
||||
text = str(path or "").strip()
|
||||
item: dict[str, Any] = {
|
||||
"label": label,
|
||||
"role": role,
|
||||
"path": text,
|
||||
"path_kind": _path_kind(text),
|
||||
"exists": bool(text and os.path.exists(text)),
|
||||
"probe_path": "",
|
||||
"total_gb": None,
|
||||
"used_gb": None,
|
||||
"free_gb": None,
|
||||
"free_ratio": None,
|
||||
"status": "missing" if text else "empty",
|
||||
"message": "",
|
||||
}
|
||||
if not text:
|
||||
item["message"] = "Path is not configured."
|
||||
return item
|
||||
if item["path_kind"] == "unc":
|
||||
item["status"] = "blocked"
|
||||
item["message"] = "UNC paths are not allowed for active local production."
|
||||
return item
|
||||
|
||||
probe = _nearest_existing_path(text)
|
||||
if not probe or not os.path.exists(probe):
|
||||
item["message"] = "No existing parent path found."
|
||||
return item
|
||||
item["probe_path"] = probe
|
||||
try:
|
||||
total, used, free = shutil.disk_usage(probe)
|
||||
except OSError as exc:
|
||||
item["status"] = "error"
|
||||
item["message"] = str(exc)
|
||||
return item
|
||||
|
||||
gb = 1024 ** 3
|
||||
free_ratio = float(free) / float(total or 1)
|
||||
item.update(
|
||||
{
|
||||
"total_gb": round(total / gb, 2),
|
||||
"used_gb": round(used / gb, 2),
|
||||
"free_gb": round(free / gb, 2),
|
||||
"free_ratio": round(free_ratio, 4),
|
||||
"status": "ok",
|
||||
"message": "",
|
||||
}
|
||||
)
|
||||
if not item["exists"]:
|
||||
item["status"] = "missing"
|
||||
item["message"] = f"Path does not exist; disk usage probed from {probe}."
|
||||
elif free_ratio < 0.10:
|
||||
item["status"] = "critical"
|
||||
item["message"] = "Free space is below 10%."
|
||||
elif free_ratio < 0.20:
|
||||
item["status"] = "warning"
|
||||
item["message"] = "Free space is below 20%."
|
||||
return item
|
||||
|
||||
|
||||
def _add_storage_roots(rows: list[dict[str, str]], label: str, paths: Iterable[str], role: str) -> None:
|
||||
for path in paths:
|
||||
text = str(path or "").strip()
|
||||
if text:
|
||||
rows.append({"label": label, "path": text, "role": role})
|
||||
|
||||
|
||||
def _storage_group_identity(item: dict[str, Any]) -> tuple[str, str, str]:
|
||||
path = str(item.get("path") or "").strip()
|
||||
path_kind = str(item.get("path_kind") or "")
|
||||
if path_kind == "windows":
|
||||
base = str(item.get("probe_path") or path).strip()
|
||||
drive, _tail = os.path.splitdrive(os.path.normpath(base))
|
||||
if drive:
|
||||
drive = drive.upper()
|
||||
volume_path = f"{drive}\\"
|
||||
return f"windows:{drive}", volume_path, f"本机磁盘 {volume_path}"
|
||||
if path_kind == "unc":
|
||||
key = os.path.normcase(os.path.normpath(path))
|
||||
return f"unc:{key}", path, "UNC path"
|
||||
if path_kind in {"posix", "wsl_mount"}:
|
||||
base = str(item.get("probe_path") or path or "/").strip()
|
||||
root = "/mnt/" + base.split("/")[2] if base.startswith("/mnt/") and len(base.split("/")) > 2 else "/"
|
||||
return f"{path_kind}:{root}", root, root
|
||||
key = os.path.normcase(os.path.normpath(path))
|
||||
return f"path:{key}", path, str(item.get("label") or item.get("role") or path or "Storage")
|
||||
|
||||
|
||||
def _capacity_status(free_ratio: Any) -> str:
|
||||
if free_ratio is None:
|
||||
return "missing"
|
||||
try:
|
||||
ratio = float(free_ratio)
|
||||
except (TypeError, ValueError):
|
||||
return "missing"
|
||||
if ratio < 0.10:
|
||||
return "critical"
|
||||
if ratio < 0.20:
|
||||
return "warning"
|
||||
return "ok"
|
||||
|
||||
|
||||
def _group_storage_statuses(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
groups: dict[str, dict[str, Any]] = {}
|
||||
order: list[str] = []
|
||||
|
||||
for item in items:
|
||||
group_key, volume_path, volume_label = _storage_group_identity(item)
|
||||
group = groups.get(group_key)
|
||||
if group is None:
|
||||
group = {
|
||||
"label": volume_label,
|
||||
"role": "storage_volume",
|
||||
"path": volume_path,
|
||||
"path_kind": item.get("path_kind"),
|
||||
"exists": False,
|
||||
"probe_path": item.get("probe_path") or "",
|
||||
"total_gb": None,
|
||||
"used_gb": None,
|
||||
"free_gb": None,
|
||||
"free_ratio": None,
|
||||
"status": "missing",
|
||||
"message": "",
|
||||
"paths": [],
|
||||
"configured_path_count": 0,
|
||||
"existing_path_count": 0,
|
||||
"missing_path_count": 0,
|
||||
"blocked_path_count": 0,
|
||||
"error_path_count": 0,
|
||||
}
|
||||
groups[group_key] = group
|
||||
order.append(group_key)
|
||||
|
||||
group["configured_path_count"] += 1
|
||||
if item.get("exists"):
|
||||
group["existing_path_count"] += 1
|
||||
if item.get("status") == "missing":
|
||||
group["missing_path_count"] += 1
|
||||
elif item.get("status") == "blocked":
|
||||
group["blocked_path_count"] += 1
|
||||
elif item.get("status") == "error":
|
||||
group["error_path_count"] += 1
|
||||
|
||||
group["paths"].append(
|
||||
{
|
||||
"label": item.get("label"),
|
||||
"role": item.get("role"),
|
||||
"path": item.get("path"),
|
||||
"exists": item.get("exists"),
|
||||
"status": item.get("status"),
|
||||
"message": item.get("message"),
|
||||
}
|
||||
)
|
||||
|
||||
if item.get("total_gb") is not None:
|
||||
group["exists"] = True
|
||||
group["probe_path"] = item.get("probe_path") or group["probe_path"]
|
||||
group["total_gb"] = item.get("total_gb")
|
||||
group["used_gb"] = item.get("used_gb")
|
||||
group["free_gb"] = item.get("free_gb")
|
||||
group["free_ratio"] = item.get("free_ratio")
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
for key in order:
|
||||
group = groups[key]
|
||||
status = _capacity_status(group.get("free_ratio"))
|
||||
if group["blocked_path_count"]:
|
||||
status = "blocked"
|
||||
elif group["error_path_count"]:
|
||||
status = "error"
|
||||
elif group["missing_path_count"] and status == "ok":
|
||||
status = "partial"
|
||||
group["status"] = status
|
||||
|
||||
messages = [f"{group['configured_path_count']} 个配置路径"]
|
||||
if group["missing_path_count"]:
|
||||
messages.append(f"{group['missing_path_count']} 个路径缺失")
|
||||
if group["blocked_path_count"]:
|
||||
messages.append(f"{group['blocked_path_count']} 个 UNC 被禁用")
|
||||
if group["error_path_count"]:
|
||||
messages.append(f"{group['error_path_count']} 个探测失败")
|
||||
if status == "critical":
|
||||
messages.append("剩余空间低于 10%")
|
||||
elif status == "warning" and not group["missing_path_count"]:
|
||||
messages.append("剩余空间低于 20%")
|
||||
group["message"] = ";".join(messages) + "。"
|
||||
result.append(group)
|
||||
return result
|
||||
|
||||
|
||||
def _collect_storage_roots() -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
_add_storage_roots(rows, "Task_Pool", [settings.TASK_POOL_ROOT], "task_pool")
|
||||
_add_storage_roots(rows, "D-InSAR Task_Pool", [settings.DINSAR_TASK_POOL_ROOT], "dinsar_task_pool")
|
||||
_add_storage_roots(rows, "SBAS Task_Pool", [settings.SBAS_TASK_POOL_ROOT], "sbas_task_pool")
|
||||
_add_storage_roots(rows, "Data distribution root", [settings.DATA_DISTRIBUTION_ROOT], "data_distribution")
|
||||
source_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
|
||||
source_path_keys = {os.path.normcase(os.path.normpath(path)) for path in source_paths}
|
||||
s1_storage_paths = [
|
||||
path
|
||||
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
|
||||
if os.path.normcase(os.path.normpath(path)) not in source_path_keys
|
||||
]
|
||||
_add_storage_roots(rows, "LT/S1 local source pools", source_paths, "source_local")
|
||||
_add_storage_roots(rows, "Sentinel-1 local source pool", s1_storage_paths, "source_local")
|
||||
_add_storage_roots(rows, "Orbit source pool", split_env_paths(settings.ORBIT_SOURCE_DIRS), "orbit_source")
|
||||
_add_storage_roots(rows, "LT-1 radar scan pool", split_env_paths(settings.MONITOR_RADAR_DIRS), "lt1_storage")
|
||||
_add_storage_roots(rows, "D-InSAR product root", [settings.DINSAR_PRODUCT_DIR], "dinsar_product")
|
||||
_add_storage_roots(rows, "GF3 native _geo", split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), "gf3_native")
|
||||
_add_storage_roots(rows, "GF3 standard/index", split_env_paths(settings.GF3_STORAGE_DIRS), "gf3_storage")
|
||||
_add_storage_roots(rows, "GF3 task/runtime pool", [settings.GF3_TASK_POOL_ROOT], "gf3_task_pool")
|
||||
_add_storage_roots(rows, "GF3 SARscape runtime", [settings.GF3_SARSCAPE_RUNTIME_DIR], "gf3_runtime")
|
||||
_add_storage_roots(rows, "Result publish root", [settings.RESULT_PUBLISH_ROOT], "result_publish")
|
||||
_add_storage_roots(rows, "SBAS work root", [settings.GAMMA_SBAS_WORK_ROOT], "sbas_work")
|
||||
_add_storage_roots(rows, "SBAS product root", [settings.GAMMA_SBAS_PRODUCT_ROOT], "sbas_product")
|
||||
_add_storage_roots(rows, "SAR analysis ready", [settings.SAR_ANALYSIS_READY_ROOT], "analysis_ready")
|
||||
_add_storage_roots(rows, "SAR analysis work", [settings.SAR_ANALYSIS_WORK_ROOT], "analysis_work")
|
||||
|
||||
seen: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
key = os.path.normcase(os.path.normpath(row["path"]))
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(_storage_status(row["label"], row["path"], row["role"]))
|
||||
return _group_storage_statuses(result)
|
||||
|
||||
|
||||
class MonitorConfig(BaseModel):
|
||||
radar_dirs: List[str] = []
|
||||
orbit_dir: Optional[str] = None
|
||||
orbit_source_dirs: List[str] = []
|
||||
orbit_production_txt_pool: Optional[str] = None
|
||||
dinsar_dirs: List[str] = []
|
||||
dinsar_product_dir: Optional[str] = None
|
||||
sbas_product_root: Optional[str] = None
|
||||
task_pool_root: Optional[str] = None
|
||||
dinsar_task_pool_root: Optional[str] = None
|
||||
sbas_task_pool_root: Optional[str] = None
|
||||
gf3_task_pool_root: Optional[str] = None
|
||||
data_distribution_root: Optional[str] = None
|
||||
gf3_archive_source_dirs: List[str] = []
|
||||
gf3_source_dirs: List[str] = []
|
||||
gf3_legacy_gdal_enabled: bool = False
|
||||
@@ -53,6 +327,7 @@ class MonitorConfig(BaseModel):
|
||||
gf3_sarscape_polarizations: Optional[str] = None
|
||||
gf3_sarscape_auto_standardize: bool = True
|
||||
gf3_sarscape_clean_after_success: bool = True
|
||||
storage_roots: List[dict[str, Any]] = []
|
||||
# Manual-only: config is read from .env
|
||||
|
||||
|
||||
@@ -70,6 +345,13 @@ class GF3UnpackRunRequest(BaseModel):
|
||||
class GF3SarscapeSyncRequest(BaseModel):
|
||||
force: bool = False
|
||||
register: bool = True
|
||||
quicklook_only: bool = False
|
||||
native_dirs: List[str] = []
|
||||
|
||||
|
||||
class GF3QuicklookWebpRequest(BaseModel):
|
||||
force: bool = False
|
||||
max_records: Optional[int] = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class GF3SarscapeProduceRequest(BaseModel):
|
||||
@@ -91,7 +373,7 @@ class GF3SarscapeCleanRequest(BaseModel):
|
||||
@router.post("/monitor/config")
|
||||
async def update_monitor_config(config: MonitorConfig):
|
||||
"""
|
||||
更新数据监控配置(雷达、精轨和Dinsar结果)。
|
||||
鏇存柊鏁版嵁鐩戞帶閰嶇疆锛堥浄杈俱€佺簿杞ㄥ拰Dinsar缁撴灉锛夈€?
|
||||
"""
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
@@ -102,8 +384,8 @@ async def update_monitor_config(config: MonitorConfig):
|
||||
@router.post("/monitor/run-now")
|
||||
async def run_monitor_now(target: Optional[str] = None, background_tasks: BackgroundTasks = None, admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
"""
|
||||
手动触发一次监控任务(扫描所有配置的目录)。
|
||||
target: 'radar', 'orbit', 'dinsar' 或 None (全部)
|
||||
鎵嬪姩瑙﹀彂涓€娆$洃鎺т换鍔★紙鎵弿鎵€鏈夐厤缃殑鐩綍锛夈€?
|
||||
target: 'radar', 'orbit', 'dinsar' 鎴?None (鍏ㄩ儴)
|
||||
"""
|
||||
normalized_target = (target or "").strip().lower() or None
|
||||
if normalized_target and normalized_target not in MONITOR_ALLOWED_TARGETS:
|
||||
@@ -127,7 +409,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
|
||||
raise HTTPException(status_code=400, detail="Monitor paths are not configured in .env.")
|
||||
|
||||
task_type = "SCAN_DATA" if normalized_target in ["radar", "orbit", "gf3", None] else "SCAN_DINSAR"
|
||||
task_name = f"手动触发扫描 ({normalized_target or '全部'})"
|
||||
task_name = f"Manual scan ({normalized_target or 'all'})"
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name)
|
||||
@@ -135,7 +417,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
|
||||
payload = {"dirs": dinsar_dirs}
|
||||
job_type = "SCAN_DINSAR"
|
||||
elif normalized_target == "gf3":
|
||||
# GF3 扫描:用 gf3_storage_dirs 作为 radar_dirs
|
||||
# GF3 scan uses gf3_storage_dirs as radar_dirs.
|
||||
payload = {
|
||||
"radar_dirs": gf3_storage_dirs,
|
||||
"target": "radar",
|
||||
@@ -150,7 +432,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
|
||||
await job_queue_service.create_job(job_type, payload=payload, task_id=task_id)
|
||||
|
||||
return {
|
||||
"message": f"已触发{normalized_target or '全部'}手动扫描任务(已进入队列)",
|
||||
"message": f"Manual scan queued for {normalized_target or 'all'}.",
|
||||
"task_id": task_id
|
||||
}
|
||||
except ValueError as e:
|
||||
@@ -160,14 +442,14 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
|
||||
@router.post("/monitor/gf3-process")
|
||||
async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
"""
|
||||
批量 GF3 L1A→L2 处理:扫描 GF3_SOURCE_DIRS,过滤已处理的,逐个处理并自动入库。
|
||||
Batch GF3 legacy L1A to L2 processing.
|
||||
"""
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Legacy GF3 Python/GDAL preprocessing is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
),
|
||||
)
|
||||
gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or []
|
||||
@@ -175,7 +457,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
|
||||
raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.")
|
||||
|
||||
task_type = "GF3_BATCH_PROCESS"
|
||||
task_name = "GF3 批量 L1A→L2 处理"
|
||||
task_name = "GF3 legacy batch process"
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name)
|
||||
@@ -185,7 +467,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
|
||||
task_id=task_id,
|
||||
)
|
||||
return {
|
||||
"message": "GF3 批量处理任务已提交",
|
||||
"message": "GF3 batch process task submitted",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
@@ -198,147 +480,100 @@ async def run_gf3_sarscape_sync(
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
扫描 GF3 SARscape 原生 _geo 二进制池,转换为标准 GeoTIFF,并登记入库。
|
||||
Scan GF3 SARscape native _geo outputs.
|
||||
"""
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
options = request_data or GF3SarscapeSyncRequest()
|
||||
gf3_native_dirs = options.native_dirs or MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
if not gf3_storage_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeSyncRequest()
|
||||
task_type = "GF3_SARSCAPE_SYNC"
|
||||
task_name = "GF3 SARscape 原生结果标准化"
|
||||
task_name = "GF3 SARscape native result inventory" if options.quicklook_only else "GF3 SARscape native standardize"
|
||||
payload = {
|
||||
"native_dirs": gf3_native_dirs,
|
||||
"storage_root": gf3_storage_dirs[0],
|
||||
"force": bool(options.force),
|
||||
"register": bool(options.register),
|
||||
"quicklook_only": bool(options.quicklook_only),
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 SARscape 原生结果标准化任务已提交",
|
||||
"message": (
|
||||
"GF3 SARscape native result inventory task submitted"
|
||||
if options.quicklook_only
|
||||
else "GF3 SARscape native standardize task submitted"
|
||||
),
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-quicklook-webp", status_code=202)
|
||||
async def run_gf3_quicklook_webp(
|
||||
request_data: GF3QuicklookWebpRequest | None = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
Generate local WebP cache files from registered GF3 SARscape native _geo records.
|
||||
"""
|
||||
options = request_data or GF3QuicklookWebpRequest()
|
||||
task_type = "GF3_QUICKLOOK_WEBP"
|
||||
task_name = "GF3 native _geo WebP cache"
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
payload = {
|
||||
"force": bool(options.force),
|
||||
"max_records": int(options.max_records or 0),
|
||||
"native_dirs": gf3_native_dirs,
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 native _geo WebP cache task submitted",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-produce", status_code=202)
|
||||
async def run_gf3_sarscape_produce(
|
||||
request_data: GF3SarscapeProduceRequest | None = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates.
|
||||
GF3 SARscape production is disabled on this management machine.
|
||||
"""
|
||||
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
|
||||
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
|
||||
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
|
||||
if not gf3_archive_source_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
if not gf3_native_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
|
||||
if not gf3_storage_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
|
||||
if not settings.GF3_SARSCAPE_WRAPPER_EXE:
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.")
|
||||
if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH):
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeProduceRequest()
|
||||
selected_dates = []
|
||||
for raw_date in options.selected_dates or []:
|
||||
text = str(raw_date or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
normalized = text.replace("-", "").replace("_", "")
|
||||
if len(normalized) != 8 or not normalized.isdigit():
|
||||
raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}")
|
||||
if normalized not in selected_dates:
|
||||
selected_dates.append(normalized)
|
||||
task_type = "GF3_SARSCAPE_PRODUCE"
|
||||
task_name = "GF3 SARscape production"
|
||||
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
|
||||
clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success)
|
||||
payload = {
|
||||
"source_dirs": gf3_archive_source_dirs,
|
||||
"native_dirs": gf3_native_dirs,
|
||||
"native_root": gf3_native_dirs[0],
|
||||
"storage_root": gf3_storage_dirs[0],
|
||||
"wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
|
||||
"idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
|
||||
"dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
|
||||
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
|
||||
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
|
||||
"max_archives_per_run": int(options.max_archives_per_run or 0),
|
||||
"selected_dates": selected_dates,
|
||||
"local_staging_root": settings.GF3_TASK_POOL_ROOT,
|
||||
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
|
||||
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
|
||||
"auto_standardize": bool(auto_standardize),
|
||||
"clean_after_success": bool(clean_after_success),
|
||||
"force_standardize": bool(options.force_standardize),
|
||||
"register": bool(options.register),
|
||||
"cleanup_require_standardized": True,
|
||||
"cleanup_dry_run": bool(options.cleanup_dry_run),
|
||||
}
|
||||
|
||||
try:
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": (
|
||||
f"GF3 SARscape production task submitted for {', '.join(selected_dates)}"
|
||||
if selected_dates
|
||||
else "GF3 SARscape production task submitted"
|
||||
),
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"GF3 SARscape production is disabled on this management machine. "
|
||||
"Run GF3 production on the SARscape host, copy completed _geo results to local "
|
||||
"GF3_SARSCAPE_NATIVE_DIRS, then use GF3 native result registration and GF3 _geo WebP generation."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/monitor/gf3-sarscape-dates")
|
||||
async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
"""
|
||||
List available GF3 SARscape source dates from configured raw archive roots.
|
||||
GF3 SARscape production date selection is disabled on this management machine.
|
||||
"""
|
||||
from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs
|
||||
|
||||
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
|
||||
if not gf3_archive_source_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
|
||||
discovery = discover_gf3_sarscape_inputs(
|
||||
gf3_archive_source_dirs,
|
||||
archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS),
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="GF3 SARscape production date selection is disabled. Register local _geo native results instead.",
|
||||
)
|
||||
by_date: dict[str, dict[str, object]] = {}
|
||||
undated = 0
|
||||
for item in discovery.get("inputs") or []:
|
||||
scene_name = str(item.get("scene_name") or "")
|
||||
date_text = str(item.get("scene_date") or "")
|
||||
if not date_text:
|
||||
undated += 1
|
||||
continue
|
||||
bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []})
|
||||
bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1
|
||||
scenes = bucket.get("scenes")
|
||||
if isinstance(scenes, list) and len(scenes) < 20:
|
||||
scenes.append(scene_name)
|
||||
|
||||
dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True)
|
||||
return {
|
||||
"dates": dates,
|
||||
"input_count": discovery.get("input_count") or 0,
|
||||
"undated_count": undated,
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
|
||||
@@ -394,14 +629,14 @@ async def run_gf3_unpack(
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
"""
|
||||
将 GF3 压缩包池解包到 GF3_SOURCE_DIRS,作为后续 L1A→L2 预处理输入。
|
||||
Unpack GF3 archives into GF3_SOURCE_DIRS for the legacy pipeline.
|
||||
"""
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
"Legacy GF3 archive unpack is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
),
|
||||
)
|
||||
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
|
||||
@@ -416,7 +651,7 @@ async def run_gf3_unpack(
|
||||
max_files = max(0, int(request_data.max_files_per_run))
|
||||
|
||||
task_type = "GF3_UNPACK"
|
||||
task_name = "GF3 压缩包解包"
|
||||
task_name = "GF3 archive unpack"
|
||||
payload = {
|
||||
"source_dirs": gf3_archive_source_dirs,
|
||||
"target_dirs": gf3_source_dirs,
|
||||
@@ -429,7 +664,7 @@ async def run_gf3_unpack(
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 解包任务已提交",
|
||||
"message": "GF3 unpack task submitted",
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
@@ -439,9 +674,24 @@ async def run_gf3_unpack(
|
||||
@router.get("/monitor/status")
|
||||
async def get_monitor_status():
|
||||
"""
|
||||
获取当前监控状态。
|
||||
鑾峰彇褰撳墠鐩戞帶鐘舵€併€?
|
||||
"""
|
||||
return MONITOR_CONFIG
|
||||
config = dict(MONITOR_CONFIG)
|
||||
config.update(
|
||||
{
|
||||
"task_pool_root": settings.TASK_POOL_ROOT,
|
||||
"dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT,
|
||||
"sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT,
|
||||
"gf3_task_pool_root": settings.GF3_TASK_POOL_ROOT,
|
||||
"data_distribution_root": settings.DATA_DISTRIBUTION_ROOT,
|
||||
"dinsar_product_dir": settings.DINSAR_PRODUCT_DIR,
|
||||
"sbas_product_root": settings.GAMMA_SBAS_PRODUCT_ROOT,
|
||||
"orbit_source_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS) or split_env_paths(settings.MONITOR_ORBIT_DIR),
|
||||
"orbit_production_txt_pool": settings.ORBIT_POOL_ENVI,
|
||||
"storage_roots": _collect_storage_roots(),
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
@router.get("/monitor/logs")
|
||||
@@ -451,7 +701,7 @@ async def get_monitor_logs(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
获取最新的监控日志 (来自任务日志表)。
|
||||
鑾峰彇鏈€鏂扮殑鐩戞帶鏃ュ織 (鏉ヨ嚜浠诲姟鏃ュ織琛?銆?
|
||||
"""
|
||||
safe_limit = min(MONITOR_LOG_MAX_LIMIT, max(1, int(limit or MONITOR_LOG_DEFAULT_LIMIT)))
|
||||
safe_offset = min(MONITOR_LOG_MAX_OFFSET, max(0, int(offset or 0)))
|
||||
@@ -478,3 +728,42 @@ async def get_monitor_logs(
|
||||
"count": len(logs),
|
||||
"logs": logs,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/monitor/scan-task-history")
|
||||
async def clear_monitor_scan_task_history(
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""
|
||||
Clear finished scan task records and their logs from the monitor panel.
|
||||
|
||||
Running and pending tasks are intentionally preserved.
|
||||
"""
|
||||
task_rows = await db.execute(
|
||||
select(SystemTaskORM.task_id)
|
||||
.where(SystemTaskORM.task_type.in_(MONITOR_SCAN_TASK_TYPES))
|
||||
.where(SystemTaskORM.status.in_(MONITOR_TERMINAL_TASK_STATUSES))
|
||||
)
|
||||
task_ids = [str(task_id) for task_id in task_rows.scalars().all() if task_id]
|
||||
if not task_ids:
|
||||
return {
|
||||
"deleted_task_count": 0,
|
||||
"deleted_log_count": 0,
|
||||
"preserved_active": True,
|
||||
}
|
||||
|
||||
log_count_result = await db.execute(
|
||||
select(func.count(TaskLogORM.id)).where(TaskLogORM.task_id.in_(task_ids))
|
||||
)
|
||||
deleted_log_count = int(log_count_result.scalar_one() or 0)
|
||||
|
||||
await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids)))
|
||||
task_delete_result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids)))
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"deleted_task_count": int(task_delete_result.rowcount or len(task_ids)),
|
||||
"deleted_log_count": deleted_log_count,
|
||||
"preserved_active": True,
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ class OrbitPoolActionRequest(BaseModel):
|
||||
async def _build_orbit_database_stats(
|
||||
db: AsyncSession,
|
||||
pool_inventory: Dict[str, Any],
|
||||
*,
|
||||
isce2_enabled: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
total_radar_count = (
|
||||
await db.execute(select(func.count(RadarDataORM.id)))
|
||||
@@ -94,9 +96,9 @@ async def _build_orbit_database_stats(
|
||||
}
|
||||
|
||||
envi_stems = set(pool_inventory["envi"]["files"].keys())
|
||||
isce2_stems = set(pool_inventory["isce2"]["files"].keys())
|
||||
isce2_stems = set(pool_inventory["isce2"]["files"].keys()) if isce2_enabled else set()
|
||||
missing_in_envi = sorted(db_expected_stems - envi_stems)
|
||||
missing_in_isce2 = sorted(db_expected_stems - isce2_stems)
|
||||
missing_in_isce2 = sorted(db_expected_stems - isce2_stems) if isce2_enabled else []
|
||||
|
||||
return {
|
||||
"total_radar_count": int(total_radar_count or 0),
|
||||
@@ -110,6 +112,7 @@ async def _build_orbit_database_stats(
|
||||
"db_expected_stem_count": len(db_expected_stems),
|
||||
"stems_missing_in_envi_count": len(missing_in_envi),
|
||||
"stems_missing_in_isce2_count": len(missing_in_isce2),
|
||||
"isce2_enabled": bool(isce2_enabled),
|
||||
"sample_missing_in_envi": missing_in_envi[:20],
|
||||
"sample_missing_in_isce2": missing_in_isce2[:20],
|
||||
"path_errors": db_path_errors,
|
||||
@@ -123,26 +126,27 @@ async def get_orbit_status(
|
||||
):
|
||||
"""Return source, pool, consistency, and database orbit status."""
|
||||
source_dir = settings.MONITOR_ORBIT_DIR
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
source_stats = await asyncio.to_thread(scan_orbit_dir, source_dir)
|
||||
pool_inventory = await asyncio.to_thread(
|
||||
get_orbit_pool_inventory,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
True,
|
||||
)
|
||||
consistency = await asyncio.to_thread(
|
||||
check_orbit_consistency,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
)
|
||||
source_gap_summary = await asyncio.to_thread(
|
||||
summarize_source_orbit_gaps,
|
||||
source_dir,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
settings.ORBIT_QUARANTINE_DIR,
|
||||
)
|
||||
database_stats = await _build_orbit_database_stats(db, pool_inventory)
|
||||
database_stats = await _build_orbit_database_stats(db, pool_inventory, isce2_enabled=bool(settings.ISCE2_ENABLED))
|
||||
|
||||
return {
|
||||
"orbit_root": source_stats.orbit_root,
|
||||
@@ -184,7 +188,8 @@ async def get_orbit_status(
|
||||
"errors": pool_inventory["envi"]["errors"],
|
||||
},
|
||||
"isce2": {
|
||||
"path": settings.ORBIT_POOL_ISCE2,
|
||||
"path": isce2_pool,
|
||||
"enabled": bool(settings.ISCE2_ENABLED),
|
||||
"total": pool_inventory["isce2"]["total"],
|
||||
"duplicate_count": pool_inventory["isce2"]["duplicate_count"],
|
||||
"errors": pool_inventory["isce2"]["errors"],
|
||||
@@ -207,25 +212,28 @@ async def sync_orbit_pool_action(
|
||||
Check pool consistency, or repair missing entries when repair=true.
|
||||
"""
|
||||
if payload and payload.quarantine_bad:
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
return await asyncio.to_thread(
|
||||
quarantine_bad_orbits,
|
||||
settings.MONITOR_ORBIT_DIR,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
settings.ORBIT_QUARANTINE_DIR,
|
||||
)
|
||||
if payload and payload.repair:
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
return await asyncio.to_thread(
|
||||
repair_orbit_pools,
|
||||
settings.MONITOR_ORBIT_DIR,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
settings.ORBIT_POOL_LANDSAR,
|
||||
)
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
return await asyncio.to_thread(
|
||||
check_orbit_consistency,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -40,10 +40,10 @@ router = APIRouter()
|
||||
|
||||
def get_pairing_request_from_form(
|
||||
time_baseline_min: int = Form(1),
|
||||
time_baseline_max: int = Form(90),
|
||||
time_baseline_max: int = Form(30),
|
||||
overlap_threshold: float = Form(0.5),
|
||||
spatial_baseline_max_meters: int = Form(3000),
|
||||
limit_footprint_center_distance: bool = Form(False),
|
||||
spatial_baseline_max_meters: int = Form(5000),
|
||||
limit_footprint_center_distance: bool = Form(True),
|
||||
max_temporal_baseline_days: Optional[int] = Form(None),
|
||||
pair_footprint_overlap_min_ratio: Optional[float] = Form(None),
|
||||
footprint_center_distance_max_meters: Optional[int] = Form(None),
|
||||
@@ -57,7 +57,7 @@ def get_pairing_request_from_form(
|
||||
master_date_to: Optional[str] = Form(None),
|
||||
slave_date_from: Optional[str] = Form(None),
|
||||
slave_date_to: Optional[str] = Form(None),
|
||||
strategy: str = Form("all"),
|
||||
strategy: str = Form("dinsar_production"),
|
||||
num_connections: int = Form(1),
|
||||
reference_image_id: Optional[int] = Form(None),
|
||||
allowed_satellites: Optional[str] = Form(None), # JSON string
|
||||
@@ -91,7 +91,7 @@ def get_pairing_request_from_form(
|
||||
master_date_to=master_date_to,
|
||||
slave_date_from=slave_date_from,
|
||||
slave_date_to=slave_date_to,
|
||||
strategy=strategy,
|
||||
strategy="dinsar_production",
|
||||
num_connections=num_connections,
|
||||
reference_image_id=reference_image_id,
|
||||
allowed_satellites=satellites_list,
|
||||
|
||||
@@ -41,6 +41,7 @@ from .dependencies import (
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
GF3_NATIVE_PREVIEW_SOURCE_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
|
||||
LIST_QUERY_MAX_LIMIT = read_int_env(
|
||||
"LIST_QUERY_MAX_LIMIT",
|
||||
2000,
|
||||
@@ -188,6 +189,38 @@ def _radar_preview_paths(record: RadarDataORM) -> Tuple[str, str]:
|
||||
return raw_cache_path, geo_cache_path
|
||||
|
||||
|
||||
def _is_gf3_native_preview_record(record: RadarDataORM) -> bool:
|
||||
return str(record.source_format or "") == GF3_NATIVE_PREVIEW_SOURCE_FORMAT
|
||||
|
||||
|
||||
def _build_gf3_native_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo:
|
||||
preview_path = str(record.preview_cache_path or "")
|
||||
has_native_cache = (
|
||||
(record.preview_cache_status or "NONE") == "READY"
|
||||
and preview_path.lower().endswith(".webp")
|
||||
and os.path.exists(preview_path)
|
||||
)
|
||||
metadata = record.metadata_json or {}
|
||||
source_path = str(metadata.get("default_native_path") or "")
|
||||
source_found = bool(source_path and os.path.exists(source_path))
|
||||
return RadarPreviewStatusInfo(
|
||||
radar_id=record.id,
|
||||
status="READY" if has_native_cache else (record.preview_cache_status or "NONE"),
|
||||
cache_version=record.preview_cache_version,
|
||||
cache_updated_at=record.preview_cache_updated_at,
|
||||
has_geo_cache=has_native_cache,
|
||||
has_raw_cache=has_native_cache,
|
||||
source_found=source_found,
|
||||
fallback_in_use=False,
|
||||
message=(
|
||||
"GF3 native _geo WebP cache is available."
|
||||
if has_native_cache
|
||||
else "GF3 native _geo WebP cache has not been generated."
|
||||
),
|
||||
error=None if has_native_cache else record.preview_cache_error,
|
||||
)
|
||||
|
||||
|
||||
def _build_radar_preview_status(
|
||||
record: RadarDataORM,
|
||||
source_found: bool,
|
||||
@@ -228,6 +261,9 @@ async def _build_radar_preview_cache(
|
||||
db: AsyncSession,
|
||||
force: bool = False,
|
||||
) -> RadarPreviewStatusInfo:
|
||||
if _is_gf3_native_preview_record(record):
|
||||
return _build_gf3_native_preview_status(record)
|
||||
|
||||
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
|
||||
has_geo_cache = os.path.exists(geo_cache_path)
|
||||
has_raw_cache = os.path.exists(raw_cache_path)
|
||||
@@ -355,7 +391,33 @@ async def _get_cached_radar_preview(data_id: int, db: AsyncSession):
|
||||
if not record:
|
||||
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
|
||||
|
||||
if _is_gf3_native_preview_record(record):
|
||||
preview_path = str(record.preview_cache_path or "")
|
||||
if (
|
||||
(record.preview_cache_status or "NONE") == "READY"
|
||||
and preview_path.lower().endswith(".webp")
|
||||
and os.path.exists(preview_path)
|
||||
):
|
||||
return FileResponse(
|
||||
preview_path,
|
||||
media_type="image/webp",
|
||||
headers={"Cache-Control": "public, max-age=31536000"},
|
||||
)
|
||||
raise HTTPException(status_code=404, detail="GF3 native _geo WebP cache has not been generated.")
|
||||
|
||||
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
|
||||
if (
|
||||
(record.preview_cache_status or "NONE") == "READY"
|
||||
and record.preview_cache_path
|
||||
and str(record.preview_cache_path).lower().endswith(".webp")
|
||||
and os.path.exists(record.preview_cache_path)
|
||||
):
|
||||
return FileResponse(
|
||||
record.preview_cache_path,
|
||||
media_type="image/webp",
|
||||
headers={"Cache-Control": "public, max-age=31536000"},
|
||||
)
|
||||
|
||||
if os.path.exists(geo_cache_path):
|
||||
return FileResponse(
|
||||
geo_cache_path,
|
||||
@@ -664,6 +726,9 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep
|
||||
if not record:
|
||||
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
|
||||
|
||||
if _is_gf3_native_preview_record(record):
|
||||
return _build_gf3_native_preview_status(record)
|
||||
|
||||
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
|
||||
has_geo_cache = os.path.exists(geo_cache_path)
|
||||
has_raw_cache = os.path.exists(raw_cache_path)
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import read_int_env
|
||||
from ..config import read_int_env, settings
|
||||
from ..database import get_db
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.task_service import TASK_LOG_DEFAULT_LIMIT, TASK_LOG_MAX_LIMIT, TASK_QUERY_MAX_OFFSET, task_service
|
||||
@@ -33,12 +35,14 @@ COPY_BATCH_MAX_COPY_ITEMS = read_int_env(
|
||||
minimum=1,
|
||||
maximum=200000,
|
||||
)
|
||||
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "task_zip", "source_bundle"}
|
||||
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "source_bundle"}
|
||||
_COPY_TARGET_NAME_RE = re.compile(r"[^0-9A-Za-z\u4e00-\u9fff._-]+")
|
||||
|
||||
|
||||
class CopyBatchRequest(BaseModel):
|
||||
batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
|
||||
dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
|
||||
dest_dir: str = Field(default="", max_length=COPY_BATCH_TEXT_MAX_LENGTH)
|
||||
target_name: Optional[str] = Field(default=None, max_length=255)
|
||||
copy_statuses: Optional[List[str]] = None
|
||||
include_orbit_files: bool = False
|
||||
export_zip: bool = False
|
||||
@@ -46,7 +50,7 @@ class CopyBatchRequest(BaseModel):
|
||||
skip_existing: bool = True
|
||||
max_items: Optional[int] = None
|
||||
|
||||
@field_validator("batch_id", "dest_dir", mode="before")
|
||||
@field_validator("batch_id", mode="before")
|
||||
@classmethod
|
||||
def _normalize_required_text(cls, value):
|
||||
normalized = str(value or "").strip()
|
||||
@@ -54,6 +58,17 @@ class CopyBatchRequest(BaseModel):
|
||||
raise ValueError("Field must not be empty.")
|
||||
return normalized
|
||||
|
||||
@field_validator("dest_dir", mode="before")
|
||||
@classmethod
|
||||
def _normalize_optional_dest_dir(cls, value):
|
||||
return str(value or "").strip()
|
||||
|
||||
@field_validator("target_name", mode="before")
|
||||
@classmethod
|
||||
def _normalize_optional_target_name(cls, value):
|
||||
normalized = str(value or "").strip()
|
||||
return normalized or None
|
||||
|
||||
@field_validator("copy_statuses", mode="before")
|
||||
@classmethod
|
||||
def _validate_copy_statuses_length(cls, value):
|
||||
@@ -88,6 +103,10 @@ class CopyBatchRequest(BaseModel):
|
||||
@classmethod
|
||||
def _normalize_package_mode(cls, value):
|
||||
normalized = str(value or "task_folder").strip().lower()
|
||||
if normalized == "task_zip":
|
||||
normalized = "task_folder"
|
||||
if normalized in {"bundle", "dedupe_source"}:
|
||||
normalized = "source_bundle"
|
||||
if normalized not in COPY_DINSAR_PACKAGE_MODES:
|
||||
raise ValueError(
|
||||
f"package_mode must be one of: {sorted(COPY_DINSAR_PACKAGE_MODES)}."
|
||||
@@ -115,6 +134,28 @@ def _normalize_copy_batch_statuses(copy_statuses: Optional[List[str]]) -> List[s
|
||||
return normalized or ["COMPLETED"]
|
||||
|
||||
|
||||
def _safe_copy_target_name(value: Optional[str]) -> str:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="target_name 不能为空")
|
||||
if os.path.isabs(raw) or os.path.splitdrive(raw)[0] or "\\" in raw or "/" in raw:
|
||||
raise HTTPException(status_code=400, detail="target_name 只能是任务名,不能包含路径")
|
||||
normalized = _COPY_TARGET_NAME_RE.sub("_", raw).strip("._ ")
|
||||
if not normalized:
|
||||
raise HTTPException(status_code=400, detail="target_name 不合法")
|
||||
if normalized.upper() in {"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "LPT1", "LPT2", "LPT3"}:
|
||||
raise HTTPException(status_code=400, detail="target_name 是 Windows 保留名称")
|
||||
return normalized[:120]
|
||||
|
||||
|
||||
def _server_copy_destination(package_mode: str, target_name: Optional[str]) -> str:
|
||||
safe_name = _safe_copy_target_name(target_name)
|
||||
root = settings.DINSAR_TASK_POOL_ROOT if package_mode == "task_folder" else settings.DATA_DISTRIBUTION_ROOT
|
||||
if not root:
|
||||
raise HTTPException(status_code=500, detail="服务器目标根目录未配置")
|
||||
return os.path.normpath(os.path.join(root, safe_name))
|
||||
|
||||
|
||||
@router.post("/tools/copy-ps-stack")
|
||||
async def copy_ps_stack_endpoint(
|
||||
request: CopyBatchRequest,
|
||||
@@ -124,20 +165,26 @@ async def copy_ps_stack_endpoint(
|
||||
"""
|
||||
Start PS-InSAR copy task from a batch.
|
||||
"""
|
||||
_validate_export_path(request.dest_dir, "dest_dir")
|
||||
try:
|
||||
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
|
||||
if request.target_name:
|
||||
dest_dir = _server_copy_destination("source_bundle", request.target_name)
|
||||
else:
|
||||
dest_dir = request.dest_dir
|
||||
_validate_export_path(dest_dir, "dest_dir")
|
||||
params = {
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"file_type": "PS_STACK",
|
||||
"batch_id": request.batch_id,
|
||||
"copy_statuses": copy_statuses,
|
||||
}
|
||||
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.dest_dir}", params=params)
|
||||
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.target_name or dest_dir}", params=params)
|
||||
|
||||
payload = {
|
||||
"file_type": "PS_STACK",
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"batch_id": request.batch_id,
|
||||
"copy_statuses": copy_statuses,
|
||||
}
|
||||
@@ -150,7 +197,8 @@ async def copy_ps_stack_endpoint(
|
||||
detail={
|
||||
"task_id": task_id,
|
||||
"batch_id": request.batch_id,
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"copy_statuses": copy_statuses,
|
||||
},
|
||||
)
|
||||
@@ -169,32 +217,41 @@ async def copy_dinsar_pairs_endpoint(
|
||||
"""
|
||||
Start D-InSAR copy task from a batch.
|
||||
"""
|
||||
_validate_export_path(request.dest_dir, "dest_dir")
|
||||
try:
|
||||
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
|
||||
package_mode = request.package_mode
|
||||
if bool(request.export_zip) and package_mode == "task_folder":
|
||||
package_mode = "task_zip"
|
||||
if request.target_name:
|
||||
dest_dir = _server_copy_destination(package_mode, request.target_name)
|
||||
else:
|
||||
dest_dir = request.dest_dir
|
||||
_validate_export_path(dest_dir, "dest_dir")
|
||||
params = {
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"file_type": "DINSAR_PAIRS",
|
||||
"batch_id": request.batch_id,
|
||||
"copy_statuses": copy_statuses,
|
||||
"include_orbit_files": bool(request.include_orbit_files),
|
||||
"export_zip": package_mode == "task_zip",
|
||||
"export_zip": False,
|
||||
"package_mode": package_mode,
|
||||
"skip_existing": bool(request.skip_existing),
|
||||
"max_items": request.max_items,
|
||||
}
|
||||
task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params)
|
||||
task_name = (
|
||||
f"D-InSAR 生产数据准备: {request.target_name or dest_dir}"
|
||||
if package_mode == "task_folder"
|
||||
else f"D-InSAR 数据分发: {request.target_name or dest_dir}"
|
||||
)
|
||||
task_id = await task_service.create_task("COPY_DATA", task_name, params=params)
|
||||
|
||||
payload = {
|
||||
"file_type": "DINSAR_PAIRS",
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"batch_id": request.batch_id,
|
||||
"copy_statuses": copy_statuses,
|
||||
"include_orbit_files": bool(request.include_orbit_files),
|
||||
"export_zip": package_mode == "task_zip",
|
||||
"export_zip": False,
|
||||
"package_mode": package_mode,
|
||||
"skip_existing": bool(request.skip_existing),
|
||||
"max_items": request.max_items,
|
||||
@@ -208,10 +265,11 @@ async def copy_dinsar_pairs_endpoint(
|
||||
detail={
|
||||
"task_id": task_id,
|
||||
"batch_id": request.batch_id,
|
||||
"dest_dir": request.dest_dir,
|
||||
"dest_dir": dest_dir,
|
||||
"target_name": request.target_name,
|
||||
"copy_statuses": copy_statuses,
|
||||
"include_orbit_files": bool(request.include_orbit_files),
|
||||
"export_zip": package_mode == "task_zip",
|
||||
"export_zip": False,
|
||||
"package_mode": package_mode,
|
||||
"skip_existing": bool(request.skip_existing),
|
||||
"max_items": request.max_items,
|
||||
|
||||
@@ -13,8 +13,11 @@ MONITOR_CONFIG = {
|
||||
# LT-1 链路
|
||||
"radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS),
|
||||
"orbit_dir": settings.MONITOR_ORBIT_DIR,
|
||||
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS),
|
||||
# Sentinel-1 链路
|
||||
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS) or [settings.DINSAR_PRODUCT_DIR],
|
||||
"task_pool_root": settings.TASK_POOL_ROOT,
|
||||
"dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT,
|
||||
"sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT,
|
||||
# LT-1 / Sentinel-1 source inventory
|
||||
"s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS),
|
||||
"s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS),
|
||||
"s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,8 @@ import json
|
||||
import asyncio
|
||||
import hashlib
|
||||
import re
|
||||
import tarfile
|
||||
import zipfile
|
||||
from typing import List, Dict, Optional, Tuple, Any
|
||||
from datetime import datetime
|
||||
from shapely.geometry import Point, Polygon, shape, mapping
|
||||
@@ -240,6 +242,8 @@ def _chunked(items: List[Any], size: int):
|
||||
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
_RADAR_ARCHIVE_SUFFIXES = (".zip", ".tar", ".tgz", ".tar.gz")
|
||||
_RADAR_ARCHIVE_PREVIEW_MAX_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def _sanitize_cache_name(name: str) -> str:
|
||||
@@ -265,6 +269,208 @@ def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_versio
|
||||
return f"RGID_{safe_version}_{digest}_{base}.webp"
|
||||
|
||||
|
||||
def _has_radar_archive_suffix(path: str) -> bool:
|
||||
return str(path or "").lower().endswith(_RADAR_ARCHIVE_SUFFIXES)
|
||||
|
||||
|
||||
def _archive_product_stem(path: str) -> str:
|
||||
base = os.path.basename(str(path or ""))
|
||||
lower_base = base.lower()
|
||||
for suffix in (".tar.gz", ".tgz", ".tar", ".zip"):
|
||||
if lower_base.endswith(suffix):
|
||||
return base[: -len(suffix)]
|
||||
return os.path.splitext(base)[0]
|
||||
|
||||
|
||||
def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> Optional[int]:
|
||||
product_stem = _archive_product_stem(archive_path)
|
||||
if not product_stem:
|
||||
return None
|
||||
|
||||
member = str(member_name or "").replace("\\", "/").strip("/")
|
||||
member_lower = member.lower()
|
||||
product_lower = product_stem.lower()
|
||||
expected_names = [
|
||||
f"{product_lower}/{product_lower}.browse.jpg",
|
||||
f"{product_lower}/{product_lower}.browse.jpeg",
|
||||
f"{product_lower}/{product_lower}.browse.png",
|
||||
f"{product_lower}/{product_lower}.quicklook.jpg",
|
||||
f"{product_lower}/{product_lower}.quicklook.png",
|
||||
f"{product_lower}/{product_lower}.quick-look.png",
|
||||
f"{product_lower}/{product_lower}.thumb.jpg",
|
||||
f"{product_lower}/{product_lower}.thumb.jpeg",
|
||||
f"{product_lower}/{product_lower}.thumb.png",
|
||||
f"{product_lower}/preview/quick-look.png",
|
||||
f"{product_lower}.browse.jpg",
|
||||
f"{product_lower}.browse.jpeg",
|
||||
f"{product_lower}.browse.png",
|
||||
f"{product_lower}.thumb.jpg",
|
||||
f"{product_lower}.thumb.jpeg",
|
||||
f"{product_lower}.thumb.png",
|
||||
]
|
||||
try:
|
||||
return expected_names.index(member_lower)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]:
|
||||
lower_name = str(member_name or "").replace("\\", "/").lower()
|
||||
base_name = os.path.basename(lower_name)
|
||||
if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
|
||||
return None
|
||||
if size_bytes and size_bytes > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
|
||||
return None
|
||||
|
||||
has_keyword = any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS)
|
||||
if base_name.endswith((".tif", ".tiff")) and not has_keyword:
|
||||
return None
|
||||
|
||||
keyword_score = 0 if has_keyword else 2
|
||||
if base_name == "quick-look.png":
|
||||
keyword_score = -4
|
||||
elif base_name == "quicklook.png":
|
||||
keyword_score = -3
|
||||
elif base_name.startswith("quick-look.") or base_name.startswith("quicklook."):
|
||||
keyword_score = min(keyword_score, -2)
|
||||
if "/preview/" in lower_name:
|
||||
keyword_score -= 1
|
||||
if "/icons/" in lower_name:
|
||||
keyword_score += 2
|
||||
if base_name.endswith((".jpg", ".jpeg", ".png", ".webp")):
|
||||
ext_score = 0
|
||||
elif base_name.endswith(".bmp"):
|
||||
ext_score = 1
|
||||
else:
|
||||
ext_score = 2
|
||||
depth = lower_name.count("/")
|
||||
size_score = -int(size_bytes or 0)
|
||||
return (keyword_score, depth, ext_score, size_score, member_name)
|
||||
|
||||
|
||||
def _radar_archive_preview_cache_path(archive_path: str, member_name: str) -> str:
|
||||
digest = hashlib.sha1(
|
||||
f"{archive_path}|{member_name}".encode("utf-8", errors="ignore")
|
||||
).hexdigest()[:16]
|
||||
raw_base = os.path.basename(str(member_name or "preview"))
|
||||
base = _sanitize_cache_name(raw_base)
|
||||
_, ext = os.path.splitext(raw_base)
|
||||
ext = _RADAR_CACHE_NAME_RE.sub("", ext.lower())[:12]
|
||||
if ext and not base.lower().endswith(ext):
|
||||
base = f"{base[:max(1, 48 - len(ext))]}{ext}"
|
||||
return os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources", f"APS_{digest}_{base}")
|
||||
|
||||
|
||||
def _write_archive_preview_cache(target_path: str, source_obj: Any, archive_mtime: float) -> Optional[str]:
|
||||
try:
|
||||
os.makedirs(os.path.dirname(target_path), exist_ok=True)
|
||||
if os.path.exists(target_path) and _safe_mtime(target_path) >= archive_mtime:
|
||||
return target_path
|
||||
tmp_path = f"{target_path}.tmp"
|
||||
with open(tmp_path, "wb") as target:
|
||||
while True:
|
||||
chunk = source_obj.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
target.write(chunk)
|
||||
os.replace(tmp_path, target_path)
|
||||
if archive_mtime:
|
||||
os.utime(target_path, (archive_mtime, archive_mtime))
|
||||
return target_path
|
||||
except Exception:
|
||||
try:
|
||||
if "tmp_path" in locals() and os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _find_radar_archive_preview_source(archive_path: str) -> Optional[str]:
|
||||
if not archive_path or not os.path.isfile(archive_path) or not _has_radar_archive_suffix(archive_path):
|
||||
return None
|
||||
|
||||
archive_mtime = _safe_mtime(archive_path)
|
||||
lower_path = archive_path.lower()
|
||||
try:
|
||||
if lower_path.endswith(".zip"):
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
candidates = []
|
||||
info_by_name = {}
|
||||
for info in archive.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
score = _radar_archive_preview_score(info.filename, int(info.file_size or 0))
|
||||
if score is None:
|
||||
continue
|
||||
candidates.append(score)
|
||||
info_by_name[info.filename] = info
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort()
|
||||
member_name = candidates[0][-1]
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
|
||||
with archive.open(info_by_name[member_name], "r") as source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
|
||||
with tarfile.open(archive_path, "r:*") as archive:
|
||||
candidates = []
|
||||
member_by_name = {}
|
||||
best_rank: Optional[int] = None
|
||||
best_member: Optional[tarfile.TarInfo] = None
|
||||
for member in archive:
|
||||
if not member.isfile():
|
||||
continue
|
||||
expected_rank = _radar_archive_expected_preview_rank(archive_path, member.name)
|
||||
if expected_rank is not None:
|
||||
if best_rank is None or expected_rank < best_rank:
|
||||
best_rank = expected_rank
|
||||
best_member = member
|
||||
if expected_rank == 0:
|
||||
source_obj = archive.extractfile(member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
continue
|
||||
|
||||
score = _radar_archive_preview_score(member.name, int(member.size or 0))
|
||||
if score is None:
|
||||
if best_member is not None and int(member.size or 0) > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES:
|
||||
break
|
||||
continue
|
||||
candidates.append(score)
|
||||
member_by_name[member.name] = member
|
||||
if os.path.basename(str(member.name or "").lower()) in {"quick-look.png", "quicklook.png"}:
|
||||
source_obj = archive.extractfile(member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
if best_member is not None:
|
||||
source_obj = archive.extractfile(best_member)
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, best_member.name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort()
|
||||
member_name = candidates[0][-1]
|
||||
source_obj = archive.extractfile(member_by_name[member_name])
|
||||
if source_obj is None:
|
||||
return None
|
||||
target_path = _radar_archive_preview_cache_path(archive_path, member_name)
|
||||
with source_obj:
|
||||
return _write_archive_preview_cache(target_path, source_obj, archive_mtime)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]:
|
||||
"""Extract coverage polygon from a GeoTIFF file using GDAL.
|
||||
|
||||
@@ -380,6 +586,8 @@ class DataService:
|
||||
|
||||
@staticmethod
|
||||
def find_radar_preview_source(scene_dir: str) -> Optional[str]:
|
||||
if scene_dir and os.path.isfile(scene_dir) and _has_radar_archive_suffix(scene_dir):
|
||||
return _find_radar_archive_preview_source(scene_dir)
|
||||
if not scene_dir or not os.path.isdir(scene_dir):
|
||||
return None
|
||||
|
||||
@@ -520,21 +728,26 @@ class DataService:
|
||||
if orbit_dir and orbit_files_map:
|
||||
update_progress("正在同步精轨到本地引擎池...", 8)
|
||||
try:
|
||||
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
|
||||
sync_result = await asyncio.to_thread(
|
||||
sync_orbit_pools,
|
||||
orbit_dir,
|
||||
settings.ORBIT_POOL_ENVI,
|
||||
settings.ORBIT_POOL_ISCE2,
|
||||
isce2_pool,
|
||||
settings.ORBIT_POOL_LANDSAR,
|
||||
bool(settings.ISCE2_ENABLED),
|
||||
)
|
||||
envi_new = len(sync_result.get("envi", {}).get("copied", []))
|
||||
envi_updated = len(sync_result.get("envi", {}).get("updated", []))
|
||||
isce2_new = len(sync_result.get("isce2", {}).get("converted", []))
|
||||
isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", []))
|
||||
print(
|
||||
f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated},"
|
||||
f"ISCE2 转换 {isce2_new}、重转 {isce2_updated}"
|
||||
)
|
||||
if settings.ISCE2_ENABLED:
|
||||
print(
|
||||
f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},"
|
||||
f"ISCE2 XML 转换 {isce2_new}、重转 {isce2_updated}"
|
||||
)
|
||||
else:
|
||||
print(f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated},ISCE2 已停用")
|
||||
invalid_orbit_stems = {
|
||||
item.get("name")
|
||||
for item in sync_result.get("invalid_sources", [])
|
||||
@@ -550,7 +763,11 @@ class DataService:
|
||||
sync_errors = (
|
||||
sync_result.get("source", {}).get("errors", [])
|
||||
+ [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])]
|
||||
+ [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
|
||||
+ (
|
||||
[item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])]
|
||||
if settings.ISCE2_ENABLED
|
||||
else []
|
||||
)
|
||||
)
|
||||
if task_id and sync_errors:
|
||||
await task_service.add_log(
|
||||
@@ -560,15 +777,16 @@ class DataService:
|
||||
"精轨同步存在异常: "
|
||||
f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, "
|
||||
f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, "
|
||||
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))} 项"
|
||||
f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', [])) if settings.ISCE2_ENABLED else 0} 项"
|
||||
),
|
||||
)
|
||||
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
"WARN",
|
||||
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
|
||||
)
|
||||
if settings.ISCE2_ENABLED:
|
||||
for item in sync_result.get("isce2", {}).get("errors", [])[:20]:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
"WARN",
|
||||
f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}",
|
||||
)
|
||||
if task_id and invalid_orbit_stems:
|
||||
await task_service.add_log(
|
||||
task_id,
|
||||
|
||||
@@ -17,10 +17,13 @@ from ..utils import parse_gf3_l2_dirname
|
||||
|
||||
NATIVE_MANIFEST_NAME = "gf3_native_manifest.json"
|
||||
NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1"
|
||||
FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes"
|
||||
POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH")
|
||||
SKIP_DIR_NAMES = {
|
||||
".git",
|
||||
FLAT_SCENE_CATALOG_DIR_NAME,
|
||||
".gf3_extract",
|
||||
".gf3_runtime",
|
||||
".sarmap",
|
||||
"__pycache__",
|
||||
"temp",
|
||||
@@ -87,6 +90,16 @@ def _polarization_from_geo_name(name: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _scene_name_from_geo_name(name: str) -> str:
|
||||
text = str(name or "").strip()
|
||||
match = re.match(r"^(?P<scene>.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE)
|
||||
if match:
|
||||
return match.group("scene")
|
||||
if text.lower().endswith("_geo"):
|
||||
return text[:-4]
|
||||
return Path(text).stem
|
||||
|
||||
|
||||
def _is_geo_native_data_file(path: Path) -> bool:
|
||||
return path.is_file() and path.name.lower().endswith("_geo")
|
||||
|
||||
@@ -128,6 +141,62 @@ def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict
|
||||
return metadata
|
||||
|
||||
|
||||
def _native_asset_from_base(base: Path) -> dict[str, Any]:
|
||||
hdr = Path(str(base) + ".hdr")
|
||||
sml = Path(str(base) + ".sml")
|
||||
aux_xml = Path(str(base) + ".aux.xml")
|
||||
ovr = Path(str(base) + ".ovr")
|
||||
kml = Path(str(base) + ".kml")
|
||||
quicklook = base.with_name(base.name + "_ql.tif")
|
||||
ql_kml = base.with_name(base.name + "_ql.kml")
|
||||
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
||||
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
||||
|
||||
return {
|
||||
"polarization": polarization,
|
||||
"role": "geo_native",
|
||||
"path": str(base),
|
||||
"hdr": str(hdr) if hdr.exists() else None,
|
||||
"sml": str(sml) if sml.exists() else None,
|
||||
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
||||
"ovr": str(ovr) if ovr.exists() else None,
|
||||
"quicklook": str(quicklook) if quicklook.exists() else None,
|
||||
"kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None),
|
||||
"complete": bool(complete),
|
||||
"source": _safe_stat(base),
|
||||
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
||||
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
||||
}
|
||||
|
||||
|
||||
def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"polarization": str(asset.get("polarization") or "").upper(),
|
||||
"path": asset.get("path"),
|
||||
"complete": bool(asset.get("complete")),
|
||||
"source": asset.get("source"),
|
||||
"hdr": asset.get("hdr_info"),
|
||||
"sml": asset.get("sml_info"),
|
||||
}
|
||||
|
||||
|
||||
def _native_fingerprint(assets: list[dict[str, Any]]) -> str:
|
||||
payload = [
|
||||
_asset_fingerprint_part(asset)
|
||||
for asset in sorted(
|
||||
assets,
|
||||
key=lambda item: (
|
||||
str(item.get("polarization") or ""),
|
||||
str(item.get("path") or ""),
|
||||
),
|
||||
)
|
||||
]
|
||||
text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
|
||||
import hashlib
|
||||
|
||||
return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest()
|
||||
|
||||
|
||||
def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
||||
assets: list[dict[str, Any]] = []
|
||||
try:
|
||||
@@ -139,43 +208,25 @@ def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]:
|
||||
if not _is_geo_native_data_file(path):
|
||||
continue
|
||||
|
||||
base = path
|
||||
hdr = Path(str(base) + ".hdr")
|
||||
sml = Path(str(base) + ".sml")
|
||||
aux_xml = Path(str(base) + ".aux.xml")
|
||||
ovr = Path(str(base) + ".ovr")
|
||||
kml = Path(str(base) + ".kml")
|
||||
quicklook = base.with_name(base.name + "_ql.tif")
|
||||
polarization = _polarization_from_geo_name(base.name) or "UNKNOWN"
|
||||
complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml)
|
||||
|
||||
asset: dict[str, Any] = {
|
||||
"polarization": polarization,
|
||||
"role": "geo_native",
|
||||
"path": str(base),
|
||||
"hdr": str(hdr) if hdr.exists() else None,
|
||||
"sml": str(sml) if sml.exists() else None,
|
||||
"aux_xml": str(aux_xml) if aux_xml.exists() else None,
|
||||
"ovr": str(ovr) if ovr.exists() else None,
|
||||
"quicklook": str(quicklook) if quicklook.exists() else None,
|
||||
"kml": str(kml) if kml.exists() else None,
|
||||
"complete": bool(complete),
|
||||
"source": _safe_stat(base),
|
||||
"hdr_info": _safe_stat(hdr) if hdr.exists() else None,
|
||||
"sml_info": _safe_stat(sml) if sml.exists() else None,
|
||||
}
|
||||
assets.append(asset)
|
||||
assets.append(_native_asset_from_base(path))
|
||||
|
||||
return assets
|
||||
|
||||
|
||||
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
||||
assets = _collect_native_assets(scene_dir)
|
||||
def _build_scene_manifest(
|
||||
root: Path,
|
||||
*,
|
||||
scene_name: str,
|
||||
native_dir: Path,
|
||||
manifest_dir: Path,
|
||||
assets: list[dict[str, Any]],
|
||||
source_dir: Path,
|
||||
storage_layout: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if not assets:
|
||||
return None
|
||||
|
||||
scene_name = scene_dir.name
|
||||
batch_name = _scene_batch_name(root, scene_dir, scene_name)
|
||||
batch_name = _scene_batch_name(root, source_dir, scene_name)
|
||||
complete_assets = [asset for asset in assets if asset.get("complete")]
|
||||
complete_pols = [
|
||||
pol
|
||||
@@ -200,25 +251,30 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
|
||||
|
||||
logs = []
|
||||
for name in ("gf3_sarscape_cli.log",):
|
||||
log_path = scene_dir / name
|
||||
log_path = source_dir / name
|
||||
if log_path.is_file():
|
||||
logs.append(str(log_path))
|
||||
try:
|
||||
logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
||||
logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
metadata = _parse_scene_metadata(scene_name, assets)
|
||||
manifest_path = scene_dir / NATIVE_MANIFEST_NAME
|
||||
native_fingerprint = _native_fingerprint(assets)
|
||||
metadata["native_fingerprint"] = native_fingerprint
|
||||
manifest_path = manifest_dir / NATIVE_MANIFEST_NAME
|
||||
return {
|
||||
"schema": NATIVE_MANIFEST_SCHEMA,
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_name,
|
||||
"batch_name": batch_name,
|
||||
"native_root": str(root),
|
||||
"native_dir": str(scene_dir),
|
||||
"native_dir": str(native_dir),
|
||||
"source_dir": str(source_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"source_archive": None,
|
||||
"storage_layout": storage_layout,
|
||||
"native_fingerprint": native_fingerprint,
|
||||
"status": status,
|
||||
"polarizations": complete_pols,
|
||||
"metadata": metadata,
|
||||
@@ -227,6 +283,49 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non
|
||||
}
|
||||
|
||||
|
||||
def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None:
|
||||
assets = _collect_native_assets(scene_dir)
|
||||
return _build_scene_manifest(
|
||||
root,
|
||||
scene_name=scene_dir.name,
|
||||
native_dir=scene_dir,
|
||||
manifest_dir=scene_dir,
|
||||
assets=assets,
|
||||
source_dir=scene_dir,
|
||||
storage_layout="scene_dir",
|
||||
)
|
||||
|
||||
|
||||
def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]:
|
||||
try:
|
||||
entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower())
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
grouped_assets: dict[str, list[dict[str, Any]]] = {}
|
||||
for path in entries:
|
||||
if not _is_geo_native_data_file(path):
|
||||
continue
|
||||
scene_name = _scene_name_from_geo_name(path.name)
|
||||
grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path))
|
||||
|
||||
manifests: list[dict[str, Any]] = []
|
||||
for scene_name in sorted(grouped_assets):
|
||||
scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name
|
||||
manifest = _build_scene_manifest(
|
||||
root,
|
||||
scene_name=scene_name,
|
||||
native_dir=scene_catalog_dir,
|
||||
manifest_dir=scene_catalog_dir,
|
||||
assets=grouped_assets[scene_name],
|
||||
source_dir=source_dir,
|
||||
storage_layout="flat_files",
|
||||
)
|
||||
if manifest:
|
||||
manifests.append(manifest)
|
||||
return manifests
|
||||
|
||||
|
||||
def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
@@ -251,6 +350,7 @@ def scan_gf3_sarscape_native_roots(
|
||||
native_dirs: list[str] | tuple[str, ...] | None,
|
||||
*,
|
||||
write_manifest: bool = True,
|
||||
scene_dirs_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan configured native roots and return discovered scene manifests."""
|
||||
roots, missing_roots = _normalize_roots(native_dirs)
|
||||
@@ -258,12 +358,56 @@ def scan_gf3_sarscape_native_roots(
|
||||
seen_scene_dirs: set[str] = set()
|
||||
write_errors: list[dict[str, str]] = []
|
||||
|
||||
def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None:
|
||||
seen_scene_dirs.add(scene_key)
|
||||
if write_manifest:
|
||||
try:
|
||||
_write_json(Path(manifest["manifest_path"]), manifest)
|
||||
except OSError as exc:
|
||||
write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)})
|
||||
scenes.append(manifest)
|
||||
|
||||
for root in roots:
|
||||
if scene_dirs_only:
|
||||
try:
|
||||
candidate_dirs = [root] + [
|
||||
child
|
||||
for child in sorted(root.iterdir(), key=lambda item: item.name.lower())
|
||||
if child.is_dir()
|
||||
and child.name not in SKIP_DIR_NAMES
|
||||
and not child.name.startswith(".SARscape")
|
||||
and not child.name.startswith(".gf3_")
|
||||
]
|
||||
except OSError:
|
||||
candidate_dirs = [root]
|
||||
|
||||
for scene_dir in candidate_dirs:
|
||||
scene_key = str(scene_dir).lower()
|
||||
if scene_key in seen_scene_dirs:
|
||||
continue
|
||||
|
||||
if not scene_dir.name.upper().startswith("GF3_"):
|
||||
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
||||
if flat_manifests:
|
||||
for manifest in flat_manifests:
|
||||
flat_key = str(manifest.get("native_dir") or "").lower()
|
||||
if flat_key and flat_key not in seen_scene_dirs:
|
||||
append_manifest(manifest, flat_key, scene_dir)
|
||||
continue
|
||||
|
||||
manifest = _collect_scene_manifest(root, scene_dir)
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
append_manifest(manifest, scene_key, scene_dir)
|
||||
continue
|
||||
|
||||
for current_dir, dir_names, _file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
||||
and not name.startswith(".gf3_")
|
||||
]
|
||||
scene_dir = Path(current_dir)
|
||||
scene_key = str(scene_dir).lower()
|
||||
@@ -271,17 +415,20 @@ def scan_gf3_sarscape_native_roots(
|
||||
dir_names[:] = []
|
||||
continue
|
||||
|
||||
if not scene_dir.name.upper().startswith("GF3_"):
|
||||
flat_manifests = _collect_flat_scene_manifests(root, scene_dir)
|
||||
if flat_manifests:
|
||||
for manifest in flat_manifests:
|
||||
flat_key = str(manifest.get("native_dir") or "").lower()
|
||||
if flat_key and flat_key not in seen_scene_dirs:
|
||||
append_manifest(manifest, flat_key, scene_dir)
|
||||
continue
|
||||
|
||||
manifest = _collect_scene_manifest(root, scene_dir)
|
||||
if not manifest:
|
||||
continue
|
||||
|
||||
seen_scene_dirs.add(scene_key)
|
||||
if write_manifest:
|
||||
try:
|
||||
_write_json(Path(manifest["manifest_path"]), manifest)
|
||||
except OSError as exc:
|
||||
write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)})
|
||||
scenes.append(manifest)
|
||||
append_manifest(manifest, scene_key, scene_dir)
|
||||
dir_names[:] = []
|
||||
|
||||
native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY")
|
||||
|
||||
@@ -15,12 +15,13 @@ from shapely.geometry import Polygon
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..config import settings
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM
|
||||
from .data_service import extract_geotiff_bounds
|
||||
from .data_service import DataService, extract_geotiff_bounds
|
||||
from .gf3_native_inventory_service import (
|
||||
NATIVE_MANIFEST_NAME,
|
||||
POLARIZATION_PRIORITY,
|
||||
SKIP_DIR_NAMES,
|
||||
scan_gf3_sarscape_native_roots,
|
||||
)
|
||||
from .image_service import image_service
|
||||
@@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1"
|
||||
CONVERTER_NAME = "gf3_sarscape_geo_to_tif"
|
||||
CONVERTER_VERSION = "v1"
|
||||
SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2"
|
||||
SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK"
|
||||
SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
|
||||
QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2"
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
@@ -72,6 +76,112 @@ def _path_kind(path: str) -> str:
|
||||
return "relative"
|
||||
|
||||
|
||||
def _path_key(path: str | Path) -> str:
|
||||
return os.path.normcase(os.path.normpath(str(path))).lower()
|
||||
|
||||
|
||||
def _has_native_geo_assets(path: Path) -> bool:
|
||||
try:
|
||||
return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir())
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]:
|
||||
"""Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning.
|
||||
|
||||
Operators usually configure a durable GF3 root rather than a date folder.
|
||||
Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer
|
||||
direct ``*_geo`` children and fall back to recursive discovery.
|
||||
"""
|
||||
roots: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(path: str | Path) -> None:
|
||||
text = os.path.normpath(str(path))
|
||||
key = _path_key(text)
|
||||
if key and key not in seen:
|
||||
seen.add(key)
|
||||
roots.append(text)
|
||||
|
||||
for raw in native_dirs or []:
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
if any(ch in text for ch in "*?"):
|
||||
glob_root = Path(os.path.normpath(text))
|
||||
try:
|
||||
matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower())
|
||||
except OSError:
|
||||
matches = []
|
||||
if matches:
|
||||
for match in matches:
|
||||
if match.is_dir():
|
||||
add(match)
|
||||
continue
|
||||
add(glob_root)
|
||||
continue
|
||||
|
||||
root = Path(os.path.normpath(text))
|
||||
try:
|
||||
is_dir = root.is_dir()
|
||||
except OSError:
|
||||
is_dir = False
|
||||
if not is_dir:
|
||||
add(root)
|
||||
continue
|
||||
|
||||
if root.name.lower().endswith("_geo"):
|
||||
add(root)
|
||||
|
||||
try:
|
||||
immediate_geo_roots = sorted(
|
||||
(
|
||||
child
|
||||
for child in root.iterdir()
|
||||
if child.is_dir() and child.name.lower().endswith("_geo")
|
||||
),
|
||||
key=lambda item: item.name.lower(),
|
||||
)
|
||||
except OSError:
|
||||
immediate_geo_roots = []
|
||||
|
||||
if immediate_geo_roots:
|
||||
for child in immediate_geo_roots:
|
||||
add(child)
|
||||
continue
|
||||
|
||||
if _has_native_geo_assets(root):
|
||||
add(root)
|
||||
continue
|
||||
|
||||
found_recursive = 0
|
||||
for current_dir, dir_names, _file_names in os.walk(root):
|
||||
dir_names[:] = [
|
||||
name
|
||||
for name in dir_names
|
||||
if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape")
|
||||
and not name.startswith(".gf3_")
|
||||
]
|
||||
selected: list[str] = []
|
||||
remaining: list[str] = []
|
||||
for name in dir_names:
|
||||
if name.lower().endswith("_geo"):
|
||||
selected.append(name)
|
||||
else:
|
||||
remaining.append(name)
|
||||
for name in selected:
|
||||
add(Path(current_dir) / name)
|
||||
found_recursive += len(selected)
|
||||
dir_names[:] = remaining
|
||||
|
||||
if found_recursive == 0:
|
||||
add(root)
|
||||
|
||||
return roots
|
||||
|
||||
|
||||
def _source_asset_uid(path: str) -> str:
|
||||
normalized = os.path.normpath(str(path or "").strip())
|
||||
digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest()
|
||||
@@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
|
||||
def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None:
|
||||
polygon = extract_geotiff_bounds(str(path))
|
||||
if polygon and len(polygon) >= 4:
|
||||
return polygon
|
||||
@@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None:
|
||||
if not asset:
|
||||
return None
|
||||
for key in ("path", "quicklook"):
|
||||
text = _path_text(asset.get(key))
|
||||
if not text:
|
||||
continue
|
||||
path = Path(text)
|
||||
if not path.is_file():
|
||||
continue
|
||||
polygon = _polygon_from_raster(path)
|
||||
if polygon and len(polygon) >= 4:
|
||||
return polygon
|
||||
return None
|
||||
|
||||
|
||||
def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]:
|
||||
if not polygon:
|
||||
return None, None
|
||||
@@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None
|
||||
return assets[0] if assets else None
|
||||
|
||||
|
||||
def _is_quicklook_raster_path(path: Any) -> bool:
|
||||
text = _path_text(path).lower()
|
||||
return text.endswith("_ql.tif") or text.endswith("_ql.tiff")
|
||||
|
||||
|
||||
def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None:
|
||||
candidates = [
|
||||
asset
|
||||
for asset in assets
|
||||
if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path"))
|
||||
]
|
||||
complete = [asset for asset in candidates if asset.get("complete") is not False]
|
||||
return _select_default_asset(complete or candidates)
|
||||
|
||||
|
||||
def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
output_assets: list[dict[str, Any]] = []
|
||||
for asset in scene_manifest.get("assets") or []:
|
||||
native_path = _path_text(asset.get("path"))
|
||||
if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True:
|
||||
continue
|
||||
output_assets.append(
|
||||
{
|
||||
"polarization": str(asset.get("polarization") or "UNKNOWN").upper(),
|
||||
"role": "native_geo",
|
||||
"path": native_path,
|
||||
"hdr": _path_text(asset.get("hdr")) or None,
|
||||
"sml": _path_text(asset.get("sml")) or None,
|
||||
"quicklook": _path_text(asset.get("quicklook")) or None,
|
||||
"complete": bool(asset.get("complete")),
|
||||
"status": "registered_native_geo",
|
||||
}
|
||||
)
|
||||
return output_assets
|
||||
|
||||
|
||||
def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]:
|
||||
root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve()
|
||||
out_dir = _standard_scene_dir(scene_manifest, root)
|
||||
manifest_path = out_dir / "gf3_native_preview_manifest.json"
|
||||
assets = _quicklook_assets_from_scene_manifest(scene_manifest)
|
||||
manifest = {
|
||||
"schema": "gf3_sarscape_native_preview.v1",
|
||||
"generated_at": _utc_now(),
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"batch_name": scene_manifest.get("batch_name"),
|
||||
"native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME),
|
||||
"native_dir": scene_manifest.get("native_dir"),
|
||||
"standard_dir": str(out_dir),
|
||||
"manifest_path": str(manifest_path),
|
||||
"status": "NATIVE_READY" if assets else "FAILED",
|
||||
"assets": assets,
|
||||
"summary": {
|
||||
"native_assets": len(assets),
|
||||
"quicklook_assets": 0,
|
||||
"full_raster_materialized": True,
|
||||
"webp_source": "native_geo",
|
||||
},
|
||||
"errors": [] if assets else [{"error": "no native _geo assets found"}],
|
||||
}
|
||||
_write_json(manifest_path, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool:
|
||||
"""Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``."""
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
from rasterio.enums import Resampling
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
try:
|
||||
target = Path(target_path)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600)
|
||||
with rasterio.open(source_path) as src:
|
||||
scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0)
|
||||
out_width = max(1, int(src.width * scale))
|
||||
out_height = max(1, int(src.height * scale))
|
||||
band = src.read(
|
||||
1,
|
||||
out_shape=(out_height, out_width),
|
||||
masked=True,
|
||||
resampling=Resampling.bilinear,
|
||||
)
|
||||
data = band.filled(0)
|
||||
if data.dtype != np.uint8:
|
||||
valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)]
|
||||
if valid.size:
|
||||
p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98])
|
||||
scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1)
|
||||
data = (scaled * 255).astype("uint8")
|
||||
else:
|
||||
data = np.zeros(data.shape, dtype="uint8")
|
||||
else:
|
||||
data = data.astype("uint8", copy=False)
|
||||
|
||||
mask = getattr(band, "mask", None)
|
||||
if mask is None or np.ndim(mask) == 0:
|
||||
alpha = np.full(data.shape, 255, dtype="uint8")
|
||||
else:
|
||||
alpha = np.where(mask, 0, 255).astype("uint8")
|
||||
rgba = np.stack([data, data, data, alpha], axis=-1)
|
||||
|
||||
image = Image.fromarray(rgba, "RGBA")
|
||||
image = image_service.make_edge_dark_transparent(image)
|
||||
image_service.save_image_as_webp(image, target_path, quality=82)
|
||||
return target.exists() and target.stat().st_size > 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str:
|
||||
candidates: list[Any] = [metadata.get("default_native_path")]
|
||||
for key in ("native_assets", "quicklook_assets"):
|
||||
for asset in metadata.get(key) or []:
|
||||
if isinstance(asset, dict):
|
||||
candidates.append(asset.get("path"))
|
||||
for candidate in candidates:
|
||||
text = _path_text(candidate)
|
||||
if text and not _is_quicklook_raster_path(text):
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _path_is_under_any(path: str, roots: list[str]) -> bool:
|
||||
target = _path_key(path)
|
||||
if not target:
|
||||
return False
|
||||
for root in roots:
|
||||
root_key = _path_key(root)
|
||||
if root_key and (target == root_key or target.startswith(root_key + os.sep)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]:
|
||||
roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
|
||||
if not roots:
|
||||
roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
return [os.path.normpath(root) for root in roots if str(root or "").strip()]
|
||||
|
||||
|
||||
def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str:
|
||||
return str(
|
||||
scene_manifest.get("native_fingerprint")
|
||||
or (scene_manifest.get("metadata") or {}).get("native_fingerprint")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def _record_has_bounds(record: RadarDataORM) -> bool:
|
||||
return (
|
||||
record.coverage_polygon is not None
|
||||
and record.min_lon is not None
|
||||
and record.min_lat is not None
|
||||
and record.max_lon is not None
|
||||
and record.max_lat is not None
|
||||
)
|
||||
|
||||
|
||||
async def _find_existing_quicklook_radar(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
scene_manifest: dict[str, Any],
|
||||
) -> RadarDataORM | None:
|
||||
native_dir = str(scene_manifest.get("native_dir") or "")
|
||||
scene_name = scene_manifest.get("scene_name") or Path(native_dir).name
|
||||
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
|
||||
result = await db.execute(
|
||||
select(RadarDataORM).where(
|
||||
or_(
|
||||
RadarDataORM.unique_id == unique_id,
|
||||
RadarDataORM.file_path == native_dir,
|
||||
)
|
||||
)
|
||||
)
|
||||
return result.scalars().first()
|
||||
|
||||
|
||||
async def _can_skip_unchanged_quicklook_radar(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
scene_manifest: dict[str, Any],
|
||||
) -> tuple[bool, int | None, str]:
|
||||
native_fingerprint = _scene_native_fingerprint(scene_manifest)
|
||||
if not native_fingerprint:
|
||||
return False, None, "missing_native_fingerprint"
|
||||
|
||||
record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest)
|
||||
if record is None:
|
||||
return False, None, "missing_record"
|
||||
|
||||
metadata = record.metadata_json or {}
|
||||
if str(metadata.get("native_fingerprint") or "") != native_fingerprint:
|
||||
return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed"
|
||||
if str(metadata.get("registration_mode") or "") != "native_preview":
|
||||
return False, int(record.id) if record.id is not None else None, "registration_mode_changed"
|
||||
if not _record_has_bounds(record):
|
||||
return False, int(record.id) if record.id is not None else None, "missing_bounds"
|
||||
|
||||
return True, int(record.id) if record.id is not None else None, "unchanged"
|
||||
|
||||
|
||||
async def generate_gf3_quicklook_webp_cache(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
force: bool = False,
|
||||
max_records: int | None = None,
|
||||
native_dirs: list[str] | tuple[str, ...] | None = None,
|
||||
progress_callback: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters."""
|
||||
stmt = (
|
||||
select(RadarDataORM)
|
||||
.where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT)
|
||||
.order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc())
|
||||
)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
configured_roots = _configured_gf3_native_roots(native_dirs)
|
||||
records = [
|
||||
record
|
||||
for record in result.scalars().all()
|
||||
if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots)
|
||||
]
|
||||
if max_records and max_records > 0:
|
||||
records = records[: int(max_records)]
|
||||
total = len(records)
|
||||
generated = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
record_results: list[dict[str, Any]] = []
|
||||
|
||||
for idx, record in enumerate(records):
|
||||
metadata = record.metadata_json or {}
|
||||
source_path = _native_preview_source_from_metadata(metadata)
|
||||
cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path)
|
||||
if progress_callback:
|
||||
pct = 5 + int((idx / max(total, 1)) * 90)
|
||||
progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}")
|
||||
|
||||
if (
|
||||
not force
|
||||
and (record.preview_cache_status or "NONE") == "READY"
|
||||
and record.preview_cache_path
|
||||
and os.path.exists(record.preview_cache_path)
|
||||
):
|
||||
skipped += 1
|
||||
record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path})
|
||||
continue
|
||||
|
||||
if not source_path:
|
||||
failed += 1
|
||||
record.preview_cache_status = "FAILED"
|
||||
record.preview_cache_path = None
|
||||
record.preview_cache_error = "default_native_path_missing"
|
||||
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
record.preview_cache_updated_at = _db_now()
|
||||
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path)
|
||||
record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
record.preview_cache_updated_at = _db_now()
|
||||
if ok and os.path.exists(cache_path):
|
||||
generated += 1
|
||||
record.preview_cache_status = "READY"
|
||||
record.preview_cache_path = cache_path
|
||||
record.preview_cache_error = None
|
||||
record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path})
|
||||
else:
|
||||
failed += 1
|
||||
record.preview_cache_status = "FAILED"
|
||||
record.preview_cache_path = None
|
||||
record.preview_cache_error = "native_webp_build_failed"
|
||||
record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error})
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"ok": failed == 0,
|
||||
"mode": "gf3_native_webp",
|
||||
"total": total,
|
||||
"generated": generated,
|
||||
"skipped": skipped,
|
||||
"failed": failed,
|
||||
"records": record_results,
|
||||
}
|
||||
|
||||
|
||||
async def _upsert_quicklook_source_product_asset(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
quicklook_manifest: dict[str, Any],
|
||||
) -> int | None:
|
||||
native_dir_text = _path_text(scene_manifest.get("native_dir"))
|
||||
if not native_dir_text:
|
||||
return None
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
|
||||
acquisition_start = _date_to_naive_utc(imaging_date)
|
||||
scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name
|
||||
now = _db_now()
|
||||
root = await _find_managed_root_for_path(db, native_dir_text)
|
||||
asset_metadata = _json_safe(
|
||||
{
|
||||
"source": "GF3 SARscape native _geo",
|
||||
"native_dir": native_dir_text,
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"native_assets": quicklook_manifest.get("assets") or [],
|
||||
"full_raster_materialized": True,
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
}
|
||||
)
|
||||
data = {
|
||||
"asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"),
|
||||
"logical_product_uid": scene_name,
|
||||
"satellite_family": "GF3",
|
||||
"satellite": "GF3",
|
||||
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
|
||||
"product_type": metadata.get("product_type") or "SARSCAPE_NATIVE",
|
||||
"product_level": "L2_NATIVE",
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": metadata.get("polarization"),
|
||||
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
|
||||
"relative_orbit": metadata.get("relative_orbit"),
|
||||
"orbit_direction": metadata.get("orbit_direction"),
|
||||
"acquisition_start_time_utc": acquisition_start,
|
||||
"acquisition_stop_time_utc": None,
|
||||
"imaging_date": imaging_date,
|
||||
"root_ref_id": root.id if root else None,
|
||||
"root_path": root.path if root else str(Path(native_dir_text).parent),
|
||||
"file_path": native_dir_text,
|
||||
"archive_path": scene_manifest.get("source_archive"),
|
||||
"path_kind": _path_kind(native_dir_text),
|
||||
"file_name": Path(native_dir_text).name,
|
||||
"file_stem": Path(native_dir_text).name,
|
||||
"file_ext": "",
|
||||
"size_bytes": None,
|
||||
"mtime_epoch": None,
|
||||
"checksum_status": "NOT_COMPUTED",
|
||||
"parser_name": "gf3_sarscape_native_preview_manifest",
|
||||
"parser_version": CONVERTER_VERSION,
|
||||
"parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED",
|
||||
"parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None,
|
||||
"parsed_at": now,
|
||||
"metadata_json": asset_metadata,
|
||||
"is_active": True,
|
||||
"missing_since": None,
|
||||
"updated_at": now,
|
||||
}
|
||||
result = await db.execute(
|
||||
select(SourceProductAssetORM).where(
|
||||
or_(
|
||||
SourceProductAssetORM.asset_uid == data["asset_uid"],
|
||||
SourceProductAssetORM.file_path == native_dir_text,
|
||||
)
|
||||
)
|
||||
)
|
||||
asset = result.scalars().first()
|
||||
if asset is None:
|
||||
asset = SourceProductAssetORM(**data)
|
||||
db.add(asset)
|
||||
else:
|
||||
for key, value in data.items():
|
||||
setattr(asset, key, value)
|
||||
await db.flush()
|
||||
return int(asset.id) if asset.id is not None else None
|
||||
|
||||
|
||||
async def _upsert_quicklook_radar_data(
|
||||
db: AsyncSession,
|
||||
scene_manifest: dict[str, Any],
|
||||
quicklook_manifest: dict[str, Any],
|
||||
source_product_ref_id: int | None = None,
|
||||
) -> int | None:
|
||||
assets = quicklook_manifest.get("assets") or []
|
||||
default_asset = _select_default_native_asset(assets)
|
||||
if not default_asset:
|
||||
return None
|
||||
metadata = scene_manifest.get("metadata") or {}
|
||||
imaging_date = str(metadata.get("imaging_date") or "").strip() or None
|
||||
acquisition_start = _date_to_naive_utc(imaging_date)
|
||||
scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name
|
||||
native_dir = str(scene_manifest.get("native_dir") or "")
|
||||
unique_id = f"gf3_sarscape_native_preview:{scene_name}"
|
||||
default_native_path = str(default_asset.get("path") or "")
|
||||
default_quicklook_path = str(default_asset.get("quicklook") or "")
|
||||
polygon = _polygon_from_native_asset(default_asset)
|
||||
min_lon = min_lat = max_lon = max_lat = None
|
||||
geom = None
|
||||
if polygon:
|
||||
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
|
||||
center_lon, center_lat = _scene_center_from_polygon(polygon)
|
||||
geom = _geom_from_polygon(polygon)
|
||||
else:
|
||||
center_lon = metadata.get("scene_center_lon")
|
||||
center_lat = metadata.get("scene_center_lat")
|
||||
preview_cache_path = None
|
||||
preview_cache_kind = "deferred"
|
||||
preview_cache_error = "native_webp_not_generated"
|
||||
radar_metadata = _json_safe(
|
||||
{
|
||||
**metadata,
|
||||
"native_dir": native_dir,
|
||||
"native_manifest": scene_manifest.get("manifest_path"),
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"native_assets": assets,
|
||||
"quicklook_assets": [],
|
||||
"default_native_path": default_native_path,
|
||||
"default_quicklook_path": default_quicklook_path,
|
||||
"coverage_source": "native_geo_or_quicklook" if polygon else "scene_name",
|
||||
"preview_cache_kind": preview_cache_kind,
|
||||
"analysis_engine": "gf3_sarscape",
|
||||
"registration_mode": "native_preview",
|
||||
"full_raster_materialized": True,
|
||||
}
|
||||
)
|
||||
data_to_upsert = {
|
||||
"unique_id": unique_id,
|
||||
"satellite": "GF3",
|
||||
"satellite_family": "GF3",
|
||||
"imaging_date": imaging_date,
|
||||
"imaging_mode": metadata.get("imaging_mode"),
|
||||
"polarization": ",".join(
|
||||
pol
|
||||
for pol in POLARIZATION_PRIORITY
|
||||
if any(str(asset.get("polarization") or "").upper() == pol for asset in assets)
|
||||
)
|
||||
or metadata.get("polarization"),
|
||||
"scene_center_lon": center_lon,
|
||||
"scene_center_lat": center_lat,
|
||||
"acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None,
|
||||
"product_level": "L2_NATIVE",
|
||||
"product_unique_id": metadata.get("product_unique_id") or scene_name,
|
||||
"source_product_token": scene_name,
|
||||
"acquisition_start_time_utc": acquisition_start,
|
||||
"acquisition_stop_time_utc": None,
|
||||
"absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"),
|
||||
"relative_orbit": metadata.get("relative_orbit"),
|
||||
"source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT,
|
||||
"source_product_ref_id": source_product_ref_id,
|
||||
"image_data_format": "ENVI_NATIVE",
|
||||
"geocoded_flag": True,
|
||||
"metadata_json": radar_metadata,
|
||||
"file_path": native_dir,
|
||||
"has_orbit_data": False,
|
||||
"orbit_file_path": None,
|
||||
"is_envi_processed": True,
|
||||
"coverage_polygon": polygon,
|
||||
"geom": geom,
|
||||
"min_lon": min_lon,
|
||||
"min_lat": min_lat,
|
||||
"max_lon": max_lon,
|
||||
"max_lat": max_lat,
|
||||
"preview_cache_status": "NONE",
|
||||
"preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION,
|
||||
"preview_cache_path": preview_cache_path,
|
||||
"preview_cache_updated_at": _db_now(),
|
||||
"preview_cache_error": preview_cache_error,
|
||||
}
|
||||
result = await db.execute(
|
||||
select(RadarDataORM).where(
|
||||
or_(
|
||||
RadarDataORM.unique_id == unique_id,
|
||||
RadarDataORM.file_path == native_dir,
|
||||
)
|
||||
)
|
||||
)
|
||||
radar = result.scalars().first()
|
||||
if (
|
||||
radar is not None
|
||||
and (radar.preview_cache_status or "").upper() == "READY"
|
||||
and radar.preview_cache_path
|
||||
and os.path.exists(radar.preview_cache_path)
|
||||
and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path
|
||||
):
|
||||
data_to_upsert["preview_cache_status"] = radar.preview_cache_status
|
||||
data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION
|
||||
data_to_upsert["preview_cache_path"] = radar.preview_cache_path
|
||||
data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at
|
||||
data_to_upsert["preview_cache_error"] = radar.preview_cache_error
|
||||
if radar is None:
|
||||
radar = RadarDataORM(**data_to_upsert)
|
||||
db.add(radar)
|
||||
else:
|
||||
for key, value in data_to_upsert.items():
|
||||
setattr(radar, key, value)
|
||||
await db.flush()
|
||||
return int(radar.id) if radar.id is not None else None
|
||||
|
||||
|
||||
def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
metadata = dict(scene_manifest.get("metadata") or {})
|
||||
metadata.update(
|
||||
@@ -635,7 +1260,7 @@ async def _upsert_radar_data(
|
||||
if not default_asset:
|
||||
return None
|
||||
|
||||
polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path"))))
|
||||
polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path"))))
|
||||
min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon)
|
||||
center_lon, center_lat = _scene_center_from_polygon(polygon)
|
||||
geom = _geom_from_polygon(polygon)
|
||||
@@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots(
|
||||
storage_root: str | None = None,
|
||||
force: bool = False,
|
||||
register: bool = True,
|
||||
quicklook_only: bool = False,
|
||||
progress_callback: Any | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Scan native roots, convert complete assets, and register standard scenes."""
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
native_dirs,
|
||||
write_manifest=True,
|
||||
)
|
||||
requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()]
|
||||
if quicklook_only:
|
||||
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
scan_roots,
|
||||
write_manifest=True,
|
||||
)
|
||||
else:
|
||||
scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs)
|
||||
inventory = await asyncio.to_thread(
|
||||
scan_gf3_sarscape_native_roots,
|
||||
scan_roots,
|
||||
write_manifest=True,
|
||||
)
|
||||
scenes = inventory.get("scenes") or []
|
||||
ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}]
|
||||
|
||||
if quicklook_only:
|
||||
registered = 0
|
||||
quicklook_assets = 0
|
||||
failed_scenes = 0
|
||||
skipped_unchanged = 0
|
||||
scene_results: list[dict[str, Any]] = []
|
||||
total = len(ready_scenes)
|
||||
for idx, scene_manifest in enumerate(ready_scenes):
|
||||
if progress_callback:
|
||||
pct = 10 + int((idx / max(total, 1)) * 80)
|
||||
progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
|
||||
|
||||
quicklook_manifest = await asyncio.to_thread(
|
||||
_quicklook_manifest_for_scene,
|
||||
scene_manifest,
|
||||
storage_root,
|
||||
)
|
||||
assets = quicklook_manifest.get("assets") or []
|
||||
quicklook_assets += len(assets)
|
||||
source_asset_id = None
|
||||
radar_id = None
|
||||
skipped_reason = None
|
||||
if register and assets:
|
||||
can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar(
|
||||
db,
|
||||
scene_manifest=scene_manifest,
|
||||
)
|
||||
if can_skip:
|
||||
skipped_unchanged += 1
|
||||
radar_id = existing_radar_id
|
||||
skipped_reason = skip_reason
|
||||
if progress_callback:
|
||||
pct = 10 + int(((idx + 1) / max(total, 1)) * 80)
|
||||
progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}")
|
||||
else:
|
||||
skipped_reason = skip_reason
|
||||
source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest)
|
||||
radar_id = await _upsert_quicklook_radar_data(
|
||||
db,
|
||||
scene_manifest,
|
||||
quicklook_manifest,
|
||||
source_product_ref_id=source_asset_id,
|
||||
)
|
||||
if radar_id:
|
||||
registered += 1
|
||||
await db.commit()
|
||||
if not assets:
|
||||
failed_scenes += 1
|
||||
scene_results.append(
|
||||
{
|
||||
"scene_name": scene_manifest.get("scene_name"),
|
||||
"native_status": scene_manifest.get("status"),
|
||||
"standard_status": quicklook_manifest.get("status"),
|
||||
"native_preview_manifest": quicklook_manifest.get("manifest_path"),
|
||||
"source_asset_id": source_asset_id,
|
||||
"radar_id": radar_id,
|
||||
"skipped_unchanged": bool(skipped_reason == "unchanged"),
|
||||
"skip_reason": skipped_reason,
|
||||
"native_fingerprint": _scene_native_fingerprint(scene_manifest),
|
||||
"summary": quicklook_manifest.get("summary") or {},
|
||||
"errors": quicklook_manifest.get("errors") or [],
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": failed_scenes == 0,
|
||||
"mode": "native_preview",
|
||||
"inventory": {
|
||||
key: value
|
||||
for key, value in inventory.items()
|
||||
if key != "scenes"
|
||||
},
|
||||
"requested_native_dirs": requested_native_dirs,
|
||||
"scan_roots": scan_roots,
|
||||
"scene_count": len(scenes),
|
||||
"ready_scene_count": len(ready_scenes),
|
||||
"converted_scenes": 0,
|
||||
"partial_scenes": 0,
|
||||
"failed_scenes": failed_scenes,
|
||||
"converted_assets": 0,
|
||||
"skipped_assets": skipped_unchanged,
|
||||
"skipped_unchanged": skipped_unchanged,
|
||||
"failed_assets": failed_scenes,
|
||||
"quicklook_assets": quicklook_assets,
|
||||
"native_assets": quicklook_assets,
|
||||
"registered": registered,
|
||||
"analysis_ready": 0,
|
||||
"scenes": scene_results,
|
||||
}
|
||||
|
||||
converted_scenes = 0
|
||||
partial_scenes = 0
|
||||
failed_scenes = 0
|
||||
@@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots(
|
||||
for key, value in inventory.items()
|
||||
if key != "scenes"
|
||||
},
|
||||
"requested_native_dirs": requested_native_dirs,
|
||||
"scan_roots": scan_roots,
|
||||
"scene_count": len(scenes),
|
||||
"ready_scene_count": len(ready_scenes),
|
||||
"converted_scenes": converted_scenes,
|
||||
|
||||
@@ -18,7 +18,7 @@ logger = logging.getLogger(__name__)
|
||||
from sqlalchemy import select
|
||||
|
||||
from .. import database
|
||||
from ..config import settings
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..scheduler import scan_data_job
|
||||
from .data_service import data_service
|
||||
@@ -88,6 +88,7 @@ JOB_TYPE_GF3_UNPACK = "GF3_UNPACK"
|
||||
JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS"
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE"
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC"
|
||||
JOB_TYPE_GF3_QUICKLOOK_WEBP = "GF3_QUICKLOOK_WEBP"
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN"
|
||||
JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
|
||||
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
@@ -97,6 +98,7 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY"
|
||||
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
|
||||
JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM"
|
||||
JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS"
|
||||
@@ -253,9 +255,12 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
families=payload.get("families") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
build_previews=bool(payload.get("build_previews", True)),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
preview = result.get("preview_cache") or {}
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
@@ -265,7 +270,42 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
f"sources={result.get('source_assets', 0)}, "
|
||||
f"orbits={result.get('orbit_assets', 0)}, "
|
||||
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}, "
|
||||
f"previews_ready={preview.get('ready', 0)}, "
|
||||
f"previews_skipped={preview.get('skipped_ready', 0)}, "
|
||||
f"previews_failed={preview.get('failed', 0)}, "
|
||||
f"previews_missing={preview.get('missing_source', 0)}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_archive_integrity_audit(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("AUDIT_SOURCE_ARCHIVE_INTEGRITY requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
await task_service.start_task(job.task_id, message="Source archive integrity audit started")
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await asset_inventory_service.audit_source_archive_integrity(
|
||||
db,
|
||||
families=payload.get("families") or None,
|
||||
source_formats=payload.get("source_formats") or None,
|
||||
asset_ids=payload.get("asset_ids") or None,
|
||||
force=bool(payload.get("force", False)),
|
||||
limit=payload.get("limit"),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
"Source archive integrity audit completed: "
|
||||
f"checked={result.get('checked', 0)}, "
|
||||
f"skipped={result.get('skipped', 0)}, "
|
||||
f"ok={result.get('ok', 0)}, "
|
||||
f"failed={result.get('failed', 0)}, "
|
||||
f"unsupported={result.get('unsupported', 0)}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
@@ -345,7 +385,7 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
|
||||
copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses"))
|
||||
include_orbit_files = bool(payload.get("include_orbit_files"))
|
||||
export_zip = bool(payload.get("export_zip"))
|
||||
package_mode = str(payload.get("package_mode") or ("task_zip" if export_zip else "task_folder")).strip().lower()
|
||||
package_mode = str(payload.get("package_mode") or ("source_bundle" if export_zip else "task_folder")).strip().lower()
|
||||
skip_existing = payload.get("skip_existing") is not False
|
||||
max_items = _normalize_positive_int(payload.get("max_items"))
|
||||
|
||||
@@ -445,17 +485,27 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
|
||||
skip_existing=skip_existing,
|
||||
max_items=max_items,
|
||||
)
|
||||
else:
|
||||
return
|
||||
if package_mode in {"task_folder", "task"}:
|
||||
await run_dinsar_copy_items(
|
||||
job.task_id,
|
||||
items,
|
||||
dest_dir,
|
||||
include_orbit_files=include_orbit_files,
|
||||
export_zip=(package_mode == "task_zip" or export_zip),
|
||||
export_zip=False,
|
||||
skip_existing=skip_existing,
|
||||
max_items=max_items,
|
||||
)
|
||||
return
|
||||
return
|
||||
if package_mode == "task_zip":
|
||||
raise ValueError(
|
||||
"D-InSAR Task ZIP export is not a production-preparation format. "
|
||||
"Use package_mode=task_folder for Task_Pool preparation or source_bundle for archive distribution."
|
||||
)
|
||||
raise ValueError(
|
||||
"Unsupported D-InSAR package_mode. "
|
||||
"Use task_folder for production preparation or source_bundle for archive distribution."
|
||||
)
|
||||
|
||||
raise ValueError(f"Unknown COPY_DATA file_type: {file_type}")
|
||||
|
||||
@@ -884,7 +934,9 @@ async def _handle_ai_diagnosis(job: SystemJobORM) -> None:
|
||||
from ..ai_service import analyze_map_with_vlm
|
||||
diagnosis_markdown = await analyze_map_with_vlm(
|
||||
images_base64=[img_base64],
|
||||
prompt=full_prompt
|
||||
prompt=full_prompt,
|
||||
model_name=model_name,
|
||||
raise_on_error=True,
|
||||
)
|
||||
|
||||
# 8. 解析风险等级和置信度(简单正则匹配)
|
||||
@@ -3646,7 +3698,7 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 Python/GDAL preprocessing is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
payload = job.payload or {}
|
||||
@@ -3725,7 +3777,7 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 archive unpack is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
if not job.task_id:
|
||||
@@ -3792,7 +3844,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
if not settings.GF3_LEGACY_GDAL_ENABLED:
|
||||
raise ValueError(
|
||||
"Legacy GF3 Python/GDAL preprocessing is disabled. "
|
||||
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
|
||||
)
|
||||
|
||||
payload = job.payload or {}
|
||||
@@ -3901,7 +3953,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
"""Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them."""
|
||||
"""Scan SARscape native GF3 _geo outputs and register requested assets."""
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
|
||||
if not job.task_id:
|
||||
@@ -3915,7 +3967,15 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty")
|
||||
|
||||
await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...")
|
||||
quicklook_only = bool(payload.get("quicklook_only", False))
|
||||
await task_service.start_task(
|
||||
job.task_id,
|
||||
message=(
|
||||
"按 GF3 日期/场景命名规则登记本机 SARscape _geo 原生结果..."
|
||||
if quicklook_only
|
||||
else "递归扫描 GF3 SARscape 原生 _geo 结果池..."
|
||||
),
|
||||
)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
@@ -3943,6 +4003,7 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
quicklook_only=quicklook_only,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -3954,44 +4015,44 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None:
|
||||
)
|
||||
raise
|
||||
|
||||
message = (
|
||||
"GF3 SARscape 标准化完成: "
|
||||
f"发现 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
|
||||
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
|
||||
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
|
||||
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景"
|
||||
)
|
||||
if quicklook_only:
|
||||
message = (
|
||||
"GF3 _geo 原生结果登记完成: "
|
||||
f"规则匹配 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可登记 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"原生资产 {int(result.get('native_assets') or result.get('quicklook_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景, "
|
||||
f"跳过未变化 {int(result.get('skipped_unchanged') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景"
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
"GF3 SARscape 标准化完成: "
|
||||
f"发现 {int(result.get('scene_count') or 0)} 景, "
|
||||
f"可转换 {int(result.get('ready_scene_count') or 0)} 景, "
|
||||
f"转换 {int(result.get('converted_scenes') or 0)} 景, "
|
||||
f"部分 {int(result.get('partial_scenes') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed_scenes') or 0)} 景, "
|
||||
f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, "
|
||||
f"跳过 {int(result.get('skipped_assets') or 0)} 个, "
|
||||
f"入库 {int(result.get('registered') or 0)} 景"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
"""Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain."""
|
||||
from .gf3_sarscape_production_service import (
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
run_gf3_sarscape_production,
|
||||
)
|
||||
from .gf3_standardize_service import standardize_gf3_sarscape_native_roots
|
||||
async def _handle_gf3_quicklook_webp(job: SystemJobORM) -> None:
|
||||
"""Generate local WebP previews from registered GF3 SARscape native _geo records."""
|
||||
from .gf3_standardize_service import generate_gf3_quicklook_webp_cache
|
||||
|
||||
if not job.task_id:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.")
|
||||
raise ValueError("GF3_QUICKLOOK_WEBP requires task_id for progress tracking.")
|
||||
|
||||
payload = job.payload or {}
|
||||
source_dirs = payload.get("source_dirs") or []
|
||||
native_dirs = payload.get("native_dirs") or []
|
||||
storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS
|
||||
native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "")
|
||||
if not source_dirs:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty")
|
||||
if not native_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty")
|
||||
if not storage_root:
|
||||
raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty")
|
||||
force = bool(payload.get("force", False))
|
||||
max_records = _normalize_positive_int(payload.get("max_records"))
|
||||
native_dirs = payload.get("native_dirs") or split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS)
|
||||
await task_service.start_task(job.task_id, message="开始从 GF3 _geo 原生数据生成 WebP 缓存...")
|
||||
|
||||
await task_service.start_task(job.task_id, message="GF3 SARscape production starting...")
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _progress_cb(progress: int, message: str) -> None:
|
||||
@@ -4005,143 +4066,49 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc)
|
||||
logger.warning("[GF3 native WebP] progress callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_progress_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
def _log_cb(level: str, message: str) -> None:
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
task_service.add_log(job.task_id, level, message),
|
||||
loop,
|
||||
)
|
||||
|
||||
def _swallow_log_error(fut):
|
||||
try:
|
||||
fut.result()
|
||||
except Exception as exc:
|
||||
logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc)
|
||||
|
||||
future.add_done_callback(_swallow_log_error)
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
async def _production_keepalive() -> None:
|
||||
progress = 8
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
progress = min(68, progress + 1)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=progress,
|
||||
message="GF3 SARscape production is still running...",
|
||||
)
|
||||
|
||||
try:
|
||||
production_task = asyncio.create_task(
|
||||
asyncio.to_thread(
|
||||
run_gf3_sarscape_production,
|
||||
source_dirs=source_dirs,
|
||||
native_root=native_root,
|
||||
wrapper_exe=payload.get("wrapper_exe"),
|
||||
dem_path=payload.get("dem_path"),
|
||||
idlrt_path=payload.get("idlrt_path"),
|
||||
polarizations=payload.get("polarizations"),
|
||||
archive_exts=payload.get("archive_exts") or [],
|
||||
max_archives_per_run=payload.get("max_archives_per_run"),
|
||||
selected_dates=payload.get("selected_dates") or [],
|
||||
task_id=job.task_id,
|
||||
local_staging_root=payload.get("local_staging_root") or settings.GF3_TASK_POOL_ROOT,
|
||||
timeout_seconds=payload.get("timeout_seconds"),
|
||||
keep_extracted=payload.get("keep_extracted"),
|
||||
log_callback=_log_cb,
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await generate_gf3_quicklook_webp_cache(
|
||||
db,
|
||||
force=force,
|
||||
max_records=max_records,
|
||||
native_dirs=native_dirs,
|
||||
progress_callback=_progress_cb,
|
||||
)
|
||||
)
|
||||
keepalive_task = asyncio.create_task(_production_keepalive())
|
||||
try:
|
||||
production_result = await production_task
|
||||
finally:
|
||||
keepalive_task.cancel()
|
||||
try:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
standardize_result: Dict[str, Any] = {}
|
||||
if bool(payload.get("auto_standardize", True)):
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=72,
|
||||
message="GF3 SARscape production finished; standardizing native _geo outputs...",
|
||||
)
|
||||
async with AsyncSessionLocal() as db:
|
||||
standardize_result = await standardize_gf3_sarscape_native_roots(
|
||||
db,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
force=bool(payload.get("force_standardize", False)),
|
||||
register=bool(payload.get("register", True)),
|
||||
progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg),
|
||||
)
|
||||
|
||||
cleanup_result: Dict[str, Any] = {}
|
||||
production_ok = int(production_result.get("failed_count") or 0) == 0
|
||||
standardize_ok = (
|
||||
not standardize_result
|
||||
or (
|
||||
int(standardize_result.get("failed_assets") or 0) == 0
|
||||
and int(standardize_result.get("failed_scenes") or 0) == 0
|
||||
)
|
||||
)
|
||||
if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=90,
|
||||
message="Cleaning GF3 SARscape intermediate files...",
|
||||
)
|
||||
cleanup_result = await asyncio.to_thread(
|
||||
cleanup_gf3_sarscape_native_pool,
|
||||
native_dirs=native_dirs or [native_root],
|
||||
storage_root=storage_root,
|
||||
require_standardized=bool(payload.get("cleanup_require_standardized", True)),
|
||||
dry_run=bool(payload.get("cleanup_dry_run", False)),
|
||||
max_scenes=payload.get("cleanup_max_scenes"),
|
||||
log_callback=_log_cb,
|
||||
progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg),
|
||||
)
|
||||
elif bool(payload.get("clean_after_success", True)):
|
||||
_log_cb(
|
||||
"WARNING",
|
||||
"GF3 SARscape automatic cleanup skipped because production or standardization had failures.",
|
||||
)
|
||||
except Exception as exc:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=f"GF3 SARscape production failed: {exc}",
|
||||
message=f"GF3 _geo WebP 生成失败: {exc}",
|
||||
)
|
||||
raise
|
||||
|
||||
failed_count = int(production_result.get("failed_count") or 0)
|
||||
failed_assets = int(standardize_result.get("failed_assets") or 0)
|
||||
cleanup_errors = int(cleanup_result.get("error_scene_count") or 0)
|
||||
final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED"
|
||||
message = (
|
||||
"GF3 SARscape production chain finished: "
|
||||
f"found={int(production_result.get('found_count') or 0)}, "
|
||||
f"produced={int(production_result.get('processed_count') or 0)}, "
|
||||
f"skipped={int(production_result.get('skipped_count') or 0)}, "
|
||||
f"failed={failed_count}, "
|
||||
f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, "
|
||||
f"registered={int(standardize_result.get('registered') or 0)}, "
|
||||
f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, "
|
||||
f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}"
|
||||
"GF3 _geo WebP 生成完成: "
|
||||
f"总数 {int(result.get('total') or 0)} 景, "
|
||||
f"生成 {int(result.get('generated') or 0)} 景, "
|
||||
f"跳过 {int(result.get('skipped') or 0)} 景, "
|
||||
f"失败 {int(result.get('failed') or 0)} 景"
|
||||
)
|
||||
await task_service.update_task(job.task_id, status=final_status, progress=100, message=message)
|
||||
await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
"""Reject GF3 production jobs on this management machine."""
|
||||
message = (
|
||||
"GF3 SARscape production is disabled on this management machine. "
|
||||
"Run production on the SARscape host and register local _geo native results here."
|
||||
)
|
||||
if job.task_id:
|
||||
await task_service.update_task(job.task_id, status="FAILED", progress=100, message=message)
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
|
||||
@@ -5263,6 +5230,7 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None:
|
||||
_HANDLERS = {
|
||||
JOB_TYPE_SCAN_DATA: _handle_scan_data,
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
|
||||
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit,
|
||||
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
|
||||
@@ -5301,6 +5269,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process,
|
||||
JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce,
|
||||
JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync,
|
||||
JOB_TYPE_GF3_QUICKLOOK_WEBP: _handle_gf3_quicklook_webp,
|
||||
JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean,
|
||||
JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration,
|
||||
JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem,
|
||||
|
||||
@@ -24,6 +24,8 @@ from ..dinsar_engines.landsar_engine import (
|
||||
_norm_path,
|
||||
_path_search_dirs,
|
||||
_summarize_landsar_failure,
|
||||
derive_landsar_dem_bbox_from_xml_paths,
|
||||
prepare_landsar_dem_crop,
|
||||
)
|
||||
|
||||
|
||||
@@ -633,6 +635,16 @@ class LandsarSbasService:
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def _derive_task_dem_bbox(self, task: dict[str, Any], input_dir: str) -> tuple[float, float, float, float] | None:
|
||||
xml_paths = [
|
||||
str(scene.get("xml") or "")
|
||||
for scene in (task.get("scenes") or [])
|
||||
if str(scene.get("xml") or "").strip()
|
||||
]
|
||||
if not xml_paths and input_dir and os.path.isdir(input_dir):
|
||||
xml_paths = [str(path) for path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower())]
|
||||
return derive_landsar_dem_bbox_from_xml_paths(xml_paths, fallback_paths=xml_paths)
|
||||
|
||||
def materialize_stack(
|
||||
self,
|
||||
*,
|
||||
@@ -949,7 +961,7 @@ class LandsarSbasService:
|
||||
params_payload = self.normalize_params(params or {})
|
||||
normalized_dem = _norm_path(dem_path or self.default_dem_path)
|
||||
if not normalized_dem or not os.path.isfile(normalized_dem):
|
||||
raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or '<empty>'}")
|
||||
raise ValueError(f"LandSAR SBAS DEM source file is missing: {normalized_dem or '<empty>'}")
|
||||
|
||||
from .sbas_insar_production_service import sbas_insar_production_service
|
||||
|
||||
@@ -1035,6 +1047,8 @@ class LandsarSbasService:
|
||||
"input_task_root": import_dest_root,
|
||||
"publish_root": str(publish_root),
|
||||
"dem_path": normalized_dem,
|
||||
"dem_source_path": normalized_dem,
|
||||
"dem_role": "global_prepared_source",
|
||||
"params": params_payload,
|
||||
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
"import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
@@ -1541,9 +1555,9 @@ class LandsarSbasService:
|
||||
if not bool(settings.LANDSAR_SBAS_ENABLED):
|
||||
raise ValueError("LandSAR SBAS is disabled.")
|
||||
params = self.normalize_params(extra or {})
|
||||
dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
|
||||
if not dem_path or not os.path.isfile(dem_path):
|
||||
raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or '<empty>'}")
|
||||
dem_source_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path)
|
||||
if not dem_source_path or not os.path.isfile(dem_source_path):
|
||||
raise ValueError(f"LandSAR SBAS DEM source file is missing: {dem_source_path or '<empty>'}")
|
||||
validation = self.validate_root_dir(
|
||||
root_dir,
|
||||
min_scenes=min_scenes,
|
||||
@@ -1585,7 +1599,9 @@ class LandsarSbasService:
|
||||
"work_root_strategy": "short_landsar_execution_path",
|
||||
"native_root": str(native_root),
|
||||
"publish_root": str(publish_root),
|
||||
"dem_path": dem_path,
|
||||
"dem_path": dem_source_path,
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_role": "global_prepared_source",
|
||||
"params": params,
|
||||
"timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)),
|
||||
"min_scenes": validation.get("min_scenes"),
|
||||
@@ -1716,10 +1732,56 @@ class LandsarSbasService:
|
||||
native_output_dir = native_root / task_alias / "Output_Data"
|
||||
native_output_dir.mkdir(parents=True, exist_ok=True)
|
||||
project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias
|
||||
dem_source_path = _norm_path(str(manifest.get("dem_path") or ""))
|
||||
effective_dem_path = dem_source_path
|
||||
dem_crop_info: dict[str, Any] = {}
|
||||
try:
|
||||
dem_bbox = self._derive_task_dem_bbox(task, input_dir)
|
||||
if not dem_bbox:
|
||||
raise ValueError("cannot derive DEM crop bbox from LandSAR SBAS Input_Data XML corner coordinates")
|
||||
dem_crop_info = prepare_landsar_dem_crop(
|
||||
dem_source_path,
|
||||
str(native_root / task_alias / "dem_crop"),
|
||||
dem_bbox,
|
||||
label=task_alias,
|
||||
)
|
||||
effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path)
|
||||
self._emit(
|
||||
progress_callback,
|
||||
"INFO",
|
||||
f"[{index}/{len(tasks)}] LandSAR SBAS DEM crop ready: {effective_dem_path}",
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_count += 1
|
||||
error = f"LandSAR SBAS DEM crop failed: {exc}"
|
||||
self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] {task_name} failed: {error}")
|
||||
task_results.append(
|
||||
{
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"input_data_dir": input_dir,
|
||||
"native_output_dir": str(native_output_dir),
|
||||
"param_file": "",
|
||||
"process_name": SBAS_PROCESS_NAME,
|
||||
"command": "",
|
||||
"returncode": -2,
|
||||
"success": False,
|
||||
"timed_out": False,
|
||||
"failure_kind": "dem_crop_failed",
|
||||
"error": error,
|
||||
"stdout_tail": "",
|
||||
"native_logs": {},
|
||||
"publish": {},
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_path": effective_dem_path,
|
||||
"dem_crop": dem_crop_info,
|
||||
}
|
||||
)
|
||||
continue
|
||||
param_file = _generate_sbas_param_file(
|
||||
str(native_output_dir / f"{SBAS_PROID}.txt"),
|
||||
slc_folder=input_dir,
|
||||
dem_path=str(manifest.get("dem_path") or ""),
|
||||
dem_path=effective_dem_path,
|
||||
output_dir=str(native_output_dir),
|
||||
project_name=project_name,
|
||||
params=dict(manifest.get("params") or {}),
|
||||
@@ -1784,6 +1846,9 @@ class LandsarSbasService:
|
||||
"stdout_tail": _collect_tail(stdout_text, 4000),
|
||||
"native_logs": log_publish_result,
|
||||
"publish": publish_result,
|
||||
"dem_source_path": dem_source_path,
|
||||
"dem_path": effective_dem_path,
|
||||
"dem_crop": dem_crop_info,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""轨道文件管理与格式转换服务。
|
||||
|
||||
目录约定:
|
||||
MONITOR_ORBIT_DIR/ 源精轨目录(可为 UNC),平铺 .txt
|
||||
MONITOR_ORBIT_DIR/ 本机源精轨目录,平铺 .txt
|
||||
ORBIT_POOL_ENVI/ ENVI 本地精轨池
|
||||
LT1A/ LT-1A 精轨 .txt
|
||||
LT1B/ LT-1B 精轨 .txt
|
||||
@@ -320,16 +320,17 @@ def summarize_source_orbit_gaps(
|
||||
|
||||
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
|
||||
source_stems = set(source_files)
|
||||
envi_stems = set(envi_files)
|
||||
isce2_stems = set(isce2_files)
|
||||
|
||||
source_without_isce2 = sorted(source_stems - isce2_stems)
|
||||
source_without_isce2 = sorted(source_stems - isce2_stems) if isce2_enabled else []
|
||||
source_without_envi = sorted(source_stems - envi_stems)
|
||||
envi_without_source = sorted(envi_stems - source_stems)
|
||||
isce2_without_source = sorted(isce2_stems - source_stems)
|
||||
isce2_without_source = sorted(isce2_stems - source_stems) if isce2_enabled else []
|
||||
sample_limit = 20
|
||||
|
||||
def _make_samples(
|
||||
@@ -349,7 +350,7 @@ def summarize_source_orbit_gaps(
|
||||
)
|
||||
return samples
|
||||
|
||||
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
|
||||
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) if isce2_enabled else []
|
||||
bad_source_samples = [
|
||||
item for item in suspect_bad_samples
|
||||
if item.get("has_corruption_signal")
|
||||
@@ -358,6 +359,7 @@ def summarize_source_orbit_gaps(
|
||||
return {
|
||||
"sample_limit": sample_limit,
|
||||
"quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"source_without_isce2_count": len(source_without_isce2),
|
||||
"source_without_envi_count": len(source_without_envi),
|
||||
"envi_without_source_count": len(envi_without_source),
|
||||
@@ -809,6 +811,7 @@ def sync_orbit_pools(
|
||||
envi_pool: str = "",
|
||||
isce2_pool: str = "",
|
||||
landsar_pool: str = "",
|
||||
validate_without_isce2: bool = True,
|
||||
) -> Dict:
|
||||
"""从 source_dir 扫描平铺 .txt 精轨,同步到各引擎本地池。
|
||||
|
||||
@@ -863,11 +866,16 @@ def sync_orbit_pools(
|
||||
xml_needs_refresh = bool(isce2_pool) and (
|
||||
not os.path.exists(xml_path) or _needs_generated_refresh(fpath, xml_path)
|
||||
)
|
||||
requires_validation = xml_needs_refresh or (not isce2_pool and envi_needs_refresh)
|
||||
requires_validation = xml_needs_refresh or (
|
||||
bool(validate_without_isce2) and not isce2_pool and envi_needs_refresh
|
||||
)
|
||||
|
||||
staged_dir = ""
|
||||
staged_xml = ""
|
||||
try:
|
||||
health = _inspect_orbit_txt_health(fpath)
|
||||
if _has_orbit_corruption_signal(health):
|
||||
raise RuntimeError("Source TXT contains NUL bytes or cannot be read as a healthy LT-1 orbit text file.")
|
||||
if requires_validation:
|
||||
staged_dir, staged_xml = _stage_isce2_xml(
|
||||
fpath,
|
||||
@@ -924,8 +932,15 @@ def repair_orbit_pools(
|
||||
isce2_pool: str = "",
|
||||
landsar_pool: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
before = check_orbit_consistency(envi_pool, isce2_pool)
|
||||
sync_result = sync_orbit_pools(source_dir, envi_pool, isce2_pool, landsar_pool)
|
||||
sync_result = sync_orbit_pools(
|
||||
source_dir,
|
||||
envi_pool,
|
||||
isce2_pool,
|
||||
landsar_pool,
|
||||
validate_without_isce2=isce2_enabled,
|
||||
)
|
||||
|
||||
repaired_from_envi: List[str] = []
|
||||
repair_errors: List[Dict[str, str]] = []
|
||||
@@ -934,7 +949,7 @@ def repair_orbit_pools(
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
|
||||
if isce2_pool:
|
||||
if isce2_enabled:
|
||||
os.makedirs(isce2_pool, exist_ok=True)
|
||||
for stem in sorted(set(envi_files) - set(isce2_files)):
|
||||
txt_path = envi_files[stem]["path"]
|
||||
@@ -952,6 +967,7 @@ def repair_orbit_pools(
|
||||
"before": before,
|
||||
"after": after,
|
||||
"sync_result": sync_result,
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"repaired_from_envi": repaired_from_envi,
|
||||
"repair_error_count": len(repair_errors),
|
||||
"repair_errors": repair_errors,
|
||||
@@ -968,17 +984,19 @@ def quarantine_bad_orbits(
|
||||
isce2_pool: str = "",
|
||||
quarantine_root: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
source_inventory = get_source_orbit_inventory(source_dir, recursive=True)
|
||||
pool_inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
|
||||
|
||||
source_files: Dict[str, Dict[str, str]] = source_inventory["files"]
|
||||
envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
|
||||
quarantine_dir = _default_quarantine_root(source_dir, quarantine_root)
|
||||
candidate_stems = sorted(set(source_files) - set(isce2_files))
|
||||
candidate_stems = sorted(set(source_files) - set(isce2_files)) if isce2_enabled else []
|
||||
|
||||
result: Dict[str, Any] = {
|
||||
"isce2_enabled": isce2_enabled,
|
||||
"quarantine_root": quarantine_dir,
|
||||
"candidate_count": len(candidate_stems),
|
||||
"validated_count": 0,
|
||||
@@ -1083,14 +1101,18 @@ def check_orbit_consistency(
|
||||
"healthy": bool,
|
||||
}
|
||||
"""
|
||||
isce2_enabled = bool(str(isce2_pool or "").strip())
|
||||
inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True)
|
||||
envi_files: Dict[str, Dict[str, str]] = inventory["envi"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"]
|
||||
isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] if isce2_enabled else {}
|
||||
by_satellite: Dict[str, int] = inventory["envi"]["by_satellite"]
|
||||
|
||||
# 对比
|
||||
envi_stems = set(envi_files)
|
||||
isce2_stems = set(isce2_files)
|
||||
if not isce2_enabled:
|
||||
envi_stems = set()
|
||||
isce2_stems = set()
|
||||
mismatches = []
|
||||
for stem in envi_stems - isce2_stems:
|
||||
mismatches.append(
|
||||
@@ -1112,6 +1134,9 @@ def check_orbit_consistency(
|
||||
)
|
||||
|
||||
mismatches.sort(key=lambda item: item["name"])
|
||||
errors = list(inventory["envi"]["errors"])
|
||||
if isce2_enabled:
|
||||
errors.extend(inventory["isce2"]["errors"])
|
||||
|
||||
return {
|
||||
"envi": {
|
||||
@@ -1121,11 +1146,12 @@ def check_orbit_consistency(
|
||||
},
|
||||
"isce2": {
|
||||
"path": isce2_pool,
|
||||
"enabled": isce2_enabled,
|
||||
"total": len(isce2_files),
|
||||
},
|
||||
"error_count": len(inventory["envi"]["errors"]) + len(inventory["isce2"]["errors"]),
|
||||
"errors": list(inventory["envi"]["errors"]) + list(inventory["isce2"]["errors"]),
|
||||
"error_count": len(errors),
|
||||
"errors": errors,
|
||||
"mismatch_count": len(mismatches),
|
||||
"mismatches": mismatches,
|
||||
"healthy": len(mismatches) == 0,
|
||||
"healthy": len(mismatches) == 0 and len(errors) == 0,
|
||||
}
|
||||
|
||||
@@ -52,6 +52,15 @@ def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _supported_dinsar_family_pair_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_family = _satellite_family_expr(left_alias)
|
||||
right_family = _satellite_family_expr(right_alias)
|
||||
return (
|
||||
f"(({left_family} = 'LT1' AND {right_family} = 'LT1') "
|
||||
f"OR ({left_family} = 'S1' AND {right_family} = 'S1'))"
|
||||
)
|
||||
|
||||
|
||||
def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
|
||||
return (
|
||||
f"(NULLIF({left_alias}.look_direction, '') IS NULL "
|
||||
@@ -60,6 +69,60 @@ def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _relative_orbit_value_expr(alias: str) -> str:
|
||||
return f"NULLIF(upper(trim(COALESCE({alias}.relative_orbit, ''))), '')"
|
||||
|
||||
|
||||
def _same_relative_orbit_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_rel = _relative_orbit_value_expr(left_alias)
|
||||
right_rel = _relative_orbit_value_expr(right_alias)
|
||||
return f"({left_rel} IS NOT NULL AND {right_rel} IS NOT NULL AND {left_rel} = {right_rel})"
|
||||
|
||||
|
||||
def _dinsar_quality_tier_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"CASE "
|
||||
f"WHEN {same_rel} AND {center_distance_expr} <= 5000 THEN 'A' "
|
||||
f"WHEN {center_distance_expr} <= 5000 THEN 'B' "
|
||||
f"WHEN {center_distance_expr} <= 12000 THEN 'C' "
|
||||
"ELSE 'REJECT' END"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_readiness_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
tier = _dinsar_quality_tier_expr(left_alias, right_alias, center_distance_expr)
|
||||
return (
|
||||
"CASE "
|
||||
f"WHEN {tier} IN ('A', 'B') THEN 'RECOMMENDED' "
|
||||
f"WHEN {tier} = 'C' THEN 'CANDIDATE' "
|
||||
"ELSE 'NOT_RECOMMENDED' END"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_quality_score_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"GREATEST(0.0, LEAST(1.0, "
|
||||
f"0.45 * COALESCE((ST_Area(ST_Intersection({left_alias}.geom, {right_alias}.geom)::geography) / "
|
||||
f"NULLIF(GREATEST(ST_Area({left_alias}.geom::geography), ST_Area({right_alias}.geom::geography)), 0)), 0) + "
|
||||
f"0.30 * CASE WHEN {same_rel} THEN 1 ELSE 0 END + "
|
||||
f"0.25 * GREATEST(0.0, 1.0 - ({center_distance_expr} / 12000.0))"
|
||||
"))"
|
||||
)
|
||||
|
||||
|
||||
def _dinsar_reasons_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str:
|
||||
same_rel = _same_relative_orbit_expr(left_alias, right_alias)
|
||||
return (
|
||||
"COALESCE(jsonb_path_query_array(jsonb_build_array("
|
||||
f"CASE WHEN NOT ({same_rel}) THEN 'relative_orbit_missing_or_mismatch' END, "
|
||||
f"CASE WHEN {center_distance_expr} > 5000 THEN 'center_distance_over_5km' END, "
|
||||
f"CASE WHEN {center_distance_expr} > 12000 THEN 'center_distance_over_12km' END"
|
||||
"), '$[*] ? (@ != null)')::json, '[]'::json)"
|
||||
)
|
||||
|
||||
|
||||
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
|
||||
left_uid = _scene_uid_expr(left_alias)
|
||||
right_uid = _scene_uid_expr(right_alias)
|
||||
@@ -82,6 +145,7 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
|
||||
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
|
||||
f"AND COALESCE({left_alias}.insar_source_ready, false) "
|
||||
f"AND COALESCE({right_alias}.insar_source_ready, false) "
|
||||
f"AND { _supported_dinsar_family_pair_expr(left_alias, right_alias) } "
|
||||
f"AND { _same_look_direction_expr(left_alias, right_alias) } "
|
||||
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
|
||||
)
|
||||
@@ -106,7 +170,15 @@ def _full_rebuild_insert_sql() -> str:
|
||||
spatial_baseline_meters,
|
||||
scene_center_distance_meters,
|
||||
scene_overlap_ratio,
|
||||
pair_aoi_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_relative_orbit,
|
||||
master_relative_orbit,
|
||||
slave_relative_orbit,
|
||||
dinsar_quality_tier,
|
||||
dinsar_quality_score,
|
||||
dinsar_readiness,
|
||||
dinsar_reasons_json,
|
||||
same_satellite,
|
||||
same_satellite_family,
|
||||
same_look_direction,
|
||||
@@ -144,7 +216,15 @@ def _full_rebuild_insert_sql() -> str:
|
||||
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
NULL::double precision AS pair_aoi_overlap_ratio,
|
||||
m.orbit_direction,
|
||||
{ _same_relative_orbit_expr('m', 's') } AS same_relative_orbit,
|
||||
{_relative_orbit_value_expr('m')} AS master_relative_orbit,
|
||||
{_relative_orbit_value_expr('s')} AS slave_relative_orbit,
|
||||
{ _dinsar_quality_tier_expr('m', 's', center_distance) } AS dinsar_quality_tier,
|
||||
{ _dinsar_quality_score_expr('m', 's', center_distance) } AS dinsar_quality_score,
|
||||
{ _dinsar_readiness_expr('m', 's', center_distance) } AS dinsar_readiness,
|
||||
{ _dinsar_reasons_expr('m', 's', center_distance) } AS dinsar_reasons_json,
|
||||
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
|
||||
{ _same_satellite_family_expr('m', 's') } AS same_satellite_family,
|
||||
{ _same_look_direction_expr('m', 's') } AS same_look_direction,
|
||||
@@ -201,7 +281,15 @@ def _incremental_insert_sql() -> str:
|
||||
spatial_baseline_meters,
|
||||
scene_center_distance_meters,
|
||||
scene_overlap_ratio,
|
||||
pair_aoi_overlap_ratio,
|
||||
orbit_direction,
|
||||
same_relative_orbit,
|
||||
master_relative_orbit,
|
||||
slave_relative_orbit,
|
||||
dinsar_quality_tier,
|
||||
dinsar_quality_score,
|
||||
dinsar_readiness,
|
||||
dinsar_reasons_json,
|
||||
same_satellite,
|
||||
same_satellite_family,
|
||||
same_look_direction,
|
||||
@@ -244,7 +332,15 @@ def _incremental_insert_sql() -> str:
|
||||
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
|
||||
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
|
||||
)::double precision AS scene_overlap_ratio,
|
||||
NULL::double precision AS pair_aoi_overlap_ratio,
|
||||
d.orbit_direction,
|
||||
{ _same_relative_orbit_expr('d', 'o') } AS same_relative_orbit,
|
||||
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('d')} ELSE {_relative_orbit_value_expr('o')} END AS master_relative_orbit,
|
||||
CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('o')} ELSE {_relative_orbit_value_expr('d')} END AS slave_relative_orbit,
|
||||
{ _dinsar_quality_tier_expr('d', 'o', center_distance) } AS dinsar_quality_tier,
|
||||
{ _dinsar_quality_score_expr('d', 'o', center_distance) } AS dinsar_quality_score,
|
||||
{ _dinsar_readiness_expr('d', 'o', center_distance) } AS dinsar_readiness,
|
||||
{ _dinsar_reasons_expr('d', 'o', center_distance) } AS dinsar_reasons_json,
|
||||
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
|
||||
{ _same_satellite_family_expr('d', 'o') } AS same_satellite_family,
|
||||
{ _same_look_direction_expr('d', 'o') } AS same_look_direction,
|
||||
|
||||
@@ -18,7 +18,7 @@ from ..models import (
|
||||
|
||||
|
||||
PAIRING_CACHE_SCOPE_GLOBAL = "global"
|
||||
DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1"
|
||||
DEFAULT_PAIRING_METRIC_VERSION = "2026.06.dinsar.family.v1"
|
||||
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
|
||||
|
||||
|
||||
|
||||
@@ -235,15 +235,6 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="directory_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_ARCHIVE_SOURCE_DIRS",
|
||||
paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
|
||||
root_role="source_pool_gf3_archive",
|
||||
display_prefix="GF3 Archive Pool",
|
||||
scan_mode="archive_walk",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="GF3_SOURCE_DIRS",
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy import and_, case, cast, func, or_
|
||||
from sqlalchemy import and_, case, cast, func, or_, text
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from geoalchemy2 import Geography
|
||||
@@ -24,6 +24,8 @@ from shapely.geometry import Polygon
|
||||
from shapely.ops import unary_union
|
||||
|
||||
from ..models import (
|
||||
DinsarProductionRunItemORM,
|
||||
DinsarProductionRunORM,
|
||||
HazardPoint,
|
||||
HazardPointORM,
|
||||
PairingNetworkEdgeORM,
|
||||
@@ -43,7 +45,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
|
||||
from .pairing_state_service import pairing_state_service
|
||||
|
||||
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
|
||||
PAIRING_POLICY_VERSION = "2026.06.dinsar-production.v1"
|
||||
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
|
||||
PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -145,10 +147,10 @@ class SpatialService:
|
||||
require_orbit_data=require_orbit_data,
|
||||
)
|
||||
|
||||
if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
|
||||
if len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT:
|
||||
raise RuntimeError(
|
||||
f"全部配对命中 {len(candidate_pool)} 条候选边,超过系统一次性返回上限 "
|
||||
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略,或收紧 AOI、日期范围、重叠率。"
|
||||
f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请收紧 AOI、日期范围、重叠率或中心距离阈值。"
|
||||
)
|
||||
|
||||
if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD:
|
||||
@@ -156,15 +158,21 @@ class SpatialService:
|
||||
f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}(当前: {len(candidate_pool)}),建议收紧参数或缩小 AOI。"
|
||||
)
|
||||
|
||||
selected_candidates, strategy_warnings = self._apply_strategy(
|
||||
candidate_pool,
|
||||
effective_params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
)
|
||||
selected_candidates, strategy_warnings = self._apply_dinsar_production_strategy(candidate_pool, effective_params)
|
||||
warnings.extend(strategy_warnings)
|
||||
if not selected_candidates:
|
||||
warnings.extend(
|
||||
await self._build_empty_pairing_diagnostics(
|
||||
db,
|
||||
effective_params,
|
||||
aoi_wkt=aoi_wkt,
|
||||
require_orbit_data=require_orbit_data,
|
||||
)
|
||||
)
|
||||
|
||||
for candidate in selected_candidates:
|
||||
candidate.setdefault("selection_strategy", effective_params.strategy)
|
||||
candidate["selection_strategy"] = "dinsar_production"
|
||||
self._ensure_candidate_identity(candidate)
|
||||
|
||||
network_run_id = await self._persist_network_run(
|
||||
db,
|
||||
@@ -176,6 +184,7 @@ class SpatialService:
|
||||
selected_candidates=selected_candidates,
|
||||
)
|
||||
|
||||
await self._attach_dinsar_production_summaries(db, selected_candidates)
|
||||
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
|
||||
metadata = {
|
||||
"fallback_used": False,
|
||||
@@ -227,13 +236,20 @@ class SpatialService:
|
||||
PairingMetricCacheORM.status == "READY",
|
||||
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
|
||||
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
|
||||
PairingMetricCacheORM.master_imaging_date < PairingMetricCacheORM.slave_imaging_date,
|
||||
PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold,
|
||||
PairingMetricCacheORM.same_look_direction.is_(True),
|
||||
PairingMetricCacheORM.dinsar_readiness.in_(["RECOMMENDED", "CANDIDATE"]),
|
||||
)
|
||||
)
|
||||
|
||||
if params.limit_footprint_center_distance:
|
||||
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
|
||||
stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
and_(master_family_expr == "LT1", slave_family_expr == "LT1"),
|
||||
and_(master_family_expr == "S1", slave_family_expr == "S1"),
|
||||
)
|
||||
)
|
||||
|
||||
if require_orbit_data:
|
||||
stmt = stmt.where(
|
||||
@@ -241,8 +257,7 @@ class SpatialService:
|
||||
slave_alias.has_orbit_data.is_(True),
|
||||
)
|
||||
|
||||
if not params.cross_satellite_pairing:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
|
||||
|
||||
if params.require_same_imaging_mode:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True))
|
||||
@@ -259,20 +274,19 @@ class SpatialService:
|
||||
)
|
||||
|
||||
if params.allowed_satellites:
|
||||
allowed_satellites = [
|
||||
str(item).strip().upper()
|
||||
for item in params.allowed_satellites
|
||||
if str(item).strip()
|
||||
]
|
||||
allowed_satellites = []
|
||||
for item in params.allowed_satellites:
|
||||
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
allowed_satellites.append("LT1")
|
||||
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
allowed_satellites.append("S1")
|
||||
allowed_satellites = list(dict.fromkeys(allowed_satellites))
|
||||
if not allowed_satellites:
|
||||
return []
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
func.upper(master_alias.satellite).in_(allowed_satellites),
|
||||
func.upper(master_alias.satellite_family).in_(allowed_satellites),
|
||||
),
|
||||
or_(
|
||||
func.upper(slave_alias.satellite).in_(allowed_satellites),
|
||||
func.upper(slave_alias.satellite_family).in_(allowed_satellites),
|
||||
),
|
||||
master_family_expr.in_(allowed_satellites),
|
||||
slave_family_expr.in_(allowed_satellites),
|
||||
)
|
||||
|
||||
if params.master_date_from:
|
||||
@@ -286,23 +300,23 @@ class SpatialService:
|
||||
|
||||
if aoi_wkt:
|
||||
aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326)
|
||||
aoi_geog = cast(aoi_geom, Geography)
|
||||
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
|
||||
pair_overlap_geom = ST_Intersection(master_alias.geom, slave_alias.geom)
|
||||
pair_aoi_geom = ST_Intersection(pair_overlap_geom, aoi_geom)
|
||||
pair_aoi_overlap_expr = (ST_Area(cast(pair_aoi_geom, Geography)) / aoi_area).label("pair_aoi_overlap_ratio")
|
||||
stmt = stmt.add_columns(pair_aoi_overlap_expr)
|
||||
stmt = stmt.where(
|
||||
ST_Intersects(master_alias.geom, aoi_geom),
|
||||
ST_Intersects(slave_alias.geom, aoi_geom),
|
||||
ST_Intersects(pair_overlap_geom, aoi_geom),
|
||||
)
|
||||
if params.aoi_overlap_threshold is not None:
|
||||
aoi_geog = cast(aoi_geom, Geography)
|
||||
aoi_area = func.nullif(ST_Area(aoi_geog), 0)
|
||||
master_inter_geog = cast(ST_Intersection(master_alias.geom, aoi_geom), Geography)
|
||||
slave_inter_geog = cast(ST_Intersection(slave_alias.geom, aoi_geom), Geography)
|
||||
stmt = stmt.where(
|
||||
ST_Area(master_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
|
||||
ST_Area(slave_inter_geog) / aoi_area >= params.aoi_overlap_threshold,
|
||||
)
|
||||
stmt = stmt.where(pair_aoi_overlap_expr >= params.aoi_overlap_threshold)
|
||||
|
||||
stmt = stmt.order_by(
|
||||
PairingMetricCacheORM.master_imaging_date.asc(),
|
||||
PairingMetricCacheORM.slave_imaging_date.asc(),
|
||||
PairingMetricCacheORM.dinsar_quality_tier.asc(),
|
||||
func.coalesce(PairingMetricCacheORM.dinsar_quality_score, 0).desc(),
|
||||
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
|
||||
center_distance_expr.asc(),
|
||||
PairingMetricCacheORM.pair_uid.asc(),
|
||||
@@ -310,7 +324,9 @@ class SpatialService:
|
||||
|
||||
result = await db.execute(stmt)
|
||||
candidate_pool: List[dict] = []
|
||||
for metric_row, master_row, slave_row in result.all():
|
||||
for row in result.all():
|
||||
metric_row, master_row, slave_row = row[0], row[1], row[2]
|
||||
pair_aoi_overlap_ratio = row[3] if len(row) > 3 else None
|
||||
center_distance = float(
|
||||
metric_row.scene_center_distance_meters
|
||||
if metric_row.scene_center_distance_meters is not None
|
||||
@@ -328,16 +344,54 @@ class SpatialService:
|
||||
"dist": center_distance,
|
||||
"scene_center_distance_meters": center_distance,
|
||||
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0),
|
||||
"dinsar_quality_tier": metric_row.dinsar_quality_tier,
|
||||
"dinsar_quality_score": (
|
||||
float(metric_row.dinsar_quality_score)
|
||||
if metric_row.dinsar_quality_score is not None
|
||||
else None
|
||||
),
|
||||
"dinsar_readiness": metric_row.dinsar_readiness,
|
||||
"dinsar_reasons": [
|
||||
str(item)
|
||||
for item in (metric_row.dinsar_reasons_json or [])
|
||||
if isinstance(item, str) and item.strip()
|
||||
],
|
||||
"same_relative_orbit": bool(metric_row.same_relative_orbit),
|
||||
"master_relative_orbit": metric_row.master_relative_orbit,
|
||||
"slave_relative_orbit": metric_row.slave_relative_orbit,
|
||||
"pair_aoi_overlap_ratio": (
|
||||
float(pair_aoi_overlap_ratio)
|
||||
if pair_aoi_overlap_ratio is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
)
|
||||
return candidate_pool
|
||||
|
||||
def _ensure_candidate_identity(self, candidate: dict) -> Tuple[str, str]:
|
||||
master = candidate["master"]
|
||||
slave = candidate["slave"]
|
||||
task_alias = str(candidate.get("task_alias") or "").strip() or build_task_alias(
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
)
|
||||
pair_key = str(candidate.get("pair_key") or "").strip() or build_pair_key(
|
||||
master.file_path,
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
)
|
||||
candidate["task_alias"] = task_alias
|
||||
candidate["pair_key"] = pair_key
|
||||
return task_alias, pair_key
|
||||
|
||||
def _build_radar_pairs(self, selected_candidates: List[dict]) -> List[RadarPair]:
|
||||
result_pairs: List[RadarPair] = []
|
||||
for candidate in selected_candidates:
|
||||
master = candidate["master"]
|
||||
slave = candidate["slave"]
|
||||
task_alias = build_task_alias(master.imaging_date, slave.imaging_date)
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
selection_score = candidate.get("selection_score")
|
||||
result_pairs.append(
|
||||
RadarPair(
|
||||
@@ -345,13 +399,7 @@ class SpatialService:
|
||||
slave=slave,
|
||||
task_name=task_alias,
|
||||
task_alias=task_alias,
|
||||
pair_key=build_pair_key(
|
||||
master.file_path,
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
),
|
||||
pair_key=pair_key,
|
||||
pair_uid=candidate.get("pair_uid"),
|
||||
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
|
||||
network_run_id=candidate.get("network_run_id"),
|
||||
@@ -367,10 +415,292 @@ class SpatialService:
|
||||
if candidate.get("scene_center_distance_meters") is not None
|
||||
else candidate.get("dist") or 0
|
||||
),
|
||||
scene_overlap_ratio=float(candidate.get("overlap_ratio") or 0.0),
|
||||
pair_aoi_overlap_ratio=(
|
||||
float(candidate["pair_aoi_overlap_ratio"])
|
||||
if candidate.get("pair_aoi_overlap_ratio") is not None
|
||||
else None
|
||||
),
|
||||
dinsar_quality_tier=candidate.get("dinsar_quality_tier"),
|
||||
dinsar_quality_score=(
|
||||
float(candidate["dinsar_quality_score"])
|
||||
if candidate.get("dinsar_quality_score") is not None
|
||||
else None
|
||||
),
|
||||
dinsar_readiness=candidate.get("dinsar_readiness"),
|
||||
dinsar_reasons=candidate.get("dinsar_reasons") or [],
|
||||
same_relative_orbit=bool(candidate.get("same_relative_orbit")),
|
||||
master_relative_orbit=candidate.get("master_relative_orbit"),
|
||||
slave_relative_orbit=candidate.get("slave_relative_orbit"),
|
||||
production_summary=candidate.get("production_summary"),
|
||||
)
|
||||
)
|
||||
return result_pairs
|
||||
|
||||
async def _attach_dinsar_production_summaries(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
selected_candidates: List[dict],
|
||||
) -> None:
|
||||
if not selected_candidates:
|
||||
return
|
||||
|
||||
pair_uids: set[str] = set()
|
||||
pair_keys: set[str] = set()
|
||||
aliases: set[str] = set()
|
||||
for candidate in selected_candidates:
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
pair_uid = str(candidate.get("pair_uid") or "").strip()
|
||||
if pair_uid:
|
||||
pair_uids.add(pair_uid)
|
||||
if pair_key:
|
||||
pair_keys.add(pair_key)
|
||||
if task_alias:
|
||||
aliases.add(task_alias)
|
||||
|
||||
run_conditions = []
|
||||
if pair_uids:
|
||||
run_conditions.append(DinsarProductionRunItemORM.pair_uid.in_(pair_uids))
|
||||
if pair_keys:
|
||||
run_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys))
|
||||
if aliases:
|
||||
run_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases))
|
||||
run_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases))
|
||||
|
||||
product_conditions = []
|
||||
if pair_uids:
|
||||
product_conditions.append(ResultProductORM.pair_uid.in_(pair_uids))
|
||||
if pair_keys:
|
||||
product_conditions.append(ResultProductORM.pair_key.in_(pair_keys))
|
||||
if aliases:
|
||||
product_conditions.append(ResultProductORM.task_alias.in_(aliases))
|
||||
product_conditions.append(ResultProductORM.task_name.in_(aliases))
|
||||
|
||||
run_rows = []
|
||||
if run_conditions:
|
||||
result = await db.execute(
|
||||
select(DinsarProductionRunItemORM, DinsarProductionRunORM)
|
||||
.join(DinsarProductionRunORM, DinsarProductionRunItemORM.run_id == DinsarProductionRunORM.run_id)
|
||||
.where(or_(*run_conditions))
|
||||
.order_by(
|
||||
DinsarProductionRunItemORM.updated_at.desc().nullslast(),
|
||||
DinsarProductionRunItemORM.id.desc(),
|
||||
)
|
||||
)
|
||||
run_rows = result.all()
|
||||
|
||||
products = []
|
||||
if product_conditions:
|
||||
result = await db.execute(
|
||||
select(ResultProductORM)
|
||||
.where(ResultProductORM.catalog_name == "dinsar")
|
||||
.where(or_(*product_conditions))
|
||||
.order_by(
|
||||
ResultProductORM.published_at.desc().nullslast(),
|
||||
ResultProductORM.id.desc(),
|
||||
)
|
||||
)
|
||||
products = result.scalars().all()
|
||||
|
||||
run_by_uid: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
run_by_key: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
run_by_alias: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list)
|
||||
for item, run in run_rows:
|
||||
pair_uid = str(item.pair_uid or "").strip()
|
||||
pair_key = str(item.pair_key or "").strip()
|
||||
if pair_uid:
|
||||
run_by_uid[pair_uid].append((item, run))
|
||||
if pair_key:
|
||||
run_by_key[pair_key].append((item, run))
|
||||
for alias in {str(item.task_alias or "").strip(), str(item.task_name or "").strip()}:
|
||||
if alias:
|
||||
run_by_alias[alias].append((item, run))
|
||||
|
||||
products_by_uid: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
products_by_key: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
products_by_alias: Dict[str, List[ResultProductORM]] = defaultdict(list)
|
||||
for product in products:
|
||||
pair_uid = str(product.pair_uid or "").strip()
|
||||
pair_key = str(product.pair_key or "").strip()
|
||||
if pair_uid:
|
||||
products_by_uid[pair_uid].append(product)
|
||||
if pair_key:
|
||||
products_by_key[pair_key].append(product)
|
||||
for alias in {str(product.task_alias or "").strip(), str(product.task_name or "").strip()}:
|
||||
if alias:
|
||||
products_by_alias[alias].append(product)
|
||||
|
||||
for candidate in selected_candidates:
|
||||
task_alias, pair_key = self._ensure_candidate_identity(candidate)
|
||||
pair_uid = str(candidate.get("pair_uid") or "").strip()
|
||||
exact_runs = self._dedupe_by_object_id(
|
||||
[*run_by_uid.get(pair_uid, []), *run_by_key.get(pair_key, [])],
|
||||
key=lambda row: getattr(row[0], "id", None),
|
||||
)
|
||||
alias_runs = self._dedupe_by_object_id(
|
||||
run_by_alias.get(task_alias, []),
|
||||
key=lambda row: getattr(row[0], "id", None),
|
||||
)
|
||||
exact_products = self._dedupe_by_object_id(
|
||||
[*products_by_uid.get(pair_uid, []), *products_by_key.get(pair_key, [])],
|
||||
key=lambda product: getattr(product, "id", None),
|
||||
)
|
||||
alias_products = self._dedupe_by_object_id(
|
||||
products_by_alias.get(task_alias, []),
|
||||
key=lambda product: getattr(product, "id", None),
|
||||
)
|
||||
matched_runs = exact_runs or alias_runs
|
||||
matched_products = exact_products or alias_products
|
||||
candidate["production_summary"] = self._summarize_dinsar_production(
|
||||
matched_runs,
|
||||
matched_products,
|
||||
match_level="identity" if (exact_runs or exact_products) else ("task_alias" if (alias_runs or alias_products) else "none"),
|
||||
)
|
||||
|
||||
def _summarize_dinsar_production(
|
||||
self,
|
||||
run_rows: List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]],
|
||||
products: List[ResultProductORM],
|
||||
*,
|
||||
match_level: str = "none",
|
||||
) -> Dict[str, Any]:
|
||||
latest_run_row = max(
|
||||
run_rows,
|
||||
key=lambda row: self._datetime_sort_key(
|
||||
row[0].updated_at,
|
||||
row[0].ended_at,
|
||||
row[0].started_at,
|
||||
row[0].created_at,
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
latest_product = max(
|
||||
products,
|
||||
key=lambda product: self._datetime_sort_key(
|
||||
product.published_at,
|
||||
product.produced_at,
|
||||
product.updated_at,
|
||||
product.registered_at,
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
ready_products = [product for product in products if self._is_ready_result_product(product)]
|
||||
completed_statuses = {"COMPLETED", "READY", "SUCCESS", "PUBLISHED"}
|
||||
failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"}
|
||||
completed_run_count = sum(
|
||||
1
|
||||
for item, run in run_rows
|
||||
if str(item.status or "").strip().upper() in completed_statuses
|
||||
or str(run.status or "").strip().upper() in completed_statuses
|
||||
)
|
||||
failed_run_count = sum(
|
||||
1
|
||||
for item, run in run_rows
|
||||
if str(item.status or "").strip().upper() in failed_statuses
|
||||
or str(run.status or "").strip().upper() in failed_statuses
|
||||
)
|
||||
|
||||
latest_item = latest_run_row[0] if latest_run_row else None
|
||||
latest_run = latest_run_row[1] if latest_run_row else None
|
||||
if ready_products and latest_product is not None:
|
||||
latest_status = str(latest_product.status or "").strip().upper()
|
||||
elif latest_item is not None or latest_run is not None:
|
||||
latest_status = str(
|
||||
(latest_item.status if latest_item is not None else None)
|
||||
or (latest_run.status if latest_run is not None else None)
|
||||
or ""
|
||||
).strip().upper()
|
||||
elif latest_product is not None:
|
||||
latest_status = str(latest_product.status or "").strip().upper()
|
||||
else:
|
||||
latest_status = ""
|
||||
if ready_products:
|
||||
status = "READY"
|
||||
elif completed_run_count > 0:
|
||||
status = "COMPLETED"
|
||||
elif latest_status:
|
||||
status = latest_status
|
||||
else:
|
||||
status = "MISSING"
|
||||
|
||||
engine_codes = sorted(
|
||||
{
|
||||
str(value or "").strip().lower()
|
||||
for value in [
|
||||
*(product.engine_code for product in products),
|
||||
*(run.engine_code for _, run in run_rows),
|
||||
]
|
||||
if str(value or "").strip()
|
||||
}
|
||||
)
|
||||
return {
|
||||
"has_record": bool(run_rows or products),
|
||||
"is_produced": bool(ready_products or completed_run_count > 0),
|
||||
"status": status,
|
||||
"match_level": match_level,
|
||||
"run_item_count": len(run_rows),
|
||||
"completed_run_count": completed_run_count,
|
||||
"failed_run_count": failed_run_count,
|
||||
"product_count": len(products),
|
||||
"ready_product_count": len(ready_products),
|
||||
"engine_codes": engine_codes,
|
||||
"latest_engine_code": (
|
||||
str(latest_product.engine_code or "").strip().lower()
|
||||
if latest_product is not None and latest_product.engine_code
|
||||
else (
|
||||
str(latest_run.engine_code or "").strip().lower()
|
||||
if latest_run is not None and latest_run.engine_code
|
||||
else None
|
||||
)
|
||||
),
|
||||
"latest_run_id": latest_run.run_id if latest_run is not None else None,
|
||||
"latest_run_status": latest_run.status if latest_run is not None else None,
|
||||
"latest_item_status": latest_item.status if latest_item is not None else None,
|
||||
"latest_output_dir": latest_item.latest_output_dir if latest_item is not None else None,
|
||||
"latest_product_id": latest_product.id if latest_product is not None else None,
|
||||
"latest_product_identifier": latest_product.product_id if latest_product is not None else None,
|
||||
"latest_product_status": latest_product.status if latest_product is not None else None,
|
||||
"latest_product_health": latest_product.health_status if latest_product is not None else None,
|
||||
"latest_product_published_at": latest_product.published_at if latest_product is not None else None,
|
||||
"updated_at": (
|
||||
latest_item.updated_at
|
||||
if latest_item is not None
|
||||
else (
|
||||
latest_product.updated_at
|
||||
if latest_product is not None
|
||||
else None
|
||||
)
|
||||
),
|
||||
}
|
||||
|
||||
def _is_ready_result_product(self, product: ResultProductORM) -> bool:
|
||||
status = str(product.status or "").strip().upper()
|
||||
health = str(product.health_status or "").strip().upper()
|
||||
return status in {"READY", "COMPLETED", "SUCCESS"} and health not in {"ERROR", "FAILED"}
|
||||
|
||||
def _datetime_sort_key(self, *values: Any) -> float:
|
||||
for value in values:
|
||||
if value is None:
|
||||
continue
|
||||
try:
|
||||
return float(value.timestamp())
|
||||
except Exception:
|
||||
continue
|
||||
return 0.0
|
||||
|
||||
def _dedupe_by_object_id(self, items: List[Any], *, key) -> List[Any]:
|
||||
seen: set[Any] = set()
|
||||
output: List[Any] = []
|
||||
for item in items:
|
||||
item_key = key(item)
|
||||
if item_key is None:
|
||||
item_key = id(item)
|
||||
if item_key in seen:
|
||||
continue
|
||||
seen.add(item_key)
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
async def _persist_network_run(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
@@ -451,6 +781,8 @@ class SpatialService:
|
||||
"master_scene_uid": candidate.get("master_scene_uid"),
|
||||
"slave_scene_uid": candidate.get("slave_scene_uid"),
|
||||
"pair_uid": candidate.get("pair_uid"),
|
||||
"pair_key": candidate.get("pair_key"),
|
||||
"task_alias": candidate.get("task_alias"),
|
||||
"time_baseline_days": int(candidate.get("days") or 0),
|
||||
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
|
||||
"scene_center_distance_meters": float(
|
||||
@@ -460,6 +792,11 @@ class SpatialService:
|
||||
),
|
||||
"legacy_spatial_baseline_field": "scene_center_distance_meters",
|
||||
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
|
||||
"pair_aoi_overlap_ratio": (
|
||||
float(candidate["pair_aoi_overlap_ratio"])
|
||||
if candidate.get("pair_aoi_overlap_ratio") is not None
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
def _stable_sha1(self, value: Any) -> str:
|
||||
@@ -814,6 +1151,226 @@ class SpatialService:
|
||||
"scenes": scene_payloads,
|
||||
}
|
||||
|
||||
def _apply_dinsar_production_strategy(
|
||||
self,
|
||||
candidate_pool: List[dict],
|
||||
params: PairingRequest,
|
||||
) -> Tuple[List[dict], List[str]]:
|
||||
if not candidate_pool:
|
||||
return [], []
|
||||
|
||||
warnings: List[str] = []
|
||||
rejected_count = sum(
|
||||
1
|
||||
for candidate in candidate_pool
|
||||
if str(candidate.get("dinsar_readiness") or "").upper() == "NOT_RECOMMENDED"
|
||||
)
|
||||
if rejected_count:
|
||||
warnings.append(f"{rejected_count}条候选因D-InSAR生产前置条件不足被过滤。")
|
||||
|
||||
selected = []
|
||||
for candidate in candidate_pool:
|
||||
readiness = str(candidate.get("dinsar_readiness") or "CANDIDATE").upper()
|
||||
if readiness == "NOT_RECOMMENDED":
|
||||
continue
|
||||
tier = str(candidate.get("dinsar_quality_tier") or "C").upper()
|
||||
quality_score = candidate.get("dinsar_quality_score")
|
||||
selected.append(
|
||||
{
|
||||
**candidate,
|
||||
"selection_reason": f"dinsar_{readiness.lower()}_{tier.lower()}",
|
||||
"selection_score": (
|
||||
float(quality_score)
|
||||
if quality_score is not None
|
||||
else self._score_pair_candidate(candidate, params)
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
selected.sort(
|
||||
key=lambda item: (
|
||||
{"A": 0, "B": 1, "C": 2}.get(str(item.get("dinsar_quality_tier") or "C").upper(), 9),
|
||||
-float(item.get("selection_score") or 0),
|
||||
int(item.get("days") or 0),
|
||||
float(item.get("dist") or 0),
|
||||
str(getattr(item.get("master"), "imaging_date", "") or ""),
|
||||
str(getattr(item.get("slave"), "imaging_date", "") or ""),
|
||||
)
|
||||
)
|
||||
return selected, warnings
|
||||
|
||||
async def _build_empty_pairing_diagnostics(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
params: PairingRequest,
|
||||
*,
|
||||
aoi_wkt: Optional[str],
|
||||
require_orbit_data: bool,
|
||||
) -> List[str]:
|
||||
allowed_families: List[str] = []
|
||||
for item in params.allowed_satellites or []:
|
||||
compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
allowed_families.append("LT1")
|
||||
elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
allowed_families.append("S1")
|
||||
allowed_families = list(dict.fromkeys(allowed_families))
|
||||
|
||||
sql = text(
|
||||
"""
|
||||
WITH base AS (
|
||||
SELECT
|
||||
pmc.*,
|
||||
m.satellite AS master_satellite_actual,
|
||||
s.satellite AS slave_satellite_actual,
|
||||
m.has_orbit_data AS master_has_orbit,
|
||||
s.has_orbit_data AS slave_has_orbit,
|
||||
COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters) AS center_m
|
||||
FROM pairing_metric_cache pmc
|
||||
JOIN radar_data m ON m.id = pmc.master_scene_ref_id
|
||||
JOIN radar_data s ON s.id = pmc.slave_scene_ref_id
|
||||
WHERE pmc.metric_version = :metric_version
|
||||
AND pmc.status = 'READY'
|
||||
AND pmc.master_imaging_date < pmc.slave_imaging_date
|
||||
AND pmc.same_look_direction IS TRUE
|
||||
AND pmc.same_satellite_family IS TRUE
|
||||
AND pmc.dinsar_readiness IN ('RECOMMENDED', 'CANDIDATE')
|
||||
AND (:require_orbit_data IS FALSE OR (m.has_orbit_data IS TRUE AND s.has_orbit_data IS TRUE))
|
||||
AND (:require_same_imaging_mode IS FALSE OR pmc.same_imaging_mode IS TRUE)
|
||||
AND (:require_same_polarization IS FALSE OR pmc.same_polarization IS TRUE)
|
||||
AND (
|
||||
:allowed_families_is_empty IS TRUE
|
||||
OR pmc.master_satellite_family = ANY(:allowed_families)
|
||||
OR pmc.master_satellite = ANY(:allowed_families)
|
||||
)
|
||||
AND (CAST(:master_date_from AS text) IS NULL OR pmc.master_imaging_date >= CAST(:master_date_from AS text))
|
||||
AND (CAST(:master_date_to AS text) IS NULL OR pmc.master_imaging_date <= CAST(:master_date_to AS text))
|
||||
AND (CAST(:slave_date_from AS text) IS NULL OR pmc.slave_imaging_date >= CAST(:slave_date_from AS text))
|
||||
AND (CAST(:slave_date_to AS text) IS NULL OR pmc.slave_imaging_date <= CAST(:slave_date_to AS text))
|
||||
AND (
|
||||
CAST(:aoi_wkt AS text) IS NULL
|
||||
OR ST_Intersects(
|
||||
ST_Intersection(m.geom, s.geom),
|
||||
ST_GeomFromText(CAST(:aoi_wkt AS text), 4326)
|
||||
)
|
||||
)
|
||||
),
|
||||
time_ok AS (
|
||||
SELECT * FROM base
|
||||
WHERE time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
|
||||
),
|
||||
overlap_ok AS (
|
||||
SELECT * FROM time_ok
|
||||
WHERE scene_overlap_ratio >= :overlap_threshold
|
||||
),
|
||||
center_ok AS (
|
||||
SELECT * FROM overlap_ok
|
||||
WHERE center_m <= :center_distance_max
|
||||
)
|
||||
SELECT
|
||||
(SELECT count(*) FROM base) AS base_count,
|
||||
(SELECT count(*) FROM time_ok) AS time_ok_count,
|
||||
(SELECT count(*) FROM overlap_ok) AS overlap_ok_count,
|
||||
(SELECT count(*) FROM center_ok) AS center_ok_count,
|
||||
(
|
||||
SELECT json_build_object(
|
||||
'master_date', master_imaging_date,
|
||||
'slave_date', slave_imaging_date,
|
||||
'master_satellite', master_satellite_actual,
|
||||
'slave_satellite', slave_satellite_actual,
|
||||
'time_baseline_days', time_baseline_days,
|
||||
'center_meters', center_m,
|
||||
'overlap_ratio', scene_overlap_ratio,
|
||||
'quality_tier', dinsar_quality_tier,
|
||||
'readiness', dinsar_readiness
|
||||
)
|
||||
FROM base
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max
|
||||
THEN 0 ELSE 1
|
||||
END,
|
||||
CASE WHEN scene_overlap_ratio >= :overlap_threshold THEN 0 ELSE 1 END,
|
||||
abs(time_baseline_days - :time_baseline_max),
|
||||
center_m ASC NULLS LAST
|
||||
LIMIT 1
|
||||
) AS nearest_candidate;
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{
|
||||
"metric_version": pairing_state_service.metric_version,
|
||||
"require_orbit_data": require_orbit_data,
|
||||
"require_same_imaging_mode": bool(params.require_same_imaging_mode),
|
||||
"require_same_polarization": bool(params.require_same_polarization),
|
||||
"allowed_families": allowed_families or ["__NONE__"],
|
||||
"allowed_families_is_empty": not allowed_families,
|
||||
"master_date_from": params.master_date_from or None,
|
||||
"master_date_to": params.master_date_to or None,
|
||||
"slave_date_from": params.slave_date_from or None,
|
||||
"slave_date_to": params.slave_date_to or None,
|
||||
"aoi_wkt": aoi_wkt,
|
||||
"time_baseline_min": int(params.time_baseline_min),
|
||||
"time_baseline_max": int(params.time_baseline_max),
|
||||
"overlap_threshold": float(params.overlap_threshold),
|
||||
"center_distance_max": float(params.spatial_baseline_max_meters),
|
||||
},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ["未找到满足条件的 D-InSAR 配对;诊断查询未返回统计结果。"]
|
||||
|
||||
base_count = int(row.get("base_count") or 0)
|
||||
time_ok_count = int(row.get("time_ok_count") or 0)
|
||||
overlap_ok_count = int(row.get("overlap_ok_count") or 0)
|
||||
center_ok_count = int(row.get("center_ok_count") or 0)
|
||||
if base_count <= 0:
|
||||
family_text = "、".join(allowed_families) if allowed_families else "LT-1/Sentinel-1"
|
||||
return [
|
||||
f"未找到 {family_text} 的可生产基础候选边。请检查数据体系、主从日期范围、AOI、精轨绑定和配对缓存状态。"
|
||||
]
|
||||
|
||||
messages = [
|
||||
(
|
||||
"当前筛选下基础候选 {base} 条;时间基线 {min_days}-{max_days} 天后剩 {time_ok} 条;"
|
||||
"重叠率 >= {overlap:.2f} 后剩 {overlap_ok} 条;footprint 中心距离 <= {center:.0f} 米后剩 {center_ok} 条。"
|
||||
).format(
|
||||
base=base_count,
|
||||
min_days=int(params.time_baseline_min),
|
||||
max_days=int(params.time_baseline_max),
|
||||
time_ok=time_ok_count,
|
||||
overlap=float(params.overlap_threshold),
|
||||
overlap_ok=overlap_ok_count,
|
||||
center=float(params.spatial_baseline_max_meters),
|
||||
center_ok=center_ok_count,
|
||||
)
|
||||
]
|
||||
nearest = row.get("nearest_candidate")
|
||||
if isinstance(nearest, str):
|
||||
try:
|
||||
nearest = json.loads(nearest)
|
||||
except Exception:
|
||||
nearest = None
|
||||
if isinstance(nearest, dict):
|
||||
messages.append(
|
||||
(
|
||||
"最接近的一对是 {master_satellite} {master_date} -> {slave_satellite} {slave_date},"
|
||||
"时间基线 {days} 天,中心距离 {center:.1f} 米,重叠率 {overlap:.3f},质量 {tier}/{readiness}。"
|
||||
).format(
|
||||
master_satellite=nearest.get("master_satellite") or "?",
|
||||
master_date=nearest.get("master_date") or "?",
|
||||
slave_satellite=nearest.get("slave_satellite") or "?",
|
||||
slave_date=nearest.get("slave_date") or "?",
|
||||
days=int(nearest.get("time_baseline_days") or 0),
|
||||
center=float(nearest.get("center_meters") or 0.0),
|
||||
overlap=float(nearest.get("overlap_ratio") or 0.0),
|
||||
tier=nearest.get("quality_tier") or "?",
|
||||
readiness=nearest.get("readiness") or "?",
|
||||
)
|
||||
)
|
||||
return messages
|
||||
|
||||
def _apply_strategy(
|
||||
self,
|
||||
candidate_pool: List[dict],
|
||||
|
||||
@@ -259,12 +259,15 @@ class TaskService:
|
||||
task = result.scalar_one_or_none()
|
||||
|
||||
if task:
|
||||
previous_message = task.message
|
||||
if status:
|
||||
task.status = status
|
||||
if progress is not None:
|
||||
task.progress = progress
|
||||
if message:
|
||||
task.message = message
|
||||
if message != previous_message:
|
||||
await self._add_log(task_id, "INFO", message, db=db)
|
||||
|
||||
# 如果任务结束,更新结束时间
|
||||
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
|
||||
|
||||
@@ -39,7 +39,7 @@ def get_unpack_config() -> Dict[str, Any]:
|
||||
"source_dirs": source_dirs,
|
||||
"insar_storage_dirs": insar_storage_dirs,
|
||||
"min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")),
|
||||
"delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")),
|
||||
"delete_archive": False,
|
||||
"tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"),
|
||||
"archive_exts": archive_exts,
|
||||
"scan_workers": module.parse_int(
|
||||
|
||||
+79
-1
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
import math
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple, List, Callable, Dict, Any
|
||||
from lxml import etree
|
||||
@@ -20,6 +21,80 @@ def _create_secure_xml_parser() -> etree.XMLParser:
|
||||
recover=False,
|
||||
)
|
||||
|
||||
|
||||
def _ordered_closed_polygon(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]:
|
||||
unique: List[Tuple[float, float]] = []
|
||||
for point in points or []:
|
||||
try:
|
||||
lon = float(point[0])
|
||||
lat = float(point[1])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
continue
|
||||
current = (lon, lat)
|
||||
if unique and abs(unique[-1][0] - lon) < 1e-12 and abs(unique[-1][1] - lat) < 1e-12:
|
||||
continue
|
||||
if unique and abs(unique[0][0] - lon) < 1e-12 and abs(unique[0][1] - lat) < 1e-12:
|
||||
continue
|
||||
if current not in unique:
|
||||
unique.append(current)
|
||||
if len(unique) < 3:
|
||||
return None
|
||||
if len(unique) == 4:
|
||||
center_lon = sum(item[0] for item in unique) / len(unique)
|
||||
center_lat = sum(item[1] for item in unique) / len(unique)
|
||||
ordered = sorted(
|
||||
unique,
|
||||
key=lambda item: math.atan2(item[1] - center_lat, item[0] - center_lon),
|
||||
)
|
||||
else:
|
||||
ordered = unique
|
||||
if ordered[0] != ordered[-1]:
|
||||
ordered.append(ordered[0])
|
||||
return ordered
|
||||
|
||||
|
||||
def _closed_polygon_if_valid(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]:
|
||||
ring = [(float(point[0]), float(point[1])) for point in points or []]
|
||||
if len(ring) < 3:
|
||||
return None
|
||||
if ring[0] != ring[-1]:
|
||||
ring.append(ring[0])
|
||||
return ring
|
||||
|
||||
|
||||
def _ordered_closed_polygon_from_corner_details(corner_details: Dict[str, Dict[str, Any]]) -> Optional[List[Tuple[float, float]]]:
|
||||
by_name = {str(key or "").strip().lower(): value for key, value in (corner_details or {}).items()}
|
||||
name_order = ["bottomleft", "bottomright", "topright", "topleft"]
|
||||
if all(name in by_name for name in name_order):
|
||||
return _closed_polygon_if_valid([(by_name[name]["lon"], by_name[name]["lat"]) for name in name_order])
|
||||
|
||||
entries = [
|
||||
value
|
||||
for value in (corner_details or {}).values()
|
||||
if value.get("lon") is not None
|
||||
and value.get("lat") is not None
|
||||
and value.get("ref_row") is not None
|
||||
and value.get("ref_col") is not None
|
||||
]
|
||||
if len(entries) >= 4:
|
||||
min_row = min(float(item["ref_row"]) for item in entries)
|
||||
max_row = max(float(item["ref_row"]) for item in entries)
|
||||
min_col = min(float(item["ref_col"]) for item in entries)
|
||||
max_col = max(float(item["ref_col"]) for item in entries)
|
||||
targets = [(min_row, min_col), (min_row, max_col), (max_row, max_col), (max_row, min_col)]
|
||||
remaining = list(entries)
|
||||
ordered_entries: List[Dict[str, Any]] = []
|
||||
for target_row, target_col in targets:
|
||||
chosen = min(
|
||||
remaining,
|
||||
key=lambda item: abs(float(item["ref_row"]) - target_row) + abs(float(item["ref_col"]) - target_col),
|
||||
)
|
||||
ordered_entries.append(chosen)
|
||||
remaining.remove(chosen)
|
||||
return _closed_polygon_if_valid([(item["lon"], item["lat"]) for item in ordered_entries])
|
||||
|
||||
return _ordered_closed_polygon([(value["lon"], value["lat"]) for value in (corner_details or {}).values()])
|
||||
|
||||
# --- Sentinel-1 (S1A/S1B/S1C) Parsers ---
|
||||
|
||||
def _radar_meta_base() -> Dict[str, Any]:
|
||||
@@ -573,7 +648,10 @@ def parse_xml_metadata(
|
||||
return None, None
|
||||
|
||||
if len(polygon) == 4:
|
||||
polygon.append(polygon[0])
|
||||
ordered_polygon = _ordered_closed_polygon_from_corner_details(corner_details)
|
||||
if not ordered_polygon:
|
||||
return None, None
|
||||
polygon = ordered_polygon
|
||||
corner_pixel_mapping = _build_corner_pixel_mapping(corner_details)
|
||||
meta = {
|
||||
"orbit_direction": orbit_direction,
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
-- Persist source XML/manifest documents and normalized scene geometry profiles.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_metadata_documents (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_asset_id INTEGER NOT NULL REFERENCES source_product_assets(id) ON DELETE CASCADE,
|
||||
radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
source_format VARCHAR(32) NULL,
|
||||
document_type VARCHAR(32) NOT NULL,
|
||||
member_path TEXT NOT NULL,
|
||||
content_sha256 VARCHAR(64) NOT NULL,
|
||||
content_encoding VARCHAR(16) NOT NULL DEFAULT 'gzip',
|
||||
content_bytes BYTEA NOT NULL,
|
||||
content_size_bytes BIGINT NULL,
|
||||
archive_path TEXT NULL,
|
||||
archive_mtime DOUBLE PRECISION NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parse_status VARCHAR(32) NOT NULL DEFAULT 'OK',
|
||||
parse_error TEXT NULL,
|
||||
extracted_at TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP NULL DEFAULT NOW(),
|
||||
CONSTRAINT uq_source_metadata_document_member UNIQUE (source_asset_id, document_type, member_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_asset_id
|
||||
ON source_metadata_documents (source_asset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_data_id
|
||||
ON source_metadata_documents (radar_data_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_satellite_family
|
||||
ON source_metadata_documents (satellite_family);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_format
|
||||
ON source_metadata_documents (source_format);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_document_type
|
||||
ON source_metadata_documents (document_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_content_sha256
|
||||
ON source_metadata_documents (content_sha256);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_parse_status
|
||||
ON source_metadata_documents (parse_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_asset_type
|
||||
ON source_metadata_documents (source_asset_id, document_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_type
|
||||
ON source_metadata_documents (radar_data_id, document_type);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sar_scene_geometry_profiles (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source_asset_id INTEGER NOT NULL UNIQUE REFERENCES source_product_assets(id) ON DELETE CASCADE,
|
||||
radar_data_id INTEGER NULL UNIQUE REFERENCES radar_data(id) ON DELETE CASCADE,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
satellite VARCHAR(32) NULL,
|
||||
source_format VARCHAR(32) NULL,
|
||||
imaging_mode VARCHAR(64) NULL,
|
||||
polarization VARCHAR(64) NULL,
|
||||
orbit_direction VARCHAR(32) NULL,
|
||||
look_direction VARCHAR(32) NULL,
|
||||
absolute_orbit VARCHAR(64) NULL,
|
||||
relative_orbit VARCHAR(64) NULL,
|
||||
acquisition_start_time_utc TIMESTAMP NULL,
|
||||
acquisition_stop_time_utc TIMESTAMP NULL,
|
||||
scene_center_lon DOUBLE PRECISION NULL,
|
||||
scene_center_lat DOUBLE PRECISION NULL,
|
||||
footprint_geom GEOMETRY(POLYGON, 4326) NULL,
|
||||
footprint_polygon JSON NULL,
|
||||
swath_summary_json JSON NULL,
|
||||
burst_summary_json JSON NULL,
|
||||
incidence_angle_min DOUBLE PRECISION NULL,
|
||||
incidence_angle_max DOUBLE PRECISION NULL,
|
||||
doppler_summary_json JSON NULL,
|
||||
state_vector_summary_json JSON NULL,
|
||||
metadata_quality VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN',
|
||||
production_readiness VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN',
|
||||
readiness_reasons_json JSON NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parsed_at TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_asset_id
|
||||
ON sar_scene_geometry_profiles (source_asset_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_radar_data_id
|
||||
ON sar_scene_geometry_profiles (radar_data_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite_family
|
||||
ON sar_scene_geometry_profiles (satellite_family);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite
|
||||
ON sar_scene_geometry_profiles (satellite);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_format
|
||||
ON sar_scene_geometry_profiles (source_format);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_imaging_mode
|
||||
ON sar_scene_geometry_profiles (imaging_mode);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_polarization
|
||||
ON sar_scene_geometry_profiles (polarization);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_orbit_direction
|
||||
ON sar_scene_geometry_profiles (orbit_direction);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_look_direction
|
||||
ON sar_scene_geometry_profiles (look_direction);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_absolute_orbit
|
||||
ON sar_scene_geometry_profiles (absolute_orbit);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_relative_orbit
|
||||
ON sar_scene_geometry_profiles (relative_orbit);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_acquisition_start_time_utc
|
||||
ON sar_scene_geometry_profiles (acquisition_start_time_utc);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_footprint_geom
|
||||
ON sar_scene_geometry_profiles USING GIST (footprint_geom);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_metadata_quality
|
||||
ON sar_scene_geometry_profiles (metadata_quality);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_production_readiness
|
||||
ON sar_scene_geometry_profiles (production_readiness);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_family_date
|
||||
ON sar_scene_geometry_profiles (satellite_family, acquisition_start_time_utc);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_track
|
||||
ON sar_scene_geometry_profiles (satellite_family, relative_orbit, orbit_direction);
|
||||
CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_readiness
|
||||
ON sar_scene_geometry_profiles (production_readiness, metadata_quality);
|
||||
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS pair_aoi_overlap_ratio DOUBLE PRECISION NULL;
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS same_relative_orbit BOOLEAN NOT NULL DEFAULT FALSE;
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS master_relative_orbit VARCHAR(64) NULL;
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS slave_relative_orbit VARCHAR(64) NULL;
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS dinsar_quality_tier VARCHAR(16) NOT NULL DEFAULT 'C';
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS dinsar_quality_score DOUBLE PRECISION NULL;
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS dinsar_readiness VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE';
|
||||
ALTER TABLE IF EXISTS pairing_metric_cache
|
||||
ADD COLUMN IF NOT EXISTS dinsar_reasons_json JSON NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_same_relative_orbit
|
||||
ON pairing_metric_cache (same_relative_orbit);
|
||||
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_quality_tier
|
||||
ON pairing_metric_cache (dinsar_quality_tier);
|
||||
CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_readiness
|
||||
ON pairing_metric_cache (dinsar_readiness);
|
||||
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_status VARCHAR(32) NOT NULL DEFAULT 'NOT_CHECKED';
|
||||
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_method VARCHAR(64) NULL;
|
||||
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_checked_at TIMESTAMP NULL;
|
||||
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_error TEXT NULL;
|
||||
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_version VARCHAR(32) NULL;
|
||||
|
||||
ALTER TABLE source_product_assets
|
||||
ADD COLUMN IF NOT EXISTS archive_integrity_member_count INTEGER NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_archive_integrity_status
|
||||
ON source_product_assets (archive_integrity_status);
|
||||
Reference in New Issue
Block a user