Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: update production workflow design and runtime changes

Browse Source
This commit is contained in:
Harmon
2026-06-14 18:47:00 +08:00
parent 58a87706ef
commit b1c59051b9
41 changed files with 2042 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/config.py
+90 -22
View File
@@ -42,7 +42,7 @@ def _infer_conda_env_name_from_python(path: str | None) -> str:
def _default_idl_runtime_dir() -> str:
return os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
return os.path.join(_default_runtime_root(_PROJECT_ROOT), "idl_worker")
def _resolve_idl_runtime_dir(value: str | None) -> str:
@@ -77,6 +77,26 @@ def _default_result_publish_root(project_root: str) -> str:
return os.path.join(normalized_root, "production_results")
def _default_input_root(project_root: str) -> str:
normalized_root = os.path.normpath(project_root)
drive, _tail = os.path.splitdrive(normalized_root)
if drive:
return os.path.join(drive + os.sep, "production_inputs")
return os.path.join(normalized_root, "production_inputs")
def _default_runtime_root(project_root: str) -> str:
normalized_root = os.path.normpath(project_root)
drive, _tail = os.path.splitdrive(normalized_root)
if drive:
return os.path.join(drive + os.sep, "production_runtime")
return os.path.join(normalized_root, "runtime")
def _default_runtime_dir(project_root: str, *parts: str) -> str:
return os.path.join(_default_runtime_root(project_root), *parts)
def _read_env_pairs(env_path: str) -> dict[str, str]:
env_map: dict[str, str] = {}
if not os.path.isfile(env_path):
@@ -197,7 +217,6 @@ class Settings(BaseSettings):
WATER_RESULTS_DIR: str = ""
SAR_ANALYSIS_READY_ROOT: str = ""
SAR_ANALYSIS_WORK_ROOT: str = ""
SAR_ANALYSIS_PREVIEW_ROOT: str = ""
SAR_ANALYSIS_NODATA_VALUE: float = -9999.0
SAR_ANALYSIS_OUTPUT_COG: bool = True
@@ -206,9 +225,11 @@ class Settings(BaseSettings):
GF3_ARCHIVE_SOURCE_DIRS: str = ""
GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz"
GF3_UNPACK_DELETE_ARCHIVE: bool = True
GF3_LEGACY_GDAL_ENABLED: bool = False
GF3_SOURCE_DIRS: str = ""
GF3_SARSCAPE_NATIVE_DIRS: str = ""
GF3_STORAGE_DIRS: str = ""
GF3_SARSCAPE_RUNTIME_DIR: str = ""
GF3_SARSCAPE_WRAPPER_EXE: str = ""
GF3_SARSCAPE_IDLRT_PATH: str = r"C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe"
GF3_SARSCAPE_DEM_PATH: str = ""
@@ -340,6 +361,7 @@ class Settings(BaseSettings):
GAMMA_SBAS_ENV_SCRIPT: str = ""
GAMMA_SBAS_WORK_ROOT: str = ""
GAMMA_SBAS_PRODUCT_ROOT: str = ""
GAMMA_SBAS_TRIAL_ROOT: str = ""
GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT: str = ""
GAMMA_SBAS_SOURCE_ROOTS: str = ""
GAMMA_SBAS_ORBIT_ROOTS: str = ""
@@ -420,19 +442,13 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"SAR_ANALYSIS_READY_ROOT",
os.path.join(backend_dir, "runtime", "sar_analysis_ready"),
os.path.join(_default_result_publish_root(project_root), "sar_analysis_ready"),
)
if not self.SAR_ANALYSIS_WORK_ROOT:
object.__setattr__(
self,
"SAR_ANALYSIS_WORK_ROOT",
os.path.join(backend_dir, "runtime", "sar_analysis_work"),
)
if not self.SAR_ANALYSIS_PREVIEW_ROOT:
object.__setattr__(
self,
"SAR_ANALYSIS_PREVIEW_ROOT",
os.path.join(backend_dir, "runtime", "sar_analysis_preview"),
_default_runtime_dir(project_root, "sar_analysis_work"),
)
object.__setattr__(
self,
@@ -441,11 +457,29 @@ class Settings(BaseSettings):
)
if not self.SRTM_DEM_DIR:
object.__setattr__(self, "SRTM_DEM_DIR", os.path.join(backend_dir, "dem_data"))
if not self.GF3_ARCHIVE_SOURCE_DIRS:
object.__setattr__(
self,
"GF3_ARCHIVE_SOURCE_DIRS",
os.path.join(_default_input_root(project_root), "gf3", "archives"),
)
if not self.GF3_SARSCAPE_NATIVE_DIRS:
object.__setattr__(
self,
"GF3_SARSCAPE_NATIVE_DIRS",
os.path.join(_default_result_publish_root(project_root), "gf3", "sarscape_native"),
)
if not self.GF3_STORAGE_DIRS:
object.__setattr__(
self,
"GF3_STORAGE_DIRS",
os.path.join(backend_dir, "gf3_results"),
os.path.join(_default_result_publish_root(project_root), "gf3", "standard_l2"),
)
if not self.GF3_SARSCAPE_RUNTIME_DIR:
object.__setattr__(
self,
"GF3_SARSCAPE_RUNTIME_DIR",
_default_runtime_dir(project_root, "gf3", "sarscape_runtime"),
)
if not self.ORBIT_QUARANTINE_DIR and self.MONITOR_ORBIT_DIR:
object.__setattr__(
@@ -496,7 +530,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"ISCE2_WORK_ROOT",
os.path.join(backend_dir, "runtime", "isce2_work"),
_default_runtime_dir(project_root, "isce2_work"),
)
if not self.ISCE2_OUTPUT_ROOT:
object.__setattr__(
@@ -582,7 +616,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"WSL_BROKER_JOB_ROOT",
os.path.join(backend_dir, "runtime", "wsl_jobs"),
_default_runtime_dir(project_root, "wsl_jobs"),
)
if not self.ISCE2_RUNTIME_ID:
object.__setattr__(self, "ISCE2_RUNTIME_ID", "isce2_runtime_v1")
@@ -608,13 +642,13 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"PYINT_TEMPLATE_ROOT",
os.path.join(backend_dir, "runtime", "pyint_templates"),
_default_runtime_dir(project_root, "pyint_templates"),
)
if not self.PYINT_WORK_ROOT:
object.__setattr__(
self,
"PYINT_WORK_ROOT",
os.path.join(backend_dir, "runtime", "pyint_work"),
_default_runtime_dir(project_root, "pyint_work"),
)
if not self.PYINT_OUTPUT_ROOT:
object.__setattr__(
@@ -626,7 +660,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"PYINT_DEM_ROOT",
os.path.join(backend_dir, "runtime", "pyint_dem"),
os.path.join(_default_runtime_root(project_root), "pyint_dem"),
)
pyint_dem_mode = str(self.PYINT_DEM_MODE or "local_fabdem").strip().lower() or "local_fabdem"
if pyint_dem_mode not in {"local_fabdem", "opentopo", "prepared_file"}:
@@ -743,7 +777,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"GAMMA_SBAS_WORK_ROOT",
os.path.join(backend_dir, "runtime", "sbas_insar_production"),
os.path.join(_default_runtime_root(project_root), "sbas_insar_work"),
)
if not self.GAMMA_SBAS_PRODUCT_ROOT:
object.__setattr__(
@@ -751,6 +785,12 @@ class Settings(BaseSettings):
"GAMMA_SBAS_PRODUCT_ROOT",
os.path.join(self.TIMESERIES_PRODUCT_DIR, "sbas"),
)
if not self.GAMMA_SBAS_TRIAL_ROOT:
object.__setattr__(
self,
"GAMMA_SBAS_TRIAL_ROOT",
os.path.join(_default_runtime_root(project_root), "gamma_ipta_trials"),
)
if not self.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT:
object.__setattr__(
self,
@@ -848,7 +888,7 @@ class Settings(BaseSettings):
object.__setattr__(
self,
"TIMESERIES_WORK_ROOT",
os.path.join(backend_dir, "runtime", "timeseries_work"),
_default_runtime_dir(project_root, "timeseries_work"),
)
if not self.TIMESERIES_DEM_PATH:
object.__setattr__(self, "TIMESERIES_DEM_PATH", self.ISCE2_DEM_PATH)
@@ -947,7 +987,14 @@ class Settings(BaseSettings):
os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True)
os.makedirs(settings.SAR_ANALYSIS_READY_ROOT, exist_ok=True)
os.makedirs(settings.SAR_ANALYSIS_WORK_ROOT, exist_ok=True)
os.makedirs(settings.SAR_ANALYSIS_PREVIEW_ROOT, exist_ok=True)
for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS):
os.makedirs(path, exist_ok=True)
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
os.makedirs(path, exist_ok=True)
for path in split_env_paths(settings.GF3_STORAGE_DIRS):
os.makedirs(path, exist_ok=True)
if settings.GF3_SARSCAPE_RUNTIME_DIR:
os.makedirs(settings.GF3_SARSCAPE_RUNTIME_DIR, exist_ok=True)
os.makedirs(settings.WSL_BROKER_JOB_ROOT, exist_ok=True)
os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True)
os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True)
@@ -958,6 +1005,7 @@ class Settings(BaseSettings):
if settings.GAMMA_SBAS_ENABLED:
os.makedirs(settings.GAMMA_SBAS_WORK_ROOT, exist_ok=True)
os.makedirs(settings.GAMMA_SBAS_PRODUCT_ROOT, exist_ok=True)
os.makedirs(settings.GAMMA_SBAS_TRIAL_ROOT, exist_ok=True)
os.makedirs(settings.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT, exist_ok=True)
if settings.LANDSAR_SBAS_ENABLED:
os.makedirs(settings.LANDSAR_SBAS_WORK_ROOT, exist_ok=True)
@@ -1115,7 +1163,6 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_WORK_ROOT", value=settings.SAR_ANALYSIS_WORK_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_PREVIEW_ROOT", value=settings.SAR_ANALYSIS_PREVIEW_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
for label, value in (
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
@@ -1143,10 +1190,9 @@ def validate_runtime_config() -> dict[str, Any]:
("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS),
("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS),
("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS),
("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS),
("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS),
("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS),
("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS),
("GF3_SARSCAPE_RUNTIME_DIR", settings.GF3_SARSCAPE_RUNTIME_DIR),
):
values = split_env_paths(raw_value)
if not values:
@@ -1155,6 +1201,21 @@ def validate_runtime_config() -> dict[str, Any]:
for item in values:
_check_path(label=label, value=item, errors=errors, warnings=warnings, expect_file=False)
gf3_archive_dirs = split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS)
if not gf3_archive_dirs:
warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置；无法触发 GF3 SARscape 生产，只能扫描已有原生结果。")
for item in gf3_archive_dirs:
_check_path(label="GF3_ARCHIVE_SOURCE_DIRS", value=item, errors=errors, warnings=warnings, expect_file=False)
if settings.GF3_LEGACY_GDAL_ENABLED:
legacy_dirs = split_env_paths(settings.GF3_SOURCE_DIRS)
if not legacy_dirs:
errors.append("GF3_LEGACY_GDAL_ENABLED=true but GF3_SOURCE_DIRS is not configured.")
for item in legacy_dirs:
_check_path(label="GF3_SOURCE_DIRS", value=item, errors=errors, warnings=warnings, expect_file=False)
elif split_env_paths(settings.GF3_SOURCE_DIRS):
warnings.append("GF3_SOURCE_DIRS is configured but GF3_LEGACY_GDAL_ENABLED=false; legacy GDAL preprocessing is disabled.")
if settings.ISCE2_ENABLED:
if not settings.ISCE2_WSL_DISTRO:
errors.append("ISCE2_ENABLED=true 但 ISCE2_WSL_DISTRO 未配置。")
@@ -1279,6 +1340,13 @@ def validate_runtime_config() -> dict[str, Any]:
warnings=warnings,
expect_file=False,
)
_check_path(
label="GAMMA_SBAS_TRIAL_ROOT",
value=settings.GAMMA_SBAS_TRIAL_ROOT,
errors=errors,
warnings=warnings,
expect_file=False,
)
_check_path(
label="GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT",
value=settings.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT,
backend/app/main.py
+1 -1
View File
@@ -88,7 +88,7 @@ async def lifespan(app: FastAPI):
sbas_catalog_bootstrap = await sbas_insar_catalog_service.bootstrap_catalog_on_startup_clean()
except Exception as exc:
sbas_catalog_bootstrap = {
"storage_root": settings.GAMMA_SBAS_WORK_ROOT,
"storage_root": settings.GAMMA_SBAS_PRODUCT_ROOT,
"manifest_count": 0,
"db_count": 0,
"needs_rebuild": False,
backend/app/routers/dinsar_products.py
+5 -1
View File
@@ -260,4 +260,8 @@ async def get_dinsar_product_preview(
if not preview_path or not os.path.isfile(preview_path):
raise HTTPException(status_code=404, detail="Preview not found")
media_type = mimetypes.guess_type(preview_path)[0] or "application/octet-stream"
return FileResponse(preview_path, media_type=media_type)
return FileResponse(
preview_path,
media_type=media_type,
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
backend/app/routers/monitor.py
+18
View File
@@ -43,8 +43,10 @@ class MonitorConfig(BaseModel):
dinsar_dirs: List[str] = []
gf3_archive_source_dirs: List[str] = []
gf3_source_dirs: List[str] = []
gf3_legacy_gdal_enabled: bool = False
gf3_sarscape_native_dirs: List[str] = []
gf3_storage_dirs: List[str] = []
gf3_sarscape_runtime_dir: Optional[str] = None
gf3_sarscape_wrapper_exe: Optional[str] = None
gf3_sarscape_idlrt_path: Optional[str] = None
gf3_sarscape_dem_path: Optional[str] = None
@@ -159,6 +161,14 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
"""
批量 GF3 L1A→L2 处理：扫描 GF3_SOURCE_DIRS，过滤已处理的，逐个处理并自动入库。
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or []
if not gf3_source_dirs:
raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.")
@@ -331,6 +341,14 @@ async def run_gf3_unpack(
"""
将 GF3 压缩包池解包到 GF3_SOURCE_DIRS，作为后续 L1A→L2 预处理输入。
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or []
if not gf3_archive_source_dirs:
backend/app/routers/ps_products.py
+5 -1
View File
@@ -142,4 +142,8 @@ async def get_psinsar_product_preview(
preview_path = str(detail.get("preview_path") or "").strip()
if not preview_path or not os.path.isfile(preview_path):
raise HTTPException(status_code=404, detail="Preview not found")
return FileResponse(preview_path, media_type="image/png")
return FileResponse(
preview_path,
media_type="image/png",
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
backend/app/routers/sbas_insar_production.py
+3
View File
@@ -479,6 +479,7 @@ async def get_landsar_sbas_run_artifact(run_id: str, relative_path: str):
artifact_path,
media_type=media_type,
filename=artifact_path.name,
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
@@ -1022,6 +1023,7 @@ async def get_sbas_insar_run_artifact(run_id: str, relative_path: str):
artifact_path,
media_type=media_type,
filename=artifact_path.name,
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
@@ -1054,4 +1056,5 @@ async def get_sbas_insar_artifact(trial_id: str, relative_path: str):
artifact_path,
media_type=media_type,
filename=artifact_path.name,
headers={"Cache-Control": "public, max-age=31536000, immutable"},
)
backend/app/routers/sbas_insar_products.py
+3 -1
View File
@@ -21,6 +21,7 @@ from .dependencies import _add_operation_audit_log, _get_current_user, _require_
router = APIRouter()
STATIC_ASSET_CACHE_HEADERS = {"Cache-Control": "public, max-age=31536000, immutable"}
class SbasInsarCatalogRebuildRequest(BaseModel):
@@ -189,7 +190,7 @@ async def get_sbas_insar_product_preview(
preview_path = str(detail.get("preview_path") or "").strip()
if not preview_path or not os.path.isfile(preview_path):
raise HTTPException(status_code=404, detail="Preview not found")
return FileResponse(preview_path, media_type="image/png")
return FileResponse(preview_path, media_type="image/png", headers=STATIC_ASSET_CACHE_HEADERS)
@router.get("/sbas-insar-products/{product_db_id}/assets/{asset_id}")
@@ -209,4 +210,5 @@ async def get_sbas_insar_product_asset(
asset.absolute_path,
media_type=asset.media_type or "application/octet-stream",
filename=asset.asset_name or os.path.basename(asset.absolute_path),
headers=STATIC_ASSET_CACHE_HEADERS,
)
backend/app/routers/water.py
+8
View File
@@ -902,6 +902,14 @@ async def submit_gf3_process(
admin_user: AuthUserORM = Depends(_require_admin),
):
"""提交 GF3 L1A→L2 处理任务（辐射定标 + RPC 几何校正）。"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use /monitor/gf3-sarscape-produce or /monitor/gf3-sarscape-sync."
),
)
if not os.path.exists(req.input_dir):
raise HTTPException(status_code=400, detail=f"输入路径不存在: {req.input_dir}")
backend/app/scheduler.py
+2
View File
@@ -21,8 +21,10 @@ MONITOR_CONFIG = {
# GF3 链路
"gf3_archive_source_dirs": split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS),
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
"gf3_legacy_gdal_enabled": bool(settings.GF3_LEGACY_GDAL_ENABLED),
"gf3_sarscape_native_dirs": split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS),
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
"gf3_sarscape_runtime_dir": settings.GF3_SARSCAPE_RUNTIME_DIR,
"gf3_sarscape_wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
"gf3_sarscape_idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
"gf3_sarscape_dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
backend/app/services/gf3_sarscape_production_service.py
+7 -2
View File
@@ -115,7 +115,7 @@ def _resolve_config_path(raw: Any) -> Path | None:
def _fallback_wrapper_exe() -> Path | None:
candidate = (
Path(settings.PROJECT_ROOT)
/ ".codex_tmp"
/ "third_party"
/ "GF3_L1A_To_L2_pipeline"
/ "dist"
/ "windows"
@@ -443,7 +443,12 @@ def run_gf3_sarscape_production(
"results": [],
}
runtime_dir = native_root_path / ".gf3_runtime"
configured_runtime_dir = _clean_text(getattr(settings, "GF3_SARSCAPE_RUNTIME_DIR", ""))
runtime_dir = (
Path(os.path.normpath(configured_runtime_dir)).resolve()
if configured_runtime_dir
else native_root_path / ".gf3_runtime"
)
runtime_dir.mkdir(parents=True, exist_ok=True)
config_path = runtime_dir / "gf3wrapper.json"
env = os.environ.copy()
backend/app/services/health_service.py
+11 -2
View File
@@ -969,7 +969,11 @@ async def _check_source_roots() -> Dict[str, Any]:
for path in split_env_paths(settings.GF3_SOURCE_DIRS):
status = _probe_directory_status(path)
status["role"] = "gf3_l1a_source"
status["role"] = (
"gf3_l1a_source"
if settings.GF3_LEGACY_GDAL_ENABLED
else "gf3_legacy_l1a_source_disabled"
)
items.append(status)
for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS):
@@ -982,6 +986,12 @@ async def _check_source_roots() -> Dict[str, Any]:
status["role"] = "gf3_l2_storage"
items.append(status)
runtime_dir = str(getattr(settings, "GF3_SARSCAPE_RUNTIME_DIR", "") or "").strip()
if runtime_dir:
status = _probe_directory_status(runtime_dir)
status["role"] = "gf3_sarscape_runtime"
items.append(status)
wrapper_exe = str(settings.GF3_SARSCAPE_WRAPPER_EXE or "").strip()
if wrapper_exe:
status = _probe_file_status(wrapper_exe)
@@ -1017,7 +1027,6 @@ async def _check_sar_analysis_ready() -> Dict[str, Any]:
roots = {
"ready": _probe_directory_status(settings.SAR_ANALYSIS_READY_ROOT),
"work": _probe_directory_status(settings.SAR_ANALYSIS_WORK_ROOT),
"preview": _probe_directory_status(settings.SAR_ANALYSIS_PREVIEW_ROOT),
}
for role, payload in roots.items():
payload["role"] = role
backend/app/services/job_handlers.py
+18
View File
@@ -3612,6 +3612,12 @@ async def _handle_gf3_process(job: SystemJobORM) -> None:
"""GF3 L1A→L2 处理 job handler（辐射定标 + RPC 几何校正）。"""
from .gf3_service import run_gf3_l1a_to_l2
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
processing_id = payload.get("processing_id")
if not processing_id:
@@ -3685,6 +3691,12 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None:
"""GF3 archive inbox -> persistent L1A source pool."""
from .gf3_unpack_service import run_gf3_archive_unpack
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
if not job.task_id:
raise ValueError("GF3_UNPACK requires task_id for progress tracking.")
@@ -3746,6 +3758,12 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None:
from .gf3_service import run_gf3_l1a_to_l2, register_l2_to_radar_data
from .sar_analysis_ready_service import standardize_gf3_l2_for_radar
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise ValueError(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
)
payload = job.payload or {}
source_dirs = payload.get("source_dirs") or []
if not source_dirs:
backend/app/services/root_registry_service.py
+50
View File
@@ -271,6 +271,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="scene_directory",
)
)
specs.extend(
_iter_single_root_specs(
env_var="GF3_SARSCAPE_RUNTIME_DIR",
path=settings.GF3_SARSCAPE_RUNTIME_DIR,
root_role="work_root_gf3_sarscape",
display_name="GF3 SARscape Runtime",
scan_mode="workspace",
owner_engine="sarscape",
)
)
specs.extend(
_iter_single_root_specs(
env_var="SAR_ANALYSIS_READY_ROOT",
@@ -438,6 +448,46 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
owner_engine="pyint",
)
)
specs.extend(
_iter_single_root_specs(
env_var="PYINT_DEM_ROOT",
path=settings.PYINT_DEM_ROOT,
root_role="dem_cache_root_pyint",
display_name="PyINT DEM Cache Root",
scan_mode="workspace",
owner_engine="pyint",
)
)
specs.extend(
_iter_single_root_specs(
env_var="GAMMA_SBAS_WORK_ROOT",
path=settings.GAMMA_SBAS_WORK_ROOT,
root_role="work_root_gamma_sbas",
display_name="Gamma SBAS Work Root",
scan_mode="workspace",
owner_engine="gamma",
)
)
specs.extend(
_iter_single_root_specs(
env_var="GAMMA_SBAS_PRODUCT_ROOT",
path=settings.GAMMA_SBAS_PRODUCT_ROOT,
root_role="publish_root_gamma_sbas",
display_name="Gamma SBAS Publish Root",
scan_mode="manifest_tree",
owner_engine="gamma",
)
)
specs.extend(
_iter_single_root_specs(
env_var="GAMMA_SBAS_TRIAL_ROOT",
path=settings.GAMMA_SBAS_TRIAL_ROOT,
root_role="trial_root_gamma_sbas",
display_name="Gamma SBAS Trial Root",
scan_mode="workspace",
owner_engine="gamma",
)
)
return specs
backend/app/services/sbas_insar_catalog_service.py
+83 -22
View File
@@ -289,6 +289,50 @@ def _read_gamma_int_param(path: Path, key: str) -> Optional[int]:
return _safe_int(_read_gamma_key_values(path).get(key))
def _expert_gamma_work_run_dir(run_dir: Path) -> Path:
run_id = run_dir.name
candidates = [
run_dir,
Path(settings.GAMMA_SBAS_WORK_ROOT or "") / "runs" / run_id if settings.GAMMA_SBAS_WORK_ROOT else run_dir,
]
seen: set[str] = set()
for candidate in candidates:
normalized = _normalize_path(candidate)
if normalized in seen:
continue
seen.add(normalized)
if (
(candidate / "sbas" / "disp.TS_tab").is_file()
and (candidate / "sbas" / "mli.ave.par").is_file()
and any((candidate / "dem").glob("*.lt_fine"))
):
return candidate
return run_dir
def _expert_gamma_lookup_path(work_run_dir: Path, manifest: dict[str, Any]) -> Path:
lt_path = work_run_dir / "dem" / f"{manifest.get('reference_date') or ''}.lt_fine"
if lt_path.is_file():
return lt_path
candidates = sorted((work_run_dir / "dem").glob("*.lt_fine"))
return candidates[0] if candidates else lt_path
def _resolve_expert_gamma_tab_path(raw_path: str, *, work_run_dir: Path) -> Path:
path = Path(_wsl_path_to_windows(raw_path))
if path.is_file():
return path
run_id = work_run_dir.name
parts = list(path.parts)
if run_id in parts:
index = parts.index(run_id)
relative_parts = parts[index + 1 :]
candidate = work_run_dir.joinpath(*relative_parts)
if candidate.is_file():
return candidate
return path
def _haversine_m(lon1: float, lat1: float, lon2: float, lat2: float) -> float:
radius_m = 6371008.8
phi1 = math.radians(lat1)
@@ -762,12 +806,13 @@ def _read_geo_rate_window(
def _query_expert_gamma_point_timeseries(run_dir: Path, *, lon: float, lat: float) -> dict[str, Any]:
source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif"
coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif"
lt_path = run_dir / "dem" / f"{_safe_read_json(run_dir / 'run_manifest.json').get('reference_date') or ''}.lt_fine"
if not lt_path.is_file():
candidates = sorted((run_dir / "dem").glob("*.lt_fine"))
lt_path = candidates[0] if candidates else lt_path
mli_par = run_dir / "sbas" / "mli.ave.par"
disp_tab = run_dir / "sbas" / "disp.TS_tab"
work_run_dir = _expert_gamma_work_run_dir(run_dir)
work_manifest = _safe_read_json(work_run_dir / "run_manifest.json")
if not work_manifest:
work_manifest = _safe_read_json(run_dir / "run_manifest.json")
lt_path = _expert_gamma_lookup_path(work_run_dir, work_manifest)
mli_par = work_run_dir / "sbas" / "mli.ave.par"
disp_tab = work_run_dir / "sbas" / "disp.TS_tab"
if not source_path.is_file():
raise FileNotFoundError("geo_los_def_rate.tif is missing")
if not lt_path.is_file():
@@ -852,11 +897,13 @@ def _query_expert_gamma_point_timeseries(run_dir: Path, *, lon: float, lat: floa
img_y = min(max(0, img_y), radar_lines - 1)
raw_disp_paths = [
Path(_wsl_path_to_windows(line.strip()))
_resolve_expert_gamma_tab_path(line.strip(), work_run_dir=work_run_dir)
for line in disp_tab.read_text(encoding="utf-8", errors="ignore").splitlines()
if line.strip()
]
dates = _read_expert_sbas_dates(run_dir)
dates = _read_expert_sbas_dates(work_run_dir)
if not dates:
dates = _read_expert_sbas_dates(run_dir)
displacements: list[dict[str, Any]] = []
for index, disp_path in enumerate(raw_disp_paths):
value_m = _read_radar_float32(disp_path, width=radar_width, img_x=img_x, img_y=img_y)
@@ -872,6 +919,8 @@ def _query_expert_gamma_point_timeseries(run_dir: Path, *, lon: float, lat: floa
return {
"schema": "insar.gamma-sbas-point-query/v1",
"source_tool": "disp.TS_tab_radar_pixel_sample",
"source_run_dir": str(run_dir),
"work_run_dir": str(work_run_dir),
"query": {"lon": lon, "lat": lat},
"matched": {
**valid_choice,
@@ -906,11 +955,11 @@ def _locate_radar_points_in_geocoded_product(run_dir: Path, points: list[dict[st
source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif"
coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif"
manifest = _safe_read_json(run_dir / "run_manifest.json")
lt_path = run_dir / "dem" / f"{manifest.get('reference_date') or ''}.lt_fine"
if not lt_path.is_file():
candidates = sorted((run_dir / "dem").glob("*.lt_fine"))
lt_path = candidates[0] if candidates else lt_path
work_run_dir = _expert_gamma_work_run_dir(run_dir)
manifest = _safe_read_json(work_run_dir / "run_manifest.json")
if not manifest:
manifest = _safe_read_json(run_dir / "run_manifest.json")
lt_path = _expert_gamma_lookup_path(work_run_dir, manifest)
if not source_path.is_file() or not lt_path.is_file():
return {}
@@ -1982,6 +2031,12 @@ def _stack_dates_from_manifest(stack_manifest: dict[str, Any], manifest: dict[st
class SbasInsarCatalogService:
def get_run_root(self) -> str:
root = Path(settings.GAMMA_SBAS_PRODUCT_ROOT or Path(settings.TIMESERIES_PRODUCT_DIR) / "sbas")
run_root = root / "runs"
run_root.mkdir(parents=True, exist_ok=True)
return _normalize_path(run_root)
def get_work_run_root(self) -> str:
root = Path(settings.GAMMA_SBAS_WORK_ROOT or Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production")
run_root = root / "runs"
run_root.mkdir(parents=True, exist_ok=True)
@@ -1989,6 +2044,9 @@ class SbasInsarCatalogService:
def get_run_roots(self) -> list[str]:
roots = [self.get_run_root()]
work_root = self.get_work_run_root()
if work_root not in roots:
roots.append(work_root)
try:
landsar_root = landsar_sbas_service.configured_run_root()
if landsar_root not in roots:
@@ -1999,17 +2057,17 @@ class SbasInsarCatalogService:
def _iter_run_manifest_paths(self, run_root: Optional[str] = None) -> list[str]:
roots = [run_root] if run_root else self.get_run_roots()
manifest_paths: list[str] = []
manifest_paths_by_run: dict[str, str] = {}
for raw_root in roots:
root = Path(raw_root)
if not root.is_dir():
continue
manifest_paths.extend(
_normalize_path(path)
for path in sorted(root.glob("*/run_manifest.json"))
if self._is_publish_ready(path.parent, _safe_read_json(path))
)
return sorted(dict.fromkeys(manifest_paths))
for path in sorted(root.glob("*/run_manifest.json")):
if not self._is_publish_ready(path.parent, _safe_read_json(path)):
continue
run_id = path.parent.name
manifest_paths_by_run.setdefault(run_id, _normalize_path(path))
return list(manifest_paths_by_run.values())
@staticmethod
def _is_landsar_manifest(manifest: dict[str, Any]) -> bool:
@@ -2295,9 +2353,12 @@ class SbasInsarCatalogService:
if self._is_landsar_manifest(manifest):
return self._build_landsar_product(run_dir, manifest_file, manifest)
detail = sbas_insar_production_service.get_run_detail(run_dir.name)
coverage = detail.get("geographic_coverage") or {}
stack_manifest = _safe_read_json(run_dir / "stack_manifest.json")
try:
detail = sbas_insar_production_service.get_run_detail(run_dir.name)
coverage = detail.get("geographic_coverage") or {}
except FileNotFoundError:
coverage = sbas_insar_production_service._build_run_geographic_coverage(run_dir, manifest)
monitor_summary = _safe_read_json(run_dir / "monitor_points_summary.json")
workflow_summary = _safe_read_json(run_dir / "workflow_summary.json")
is_expert_gamma = self._is_expert_gamma_manifest(manifest, run_dir)
backend/app/services/sbas_insar_production_service.py
+132 -2
View File
@@ -786,8 +786,20 @@ class SbasInsarProductionService:
}
def __init__(self) -> None:
self.trial_root = Path(settings.BACKEND_DIR) / "runtime" / "gamma_ipta_trials"
self.production_root = Path(settings.GAMMA_SBAS_WORK_ROOT or (Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production"))
project_drive = Path(settings.PROJECT_ROOT).drive
default_runtime_root = Path(f"{project_drive}\\production_runtime") if project_drive else Path(settings.PROJECT_ROOT) / "runtime"
self.trial_root = Path(settings.GAMMA_SBAS_TRIAL_ROOT or default_runtime_root / "gamma_ipta_trials")
self.production_root = Path(settings.GAMMA_SBAS_WORK_ROOT or default_runtime_root / "sbas_insar_work")
self.product_root = Path(settings.GAMMA_SBAS_PRODUCT_ROOT or Path(settings.TIMESERIES_PRODUCT_DIR) / "sbas")
def product_run_root(self) -> Path:
return self.product_root / "runs"
def product_run_dir(self, run_id: str) -> Path:
clean_id = str(run_id or "").strip()
if not clean_id or Path(clean_id).name != clean_id:
raise ValueError("invalid run id")
return self.product_run_root() / clean_id
def get_capabilities(self) -> dict[str, Any]:
return {
@@ -797,6 +809,7 @@ class SbasInsarProductionService:
"implementation_state": "expert_manifest_script_runner_primary",
"trial_root": str(self.trial_root),
"production_root": str(self.production_root),
"product_root": str(self.product_root),
"min_common_overlap_ratio": self._effective_min_common_overlap_ratio(None),
"workflow_runner": {
"enabled": bool(settings.GAMMA_SBAS_ENABLED),
@@ -3412,6 +3425,8 @@ class SbasInsarProductionService:
self._write_json(run_dir / "monitor_points_summary.json", summary)
self._write_json(manifest_path, manifest)
self._refresh_command_manifest_after_monitor_points(run_dir, manifest)
if manifest["status"] == "MONITOR_POINTS_READY":
self.sync_product_package(run_id)
return self.get_run_detail(run_id)
def list_trial_runs(self) -> dict[str, Any]:
@@ -3828,6 +3843,8 @@ class SbasInsarProductionService:
run_manifest["next_stage"] = "fix_workflow"
self._write_json(manifest_path, run_manifest)
self._write_json(run_dir / "workflow_summary.json", summary)
if run_manifest.get("status") == "WORKFLOW_COMPLETED":
self.sync_product_package(run_id)
return self.get_run_detail(run_id)
@staticmethod
@@ -5204,6 +5221,7 @@ class SbasInsarProductionService:
errors.append(f"{label} does not point to an existing Gamma DEM + .par pair: {raw_path}")
cache_roots = [
Path(settings.PYINT_DEM_ROOT),
Path(settings.BACKEND_DIR) / "runtime" / "pyint_dem",
Path(settings.BACKEND_DIR) / "runtime" / "pyint_dem_cache",
]
@@ -7138,6 +7156,118 @@ class SbasInsarProductionService:
out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
return out_path
@staticmethod
def _copy_file_if_newer(source: Path, target: Path) -> bool:
if not source.is_file():
return False
target.parent.mkdir(parents=True, exist_ok=True)
try:
source_stat = source.stat()
target_stat = target.stat() if target.exists() else None
if (
target_stat is not None
and target_stat.st_size == source_stat.st_size
and int(target_stat.st_mtime) >= int(source_stat.st_mtime)
):
return False
shutil.copy2(source, target)
return True
except OSError:
shutil.copy2(source, target)
return True
@staticmethod
def _copy_tree_files(source_dir: Path, target_dir: Path) -> tuple[int, int]:
if not source_dir.is_dir():
return 0, 0
copied = 0
skipped = 0
for source in source_dir.rglob("*"):
if not source.is_file():
continue
target = target_dir / source.relative_to(source_dir)
if SbasInsarProductionService._copy_file_if_newer(source, target):
copied += 1
else:
skipped += 1
return copied, skipped
def sync_product_package(self, run_id: str) -> dict[str, Any]:
run_dir = self._resolve_run_dir(run_id)
product_dir = self.product_run_dir(run_dir.name)
product_dir.mkdir(parents=True, exist_ok=True)
copied = 0
skipped = 0
for relative in (
"run_manifest.json",
"stack_manifest.json",
"pair_network.json",
"workflow_summary.json",
"monitor_points_summary.json",
"product_summary.json",
"quality_summary.json",
"gamma_command_manifest.json",
"expert_command_audit.json",
):
if self._copy_file_if_newer(run_dir / relative, product_dir / relative):
copied += 1
else:
skipped += 1
for dirname in ("publish",):
tree_copied, tree_skipped = self._copy_tree_files(run_dir / dirname, product_dir / dirname)
copied += tree_copied
skipped += tree_skipped
for relative in (
"diff_dir/bprep_file.png",
"diff_dir/mean.cc_mask.bmp",
"sbas/final_unw_tab",
):
if self._copy_file_if_newer(run_dir / relative, product_dir / relative):
copied += 1
else:
skipped += 1
final_tab = run_dir / "sbas" / "final_unw_tab"
diff_dir = run_dir / "diff_dir"
pair_ids: list[str] = []
if final_tab.is_file():
for line in final_tab.read_text(encoding="utf-8", errors="ignore").splitlines():
raw_path = line.strip().split()[0] if line.strip() else ""
if not raw_path:
continue
name = Path(self._path_to_windows(raw_path) or raw_path).name
pair_id = name.replace(".unw.atmsub_1", "").replace(".unw", "")
if pair_id and pair_id not in pair_ids:
pair_ids.append(pair_id)
qcs = [diff_dir / f"{pair_id}.adf.unw.bmp" for pair_id in pair_ids if (diff_dir / f"{pair_id}.adf.unw.bmp").is_file()]
if not qcs and diff_dir.is_dir():
qcs = sorted(diff_dir.glob("*.adf.unw.bmp"))
if len(qcs) > 3:
last_index = len(qcs) - 1
indexes = sorted({round(index * last_index / 2) for index in range(3)})
qcs = [qcs[index] for index in indexes]
for source in qcs:
target = product_dir / source.relative_to(run_dir)
if self._copy_file_if_newer(source, target):
copied += 1
else:
skipped += 1
marker = {
"schema": "insar.gamma-sbas-product-package/v1",
"run_id": run_dir.name,
"source_run_dir": str(run_dir),
"product_run_dir": str(product_dir),
"copied_files": copied,
"skipped_files": skipped,
"synced_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
}
self._write_json(product_dir / "product_package_manifest.json", marker)
return marker
@staticmethod
def _write_json(path: Path, payload: dict[str, Any]) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
backend/app/services/wsl_runtime_registry.py
+9 -1
View File
@@ -24,6 +24,14 @@ def _project_file_windows(*relative_parts: str) -> str:
return os.path.normpath(str(Path(settings.PROJECT_ROOT, *relative_parts)))
def _default_runtime_root_windows() -> str:
root = os.path.normpath(str(settings.PROJECT_ROOT))
drive, _tail = os.path.splitdrive(root)
if drive:
return os.path.join(drive + os.sep, "production_runtime")
return os.path.join(root, "runtime")
@dataclass(frozen=True)
class WslRuntimeDefinition:
runtime_id: str
@@ -106,7 +114,7 @@ def build_wsl_runtime_registry() -> WslRuntimeRegistry:
).strip()
broker_job_root_windows = os.path.normpath(
settings.WSL_BROKER_JOB_ROOT
or os.path.join(settings.BACKEND_DIR, "runtime", "wsl_jobs")
or os.path.join(_default_runtime_root_windows(), "wsl_jobs")
)
broker_job_root_wsl = _windows_path_to_wsl_mount(broker_job_root_windows)