chore: sync production runtime and docs
This commit is contained in:
+135
-101
@@ -1,35 +1,27 @@
|
||||
# =============================================================================
|
||||
# InSAR Management System v2 — 环境配置
|
||||
# InSAR Management System v2 - 示例环境变量
|
||||
# =============================================================================
|
||||
# 修改后需重启后端服务生效。
|
||||
# 路径使用 Windows 格式(反斜杠),UNC 路径以 \\ 开头。
|
||||
# 说明:
|
||||
# 1. 复制为 .env 后按现场路径修改。
|
||||
# 2. 根目录 .env 是后端与前端共同的事实来源。
|
||||
# 3. 路径请使用 Windows 绝对路径;WSL 中的 Python / 脚本使用 Linux 路径。
|
||||
# =============================================================================
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 数据库
|
||||
# 数据库与认证
|
||||
# -----------------------------------------------------------------------------
|
||||
# 格式: postgresql+asyncpg://用户名:密码@主机:端口/数据库名
|
||||
DATABASE_URL=postgresql+asyncpg://postgres:your_password@localhost:5432/insar_management
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 服务部署
|
||||
# -----------------------------------------------------------------------------
|
||||
PORT=8000
|
||||
BACKEND_BIND_HOST=127.0.0.1
|
||||
NGINX_PATH=C:\nginx\nginx.exe
|
||||
NGINX_HEALTH_URL=http://127.0.0.1/
|
||||
PYTHON_PATH=C:\envs\InSAR\python.exe
|
||||
CORS_ORIGINS=http://localhost
|
||||
UVICORN_LOG_LEVEL=info
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 认证与授权
|
||||
# -----------------------------------------------------------------------------
|
||||
LICENSE_PATH=
|
||||
INIT_ADMIN_USERNAME=admin
|
||||
INIT_ADMIN_PASSWORD=change_me
|
||||
INIT_ADMIN_RESET_PASSWORD=false
|
||||
|
||||
DB_SCHEMA_RESET_ON_MISMATCH=false
|
||||
DB_SCHEMA_RESET_CONFIRM=false
|
||||
|
||||
AUTH_SESSION_COOKIE_NAME=ims_session
|
||||
AUTH_COOKIE_SAMESITE=lax
|
||||
AUTH_COOKIE_SECURE=false
|
||||
AUTH_LOGIN_MAX_FAILURES=5
|
||||
AUTH_LOGIN_WINDOW_SECONDS=300
|
||||
@@ -37,46 +29,53 @@ AUTH_LOGIN_LOCK_SECONDS=600
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 数据目录 — LT-1
|
||||
# 服务启动
|
||||
# -----------------------------------------------------------------------------
|
||||
# 压缩包来源(解包输入)
|
||||
UNPACK_SOURCE_DIRS=\\server\share\archives
|
||||
# 解包后存储 / 雷达数据扫描目录(通常指向同一路径)
|
||||
INSAR_STORAGE_DIRS=\\server\share\storage
|
||||
MONITOR_RADAR_DIRS=\\server\share\storage
|
||||
# D-InSAR 结果目录
|
||||
MONITOR_DINSAR_DIRS=\\server\share\dinsar_results
|
||||
# 精轨源目录(UNC,平铺 .txt)
|
||||
MONITOR_ORBIT_DIR=\\server\share\orbit
|
||||
PORT=18000
|
||||
BACKEND_BIND_HOST=127.0.0.1
|
||||
UVICORN_LOG_LEVEL=info
|
||||
|
||||
PYTHON_PATH=C:\ProgramData\anaconda3\envs\InSAR\python.exe
|
||||
CONDA_EXE=
|
||||
CONDA_ENV_NAME=
|
||||
|
||||
NGINX_PATH=C:\nginx\nginx.exe
|
||||
NGINX_HEALTH_URL=http://127.0.0.1/
|
||||
|
||||
CORS_ORIGINS=http://127.0.0.1:5173,http://localhost
|
||||
CORS_ALLOW_CREDENTIALS=true
|
||||
CORS_STRICT_MODE=false
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 数据目录 — GF3
|
||||
# 源数据目录
|
||||
# -----------------------------------------------------------------------------
|
||||
UNPACK_SOURCE_DIRS=D:\Archives
|
||||
INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool
|
||||
MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool
|
||||
MONITOR_DINSAR_DIRS=D:\DInSARResult
|
||||
MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit
|
||||
|
||||
GF3_SOURCE_DIRS=D:\GF3_L1A_Image
|
||||
GF3_STORAGE_DIRS=D:\GF3_L2_Image
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 数据目录 — 水体监测
|
||||
# -----------------------------------------------------------------------------
|
||||
WATER_RESULTS_DIR=D:\WaterResult
|
||||
HAZARD_POINTS_DIR=D:\Code\Insar_management_system_v2\backend\Point
|
||||
HAZARD_POINTS_FILENAME=Point.shp
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# DEM
|
||||
# 结果目录
|
||||
# -----------------------------------------------------------------------------
|
||||
# D-InSAR 生产用 DEM(SARscape/ENVI 格式,不含扩展名)
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\DEM\YourDemBase
|
||||
# 水体检测用 SRTM DEM 目录(存放 .hgt 文件)
|
||||
SRTM_DEM_DIR=D:\DEM
|
||||
# GF3 几何校正用 DEM
|
||||
GF3_GEO_DEM_PATH=D:\DEM\your_dem.jp2
|
||||
RESULT_PUBLISH_ROOT=D:\production_results
|
||||
DINSAR_PRODUCT_DIR=D:\production_results\dinsar
|
||||
TIMESERIES_PRODUCT_DIR=D:\production_results\timeseries
|
||||
RESULT_QUARANTINE_ROOT=D:\production_results\_quarantine
|
||||
|
||||
RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP=true
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 精轨本地池(本地路径,供各引擎使用)
|
||||
# 通过运维自检"同步精轨池"从 MONITOR_ORBIT_DIR 同步
|
||||
# 轨道池
|
||||
# -----------------------------------------------------------------------------
|
||||
ORBIT_POOL_ENVI=D:\orbit_pools\envi
|
||||
ORBIT_POOL_ISCE2=D:\orbit_pools\isce2
|
||||
@@ -84,17 +83,25 @@ ORBIT_POOL_LANDSAR=
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ENVI / SARscape 引擎
|
||||
# DEM / 其他数据
|
||||
# -----------------------------------------------------------------------------
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape
|
||||
SRTM_DEM_DIR=D:\SRTM30m
|
||||
GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2
|
||||
WATER_RESULTS_DIR=D:\WaterResult
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ENVI / SARscape
|
||||
# -----------------------------------------------------------------------------
|
||||
IDL_EXECUTABLE=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idl.exe
|
||||
IDL_WORKBENCH_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlde.exe
|
||||
# 留空则默认使用项目 backend/runtime/idl_worker
|
||||
IDL_WORKER_RUNTIME_DIR=
|
||||
IDL_WORKER_RUNTIME_DIR=D:\Code\Insar_management_system_v2\backend\runtime\idl_worker
|
||||
|
||||
IDL_WORKER_DEFAULT_TIMEOUT_SECONDS=14400
|
||||
IDL_WORKER_MAX_TIMEOUT_SECONDS=43200
|
||||
IDL_JOB_MAX_ATTEMPTS=1
|
||||
IDL_JOB_RETRY_DELAY_SECONDS=180
|
||||
|
||||
# D-InSAR 自定义处理参数
|
||||
IDL_DINSAR_CUSTOM_TARGET_RESOLUTION_M=10.0
|
||||
IDL_DINSAR_CUSTOM_FILTER_METHOD=GOLDSTEIN
|
||||
IDL_DINSAR_CUSTOM_UNWRAP_COH_THRESHOLD=0.05
|
||||
@@ -103,59 +110,60 @@ IDL_DINSAR_CUSTOM_GCP_NUMBER=100
|
||||
IDL_DINSAR_CUSTOM_GEOCODING_COH_THRESHOLD=0.0
|
||||
IDL_DINSAR_CUSTOM_GEOCODING_PIXEL_SIZE_M=10.0
|
||||
|
||||
# ENVI 子进程超时与稳定性检测
|
||||
ENVI_TASK_TIMEOUT_SECONDS=21600
|
||||
ENVI_FILE_STALE_SECONDS=43200
|
||||
ENVI_STABILITY_CHECK_INTERVAL=15
|
||||
ENVI_STABILITY_ROUNDS=3
|
||||
ENVI_STABILITY_MAX_WAIT=3600
|
||||
ENVI_TASK_TIMEOUT_SECONDS=14400
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ISCE2 引擎(需要 WSL + ISCE2 环境)
|
||||
# WSL 共享运行时
|
||||
# -----------------------------------------------------------------------------
|
||||
WSL_DISTRO=Ubuntu-24.04
|
||||
WSL_SHARED_CONDA_ENV=insar_wsl_v1
|
||||
WSL_SHARED_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
WSL_BROKER_JOB_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\wsl_jobs
|
||||
|
||||
ISCE2_RUNTIME_ID=isce2_runtime_v1
|
||||
PYINT_RUNTIME_ID=gamma_pyint_runtime_v1
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ISCE2 D-InSAR
|
||||
# -----------------------------------------------------------------------------
|
||||
ISCE2_ENABLED=true
|
||||
ISCE2_WSL_DISTRO=Ubuntu-24.04
|
||||
ISCE2_PYTHON=/home/administrator/miniconda3/envs/isce2/bin/python
|
||||
ISCE2_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
ISCE2_PROFILE=lt1_stripmap
|
||||
ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/isce2/lib/python3.11/site-packages/isce/applications/stripmapApp.py
|
||||
ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/applications/stripmapApp.py
|
||||
ISCE2_PIPELINE_SCRIPT=
|
||||
# DEM:留空则自动从 IDL_DINSAR_DEM_BASE_FILE 派生(加 .wgs84 后缀)
|
||||
ISCE2_DEM_PATH=
|
||||
ISCE2_WORK_ROOT=D:\isce2_work
|
||||
ISCE2_OUTPUT_ROOT=D:\isce2_output
|
||||
ISCE2_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84
|
||||
ISCE2_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\isce2_work
|
||||
ISCE2_OUTPUT_ROOT=D:\production_results\dinsar
|
||||
ISCE2_PER_TASK_TIMEOUT_SECONDS=43200
|
||||
ISCE2_SMOKE_TEST_ENABLED=false
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# PyINT / Gamma 引擎(需要 WSL + PyINT + GAMMA)
|
||||
# Gamma / PyINT
|
||||
# -----------------------------------------------------------------------------
|
||||
PYINT_ENABLED=false
|
||||
# 留空时默认跟随 ISCE2_WSL_DISTRO
|
||||
PYINT_WSL_DISTRO=
|
||||
# 留空时默认跟随 ISCE2_PYTHON
|
||||
PYINT_WSL_PYTHON=
|
||||
# 留空时默认使用项目内置的 third_party\PyINT
|
||||
PYINT_HOME=
|
||||
# 留空时默认使用 PYINT_HOME\pyint\pyintApp.py
|
||||
PYINT_APP_SCRIPT=
|
||||
PYINT_ENABLED=true
|
||||
PYINT_WSL_DISTRO=Ubuntu-24.04
|
||||
PYINT_WSL_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
PYINT_HOME=D:\Code\Insar_management_system_v2\third_party\PyINT
|
||||
PYINT_APP_SCRIPT=D:\Code\Insar_management_system_v2\third_party\PyINT\pyint\pyintApp.py
|
||||
PYINT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_templates
|
||||
PYINT_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_work
|
||||
PYINT_OUTPUT_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_output
|
||||
PYINT_OUTPUT_ROOT=D:\production_results\dinsar
|
||||
PYINT_DEM_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_dem
|
||||
# 可选: local_fabdem / opentopo / prepared_file
|
||||
PYINT_DEM_MODE=local_fabdem
|
||||
PYINT_FABDEM_ROOT=
|
||||
# 现有 DEM 复用模式下可指向:
|
||||
# 1. 已有 Gamma DEM 基础文件(需同名 .par)
|
||||
# 2. 系统现有 DEM 基础文件(需至少有 .xml/.hdr/.vrt,运行时按任务范围裁剪并转换为 Gamma DEM)
|
||||
# 留空时会尝试回退到 ISCE2_DEM_PATH / IDL_DINSAR_DEM_BASE_FILE
|
||||
PYINT_PREPARED_DEM_PATH=
|
||||
PYINT_OPENTOPO_DEM_TYPE=SRTMGL1
|
||||
PYINT_DEM_STRICT=true
|
||||
PYINT_ORBIT_POLICY=require_txt
|
||||
PYINT_ORBIT_POOL_TXT=
|
||||
PYINT_ORBIT_POOL_TXT=D:\orbit_pools\envi
|
||||
PYINT_RECORD_INPUT_ASSETS=true
|
||||
PYINT_LT1_PRECISE_ORBIT_ENABLED=true
|
||||
PYINT_LT1_PRECISE_ORBIT_MODE=replace
|
||||
@@ -163,42 +171,45 @@ PYINT_LT1_PRECISE_ORBIT_STRICT=true
|
||||
PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT=false
|
||||
PYINT_LT1_PRECISE_ORBIT_BACKUP=true
|
||||
PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE=5
|
||||
# 可选:source 该脚本后应能找到 create_offset / geocode_back / LT1_import_SLC_from_zipfiles1
|
||||
PYINT_GAMMA_ENV_SCRIPT=
|
||||
PYINT_GAMMA_ENV_SCRIPT=D:\Code\Insar_management_system_v2\deploy\wsl\profiles\gamma_env.sh
|
||||
PYINT_DEFAULT_TIMEOUT_SECONDS=43200
|
||||
PYINT_SMOKE_TEST_ENABLED=false
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 前端 tile-server 接入
|
||||
# 说明:frontend/vite.config.js 会同时读取项目根 .env 与 frontend/.env,
|
||||
# 这里配置的 VITE_TILE_* 会透传给前端,且优先级高于 frontend/.env 同名项
|
||||
# 时序 InSAR
|
||||
# -----------------------------------------------------------------------------
|
||||
# Tile Server
|
||||
# -----------------------------------------------------------------------------
|
||||
VITE_TILE_SERVER_URL=http://127.0.0.1:8910
|
||||
VITE_TILE_SERVER_TOKEN=change_me
|
||||
TIMESERIES_ENABLED=true
|
||||
TIMESERIES_WSL_DISTRO=Ubuntu-24.04
|
||||
TIMESERIES_ENV_NAME=insar_wsl_v1
|
||||
TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
TIMESERIES_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\timeseries_work
|
||||
TIMESERIES_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84
|
||||
TIMESERIES_ORBIT_POOL_ISCE2=D:\orbit_pools\isce2
|
||||
TIMESERIES_EXPERIMENT_ROOT=D:\Code\Insar_management_system_v2\experiments\isce2_sbas_timeseries
|
||||
TIMESERIES_STACK_PREP_SCRIPT=
|
||||
TIMESERIES_MATERIALIZE_SCRIPT=
|
||||
TIMESERIES_PREPARE_DEM_SCRIPT=
|
||||
TIMESERIES_STACK_RUNNER_SCRIPT=
|
||||
TIMESERIES_MINTPY_SBAS_SCRIPT=
|
||||
TIMESERIES_EXPORT_PUBLISH_SCRIPT=
|
||||
TIMESERIES_STACK_WORKFLOW=interferogram
|
||||
TIMESERIES_WSL_STEP_TIMEOUT_SECONDS=7200
|
||||
TIMESERIES_ALLOW_SYNTHETIC_WATER_MASK=true
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 任务调度与 Worker
|
||||
# Worker / 健康检查
|
||||
# -----------------------------------------------------------------------------
|
||||
JOB_WORKER_HEALTH_TIMEOUT=60
|
||||
JOB_WORKER_JOB_HEARTBEAT_INTERVAL=5
|
||||
JOB_WORKER_STALE_RECOVER_INTERVAL=15
|
||||
JOB_WORKER_STALE_RUNNING_SECONDS=7200
|
||||
TASK_TIMEOUT_MINUTES=720
|
||||
JOB_WORKER_HEARTBEAT_INTERVAL=5
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# AI 服务(Ollama)
|
||||
# -----------------------------------------------------------------------------
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
OLLAMA_API_URL=http://127.0.0.1:11434/api/generate
|
||||
DEFAULT_VLM_MODEL=qwen3-vl:30b
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 解包配置
|
||||
# 解包
|
||||
# -----------------------------------------------------------------------------
|
||||
UNPACK_MIN_DISK_SPACE_GB=50
|
||||
UNPACK_SCAN_WORKERS=4
|
||||
@@ -210,7 +221,30 @@ UNPACK_TMP_SUFFIX=.unpack_tmp
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 灾害点
|
||||
# 预览与缓存
|
||||
# -----------------------------------------------------------------------------
|
||||
HAZARD_POINTS_DIR=D:\Code\Insar_management_system_v2\backend\Point
|
||||
HAZARD_POINTS_FILENAME=Point.shp
|
||||
RADAR_THUMBNAIL_MAX_SIZE=1600
|
||||
RADAR_CACHE_WORKERS=2
|
||||
RADAR_GEO_CACHE_WORKERS=2
|
||||
RADAR_GEO_CACHE_VERSION=b1
|
||||
RADAR_GEO_CACHE_QUALITY=84
|
||||
RADAR_PREVIEW_BUILD_ON_DEMAND=true
|
||||
|
||||
DINSAR_THUMBNAIL_MAX_SIZE=1600
|
||||
DINSAR_FOOTPRINT_MAX_SIZE=1000
|
||||
DINSAR_CACHE_WORKERS=4
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# AI / 其他外部服务
|
||||
# -----------------------------------------------------------------------------
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
OLLAMA_API_URL=http://127.0.0.1:11434/api/generate
|
||||
DEFAULT_VLM_MODEL=qwen3-vl:30b
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# 前端 tile server
|
||||
# -----------------------------------------------------------------------------
|
||||
VITE_TILE_SERVER_URL=http://127.0.0.1:8910
|
||||
VITE_TILE_SERVER_TOKEN=change_me
|
||||
|
||||
@@ -1,671 +1,109 @@
|
||||
# InSAR 管理系统 v2 - 技术文档
|
||||
# InSAR Management System v2
|
||||
|
||||
面向内网运行环境的 InSAR 数据管理、生产调度、结果发布与运维自检系统。
|
||||
|
||||
## 文档导航
|
||||
当前项目已经从“以 D-InSAR 结果扫描为中心”的旧形态,收口为“地图主界面 + 生产管理工作台 + 统一结果目录 + 数据库自维护”的架构。
|
||||
|
||||
### 📖 用户文档
|
||||
- [部署指南](#部署指南) - 系统安装、配置、启动
|
||||
- [配置说明](#配置说明) - `.env` 配置项详解
|
||||
- [功能说明](#功能说明) - 各模块功能介绍
|
||||
## 当前状态
|
||||
|
||||
### 🔧 开发文档
|
||||
- [架构设计](#架构设计) - 系统架构和技术栈
|
||||
- [开发规范](#开发规范) - 代码规范和最佳实践
|
||||
- [API 文档](#api-文档) - RESTful API 接口
|
||||
- 顶级生产入口已经统一为“生产管理”。
|
||||
- “生产管理”同时承载 D-InSAR 运行、时序 InSAR 运行、D-InSAR 产物、时序 InSAR 产物。
|
||||
- 前端显示名已经统一为“时序 InSAR”;当前默认接入的是 SBAS 流程,后续可继续扩展 PS-InSAR、SBAS-InSAR 等子类型。
|
||||
- D-InSAR 当前支持 `sarscape` 与 `isce2` 两类引擎;`gamma / pyint` 已按共享 WSL 运行时模型预留接口。
|
||||
- 结果目录已经统一发布到 `RESULT_PUBLISH_ROOT`,默认根目录为 `D:\production_results`。
|
||||
|
||||
### 🔒 安全文档
|
||||
- [安全审计](#安全审计) - 安全问题和修复记录
|
||||
- [部署安全](#部署安全) - 生产环境安全配置
|
||||
## 当前有效目录约定
|
||||
|
||||
---
|
||||
- D-InSAR 发布根目录:`D:\production_results\dinsar`
|
||||
- 时序 InSAR 发布根目录:`D:\production_results\timeseries`
|
||||
- 隔离目录:`D:\production_results\_quarantine`
|
||||
- WSL Broker 作业目录:`backend\runtime\wsl_jobs`
|
||||
|
||||
## 部署指南
|
||||
说明:
|
||||
|
||||
### 系统要求
|
||||
- 现在的文件系统事实来源是 `RESULT_PUBLISH_ROOT` 及其下属目录,不再是历史性的 `backend\result_products` 目录。
|
||||
- 数据库里的 `result_products / result_assets / result_issues` 表仍然保留,并继续作为结果登记与检索的核心表结构。
|
||||
|
||||
**硬件要求**:
|
||||
- CPU:4 核及以上
|
||||
- 内存:16GB 及以上
|
||||
- 硬盘:500GB 及以上(SSD 推荐)
|
||||
## 启动时自维护链路
|
||||
|
||||
**软件要求**:
|
||||
- 操作系统:Windows 10/11 或 Windows Server 2019+
|
||||
- Python:3.9+ (推荐使用 Conda 环境)
|
||||
- PostgreSQL:14+ (含 PostGIS 扩展)
|
||||
- Nginx:1.20+
|
||||
- IDL/ENVI:8.8+ / 5.6+ (可选,用于 D-InSAR 处理)
|
||||
- Ollama:最新版 (可选,用于 AI 诊断)
|
||||
后端启动时会按固定顺序做自维护:
|
||||
|
||||
### 快速开始
|
||||
1. `ensure_database_ready(...)`
|
||||
2. `database.init_db()`
|
||||
3. `root_registry_service.sync_from_settings()`
|
||||
4. `manifest_inventory_service.sync_manifest_roots()`
|
||||
5. `result_catalog_service.bootstrap_catalog_on_startup_clean()`
|
||||
6. `psinsar_catalog_service.bootstrap_catalog_on_startup_clean()`
|
||||
7. `pairing_state_service.bootstrap_pairing_cache_state()`
|
||||
8. `get_health_status(include_external=False)`
|
||||
|
||||
#### 1. 环境准备
|
||||
这意味着系统启动后会自动完成数据库结构校验、根目录登记、结果 catalog 自举、配对缓存状态恢复和一次启动健康检查。
|
||||
|
||||
```bash
|
||||
# 创建 Conda 环境
|
||||
conda create -n InSAR python=3.9
|
||||
conda activate InSAR
|
||||
## 数据库自维护边界
|
||||
|
||||
# 安装依赖
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
当前数据库自维护是“增量自愈”模型,不是完整迁移框架。
|
||||
|
||||
#### 2. 数据库配置
|
||||
- 会自动创建缺失表。
|
||||
- 会自动补齐缺失列。
|
||||
- 会自动执行 `backend/migrations/001` 到 `006` 的 SQL 文件。
|
||||
- 只有同时设置 `DB_SCHEMA_RESET_ON_MISMATCH=true` 和 `DB_SCHEMA_RESET_CONFIRM=true`,才允许破坏性重建。
|
||||
|
||||
```bash
|
||||
# 创建数据库
|
||||
createdb -U postgres insar_management
|
||||
不会自动处理的情况:
|
||||
|
||||
# 启用 PostGIS 扩展(系统会自动执行)
|
||||
```
|
||||
- 字段改名
|
||||
- 字段类型变更
|
||||
- 可空性收紧
|
||||
- 索引或约束漂移
|
||||
- 删除列 / 删除表
|
||||
|
||||
#### 3. 配置文件
|
||||
## 快速启动
|
||||
|
||||
复制 `.env.example` 为 `.env`,修改以下配置:
|
||||
1. 复制 `.env.example` 为 `.env`,按现场路径修改数据库、源数据目录、结果目录、IDL/ENVI、WSL 运行时配置。
|
||||
2. 准备 PostgreSQL + PostGIS。
|
||||
3. 准备 Windows Python 运行环境。
|
||||
4. 如需 ISCE2 / Gamma / 时序 InSAR,准备 WSL2 Ubuntu 运行时与共享 conda 环境。
|
||||
5. 启动后端与 Worker。
|
||||
|
||||
```bash
|
||||
# 数据库连接
|
||||
DATABASE_URL=postgresql+asyncpg://postgres:your_password@localhost:5432/insar_management
|
||||
常用启动方式:
|
||||
|
||||
# 数据目录(根据实际环境修改)
|
||||
MONITOR_RADAR_DIRS=\\\\server\\share\\RadarData
|
||||
MONITOR_ORBIT_DIR=\\\\server\\share\\Orbit
|
||||
MONITOR_DINSAR_DIRS=\\\\server\\share\\DinsarResults
|
||||
|
||||
# IDL/ENVI 路径(如果使用 D-InSAR 功能)
|
||||
IDL_EXECUTABLE=C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idl.exe
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\\SRTM30m\\SRTMDEM_RSP_SARscape
|
||||
|
||||
# 服务端口
|
||||
PORT=8000
|
||||
```
|
||||
|
||||
#### 4. 启动系统
|
||||
|
||||
**方式 1:使用启动脚本(推荐)**
|
||||
```bash
|
||||
# Windows
|
||||
```powershell
|
||||
start_system.bat
|
||||
```
|
||||
|
||||
**方式 2:手动启动**
|
||||
```bash
|
||||
# 启动后端
|
||||
或分别启动:
|
||||
|
||||
```powershell
|
||||
python run_backend.py
|
||||
|
||||
# 启动 Nginx(另一个终端)
|
||||
nginx.exe
|
||||
|
||||
# 启动 Job Worker(另一个终端)
|
||||
python -m backend.app.job_worker
|
||||
python run_worker.py
|
||||
```
|
||||
|
||||
#### 5. 访问系统
|
||||
说明:
|
||||
|
||||
打开浏览器访问:`http://localhost`
|
||||
- Worker 必须常驻,否则生产任务、扫描任务、解包任务不会执行。
|
||||
- 当前扫描模式默认是 manual-only,不会自动开启后台定时扫描。
|
||||
|
||||
默认管理员账号:
|
||||
- 用户名:`admin`
|
||||
- 密码:`.env` 中的 `INIT_ADMIN_PASSWORD`
|
||||
## 文档导航
|
||||
|
||||
---
|
||||
- [当前状态快照](docs/CURRENT_STATUS_20260425.md)
|
||||
- [部署与运行说明](docs/DEPLOYMENT.md)
|
||||
- [数据库自维护审计](docs/DATABASE_SELF_MAINTENANCE_AUDIT_20260425.md)
|
||||
- [文档索引](docs/INDEX.md)
|
||||
|
||||
## 配置说明
|
||||
当前仍在生效的专题设计文档:
|
||||
|
||||
### 核心配置
|
||||
- [多引擎结果目录设计](docs/PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md)
|
||||
- [WSL 共享运行时重构](docs/WSL_RUNTIME_REFACTOR_DESIGN_20260422.md)
|
||||
- [ISCE2 托管 D-InSAR 实施说明](docs/ISCE2_MANAGED_DINSAR_IMPLEMENTATION_20260424.md)
|
||||
- [ISCE2 生产可靠性加固设计](docs/ISCE2_PRODUCTION_RELIABILITY_HARDENING_DESIGN_20260424.md)
|
||||
|
||||
#### 数据库配置
|
||||
## 2026-04-25 运行态摘要
|
||||
|
||||
```bash
|
||||
# PostgreSQL 连接字符串
|
||||
DATABASE_URL=postgresql+asyncpg://用户名:密码@主机:端口/数据库名
|
||||
```
|
||||
基于当前代码和现场数据库检查结果:
|
||||
|
||||
#### 数据目录配置
|
||||
- 数据库 schema 与 ORM 一致,无缺表、缺列、类型漂移、可空性漂移。
|
||||
- D-InSAR catalog 已登记 19 个产品。
|
||||
- 时序 InSAR catalog 当前为 0 个产品。
|
||||
- 产品包 schema 已统一为 `insar.product-package/v1`。
|
||||
- WSL 共享运行时健康,当前共享环境为 `insar_wsl_v1`。
|
||||
|
||||
```bash
|
||||
# 雷达数据监控目录(支持 UNC 路径,多个路径用逗号分隔)
|
||||
MONITOR_RADAR_DIRS=\\\\server1\\share\\data,\\\\server2\\share\\data
|
||||
|
||||
# 轨道数据目录
|
||||
MONITOR_ORBIT_DIR=\\\\server\\share\\orbit
|
||||
|
||||
# D-InSAR 结果目录
|
||||
MONITOR_DINSAR_DIRS=\\\\server\\share\\results
|
||||
|
||||
# 水体监测结果目录
|
||||
WATER_RESULTS_DIR=\\\\server\\share\\water
|
||||
```
|
||||
|
||||
#### 服务配置
|
||||
|
||||
```bash
|
||||
# 后端服务端口
|
||||
PORT=8000
|
||||
|
||||
# 后端绑定地址(127.0.0.1 仅本地,0.0.0.0 允许外部访问)
|
||||
BACKEND_BIND_HOST=127.0.0.1
|
||||
|
||||
# Ollama 服务地址(用于 AI 诊断)
|
||||
OLLAMA_BASE_URL=http://127.0.0.1:11434
|
||||
|
||||
# Nginx 健康检查地址
|
||||
NGINX_HEALTH_URL=http://127.0.0.1/
|
||||
```
|
||||
|
||||
#### PROJ 数据库配置
|
||||
|
||||
```bash
|
||||
# PROJ 数据库路径(用于地理空间坐标转换)
|
||||
# 留空则自动使用 pyproj 的 PROJ 数据库(推荐)
|
||||
# PROJ_LIB=C:\\path\\to\\proj\\data
|
||||
```
|
||||
|
||||
**重要提示**:
|
||||
- PostgreSQL PostGIS 的 PROJ 数据库版本较旧(v2),可能导致坐标转换错误
|
||||
- 推荐使用 pyproj 的 PROJ 数据库(v9.2+)
|
||||
- 详见:[PROJ 配置指南](#proj-配置指南)
|
||||
|
||||
#### IDL/ENVI 配置
|
||||
|
||||
```bash
|
||||
# IDL 可执行文件路径
|
||||
IDL_EXECUTABLE=C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idl.exe
|
||||
|
||||
# ENVI Workbench 路径
|
||||
IDL_WORKBENCH_PATH=C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idlde.exe
|
||||
|
||||
# DEM 文件路径(D-InSAR 处理必需)
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\\SRTM30m\\SRTMDEM_RSP_SARscape
|
||||
|
||||
# 工作目录
|
||||
IDL_WORKER_RUNTIME_DIR=<项目根目录>\\backend\\runtime\\idl_worker
|
||||
|
||||
# 超时配置
|
||||
IDL_WORKER_DEFAULT_TIMEOUT_SECONDS=14400
|
||||
IDL_WORKER_MAX_TIMEOUT_SECONDS=43200
|
||||
```
|
||||
|
||||
#### 解包配置
|
||||
|
||||
```bash
|
||||
# 压缩包源目录(支持 UNC 路径)
|
||||
UNPACK_SOURCE_DIRS=\\\\server\\share\\archives
|
||||
|
||||
# 解包目标目录
|
||||
INSAR_STORAGE_DIRS=\\\\server\\share\\storage
|
||||
|
||||
# 最小剩余空间(GB)
|
||||
UNPACK_MIN_DISK_SPACE_GB=50
|
||||
|
||||
# 扫描压缩包时的并发数
|
||||
UNPACK_SCAN_WORKERS=4
|
||||
|
||||
# 多压缩包并行解包数(单包仍为单线程)
|
||||
UNPACK_EXTRACT_WORKERS=4
|
||||
|
||||
# 解包后是否删除压缩包
|
||||
UNPACK_DELETE_ARCHIVE=true
|
||||
```
|
||||
|
||||
#### 解包任务说明
|
||||
|
||||
- 前端“解包”按钮和 `POST /api/unpack/run` 都会创建后台任务,实际执行依赖 `run_worker.py`。
|
||||
- 后端当前加载的解包实现为 `scripts/unpack_archives_parallel.py`;旧的 `scripts/unpack_archives.py` 保留作历史实现参考,不再作为默认入口。
|
||||
- 当前并发模型为“扫描并发 + 多压缩包并行解包”。单个 `.tar.gz` 仍按单包单线程处理,不会把一个压缩包拆成多核并行。
|
||||
- `UNPACK_SCAN_WORKERS` 控制源目录扫描并发,适合目录层级深、来源路径多的场景。
|
||||
- `UNPACK_EXTRACT_WORKERS` 控制同时解包的压缩包数量。建议先从 `4` 起步,根据源盘/目标盘吞吐逐步调整到 `6` 或 `8`;如果压缩包和目标目录都在同一台 NAS,上限通常受 I/O 而不是 CPU 限制。
|
||||
- 并行解包会在任务内部做目标盘空间预留,避免多个压缩包同时判断“空间足够”后把盘写满。
|
||||
- 运行日志与进度文件位于 `logs/tasks/unpacker/`;并行实现使用 `unpacker_parallel_YYYYMMDD.*` 文件名。
|
||||
|
||||
### 部署场景配置
|
||||
|
||||
#### 场景 1:本地开发
|
||||
|
||||
```bash
|
||||
DATABASE_URL=postgresql+asyncpg://postgres:password@localhost:5432/insar_management
|
||||
BACKEND_BIND_HOST=127.0.0.1
|
||||
PORT=8000
|
||||
```
|
||||
|
||||
#### 场景 2:Docker 容器
|
||||
|
||||
```bash
|
||||
DATABASE_URL=postgresql+asyncpg://postgres:password@db:5432/insar_management
|
||||
BACKEND_BIND_HOST=0.0.0.0
|
||||
PORT=8000
|
||||
```
|
||||
|
||||
#### 场景 3:远程 Ollama
|
||||
|
||||
```bash
|
||||
OLLAMA_BASE_URL=http://192.168.1.100:11434
|
||||
```
|
||||
|
||||
#### 场景 4:客户现场
|
||||
|
||||
```bash
|
||||
# 根据客户环境修改所有路径和地址
|
||||
DATABASE_URL=postgresql+asyncpg://user:pass@192.168.1.10:5432/insar
|
||||
MONITOR_RADAR_DIRS=D:\\InSAR_Data\\Radar
|
||||
MONITOR_ORBIT_DIR=D:\\InSAR_Data\\Orbit
|
||||
MONITOR_DINSAR_DIRS=D:\\InSAR_Data\\Results
|
||||
IDL_EXECUTABLE=C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idl.exe
|
||||
IDL_DINSAR_DEM_BASE_FILE=D:\\DEM\\SRTM30m\\SRTMDEM_RSP_SARscape
|
||||
BACKEND_BIND_HOST=127.0.0.1
|
||||
PORT=8080
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 功能说明
|
||||
|
||||
### 数据管理
|
||||
|
||||
#### 雷达数据管理
|
||||
- 自动扫描网络共享目录
|
||||
- 解析影像元数据(时间、位置、极化方式)
|
||||
- 生成预览缩略图
|
||||
- 空间范围可视化
|
||||
|
||||
#### 轨道数据管理
|
||||
- 轨道文件自动关联
|
||||
- 时间窗口匹配
|
||||
|
||||
#### D-InSAR 结果管理
|
||||
- 结果自动入库
|
||||
- 形变图可视化
|
||||
- 统计分析
|
||||
|
||||
### D-InSAR 处理
|
||||
|
||||
#### 自动化处理流程
|
||||
1. 影像导入(SARscape Import)
|
||||
2. 影像配准(Coregistration)
|
||||
3. 干涉处理(Interferogram Generation)
|
||||
4. 相位滤波(Goldstein Filter)
|
||||
5. 相位解缠(Phase Unwrapping)
|
||||
6. 相位转位移(Phase to Displacement)
|
||||
7. 地理编码(Geocoding)
|
||||
|
||||
#### 批量处理
|
||||
- 支持多对影像批量处理
|
||||
- 进度实时显示
|
||||
- 失败自动重试
|
||||
|
||||
### 水体监测
|
||||
|
||||
#### 单景预处理
|
||||
- 多视处理(Multilooking)
|
||||
- 地理编码(Geocoding)
|
||||
- 辐射定标
|
||||
|
||||
#### 洪涝检测
|
||||
- 两景配对
|
||||
- 洪涝分类
|
||||
- MRF 精化(可选)
|
||||
|
||||
### AI 诊断
|
||||
|
||||
#### 多模态分析
|
||||
- 支持 Ollama 本地部署
|
||||
- 推荐模型:Qwen2-VL、LLaVA、MiniCPM-V
|
||||
- 自动分析形变图
|
||||
- 生成诊断报告
|
||||
|
||||
#### Prompt 模板
|
||||
- Quick:快速分析
|
||||
- Standard:标准分析(推荐)
|
||||
- Detailed:详细分析
|
||||
|
||||
### 隐患点管理
|
||||
|
||||
- Shapefile 导入
|
||||
- 空间查询
|
||||
- 与影像范围关联
|
||||
|
||||
### 系统管理
|
||||
|
||||
#### 用户管理
|
||||
- 管理员/只读用户
|
||||
- 权限控制
|
||||
- 审计日志
|
||||
|
||||
#### 运维自检
|
||||
- 数据库健康检查
|
||||
- IDL/ENVI 状态检查
|
||||
- Ollama 状态检查
|
||||
- Nginx 状态检查
|
||||
- 一致性检测
|
||||
|
||||
#### 日志管理
|
||||
- 统一日志目录
|
||||
- 日志查看
|
||||
- 日志删除(仅管理员)
|
||||
|
||||
---
|
||||
|
||||
## 架构设计
|
||||
|
||||
### 技术栈
|
||||
|
||||
**后端**:
|
||||
- FastAPI:Web 框架
|
||||
- SQLAlchemy:ORM
|
||||
- PostgreSQL + PostGIS:数据库
|
||||
- asyncpg:异步数据库驱动
|
||||
- httpx:HTTP 客户端
|
||||
- Pydantic:数据验证
|
||||
|
||||
**前端**:
|
||||
- React:UI 框架
|
||||
- Leaflet:地图组件
|
||||
- Chart.js:图表组件
|
||||
- Axios:HTTP 客户端
|
||||
|
||||
**地理空间**:
|
||||
- GDAL/OGR:地理数据处理
|
||||
- pyproj:坐标转换
|
||||
- Shapely:几何运算
|
||||
- Rasterio:栅格数据处理
|
||||
|
||||
**IDL/ENVI**:
|
||||
- envipyengine:Python 调用 ENVI
|
||||
- SARscape:SAR 数据处理
|
||||
|
||||
**AI**:
|
||||
- Ollama:本地 LLM 部署
|
||||
- 多模态模型:Qwen2-VL、LLaVA、MiniCPM-V
|
||||
|
||||
### 系统架构
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Nginx │
|
||||
│ (反向代理 + 静态文件) │
|
||||
└────────────┬────────────────────────────────────────────┘
|
||||
│
|
||||
├─────────────────────────────────────────────┐
|
||||
│ │
|
||||
┌────────────▼──────────┐ ┌────────────▼──────────┐
|
||||
│ FastAPI Backend │ │ React Frontend │
|
||||
│ (RESTful API) │ │ (SPA) │
|
||||
└────────────┬──────────┘ └───────────────────────┘
|
||||
│
|
||||
├─────────────────────────────────────────────┐
|
||||
│ │
|
||||
┌────────────▼──────────┐ ┌────────────▼──────────┐
|
||||
│ PostgreSQL + PostGIS │ │ Job Worker │
|
||||
│ (数据存储) │ │ (后台任务) │
|
||||
└───────────────────────┘ └────────────┬──────────┘
|
||||
│
|
||||
┌────────────▼──────────┐
|
||||
│ IDL/ENVI │
|
||||
│ (D-InSAR 处理) │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
### 数据流
|
||||
|
||||
#### 雷达数据扫描流程
|
||||
```
|
||||
用户触发扫描 → FastAPI → 扫描网络目录 → 解析元数据 →
|
||||
生成缩略图 → 入库 → 返回结果
|
||||
```
|
||||
|
||||
#### D-InSAR 处理流程
|
||||
```
|
||||
用户提交任务 → FastAPI → 创建 SystemTask → Job Worker →
|
||||
调用 IDL/ENVI → 执行 SARscape 流程 → 更新进度 →
|
||||
结果入库 → 通知前端
|
||||
```
|
||||
|
||||
#### AI 诊断流程
|
||||
```
|
||||
用户创建诊断 → FastAPI → 创建 SystemTask → Job Worker →
|
||||
加载影像 → 查询隐患点 → 调用 Ollama → 解析结果 →
|
||||
保存报告 → 通知前端
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 开发规范
|
||||
|
||||
### 代码规范
|
||||
|
||||
#### Python
|
||||
- 遵循 PEP 8
|
||||
- 使用类型注解
|
||||
- 函数/类添加 docstring
|
||||
- 异步函数使用 `async/await`
|
||||
|
||||
#### JavaScript/React
|
||||
- 使用 ES6+ 语法
|
||||
- 组件使用函数式组件 + Hooks
|
||||
- 使用 PropTypes 或 TypeScript
|
||||
|
||||
### Git 规范
|
||||
|
||||
#### 提交信息格式
|
||||
```
|
||||
<type>(<scope>): <subject>
|
||||
|
||||
<body>
|
||||
|
||||
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
|
||||
```
|
||||
|
||||
**type**:
|
||||
- `feat`: 新功能
|
||||
- `fix`: Bug 修复
|
||||
- `docs`: 文档更新
|
||||
- `refactor`: 代码重构
|
||||
- `test`: 测试相关
|
||||
- `chore`: 构建/工具相关
|
||||
|
||||
**scope**:
|
||||
- `backend`: 后端
|
||||
- `frontend`: 前端
|
||||
- `db`: 数据库
|
||||
- `config`: 配置
|
||||
- `docs`: 文档
|
||||
|
||||
### 安全规范
|
||||
|
||||
1. **不允许硬编码**
|
||||
- 所有路径、IP、端口必须通过 `.env` 配置
|
||||
- 不允许在代码中硬编码敏感信息
|
||||
|
||||
2. **输入验证**
|
||||
- 所有用户输入必须验证
|
||||
- 防止 SQL 注入、XSS、路径遍历
|
||||
|
||||
3. **权限控制**
|
||||
- 敏感操作仅管理员可执行
|
||||
- API 端点添加权限检查
|
||||
|
||||
4. **日志脱敏**
|
||||
- 日志中不记录密码、Token
|
||||
- 敏感信息使用 `***` 替代
|
||||
|
||||
---
|
||||
|
||||
## PROJ 配置指南
|
||||
|
||||
### 问题背景
|
||||
|
||||
PostgreSQL PostGIS 扩展自带 PROJ 数据库 v2(旧版本),而 GDAL/pyproj 期望 PROJ v3+(新版本)。版本不匹配会导致:
|
||||
- 坐标转换警告
|
||||
- 坐标转换精度下降
|
||||
- 某些坐标系不支持
|
||||
|
||||
### 解决方案
|
||||
|
||||
系统会自动使用 pyproj 的 PROJ 数据库(v9.2+),无需手动配置。
|
||||
|
||||
启动后端时,会看到:
|
||||
```
|
||||
[*] Configuring PROJ database...
|
||||
[*] Auto-detected PROJ_LIB: C:\\Users\\...\\pyproj\\proj_dir\\share\\proj
|
||||
[*] PROJ version: 9.2.0
|
||||
```
|
||||
|
||||
### 手动配置(可选)
|
||||
|
||||
如果需要使用自定义 PROJ 数据库,在 `.env` 中设置:
|
||||
|
||||
```bash
|
||||
PROJ_LIB=C:\\path\\to\\proj\\data
|
||||
```
|
||||
|
||||
### 验证配置
|
||||
|
||||
运行诊断脚本:
|
||||
```bash
|
||||
conda activate InSAR
|
||||
python scripts/check_proj.py
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 安全审计
|
||||
|
||||
### 审计记录
|
||||
|
||||
#### 2026-03-12:当前有效安全审计记录
|
||||
- 已区分“开发机阶段可接受暴露”和“上线前必须修复的问题”
|
||||
- 当前结论以 `docs/SECURITY_AUDIT_2026-03-12.md` 为准
|
||||
- 详见:`docs/SECURITY_AUDIT_2026-03-12.md`
|
||||
|
||||
#### 历史审计归档
|
||||
- 2026-03-04 安全审计:`docs/archive/SECURITY_AUDIT_2026-03-04.md`
|
||||
- 2026-03-05 硬编码审计:`docs/archive/HARDCODE_AUDIT_2026-03-05.md`
|
||||
- 其他过程性安全文档已统一整理到:`docs/archive/`
|
||||
|
||||
### 安全配置
|
||||
|
||||
#### 生产环境建议
|
||||
|
||||
1. **数据库安全**
|
||||
```bash
|
||||
# 使用强密码
|
||||
DATABASE_URL=postgresql+asyncpg://user:strong_password@localhost:5432/insar
|
||||
|
||||
# 限制数据库访问
|
||||
# 在 PostgreSQL pg_hba.conf 中配置
|
||||
```
|
||||
|
||||
2. **CORS 配置**
|
||||
```bash
|
||||
# 生产环境使用明确的域名
|
||||
CORS_ORIGINS=https://insar.yourdomain.com
|
||||
```
|
||||
|
||||
3. **Cookie 安全**
|
||||
```bash
|
||||
# HTTPS 环境启用 Secure 标志
|
||||
AUTH_COOKIE_SECURE=true
|
||||
```
|
||||
|
||||
4. **登录限流**
|
||||
```bash
|
||||
# 防止暴力破解
|
||||
AUTH_LOGIN_MAX_FAILURES=5
|
||||
AUTH_LOGIN_WINDOW_SECONDS=300
|
||||
AUTH_LOGIN_LOCK_SECONDS=600
|
||||
```
|
||||
|
||||
5. **Nginx 安全头**
|
||||
```nginx
|
||||
# 在 nginx.conf 中添加
|
||||
add_header X-Frame-Options "SAMEORIGIN";
|
||||
add_header X-Content-Type-Options "nosniff";
|
||||
add_header X-XSS-Protection "1; mode=block";
|
||||
add_header Content-Security-Policy "default-src 'self'";
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 部署检查清单
|
||||
|
||||
### 部署前检查
|
||||
|
||||
- [ ] 所有配置项在 `.env` 中设置
|
||||
- [ ] 数据库连接测试通过
|
||||
- [ ] 数据目录路径正确且可访问
|
||||
- [ ] IDL/ENVI 路径正确(如果使用)
|
||||
- [ ] DEM 文件存在(如果使用 D-InSAR)
|
||||
- [ ] Ollama 服务运行(如果使用 AI 诊断)
|
||||
- [ ] Nginx 配置正确
|
||||
- [ ] 防火墙规则配置
|
||||
- [ ] CORS 配置为明确域名
|
||||
- [ ] Cookie Secure 标志启用(HTTPS)
|
||||
- [ ] 管理员密码已修改
|
||||
|
||||
### 部署后验证
|
||||
|
||||
- [ ] 访问前端页面正常
|
||||
- [ ] 管理员登录成功
|
||||
- [ ] 运维自检全部通过
|
||||
- [ ] 雷达数据扫描正常
|
||||
- [ ] D-InSAR 处理测试通过(如果使用)
|
||||
- [ ] AI 诊断测试通过(如果使用)
|
||||
- [ ] 日志记录正常
|
||||
- [ ] 性能测试通过
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q1: 启动时报 "数据库连接失败"
|
||||
|
||||
**A**: 检查:
|
||||
1. PostgreSQL 服务是否运行
|
||||
2. `.env` 中的 `DATABASE_URL` 是否正确
|
||||
3. 数据库是否已创建
|
||||
4. 用户名密码是否正确
|
||||
|
||||
### Q2: PROJ 警告重复出现
|
||||
|
||||
**A**:
|
||||
1. 检查启动日志是否显示 `Auto-detected PROJ_LIB`
|
||||
2. 运行 `python scripts/check_proj.py` 诊断
|
||||
3. 如果问题持续,在 `.env` 中手动设置 `PROJ_LIB`
|
||||
|
||||
### Q3: IDL/ENVI 任务失败
|
||||
|
||||
**A**: 检查:
|
||||
1. IDL_EXECUTABLE 路径是否正确
|
||||
2. IDL_DINSAR_DEM_BASE_FILE 是否存在
|
||||
3. 查看任务日志:`logs/tasks/envi/`
|
||||
4. 检查 ENVI 许可证是否有效
|
||||
|
||||
### Q4: Ollama 连接失败
|
||||
|
||||
**A**: 检查:
|
||||
1. Ollama 服务是否运行
|
||||
2. `.env` 中的 `OLLAMA_BASE_URL` 是否正确
|
||||
3. 防火墙是否阻止连接
|
||||
4. 运行 `curl http://127.0.0.1:11434/api/tags` 测试
|
||||
|
||||
### Q5: 前端无法访问后端 API
|
||||
|
||||
**A**: 检查:
|
||||
1. Nginx 是否运行
|
||||
2. Nginx 配置中的 proxy_pass 是否正确
|
||||
3. 后端服务是否运行
|
||||
4. 防火墙规则
|
||||
|
||||
### Q6: 配对候选缓存已重建,但执行配对时仍然显示“候选 0,入选 0”
|
||||
|
||||
**A**: 先区分“缓存候选总数”和“本次查询候选数”。
|
||||
1. 全量重建配对缓存只负责重建候选池,缓存里的总候选数不等于当前表单参数下的查询结果。
|
||||
2. 正式入口已迁移到“生产规划 -> 配对规划”。“修复配对基础”会自动选择增量修复或全量重建,“强制全量重建”会重算全部候选池。
|
||||
3. 如果缓存里有候选,但本次查询仍为 0,优先检查:
|
||||
- 当前是否仍勾选“仅使用有精轨数据的影像”
|
||||
- 时间基线、空间基线、重叠率阈值是否过严
|
||||
- AOI 或主从时间池是否把候选全部过滤掉
|
||||
- 卫星筛选是否与当前库中实际存在的卫星一致
|
||||
4. 当前前端会在打开配对弹窗时自动清理数据库中已不存在的卫星筛选,避免界面只显示 A 星、请求却仍残留旧的 B 星筛选。
|
||||
5. 如果需要底层状态,请在“运维自检”里查看配对系统状态;如果需要执行修复,请回到“生产规划 -> 配对规划”操作。
|
||||
|
||||
---
|
||||
详细结果见 [docs/DATABASE_SELF_MAINTENANCE_AUDIT_20260425.md](docs/DATABASE_SELF_MAINTENANCE_AUDIT_20260425.md)。
|
||||
|
||||
+140
-7
@@ -30,6 +30,17 @@ def _clean_path_text(value: str | None) -> str:
|
||||
return str(value or "").strip().strip('"').strip("'")
|
||||
|
||||
|
||||
def _infer_conda_env_name_from_python(path: str | None) -> str:
|
||||
text = _clean_path_text(path)
|
||||
marker = "/envs/"
|
||||
if not text or marker not in text:
|
||||
return ""
|
||||
tail = text.split(marker, 1)[1].strip("/")
|
||||
if not tail:
|
||||
return ""
|
||||
return tail.split("/", 1)[0].strip()
|
||||
|
||||
|
||||
def _default_idl_runtime_dir() -> str:
|
||||
return os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
|
||||
|
||||
@@ -58,6 +69,14 @@ def _resolve_idl_runtime_dir(value: str | None) -> str:
|
||||
return os.path.normpath(os.path.abspath(normalized))
|
||||
|
||||
|
||||
def _default_result_publish_root(project_root: str) -> str:
|
||||
normalized_root = os.path.normpath(project_root)
|
||||
drive, _tail = os.path.splitdrive(normalized_root)
|
||||
if drive:
|
||||
return os.path.join(drive + os.sep, "production_results")
|
||||
return os.path.join(normalized_root, "production_results")
|
||||
|
||||
|
||||
def _read_env_pairs(env_path: str) -> dict[str, str]:
|
||||
env_map: dict[str, str] = {}
|
||||
if not os.path.isfile(env_path):
|
||||
@@ -208,10 +227,18 @@ class Settings(BaseSettings):
|
||||
|
||||
RESULT_PUBLISH_ROOT: str = ""
|
||||
DINSAR_PRODUCT_DIR: str = ""
|
||||
TIMESERIES_PRODUCT_DIR: str = ""
|
||||
PSINSAR_PRODUCT_DIR: str = ""
|
||||
RESULT_QUARANTINE_ROOT: str = ""
|
||||
RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP: bool = True
|
||||
|
||||
WSL_DISTRO: str = ""
|
||||
WSL_SHARED_CONDA_ENV: str = ""
|
||||
WSL_SHARED_PYTHON: str = ""
|
||||
WSL_BROKER_JOB_ROOT: str = ""
|
||||
ISCE2_RUNTIME_ID: str = ""
|
||||
PYINT_RUNTIME_ID: str = ""
|
||||
|
||||
ISCE2_ENABLED: bool = False
|
||||
ISCE2_WSL_DISTRO: str = "Ubuntu-24.04"
|
||||
ISCE2_PYTHON: str = "/home/administrator/miniconda3/envs/isce2/bin/python"
|
||||
@@ -325,7 +352,7 @@ class Settings(BaseSettings):
|
||||
object.__setattr__(
|
||||
self,
|
||||
"RESULT_PUBLISH_ROOT",
|
||||
os.path.join(backend_dir, "result_products"),
|
||||
_default_result_publish_root(project_root),
|
||||
)
|
||||
if not self.DINSAR_PRODUCT_DIR:
|
||||
object.__setattr__(
|
||||
@@ -333,11 +360,21 @@ class Settings(BaseSettings):
|
||||
"DINSAR_PRODUCT_DIR",
|
||||
os.path.join(self.RESULT_PUBLISH_ROOT, "dinsar"),
|
||||
)
|
||||
if not self.PSINSAR_PRODUCT_DIR:
|
||||
timeseries_product_dir = (
|
||||
_clean_path_text(self.TIMESERIES_PRODUCT_DIR)
|
||||
or _clean_path_text(self.PSINSAR_PRODUCT_DIR)
|
||||
or os.path.join(self.RESULT_PUBLISH_ROOT, "timeseries")
|
||||
)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"TIMESERIES_PRODUCT_DIR",
|
||||
os.path.normpath(timeseries_product_dir),
|
||||
)
|
||||
if not self.PSINSAR_PRODUCT_DIR or _clean_path_text(self.PSINSAR_PRODUCT_DIR) != self.TIMESERIES_PRODUCT_DIR:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PSINSAR_PRODUCT_DIR",
|
||||
os.path.join(self.RESULT_PUBLISH_ROOT, "psinsar"),
|
||||
self.TIMESERIES_PRODUCT_DIR,
|
||||
)
|
||||
if not self.RESULT_QUARANTINE_ROOT:
|
||||
object.__setattr__(
|
||||
@@ -345,6 +382,18 @@ class Settings(BaseSettings):
|
||||
"RESULT_QUARANTINE_ROOT",
|
||||
os.path.join(self.RESULT_PUBLISH_ROOT, "_quarantine"),
|
||||
)
|
||||
if not self.ISCE2_WORK_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"ISCE2_WORK_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "isce2_work"),
|
||||
)
|
||||
if not self.ISCE2_OUTPUT_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"ISCE2_OUTPUT_ROOT",
|
||||
self.DINSAR_PRODUCT_DIR,
|
||||
)
|
||||
if not self.ISCE2_PIPELINE_SCRIPT:
|
||||
local_pipeline = os.path.join(
|
||||
project_root,
|
||||
@@ -358,10 +407,45 @@ class Settings(BaseSettings):
|
||||
"ISCE2_PIPELINE_SCRIPT",
|
||||
_windows_path_to_wsl_mount(local_pipeline),
|
||||
)
|
||||
if not self.WSL_DISTRO:
|
||||
fallback_distro = str(
|
||||
self.ISCE2_WSL_DISTRO
|
||||
or self.PYINT_WSL_DISTRO
|
||||
or "Ubuntu-24.04"
|
||||
).strip()
|
||||
object.__setattr__(self, "WSL_DISTRO", fallback_distro)
|
||||
if not self.WSL_SHARED_CONDA_ENV:
|
||||
shared_conda_env = (
|
||||
_infer_conda_env_name_from_python(self.WSL_SHARED_PYTHON)
|
||||
or _infer_conda_env_name_from_python(self.ISCE2_PYTHON)
|
||||
or _infer_conda_env_name_from_python(self.PYINT_WSL_PYTHON)
|
||||
or "insar_wsl_v1"
|
||||
)
|
||||
object.__setattr__(self, "WSL_SHARED_CONDA_ENV", shared_conda_env)
|
||||
if not self.WSL_SHARED_PYTHON:
|
||||
shared_python = (
|
||||
_clean_path_text(self.ISCE2_PYTHON)
|
||||
or _clean_path_text(self.PYINT_WSL_PYTHON)
|
||||
or (
|
||||
f"/home/administrator/miniconda3/envs/"
|
||||
f"{self.WSL_SHARED_CONDA_ENV}/bin/python"
|
||||
)
|
||||
)
|
||||
object.__setattr__(self, "WSL_SHARED_PYTHON", shared_python)
|
||||
if not self.WSL_BROKER_JOB_ROOT:
|
||||
object.__setattr__(
|
||||
self,
|
||||
"WSL_BROKER_JOB_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "wsl_jobs"),
|
||||
)
|
||||
if not self.ISCE2_RUNTIME_ID:
|
||||
object.__setattr__(self, "ISCE2_RUNTIME_ID", "isce2_runtime_v1")
|
||||
if not self.PYINT_RUNTIME_ID:
|
||||
object.__setattr__(self, "PYINT_RUNTIME_ID", "gamma_pyint_runtime_v1")
|
||||
if not self.PYINT_WSL_DISTRO:
|
||||
object.__setattr__(self, "PYINT_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
|
||||
object.__setattr__(self, "PYINT_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO)
|
||||
if not self.PYINT_WSL_PYTHON:
|
||||
object.__setattr__(self, "PYINT_WSL_PYTHON", self.ISCE2_PYTHON)
|
||||
object.__setattr__(self, "PYINT_WSL_PYTHON", self.WSL_SHARED_PYTHON or self.ISCE2_PYTHON)
|
||||
if not self.PYINT_HOME:
|
||||
object.__setattr__(
|
||||
self,
|
||||
@@ -390,7 +474,7 @@ class Settings(BaseSettings):
|
||||
object.__setattr__(
|
||||
self,
|
||||
"PYINT_OUTPUT_ROOT",
|
||||
os.path.join(backend_dir, "runtime", "pyint_output"),
|
||||
self.DINSAR_PRODUCT_DIR,
|
||||
)
|
||||
if not self.PYINT_DEM_ROOT:
|
||||
object.__setattr__(
|
||||
@@ -420,7 +504,7 @@ class Settings(BaseSettings):
|
||||
if not self.PYINT_ORBIT_POOL_TXT:
|
||||
object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI)
|
||||
if not self.TIMESERIES_WSL_DISTRO:
|
||||
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.ISCE2_WSL_DISTRO)
|
||||
object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO)
|
||||
if not self.TIMESERIES_ENV_NAME:
|
||||
object.__setattr__(self, "TIMESERIES_ENV_NAME", "isce2_mintpy_v1")
|
||||
if not self.TIMESERIES_PYTHON:
|
||||
@@ -518,8 +602,10 @@ class Settings(BaseSettings):
|
||||
os.makedirs(settings.IDL_WORKER_RUNTIME_DIR, exist_ok=True)
|
||||
os.makedirs(settings.RESULT_PUBLISH_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.DINSAR_PRODUCT_DIR, exist_ok=True)
|
||||
os.makedirs(settings.TIMESERIES_PRODUCT_DIR, exist_ok=True)
|
||||
os.makedirs(settings.PSINSAR_PRODUCT_DIR, exist_ok=True)
|
||||
os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.WSL_BROKER_JOB_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True)
|
||||
os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True)
|
||||
@@ -668,6 +754,16 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="DINSAR_PRODUCT_DIR", value=settings.DINSAR_PRODUCT_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(
|
||||
label="TIMESERIES_PRODUCT_DIR",
|
||||
value=settings.TIMESERIES_PRODUCT_DIR,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
_check_path(label="RESULT_QUARANTINE_ROOT", value=settings.RESULT_QUARANTINE_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
|
||||
for label, raw_value in (
|
||||
("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS),
|
||||
@@ -780,6 +876,43 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
expect_file=True,
|
||||
)
|
||||
|
||||
if settings.ISCE2_ENABLED or settings.PYINT_ENABLED:
|
||||
info.append(
|
||||
"WSL shared runtime: "
|
||||
f"distro={settings.WSL_DISTRO or '<empty>'}, "
|
||||
f"conda_env={settings.WSL_SHARED_CONDA_ENV or '<empty>'}, "
|
||||
f"isce2_runtime={settings.ISCE2_RUNTIME_ID}, "
|
||||
f"pyint_runtime={settings.PYINT_RUNTIME_ID}"
|
||||
)
|
||||
_check_path(
|
||||
label="WSL_BROKER_JOB_ROOT",
|
||||
value=settings.WSL_BROKER_JOB_ROOT,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
expect_file=False,
|
||||
)
|
||||
if settings.ISCE2_PYTHON and settings.WSL_SHARED_PYTHON:
|
||||
isce_python = _clean_path_text(settings.ISCE2_PYTHON)
|
||||
shared_python = _clean_path_text(settings.WSL_SHARED_PYTHON)
|
||||
if isce_python != shared_python:
|
||||
warnings.append(
|
||||
"ISCE2_PYTHON differs from WSL_SHARED_PYTHON. "
|
||||
"Legacy execution path and new shared runtime are not aligned."
|
||||
)
|
||||
if settings.PYINT_WSL_PYTHON and settings.WSL_SHARED_PYTHON:
|
||||
pyint_python = _clean_path_text(settings.PYINT_WSL_PYTHON)
|
||||
shared_python = _clean_path_text(settings.WSL_SHARED_PYTHON)
|
||||
if pyint_python != shared_python:
|
||||
warnings.append(
|
||||
"PYINT_WSL_PYTHON differs from WSL_SHARED_PYTHON. "
|
||||
"Gamma/PyINT still depends on a legacy Python path override."
|
||||
)
|
||||
if settings.PYINT_GAMMA_ENV_SCRIPT:
|
||||
warnings.append(
|
||||
"PYINT_GAMMA_ENV_SCRIPT is still configured. "
|
||||
"Gamma runtime has not been fully migrated to the fixed profile model."
|
||||
)
|
||||
|
||||
if settings.TIMESERIES_ENABLED:
|
||||
_check_path(
|
||||
label="TIMESERIES_PYTHON",
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""ISCE2 D-InSAR engine backed by a WSL pipeline script."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
@@ -15,6 +17,10 @@ from ..services.dinsar_naming import (
|
||||
find_json_sidecar,
|
||||
write_run_metadata,
|
||||
)
|
||||
from ..services.dinsar_result_layout_service import (
|
||||
normalize_isce2_run_layout,
|
||||
)
|
||||
from ..services.isce2_result_validator import validate_isce2_result_files
|
||||
|
||||
|
||||
LT1_FIXED_WAVELENGTH = 0.23793052222222222
|
||||
@@ -25,6 +31,7 @@ ORBIT_MARGIN_MIN_SEC = 60.0
|
||||
ORBIT_MARGIN_MAX_SEC = 120.0
|
||||
TARGET_GRID_SIZE_MIN_M = 5
|
||||
TARGET_GRID_SIZE_MAX_M = 100
|
||||
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
@@ -47,6 +54,60 @@ def _windows_path_to_wsl_mount(path: str) -> str:
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
def _join_argv_for_log(argv: List[str]) -> str:
|
||||
return " ".join(shlex.quote(str(item)) for item in argv if str(item))
|
||||
|
||||
|
||||
def _normalize_rerun_mode(value: Any) -> str:
|
||||
normalized = str(value or "").strip().lower()
|
||||
return normalized if normalized == RERUN_MODE_UNFINISHED_ONLY else "rerun_all"
|
||||
|
||||
|
||||
def _normalize_optional_path(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _load_json_file(path: str) -> Dict[str, Any]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _find_isce2_export_outputs(export_dir: str) -> Dict[str, str]:
|
||||
normalized_export_dir = _normalize_optional_path(export_dir)
|
||||
if not normalized_export_dir or not os.path.isdir(normalized_export_dir):
|
||||
return {}
|
||||
|
||||
outputs: Dict[str, str] = {}
|
||||
with os.scandir(normalized_export_dir) as entries:
|
||||
files = sorted(
|
||||
[entry for entry in entries if entry.is_file()],
|
||||
key=lambda entry: entry.name.lower(),
|
||||
)
|
||||
for entry in files:
|
||||
lower_name = entry.name.lower()
|
||||
if lower_name.endswith(("_disp_full.tif", "_disp_full.tiff")):
|
||||
continue
|
||||
if "disp" not in outputs and (
|
||||
lower_name in {"disp.tif", "disp.tiff"}
|
||||
or lower_name.endswith(("_disp.tif", "_disp.tiff"))
|
||||
):
|
||||
outputs["disp"] = entry.path
|
||||
continue
|
||||
if "coh" not in outputs and (
|
||||
lower_name in {"coh.tif", "coh.tiff"}
|
||||
or lower_name.endswith(("_coh.tif", "_coh.tiff"))
|
||||
):
|
||||
outputs["coh"] = entry.path
|
||||
return outputs
|
||||
|
||||
|
||||
class Isce2Engine(DinsarEngine):
|
||||
@property
|
||||
def engine_code(self) -> str:
|
||||
@@ -79,6 +140,10 @@ class Isce2Engine(DinsarEngine):
|
||||
"/home/administrator/miniconda3/envs/isce2/bin/python",
|
||||
)
|
||||
|
||||
@property
|
||||
def _runtime_id(self) -> str:
|
||||
return _read_env("ISCE2_RUNTIME_ID", settings.ISCE2_RUNTIME_ID or "isce2_runtime_v1")
|
||||
|
||||
@property
|
||||
def _dem_path(self) -> str:
|
||||
explicit = _read_env("ISCE2_DEM_PATH", "")
|
||||
@@ -315,7 +380,45 @@ class Isce2Engine(DinsarEngine):
|
||||
# Task discovery
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
def _has_completed_task_result(self, task_dir: str, profile_code: str) -> bool:
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
pointer_path = os.path.join(
|
||||
settings.DINSAR_PRODUCT_DIR,
|
||||
pair_key,
|
||||
"current",
|
||||
f"{self.engine_code}__{str(profile_code or '').strip()}.json",
|
||||
)
|
||||
if not os.path.isfile(pointer_path):
|
||||
return False
|
||||
|
||||
payload = _load_json_file(pointer_path)
|
||||
if str(payload.get("status") or "").strip().upper() != "COMPLETED":
|
||||
return False
|
||||
|
||||
primary_file = _normalize_optional_path(payload.get("primary_file"))
|
||||
source_files = payload.get("source_files") if isinstance(payload.get("source_files"), list) else []
|
||||
validation = validate_isce2_result_files(primary_file, source_files)
|
||||
if not bool(validation.get("accepted")):
|
||||
return False
|
||||
|
||||
manifest_path = _normalize_optional_path(payload.get("manifest_path"))
|
||||
if manifest_path and os.path.isfile(manifest_path):
|
||||
return True
|
||||
|
||||
output_dir = _normalize_optional_path(payload.get("output_dir"))
|
||||
if output_dir and os.path.isfile(os.path.join(output_dir, "execution_manifest.json")):
|
||||
return True
|
||||
return False
|
||||
|
||||
def validate_root_dir(
|
||||
self,
|
||||
root_dir: str,
|
||||
num_to_process: int = 0,
|
||||
rerun_mode: str = "rerun_all",
|
||||
) -> Dict[str, Any]:
|
||||
normalized_root = os.path.normpath(os.path.abspath(str(root_dir or "").strip()))
|
||||
if not root_dir or not os.path.isdir(normalized_root):
|
||||
raise ValueError(f"ISCE2 root_dir does not exist or is not a directory: {root_dir}")
|
||||
@@ -353,18 +456,95 @@ class Isce2Engine(DinsarEngine):
|
||||
f"{detail}"
|
||||
)
|
||||
|
||||
discovered_task_count = len(task_dirs)
|
||||
skipped_completed_count = 0
|
||||
selected_task_dirs = task_dirs
|
||||
if _normalize_rerun_mode(rerun_mode) == RERUN_MODE_UNFINISHED_ONLY:
|
||||
selected_task_dirs = []
|
||||
for task_dir in task_dirs:
|
||||
if self._has_completed_task_result(task_dir, "lt1_stripmap"):
|
||||
skipped_completed_count += 1
|
||||
continue
|
||||
selected_task_dirs.append(task_dir)
|
||||
|
||||
selected_count = int(num_to_process or 0)
|
||||
if selected_count > 0:
|
||||
task_dirs = task_dirs[:selected_count]
|
||||
selected_task_dirs = selected_task_dirs[:selected_count]
|
||||
|
||||
return {
|
||||
"root_dir": normalized_root,
|
||||
"mode": mode,
|
||||
"task_dirs": task_dirs,
|
||||
"task_count": len(task_dirs),
|
||||
"task_dirs": selected_task_dirs,
|
||||
"task_count": len(selected_task_dirs),
|
||||
"selected_task_count": len(selected_task_dirs),
|
||||
"discovered_task_count": discovered_task_count,
|
||||
"skipped_completed_count": skipped_completed_count,
|
||||
"invalid_candidates": invalid_candidates,
|
||||
}
|
||||
|
||||
def _build_lt1_manifest_payload(
|
||||
self,
|
||||
*,
|
||||
request: RunRequest,
|
||||
task_dir: str,
|
||||
task_name: str,
|
||||
task_alias: str,
|
||||
pair_key: str,
|
||||
run_key: str,
|
||||
work_dir: str,
|
||||
output_dir: str,
|
||||
orbit_output_dir: str,
|
||||
wsl_task_dir: str,
|
||||
wsl_work_dir: str,
|
||||
wsl_output_dir: str,
|
||||
wsl_orbit_output_dir: str,
|
||||
wsl_orbit_root: str,
|
||||
wsl_dem: str,
|
||||
force: bool,
|
||||
target_grid_size_m: int,
|
||||
bbox: str,
|
||||
coh_threshold: Any,
|
||||
bbox_margin: Any,
|
||||
wavelength: Any,
|
||||
orbit_margin_sec: Any,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
return {
|
||||
"job_id": request.job_id,
|
||||
"profile": request.profile,
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"paths": {
|
||||
"source_root_windows": os.path.normpath(
|
||||
str((request.extra or {}).get("__source_root_override") or request.root_dir)
|
||||
),
|
||||
"task_dir_windows": os.path.normpath(task_dir),
|
||||
"task_dir_wsl": wsl_task_dir,
|
||||
"work_dir_windows": work_dir,
|
||||
"work_dir_wsl": wsl_work_dir,
|
||||
"output_dir_windows": output_dir,
|
||||
"output_dir_wsl": wsl_output_dir,
|
||||
"orbit_output_dir_windows": orbit_output_dir,
|
||||
"orbit_output_dir_wsl": wsl_orbit_output_dir,
|
||||
"orbit_root_windows": self._orbit_pool_isce2,
|
||||
"orbit_root_wsl": wsl_orbit_root,
|
||||
"dem_path_windows": self._dem_path,
|
||||
"dem_path_wsl": wsl_dem,
|
||||
},
|
||||
"params": {
|
||||
"force": bool(force),
|
||||
"target_grid_size_m": int(target_grid_size_m),
|
||||
"bbox": str(bbox or "").strip(),
|
||||
"coh_threshold": coh_threshold,
|
||||
"bbox_margin": bbox_margin,
|
||||
"wavelength": wavelength,
|
||||
"orbit_margin_sec": orbit_margin_sec,
|
||||
},
|
||||
"pair_meta": dict(pair_meta or {}),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _iter_child_dirs(root_dir: str):
|
||||
with os.scandir(root_dir) as entries:
|
||||
@@ -409,16 +589,50 @@ class Isce2Engine(DinsarEngine):
|
||||
return self._run_lt1_stripmap(request)
|
||||
|
||||
def _run_lt1_stripmap(self, request: RunRequest) -> RunResult:
|
||||
from ..services.wsl_service import run_wsl_command, windows_path_to_wsl
|
||||
from ..services.wsl_broker import wsl_broker
|
||||
from ..services.wsl_runtime_registry import get_wsl_runtime
|
||||
from ..services.wsl_service import windows_path_to_wsl
|
||||
|
||||
extra = self.normalize_extra(request.extra)
|
||||
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
|
||||
validation = self.validate_root_dir(
|
||||
request.root_dir,
|
||||
request.num_to_process,
|
||||
str((request.extra or {}).get("__rerun_mode") or "rerun_all"),
|
||||
)
|
||||
task_dirs: List[str] = validation["task_dirs"]
|
||||
total_tasks = len(task_dirs)
|
||||
run_started_at = datetime.utcnow()
|
||||
run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z"
|
||||
run_key = build_run_key(self.engine_code, request.profile, started_at=run_started_at)
|
||||
run_key = str(extra.get("__managed_run_key") or "").strip() or build_run_key(
|
||||
self.engine_code,
|
||||
request.profile,
|
||||
started_at=run_started_at,
|
||||
)
|
||||
progress_callback = request.progress_callback
|
||||
runtime = get_wsl_runtime(self._runtime_id)
|
||||
managed_run_dir_override = _normalize_optional_path(extra.get("__managed_run_dir"))
|
||||
managed_native_output_dir_override = _normalize_optional_path(extra.get("__managed_native_output_dir"))
|
||||
managed_work_dir_override = _normalize_optional_path(extra.get("__managed_work_dir"))
|
||||
managed_export_dir_override = _normalize_optional_path(extra.get("__managed_export_dir"))
|
||||
managed_orbit_output_dir_override = _normalize_optional_path(extra.get("__managed_orbit_output_dir"))
|
||||
source_root_override = _normalize_optional_path(extra.get("__source_root_override")) or os.path.normpath(request.root_dir)
|
||||
has_managed_override = any(
|
||||
[
|
||||
managed_run_dir_override,
|
||||
managed_native_output_dir_override,
|
||||
managed_work_dir_override,
|
||||
managed_export_dir_override,
|
||||
managed_orbit_output_dir_override,
|
||||
]
|
||||
)
|
||||
if has_managed_override and total_tasks > 1:
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
profile=request.profile,
|
||||
job_id=request.job_id,
|
||||
error="Managed ISCE2 directory overrides require a single task request.",
|
||||
)
|
||||
|
||||
def emit_progress(event_type: str, **payload: Any) -> None:
|
||||
if not callable(progress_callback):
|
||||
@@ -439,11 +653,11 @@ class Isce2Engine(DinsarEngine):
|
||||
|
||||
wsl_isce2_pool = ""
|
||||
if self._orbit_pool_isce2:
|
||||
wsl_isce2_pool = windows_path_to_wsl(self._orbit_pool_isce2, distro=self._distro)
|
||||
wsl_isce2_pool = windows_path_to_wsl(self._orbit_pool_isce2, distro=runtime.distro)
|
||||
|
||||
wsl_dem = ""
|
||||
if self._dem_path:
|
||||
wsl_dem = windows_path_to_wsl(self._dem_path, distro=self._distro)
|
||||
wsl_dem = windows_path_to_wsl(self._dem_path, distro=runtime.distro)
|
||||
|
||||
task_results: List[Dict[str, Any]] = []
|
||||
output_dirs: List[str] = []
|
||||
@@ -455,15 +669,18 @@ class Isce2Engine(DinsarEngine):
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
work_root = self._work_root or os.path.join(task_dir, "isce2_work")
|
||||
output_root = self._output_root or os.path.join(task_dir, "isce2_output")
|
||||
work_dir = os.path.normpath(os.path.join(work_root, pair_key, run_key))
|
||||
output_dir = os.path.normpath(os.path.join(output_root, pair_key, run_key, "native"))
|
||||
orbit_output_dir = os.path.join(work_dir, "orbits")
|
||||
wsl_task_dir = windows_path_to_wsl(task_dir, distro=self._distro)
|
||||
wsl_work_dir = windows_path_to_wsl(work_dir, distro=self._distro)
|
||||
wsl_output_dir = windows_path_to_wsl(output_dir, distro=self._distro)
|
||||
wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=self._distro)
|
||||
run_dir = managed_run_dir_override or os.path.normpath(
|
||||
os.path.join(output_root, pair_key, "runs", run_key)
|
||||
)
|
||||
native_output_dir = managed_native_output_dir_override or os.path.join(run_dir, "native")
|
||||
work_dir = managed_work_dir_override or os.path.join(native_output_dir, "workflow")
|
||||
export_dir = managed_export_dir_override or os.path.join(native_output_dir, "export")
|
||||
orbit_output_dir = managed_orbit_output_dir_override or os.path.join(work_dir, "orbits")
|
||||
wsl_task_dir = windows_path_to_wsl(task_dir, distro=runtime.distro)
|
||||
wsl_work_dir = windows_path_to_wsl(work_dir, distro=runtime.distro)
|
||||
wsl_output_dir = windows_path_to_wsl(export_dir, distro=runtime.distro)
|
||||
wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=runtime.distro)
|
||||
emit_progress(
|
||||
"pair_started",
|
||||
pair_index=pair_index,
|
||||
@@ -473,7 +690,7 @@ class Isce2Engine(DinsarEngine):
|
||||
pair_key=pair_key,
|
||||
task_dir=task_dir,
|
||||
work_dir=work_dir,
|
||||
output_dir=output_dir,
|
||||
output_dir=run_dir,
|
||||
)
|
||||
if not wsl_task_dir:
|
||||
pairs_failed += 1
|
||||
@@ -496,8 +713,11 @@ class Isce2Engine(DinsarEngine):
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"run_dir": run_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"work_dir": work_dir,
|
||||
"output_dir": output_dir,
|
||||
"output_dir": run_dir,
|
||||
"export_dir": export_dir,
|
||||
"success": False,
|
||||
"returncode": -2,
|
||||
"error": error_text,
|
||||
@@ -531,8 +751,11 @@ class Isce2Engine(DinsarEngine):
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"run_dir": run_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"work_dir": work_dir,
|
||||
"output_dir": output_dir,
|
||||
"output_dir": run_dir,
|
||||
"export_dir": export_dir,
|
||||
"success": False,
|
||||
"returncode": -2,
|
||||
"error": error_text,
|
||||
@@ -546,92 +769,137 @@ class Isce2Engine(DinsarEngine):
|
||||
)
|
||||
continue
|
||||
|
||||
cmd_parts = [
|
||||
"export PROJ_DATA=/home/administrator/miniconda3/envs/isce2/share/proj",
|
||||
f"&& {self._python} '{self._pipeline_script}' '{wsl_task_dir}'",
|
||||
f"--task-name '{task_alias}'",
|
||||
f"--output-prefix '{task_alias}'",
|
||||
f"--work-dir '{wsl_work_dir}'",
|
||||
f"--output-dir '{wsl_output_dir}'",
|
||||
f"--orbit-output-dir '{wsl_orbit_output_dir}'",
|
||||
]
|
||||
if wsl_isce2_pool:
|
||||
cmd_parts.append(f"--orbit-root '{wsl_isce2_pool}'")
|
||||
if wsl_dem:
|
||||
cmd_parts.append(f"--dem '{wsl_dem}'")
|
||||
if force:
|
||||
cmd_parts.append("--force")
|
||||
if target_grid_size_m:
|
||||
cmd_parts.append(f"--target-grid-size-m {target_grid_size_m}")
|
||||
if bbox:
|
||||
cmd_parts.append(f"--bbox '{bbox}'")
|
||||
if coh_threshold is not None:
|
||||
cmd_parts.append(f"--coh-threshold {coh_threshold}")
|
||||
if bbox_margin is not None:
|
||||
cmd_parts.append(f"--bbox-margin {bbox_margin}")
|
||||
if wavelength is not None:
|
||||
cmd_parts.append(f"--wavelength {wavelength}")
|
||||
if orbit_margin_sec is not None:
|
||||
cmd_parts.append(f"--orbit-margin-sec {orbit_margin_sec}")
|
||||
|
||||
cmd = " ".join(cmd_parts)
|
||||
rc, stdout, stderr = run_wsl_command(
|
||||
cmd,
|
||||
distro=self._distro,
|
||||
timeout=timeout,
|
||||
manifest_payload = self._build_lt1_manifest_payload(
|
||||
request=request,
|
||||
task_dir=task_dir,
|
||||
task_name=task_name,
|
||||
task_alias=task_alias,
|
||||
pair_key=pair_key,
|
||||
run_key=run_key,
|
||||
work_dir=work_dir,
|
||||
output_dir=export_dir,
|
||||
orbit_output_dir=orbit_output_dir,
|
||||
wsl_task_dir=wsl_task_dir,
|
||||
wsl_work_dir=wsl_work_dir,
|
||||
wsl_output_dir=wsl_output_dir,
|
||||
wsl_orbit_output_dir=wsl_orbit_output_dir,
|
||||
wsl_orbit_root=wsl_isce2_pool,
|
||||
wsl_dem=wsl_dem,
|
||||
force=force,
|
||||
target_grid_size_m=target_grid_size_m,
|
||||
bbox=bbox,
|
||||
coh_threshold=coh_threshold,
|
||||
bbox_margin=bbox_margin,
|
||||
wavelength=wavelength,
|
||||
orbit_margin_sec=orbit_margin_sec,
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
broker_result = wsl_broker.run_manifest(
|
||||
runtime_id=runtime.runtime_id,
|
||||
operation="lt1_stripmap",
|
||||
payload=manifest_payload,
|
||||
job_id=f"{request.job_id}_{pair_key}",
|
||||
timeout_seconds=timeout,
|
||||
)
|
||||
rc = broker_result.returncode
|
||||
stdout = broker_result.stdout
|
||||
stderr = broker_result.stderr
|
||||
command = _join_argv_for_log(list(broker_result.argv))
|
||||
|
||||
success = rc == 0
|
||||
if success:
|
||||
pairs_processed += 1
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
write_run_metadata(
|
||||
output_dir,
|
||||
{
|
||||
"run_key": run_key,
|
||||
"pair_key": pair_key,
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"engine_code": self.engine_code,
|
||||
"profile_code": request.profile,
|
||||
"source_root": os.path.normpath(request.root_dir),
|
||||
"task_dir": os.path.normpath(task_dir),
|
||||
"work_dir": work_dir,
|
||||
"output_dir": output_dir,
|
||||
"orbit_output_dir": orbit_output_dir,
|
||||
"started_at": run_started_at_text,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"params": {
|
||||
"force": force,
|
||||
"target_grid_size_m": target_grid_size_m,
|
||||
"bbox": bbox,
|
||||
"coh_threshold": coh_threshold,
|
||||
"bbox_margin": bbox_margin,
|
||||
"wavelength": wavelength,
|
||||
"orbit_margin_sec": orbit_margin_sec,
|
||||
success = False
|
||||
layout_result: Dict[str, Any] = {}
|
||||
validation_result: Dict[str, Any] = {}
|
||||
error_text = stderr.strip() if stderr else ""
|
||||
if rc == 0:
|
||||
try:
|
||||
os.makedirs(run_dir, exist_ok=True)
|
||||
export_outputs = _find_isce2_export_outputs(export_dir)
|
||||
disp_path = export_outputs.get("disp", "")
|
||||
coh_path = export_outputs.get("coh", "")
|
||||
if not disp_path:
|
||||
raise FileNotFoundError(
|
||||
f"No ISCE2 displacement GeoTIFF found under export dir: {export_dir}"
|
||||
)
|
||||
|
||||
source_files = [disp_path]
|
||||
if coh_path:
|
||||
source_files.append(coh_path)
|
||||
validation_result = validate_isce2_result_files(disp_path, source_files)
|
||||
if not bool(validation_result.get("accepted")):
|
||||
issues = validation_result.get("issues") or []
|
||||
issue_text = "; ".join(str(item) for item in issues[:3]) or "unknown validation error"
|
||||
raise RuntimeError(f"ISCE2 output validation failed: {issue_text}")
|
||||
|
||||
layout_result = normalize_isce2_run_layout(
|
||||
run_dir,
|
||||
primary_file=str(validation_result.get("primary_file") or disp_path),
|
||||
source_files=list(validation_result.get("source_files") or source_files),
|
||||
rewrite_metadata=False,
|
||||
)
|
||||
write_run_metadata(
|
||||
run_dir,
|
||||
{
|
||||
"run_key": run_key,
|
||||
"pair_key": pair_key,
|
||||
"task_name": task_name,
|
||||
"task_alias": task_alias,
|
||||
"engine_code": self.engine_code,
|
||||
"profile_code": request.profile,
|
||||
"source_root": source_root_override,
|
||||
"task_dir": os.path.normpath(task_dir),
|
||||
"work_dir": work_dir,
|
||||
"export_dir": export_dir,
|
||||
"output_dir": run_dir,
|
||||
"native_output_dir": layout_result["native_output_dir"],
|
||||
"orbit_output_dir": orbit_output_dir,
|
||||
"runtime_id": runtime.runtime_id,
|
||||
"manifest_path_windows": broker_result.manifest.manifest_path_windows,
|
||||
"manifest_path_wsl": broker_result.manifest.manifest_path_wsl,
|
||||
"started_at": run_started_at_text,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"primary_file": layout_result["primary_file"],
|
||||
"source_files": layout_result["source_files"],
|
||||
"acceptance": validation_result,
|
||||
"params": {
|
||||
"force": force,
|
||||
"target_grid_size_m": target_grid_size_m,
|
||||
"bbox": bbox,
|
||||
"coh_threshold": coh_threshold,
|
||||
"bbox_margin": bbox_margin,
|
||||
"wavelength": wavelength,
|
||||
"orbit_margin_sec": orbit_margin_sec,
|
||||
},
|
||||
"master_path": pair_meta.get("master_path"),
|
||||
"slave_path": pair_meta.get("slave_path"),
|
||||
"master_satellite": pair_meta.get("master_satellite"),
|
||||
"slave_satellite": pair_meta.get("slave_satellite"),
|
||||
"master_imaging_date": pair_meta.get("master_imaging_date"),
|
||||
"slave_imaging_date": pair_meta.get("slave_imaging_date"),
|
||||
"master_imaging_mode": pair_meta.get("master_imaging_mode"),
|
||||
"slave_imaging_mode": pair_meta.get("slave_imaging_mode"),
|
||||
"master_polarization": pair_meta.get("master_polarization"),
|
||||
"slave_polarization": pair_meta.get("slave_polarization"),
|
||||
"time_baseline_days": pair_meta.get("time_baseline_days"),
|
||||
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
|
||||
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
|
||||
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
|
||||
"network_run_id": pair_meta.get("network_run_id"),
|
||||
"network_edge_id": pair_meta.get("network_edge_id"),
|
||||
"policy_version": pair_meta.get("policy_version"),
|
||||
"selection_strategy": pair_meta.get("selection_strategy"),
|
||||
},
|
||||
"master_path": pair_meta.get("master_path"),
|
||||
"slave_path": pair_meta.get("slave_path"),
|
||||
"master_satellite": pair_meta.get("master_satellite"),
|
||||
"slave_satellite": pair_meta.get("slave_satellite"),
|
||||
"master_imaging_date": pair_meta.get("master_imaging_date"),
|
||||
"slave_imaging_date": pair_meta.get("slave_imaging_date"),
|
||||
"master_imaging_mode": pair_meta.get("master_imaging_mode"),
|
||||
"slave_imaging_mode": pair_meta.get("slave_imaging_mode"),
|
||||
"master_polarization": pair_meta.get("master_polarization"),
|
||||
"slave_polarization": pair_meta.get("slave_polarization"),
|
||||
"time_baseline_days": pair_meta.get("time_baseline_days"),
|
||||
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
|
||||
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
|
||||
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
|
||||
"network_run_id": pair_meta.get("network_run_id"),
|
||||
"network_edge_id": pair_meta.get("network_edge_id"),
|
||||
"policy_version": pair_meta.get("policy_version"),
|
||||
"selection_strategy": pair_meta.get("selection_strategy"),
|
||||
},
|
||||
)
|
||||
output_dirs.append(output_dir)
|
||||
else:
|
||||
)
|
||||
output_dirs.append(run_dir)
|
||||
pairs_processed += 1
|
||||
success = True
|
||||
except Exception as exc:
|
||||
error_text = str(exc)
|
||||
if stderr:
|
||||
stderr = stderr.rstrip() + "\n" + error_text
|
||||
else:
|
||||
stderr = error_text
|
||||
|
||||
if not success:
|
||||
pairs_failed += 1
|
||||
|
||||
emit_progress(
|
||||
@@ -643,7 +911,7 @@ class Isce2Engine(DinsarEngine):
|
||||
pair_key=pair_key,
|
||||
success=success,
|
||||
returncode=rc,
|
||||
error=stderr.strip() if stderr else "",
|
||||
error=error_text,
|
||||
)
|
||||
task_results.append(
|
||||
{
|
||||
@@ -652,17 +920,30 @@ class Isce2Engine(DinsarEngine):
|
||||
"pair_key": pair_key,
|
||||
"run_key": run_key,
|
||||
"task_dir": task_dir,
|
||||
"run_dir": run_dir,
|
||||
"native_output_dir": (
|
||||
layout_result.get("native_output_dir")
|
||||
or native_output_dir
|
||||
),
|
||||
"work_dir": work_dir,
|
||||
"output_dir": output_dir,
|
||||
"output_dir": run_dir,
|
||||
"export_dir": export_dir,
|
||||
"primary_file": layout_result.get("primary_file", ""),
|
||||
"source_files": layout_result.get("source_files", []),
|
||||
"validation": validation_result,
|
||||
"wsl_task_dir": wsl_task_dir,
|
||||
"wsl_work_dir": wsl_work_dir,
|
||||
"wsl_output_dir": wsl_output_dir,
|
||||
"command": cmd,
|
||||
"runtime_id": runtime.runtime_id,
|
||||
"command": command,
|
||||
"runner_argv": list(broker_result.argv),
|
||||
"manifest_path_windows": broker_result.manifest.manifest_path_windows,
|
||||
"manifest_path_wsl": broker_result.manifest.manifest_path_wsl,
|
||||
"success": success,
|
||||
"returncode": rc,
|
||||
"stdout_tail": stdout[-3000:] if stdout else "",
|
||||
"stderr_tail": stderr[-3000:] if stderr else "",
|
||||
"error": stderr.strip() if stderr else "",
|
||||
"error": error_text,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -706,7 +987,11 @@ class Isce2Engine(DinsarEngine):
|
||||
"target_grid_size_m": target_grid_size_m,
|
||||
"wavelength": wavelength,
|
||||
"orbit_margin_sec": orbit_margin_sec,
|
||||
"runtime_id": runtime.runtime_id,
|
||||
"command": last_task_result.get("command", ""),
|
||||
"runner_argv": last_task_result.get("runner_argv", []),
|
||||
"manifest_path_windows": last_task_result.get("manifest_path_windows", ""),
|
||||
"manifest_path_wsl": last_task_result.get("manifest_path_wsl", ""),
|
||||
"stdout_tail": last_task_result.get("stdout_tail", ""),
|
||||
"stderr_tail": last_task_result.get("stderr_tail", ""),
|
||||
"wsl_task_dir": last_task_result.get("wsl_task_dir", ""),
|
||||
@@ -714,7 +999,15 @@ class Isce2Engine(DinsarEngine):
|
||||
"wsl_output_dir": last_task_result.get("wsl_output_dir", ""),
|
||||
"wsl_orbit_pool": wsl_isce2_pool,
|
||||
"wsl_dem": wsl_dem,
|
||||
"wsl_work_root": windows_path_to_wsl(self._work_root, distro=self._distro) if self._work_root else "",
|
||||
"wsl_output_root": windows_path_to_wsl(self._output_root, distro=self._distro) if self._output_root else "",
|
||||
"wsl_work_root": (
|
||||
os.path.dirname(str(last_task_result.get("wsl_work_dir") or "").strip())
|
||||
if str(last_task_result.get("wsl_work_dir") or "").strip()
|
||||
else (windows_path_to_wsl(self._work_root, distro=runtime.distro) if self._work_root else "")
|
||||
),
|
||||
"wsl_output_root": (
|
||||
os.path.dirname(str(last_task_result.get("wsl_output_dir") or "").strip())
|
||||
if str(last_task_result.get("wsl_output_dir") or "").strip()
|
||||
else (windows_path_to_wsl(self._output_root, distro=runtime.distro) if self._output_root else "")
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
@@ -25,6 +26,7 @@ from ..services.pyint_service import (
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
quote_shell,
|
||||
resolve_gamma_env_script,
|
||||
resolve_time_baseline_days,
|
||||
to_wsl_path,
|
||||
validate_pyint_root_dir,
|
||||
@@ -32,6 +34,8 @@ from ..services.pyint_service import (
|
||||
from ..services.wsl_service import run_wsl_command
|
||||
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
|
||||
|
||||
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
|
||||
|
||||
|
||||
def _read_env(name: str, default: str = "") -> str:
|
||||
return get_env_text(name, default) or default
|
||||
@@ -53,6 +57,11 @@ def _windows_path_to_wsl_mount(path: str) -> str:
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
def _normalize_rerun_mode(value: Any) -> str:
|
||||
normalized = str(value or "").strip().lower()
|
||||
return normalized if normalized == RERUN_MODE_UNFINISHED_ONLY else "rerun_all"
|
||||
|
||||
|
||||
class PyintEngine(DinsarEngine):
|
||||
@property
|
||||
def engine_code(self) -> str:
|
||||
@@ -138,7 +147,7 @@ class PyintEngine(DinsarEngine):
|
||||
|
||||
@property
|
||||
def _gamma_env_script(self) -> str:
|
||||
return _read_env("PYINT_GAMMA_ENV_SCRIPT", "")
|
||||
return resolve_gamma_env_script()
|
||||
|
||||
@property
|
||||
def _lt1_precise_orbit_enabled(self) -> bool:
|
||||
@@ -269,8 +278,70 @@ class PyintEngine(DinsarEngine):
|
||||
|
||||
return normalized
|
||||
|
||||
def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
return validate_pyint_root_dir(root_dir, num_to_process)
|
||||
def _has_completed_task_result(self, task_dir: str, profile_code: str) -> bool:
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
pair_key = task_identity["pair_key"]
|
||||
output_root = self._output_root or os.path.join(task_dir, "pyint_output")
|
||||
runs_root = os.path.join(output_root, pair_key, "runs")
|
||||
if not os.path.isdir(runs_root):
|
||||
return False
|
||||
|
||||
with os.scandir(runs_root) as entries:
|
||||
run_dirs = [entry.path for entry in entries if entry.is_dir()]
|
||||
run_dirs.sort(key=lambda path: os.path.basename(path).lower(), reverse=True)
|
||||
|
||||
for run_dir in run_dirs:
|
||||
metadata_path = os.path.join(run_dir, "native", ".dinsar_run.json")
|
||||
if not os.path.isfile(metadata_path):
|
||||
metadata_path = os.path.join(run_dir, ".dinsar_run.json")
|
||||
if not os.path.isfile(metadata_path):
|
||||
continue
|
||||
try:
|
||||
with open(metadata_path, "r", encoding="utf-8") as fp:
|
||||
metadata = json.load(fp) or {}
|
||||
except Exception:
|
||||
continue
|
||||
if str(metadata.get("engine_code") or "").strip().lower() != self.engine_code:
|
||||
continue
|
||||
if str(metadata.get("profile_code") or "").strip() != str(profile_code or "").strip():
|
||||
continue
|
||||
output_dir = str(metadata.get("output_dir") or os.path.join(run_dir, "native")).strip()
|
||||
if output_dir and os.path.isdir(output_dir):
|
||||
return True
|
||||
return False
|
||||
|
||||
def validate_root_dir(
|
||||
self,
|
||||
root_dir: str,
|
||||
num_to_process: int = 0,
|
||||
rerun_mode: str = "rerun_all",
|
||||
) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, 0)
|
||||
task_dirs: List[str] = list(validation.get("task_dirs") or [])
|
||||
discovered_task_count = len(task_dirs)
|
||||
skipped_completed_count = 0
|
||||
|
||||
if _normalize_rerun_mode(rerun_mode) == RERUN_MODE_UNFINISHED_ONLY:
|
||||
filtered_task_dirs: List[str] = []
|
||||
for task_dir in task_dirs:
|
||||
if self._has_completed_task_result(task_dir, "lt1_gamma_dinsar"):
|
||||
skipped_completed_count += 1
|
||||
continue
|
||||
filtered_task_dirs.append(task_dir)
|
||||
task_dirs = filtered_task_dirs
|
||||
|
||||
selected_count = int(num_to_process or 0)
|
||||
if selected_count > 0:
|
||||
task_dirs = task_dirs[:selected_count]
|
||||
|
||||
return {
|
||||
**validation,
|
||||
"task_dirs": task_dirs,
|
||||
"task_count": len(task_dirs),
|
||||
"selected_task_count": len(task_dirs),
|
||||
"discovered_task_count": discovered_task_count,
|
||||
"skipped_completed_count": skipped_completed_count,
|
||||
}
|
||||
|
||||
def check_available(self) -> EngineAvailability:
|
||||
report = check_pyint_environment(
|
||||
@@ -333,7 +404,11 @@ class PyintEngine(DinsarEngine):
|
||||
|
||||
def _run_lt1_gamma_dinsar(self, request: RunRequest) -> RunResult:
|
||||
extra = self.normalize_extra(request.extra)
|
||||
validation = self.validate_root_dir(request.root_dir, request.num_to_process)
|
||||
validation = self.validate_root_dir(
|
||||
request.root_dir,
|
||||
request.num_to_process,
|
||||
str((request.extra or {}).get("__rerun_mode") or "rerun_all"),
|
||||
)
|
||||
task_dirs: List[str] = validation["task_dirs"]
|
||||
total_tasks = len(task_dirs)
|
||||
run_started_at = datetime.utcnow()
|
||||
@@ -383,7 +458,7 @@ class PyintEngine(DinsarEngine):
|
||||
slave_date = task_identity["slave_date"]
|
||||
|
||||
work_run_root = os.path.normpath(os.path.join(self._work_root, pair_key, run_key))
|
||||
output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, run_key, "native"))
|
||||
output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, "runs", run_key, "native"))
|
||||
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
|
||||
project_name = build_project_name(pair_key, run_key)
|
||||
project_dir = os.path.join(work_run_root, project_name)
|
||||
|
||||
@@ -50,6 +50,7 @@ class SarscapeEngine(DinsarEngine):
|
||||
status_raw = envi_service.get_status()
|
||||
idl_ok: bool = status_raw.get("idl_installed", False)
|
||||
dem_ok: bool = status_raw.get("dem_exists", False)
|
||||
runner_ok: bool = status_raw.get("runner_ready", False)
|
||||
|
||||
checks = [
|
||||
{
|
||||
@@ -62,12 +63,30 @@ class SarscapeEngine(DinsarEngine):
|
||||
"ok": dem_ok,
|
||||
"detail": status_raw.get("dem_base_file", ""),
|
||||
},
|
||||
{
|
||||
"name": "Python Runner",
|
||||
"ok": runner_ok,
|
||||
"detail": (
|
||||
status_raw.get("runner_message")
|
||||
or status_raw.get("runner_python")
|
||||
or ""
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
if idl_ok and dem_ok:
|
||||
if idl_ok and dem_ok and runner_ok:
|
||||
status = "ok"
|
||||
available = True
|
||||
message = "ENVI/SARscape 可用"
|
||||
elif not idl_ok:
|
||||
status = "unavailable"
|
||||
available = False
|
||||
message = "IDL/ENVI 未安装或路径错误"
|
||||
elif not runner_ok:
|
||||
status = "unavailable"
|
||||
available = False
|
||||
runner_message = str(status_raw.get("runner_message") or "").strip()
|
||||
message = runner_message or "ENVI Python runner unavailable."
|
||||
elif idl_ok:
|
||||
status = "degraded"
|
||||
available = True
|
||||
@@ -75,7 +94,7 @@ class SarscapeEngine(DinsarEngine):
|
||||
else:
|
||||
status = "unavailable"
|
||||
available = False
|
||||
message = "IDL/ENVI 未安装或路径错误"
|
||||
message = "ENVI/SARscape unavailable."
|
||||
|
||||
return EngineAvailability(
|
||||
engine_code=self.engine_code,
|
||||
|
||||
@@ -29,6 +29,7 @@ class Scene:
|
||||
tiff_path: Path
|
||||
meta_path: Path
|
||||
date_yyyymmdd: str
|
||||
satellite: str
|
||||
orbit_xml_path: Path
|
||||
|
||||
|
||||
@@ -157,6 +158,16 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Delete an existing work directory before rerunning",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reference-satellite",
|
||||
default=None,
|
||||
help="Optional LT-1 satellite for the reference/master scene (LT1A or LT1B)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--secondary-satellite",
|
||||
default=None,
|
||||
help="Optional LT-1 satellite for the secondary/slave scene (LT1A or LT1B)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
@@ -209,10 +220,17 @@ def choose_scene_tiff(scene_dir: Path, scene_glob: str, prefer_scene_keyword: st
|
||||
|
||||
|
||||
def scene_meta_from_tiff(tiff_path: Path) -> Path:
|
||||
meta_path = Path(str(tiff_path).replace(".tiff", ".meta.xml"))
|
||||
if not meta_path.exists():
|
||||
raise FileNotFoundError(f"Missing meta XML for {tiff_path}: {meta_path}")
|
||||
return meta_path
|
||||
candidates = [tiff_path.with_suffix(".meta.xml")]
|
||||
legacy_path = Path(str(tiff_path).replace(".tiff", ".meta.xml"))
|
||||
if legacy_path not in candidates:
|
||||
candidates.append(legacy_path)
|
||||
|
||||
for meta_path in candidates:
|
||||
if meta_path.exists():
|
||||
return meta_path
|
||||
|
||||
searched = ", ".join(str(path) for path in candidates)
|
||||
raise FileNotFoundError(f"Missing meta XML for {tiff_path}. Searched: {searched}")
|
||||
|
||||
|
||||
def extract_scene_date(name: str) -> str:
|
||||
@@ -224,8 +242,71 @@ def extract_scene_date(name: str) -> str:
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def normalize_lt1_satellite(value: str | None) -> str:
|
||||
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
|
||||
if "LT1A" in text or text in {"A", "LTA"}:
|
||||
return "LT1A"
|
||||
if "LT1B" in text or text in {"B", "LTB"}:
|
||||
return "LT1B"
|
||||
return ""
|
||||
|
||||
|
||||
def extract_scene_satellite_from_name(name: str) -> str:
|
||||
match = re.search(r"(LT1[AB])", str(name or ""), re.IGNORECASE)
|
||||
return normalize_lt1_satellite(match.group(1) if match else "")
|
||||
|
||||
|
||||
def extract_scene_satellite_from_meta(meta_path: Path) -> str:
|
||||
try:
|
||||
root = ET.parse(meta_path).getroot()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
for element in root.iter():
|
||||
tag = str(element.tag or "").rsplit("}", 1)[-1].strip().lower()
|
||||
if tag not in {"mission", "satellite", "platform", "platformid", "missionid"}:
|
||||
continue
|
||||
satellite = normalize_lt1_satellite(element.text)
|
||||
if satellite:
|
||||
return satellite
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_scene_satellite(
|
||||
tiff_path: Path,
|
||||
meta_path: Path,
|
||||
explicit_satellite: str | None = None,
|
||||
) -> str:
|
||||
explicit = normalize_lt1_satellite(explicit_satellite)
|
||||
name_satellite = extract_scene_satellite_from_name(tiff_path.name)
|
||||
meta_satellite = extract_scene_satellite_from_meta(meta_path)
|
||||
|
||||
if explicit:
|
||||
if name_satellite and name_satellite != explicit:
|
||||
raise ValueError(
|
||||
f"Explicit satellite {explicit} does not match filename for {tiff_path.name}: {name_satellite}"
|
||||
)
|
||||
if meta_satellite and meta_satellite != explicit:
|
||||
raise ValueError(
|
||||
f"Explicit satellite {explicit} does not match metadata for {meta_path.name}: {meta_satellite}"
|
||||
)
|
||||
return explicit
|
||||
|
||||
if name_satellite and meta_satellite and name_satellite != meta_satellite:
|
||||
raise ValueError(
|
||||
f"Satellite mismatch between filename and metadata for {tiff_path.name}: "
|
||||
f"{name_satellite} vs {meta_satellite}"
|
||||
)
|
||||
if name_satellite:
|
||||
return name_satellite
|
||||
if meta_satellite:
|
||||
return meta_satellite
|
||||
raise ValueError(f"Unable to resolve LT-1 satellite from {tiff_path} / {meta_path}")
|
||||
|
||||
|
||||
def ensure_orbit_xml(
|
||||
date_yyyymmdd: str,
|
||||
satellite: str,
|
||||
annotation_xml: Path,
|
||||
orbit_root: Path,
|
||||
orbit_out_dir: Path,
|
||||
@@ -233,7 +314,7 @@ def ensure_orbit_xml(
|
||||
) -> Path:
|
||||
resolution = ensure_lt1_orbit_xml(
|
||||
date_yyyymmdd=date_yyyymmdd,
|
||||
satellite="LT1A",
|
||||
satellite=satellite,
|
||||
annotation_xml=annotation_xml,
|
||||
orbit_root=orbit_root,
|
||||
orbit_output_dir=orbit_out_dir,
|
||||
@@ -268,8 +349,14 @@ def resolve_task(
|
||||
slave_dir_name: str,
|
||||
scene_glob: str,
|
||||
prefer_scene_keyword: str,
|
||||
reference_satellite: str | None = None,
|
||||
secondary_satellite: str | None = None,
|
||||
) -> tuple[Scene, Scene]:
|
||||
scenes: list[Scene] = []
|
||||
satellite_hints = {
|
||||
"master": reference_satellite,
|
||||
"slave": secondary_satellite,
|
||||
}
|
||||
for role, subdir in (("master", master_dir_name), ("slave", slave_dir_name)):
|
||||
scene_dir = task_dir / subdir
|
||||
if not scene_dir.exists():
|
||||
@@ -278,8 +365,14 @@ def resolve_task(
|
||||
tiff_path = choose_scene_tiff(scene_dir, scene_glob, prefer_scene_keyword)
|
||||
meta_path = scene_meta_from_tiff(tiff_path)
|
||||
date_yyyymmdd = extract_scene_date(tiff_path.name)
|
||||
satellite = resolve_scene_satellite(
|
||||
tiff_path=tiff_path,
|
||||
meta_path=meta_path,
|
||||
explicit_satellite=satellite_hints.get(role),
|
||||
)
|
||||
orbit_xml_path = ensure_orbit_xml(
|
||||
date_yyyymmdd=date_yyyymmdd,
|
||||
satellite=satellite,
|
||||
annotation_xml=meta_path,
|
||||
orbit_root=orbit_root,
|
||||
orbit_out_dir=orbit_out_dir,
|
||||
@@ -291,6 +384,7 @@ def resolve_task(
|
||||
tiff_path=tiff_path,
|
||||
meta_path=meta_path,
|
||||
date_yyyymmdd=date_yyyymmdd,
|
||||
satellite=satellite,
|
||||
orbit_xml_path=orbit_xml_path,
|
||||
)
|
||||
)
|
||||
@@ -467,8 +561,8 @@ def print_summary(
|
||||
print(f"Work dir: {work_dir}")
|
||||
print(f"Output dir: {output_dir}")
|
||||
print(f"DEM: {config.dem_path}")
|
||||
print(f"Reference: {config.reference.tiff_path}")
|
||||
print(f"Secondary: {config.secondary.tiff_path}")
|
||||
print(f"Reference: {config.reference.tiff_path} [{config.reference.satellite}]")
|
||||
print(f"Secondary: {config.secondary.tiff_path} [{config.secondary.satellite}]")
|
||||
print(f"Ref orbit: {config.reference.orbit_xml_path}")
|
||||
print(f"Sec orbit: {config.secondary.orbit_xml_path}")
|
||||
print(f"BBox: {config.bbox if config.bbox is not None else 'auto'}")
|
||||
@@ -510,6 +604,8 @@ def main() -> int:
|
||||
slave_dir_name=args.slave_dir_name,
|
||||
scene_glob=args.scene_glob,
|
||||
prefer_scene_keyword=args.prefer_scene_keyword,
|
||||
reference_satellite=args.reference_satellite,
|
||||
secondary_satellite=args.secondary_satellite,
|
||||
)
|
||||
dem_path = resolve_dem(args.dem)
|
||||
bbox = parse_bbox_arg(args.bbox)
|
||||
|
||||
+12
-5
@@ -76,7 +76,7 @@ async def lifespan(app: FastAPI):
|
||||
ps_catalog_bootstrap = await psinsar_catalog_service.bootstrap_catalog_on_startup_clean()
|
||||
except Exception as exc:
|
||||
ps_catalog_bootstrap = {
|
||||
"storage_root": settings.PSINSAR_PRODUCT_DIR,
|
||||
"storage_root": settings.TIMESERIES_PRODUCT_DIR,
|
||||
"manifest_count": 0,
|
||||
"db_count": 0,
|
||||
"needs_rebuild": False,
|
||||
@@ -163,7 +163,7 @@ async def lifespan(app: FastAPI):
|
||||
if catalog_bootstrap.get("compat_error"):
|
||||
print(f">>> [Catalog Compat] Startup sync failed: {catalog_bootstrap['compat_error']}")
|
||||
print(
|
||||
">>> [PS Catalog] root={0} manifests={1} db={2} rebuild={3} queued={4}".format(
|
||||
">>> [Timeseries Catalog] root={0} manifests={1} db={2} rebuild={3} queued={4}".format(
|
||||
ps_catalog_bootstrap.get("storage_root") or "?",
|
||||
ps_catalog_bootstrap.get("manifest_count", 0),
|
||||
ps_catalog_bootstrap.get("db_count", 0),
|
||||
@@ -172,7 +172,7 @@ async def lifespan(app: FastAPI):
|
||||
)
|
||||
)
|
||||
if ps_catalog_bootstrap.get("error"):
|
||||
print(f">>> [PS Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}")
|
||||
print(f">>> [Timeseries Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}")
|
||||
print(
|
||||
">>> [Pairing] status={0} scenes={1} pairs={2} dirty={3} metric={4} rebuild={5}".format(
|
||||
pairing_bootstrap.get("status") or "?",
|
||||
@@ -195,16 +195,23 @@ async def lifespan(app: FastAPI):
|
||||
schema_ok = health.get("database", {}).get("schema_ok")
|
||||
worker_ok = health.get("worker", {}).get("ok")
|
||||
dinsar_catalog_ok = health.get("dinsar_result_catalog", {}).get("ok")
|
||||
psinsar_catalog_ok = health.get("psinsar_result_catalog", {}).get("ok")
|
||||
psinsar_catalog_ok = (
|
||||
health.get("timeseries_result_catalog", {})
|
||||
or health.get("psinsar_result_catalog", {})
|
||||
).get("ok")
|
||||
pairing_ok = health.get("pairing_system", {}).get("ok")
|
||||
idl_ok = health.get("idl", {}).get("ok")
|
||||
product_packages_ok = health.get("product_packages", {}).get("ok")
|
||||
wsl_runtime_ok = health.get("wsl_runtime", {}).get("ok")
|
||||
print(
|
||||
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} PSInSAR-Catalog:{4} Pairing:{5} IDL:{6}".format(
|
||||
">>> [Health] DB:{0} Schema:{1} Worker:{2} DInSAR-Catalog:{3} Timeseries-Catalog:{4} Packages:{5} WSL:{6} Pairing:{7} IDL:{8}".format(
|
||||
"OK" if db_ok else "FAIL",
|
||||
"OK" if schema_ok else "FAIL",
|
||||
"OK" if worker_ok else "FAIL",
|
||||
"OK" if dinsar_catalog_ok else "FAIL",
|
||||
"OK" if psinsar_catalog_ok else "FAIL",
|
||||
"OK" if product_packages_ok else "FAIL",
|
||||
"OK" if wsl_runtime_ok else "FAIL",
|
||||
"OK" if pairing_ok else "FAIL",
|
||||
"OK" if idl_ok else "FAIL",
|
||||
)
|
||||
|
||||
@@ -83,11 +83,13 @@ class ResultProductORM(Base):
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
product_id = Column(String(64), unique=True, index=True, nullable=False)
|
||||
catalog_name = Column(String(32), index=True, nullable=False, default="dinsar")
|
||||
product_family = Column(String(32), index=True, nullable=True)
|
||||
product_type = Column(String(32), index=True, nullable=False, default="dinsar")
|
||||
display_name = Column(String(255), nullable=False)
|
||||
task_name = Column(String(255), index=True, nullable=True)
|
||||
task_alias = Column(String(255), index=True, nullable=True)
|
||||
pair_key = Column(String(128), index=True, nullable=True)
|
||||
stack_key = Column(String(128), index=True, nullable=True)
|
||||
pair_uid = Column(String(64), index=True, nullable=True)
|
||||
run_key = Column(String(128), index=True, nullable=True)
|
||||
network_run_id = Column(String(64), index=True, nullable=True)
|
||||
@@ -97,12 +99,17 @@ class ResultProductORM(Base):
|
||||
profile_code = Column(String(64), index=True, nullable=True)
|
||||
engine_code = Column(String(32), index=True, nullable=False)
|
||||
engine_version = Column(String(64), nullable=True)
|
||||
package_schema = Column(String(64), nullable=True)
|
||||
package_layout = Column(String(64), nullable=True)
|
||||
processor_code = Column(String(64), nullable=True)
|
||||
runtime_id = Column(String(64), nullable=True)
|
||||
status = Column(String(32), index=True, nullable=False, default="READY")
|
||||
health_status = Column(String(16), index=True, nullable=False, default="OK")
|
||||
|
||||
publish_dir = Column(String, unique=True, nullable=False)
|
||||
manifest_path = Column(String, unique=True, nullable=False)
|
||||
source_primary_path = Column(String, nullable=True)
|
||||
native_output_dir = Column(String, nullable=True)
|
||||
preview_path = Column(String, nullable=True)
|
||||
primary_asset_path = Column(String, nullable=True)
|
||||
|
||||
@@ -260,6 +267,7 @@ class ResultCatalogStateORM(Base):
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
catalog_name = Column(String(32), unique=True, index=True, nullable=False)
|
||||
product_family = Column(String(32), index=True, nullable=True)
|
||||
storage_root = Column(String, nullable=False)
|
||||
status = Column(String(16), index=True, nullable=False, default="READY")
|
||||
needs_rebuild = Column(Boolean, nullable=False, default=False)
|
||||
@@ -789,10 +797,12 @@ class DinsarProductionRunORM(Base):
|
||||
task_id = Column(String, index=True, nullable=True)
|
||||
workflow_run_id = Column(String, index=True, nullable=True)
|
||||
|
||||
product_family = Column(String(32), index=True, nullable=True)
|
||||
engine_code = Column(String(32), index=True, nullable=False, default="sarscape")
|
||||
profile_code = Column(String(64), index=True, nullable=False, default="custom6")
|
||||
mode = Column(String(32), index=True, nullable=False, default="custom")
|
||||
source_root = Column(String, nullable=False)
|
||||
publish_root_dir = Column(String, nullable=True)
|
||||
status = Column(String(32), index=True, nullable=False, default="PENDING")
|
||||
cancel_requested = Column(Boolean, nullable=False, default=False)
|
||||
|
||||
@@ -928,11 +938,14 @@ class PsTimeseriesRunORM(Base):
|
||||
run_id = Column(String(64), unique=True, index=True, nullable=False)
|
||||
batch_id = Column(String, ForeignKey("ps_task_batches.batch_id"), index=True, nullable=False)
|
||||
|
||||
product_family = Column(String(32), index=True, nullable=True)
|
||||
run_name = Column(String(255), nullable=False)
|
||||
catalog_name = Column(String(32), index=True, nullable=False, default="psinsar")
|
||||
stack_key = Column(String(128), index=True, nullable=True)
|
||||
mode = Column(String(32), nullable=False, default="sbas")
|
||||
engine_code = Column(String(32), index=True, nullable=False, default="isce2")
|
||||
processor_code = Column(String(64), nullable=False, default="isce2_stack_mintpy")
|
||||
runtime_id = Column(String(64), nullable=True)
|
||||
env_name = Column(String(128), nullable=True)
|
||||
wsl_distro = Column(String(128), nullable=True)
|
||||
|
||||
|
||||
@@ -66,10 +66,12 @@ class DinsarResult(BaseModel):
|
||||
id: int
|
||||
product_id: Optional[str] = None
|
||||
compat_result_id: Optional[int] = None
|
||||
product_family: Optional[str] = None
|
||||
name: str
|
||||
task_name: Optional[str] = None
|
||||
task_alias: Optional[str] = None
|
||||
pair_key: Optional[str] = None
|
||||
stack_key: Optional[str] = None
|
||||
pair_uid: Optional[str] = None
|
||||
run_key: Optional[str] = None
|
||||
network_run_id: Optional[str] = None
|
||||
@@ -472,11 +474,14 @@ class PsTaskItem(BaseModel):
|
||||
class PsTimeseriesRun(BaseModel):
|
||||
run_id: str
|
||||
batch_id: str
|
||||
product_family: Optional[str] = None
|
||||
run_name: str
|
||||
catalog_name: str
|
||||
stack_key: Optional[str] = None
|
||||
mode: str
|
||||
engine_code: str
|
||||
processor_code: str
|
||||
runtime_id: Optional[str] = None
|
||||
env_name: Optional[str] = None
|
||||
wsl_distro: Optional[str] = None
|
||||
status: str
|
||||
|
||||
@@ -1,78 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
_pyint_gamma_die() {
|
||||
echo "$1" >&2
|
||||
_pyint_legacy_profile="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)/deploy/wsl/profiles/gamma_env.sh"
|
||||
if [ ! -f "${_pyint_legacy_profile}" ]; then
|
||||
echo "Gamma profile not found: ${_pyint_legacy_profile}" >&2
|
||||
return 1 2>/dev/null || exit 1
|
||||
}
|
||||
|
||||
_pyint_gamma_home=""
|
||||
if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${PYINT_GAMMA_HOME}"
|
||||
elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${GAMMA_HOME}"
|
||||
else
|
||||
for _candidate in \
|
||||
/usr/local/GAMMA_SOFTWARE-20240627 \
|
||||
/usr/local/GAMMA_SOFTWARE-* \
|
||||
/opt/GAMMA_SOFTWARE-*; do
|
||||
[ -d "${_candidate}" ] || continue
|
||||
_pyint_gamma_home="${_candidate}"
|
||||
break
|
||||
done
|
||||
fi
|
||||
|
||||
[ -n "${_pyint_gamma_home}" ] || _pyint_gamma_die "Gamma home not found."
|
||||
. "${_pyint_legacy_profile}"
|
||||
unset _pyint_legacy_profile
|
||||
|
||||
export GAMMA_HOME="${_pyint_gamma_home}"
|
||||
export MSP_HOME="${GAMMA_HOME}/MSP"
|
||||
export ISP_HOME="${GAMMA_HOME}/ISP"
|
||||
export DIFF_HOME="${GAMMA_HOME}/DIFF"
|
||||
export DISP_HOME="${GAMMA_HOME}/DISP"
|
||||
export LAT_HOME="${GAMMA_HOME}/LAT"
|
||||
export IPTA_HOME="${GAMMA_HOME}/IPTA"
|
||||
export GEO_HOME="${GAMMA_HOME}/GEO"
|
||||
|
||||
_pyint_gamma_prepend_path() {
|
||||
local _dir="$1"
|
||||
[ -d "${_dir}" ] || return 0
|
||||
case ":${PATH}:" in
|
||||
*":${_dir}:"*) ;;
|
||||
*) PATH="${_dir}:${PATH}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for _gamma_dir in \
|
||||
"${MSP_HOME}/bin" \
|
||||
"${ISP_HOME}/bin" \
|
||||
"${DIFF_HOME}/bin" \
|
||||
"${DISP_HOME}/bin" \
|
||||
"${LAT_HOME}/bin" \
|
||||
"${IPTA_HOME}/bin" \
|
||||
"${GEO_HOME}/bin" \
|
||||
"${MSP_HOME}/scripts" \
|
||||
"${ISP_HOME}/scripts" \
|
||||
"${DIFF_HOME}/scripts" \
|
||||
"${DISP_HOME}/scripts" \
|
||||
"${LAT_HOME}/scripts" \
|
||||
"${IPTA_HOME}/scripts" \
|
||||
"${GEO_HOME}/scripts"; do
|
||||
_pyint_gamma_prepend_path "${_gamma_dir}"
|
||||
done
|
||||
|
||||
export PATH
|
||||
export OS="linux64"
|
||||
export HDF5_DISABLE_VERSION_CHECK="1"
|
||||
export GNUTERM="${GNUTERM:-qt}"
|
||||
export GAMMA_RASTER="${GAMMA_RASTER:-BMP}"
|
||||
export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
_pyint_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
_pyint_script_dir="${_pyint_repo_root}/third_party/PyINT/pyint"
|
||||
_pyint_gamma_prepend_path "${_pyint_script_dir}"
|
||||
|
||||
unset _pyint_gamma_home
|
||||
unset _gamma_dir
|
||||
unset _pyint_repo_root
|
||||
unset _pyint_script_dir
|
||||
unset -f _pyint_gamma_prepend_path
|
||||
unset -f _pyint_gamma_die
|
||||
return 0 2>/dev/null || exit 0
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
@@ -43,6 +43,10 @@ class RunJobRequest(BaseModel):
|
||||
profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap / lt1_gamma_dinsar")
|
||||
root_dir: str = Field(..., description="Windows root directory")
|
||||
num_to_process: int = Field(default=0, ge=0, description="How many tasks to process; 0 means all")
|
||||
rerun_mode: Literal["unfinished_only", "rerun_all"] = Field(
|
||||
default="unfinished_only",
|
||||
description="unfinished_only = only run unfinished tasks; rerun_all = rerun everything",
|
||||
)
|
||||
timeout_seconds: Optional[int] = Field(default=None, ge=60)
|
||||
extra: Dict[str, Any] = Field(default_factory=dict, description="Engine-specific parameters")
|
||||
|
||||
@@ -178,9 +182,23 @@ async def submit_run(
|
||||
engine.validate_root_dir,
|
||||
req.root_dir,
|
||||
req.num_to_process,
|
||||
req.rerun_mode,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
if int(validation_summary.get("task_count", 0) or 0) <= 0:
|
||||
if (
|
||||
req.rerun_mode == "unfinished_only"
|
||||
and int(validation_summary.get("skipped_completed_count", 0) or 0) > 0
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"All discovered Task_* directories already have completed "
|
||||
f"{req.engine_code}/{req.profile} results under: {req.root_dir}"
|
||||
),
|
||||
)
|
||||
raise HTTPException(status_code=400, detail="No valid task directories selected.")
|
||||
|
||||
effective_timeout_seconds = req.timeout_seconds
|
||||
if effective_timeout_seconds is None:
|
||||
@@ -189,7 +207,13 @@ async def submit_run(
|
||||
effective_timeout_seconds = int(engine_default_timeout)
|
||||
|
||||
pyint_preview = None
|
||||
if req.engine_code == "pyint":
|
||||
skip_pyint_submit_preview = (
|
||||
req.engine_code == "pyint"
|
||||
and req.rerun_mode == "unfinished_only"
|
||||
and validation_summary is not None
|
||||
and int(validation_summary.get("skipped_completed_count", 0) or 0) > 0
|
||||
)
|
||||
if req.engine_code == "pyint" and not skip_pyint_submit_preview:
|
||||
try:
|
||||
pyint_preview = await asyncio.to_thread(
|
||||
build_pyint_input_preview,
|
||||
@@ -216,36 +240,43 @@ async def submit_run(
|
||||
"profile": req.profile,
|
||||
"root_dir": req.root_dir,
|
||||
"num_to_process": req.num_to_process,
|
||||
"rerun_mode": req.rerun_mode,
|
||||
"timeout_seconds": effective_timeout_seconds,
|
||||
"extra": dict(req.extra or {}),
|
||||
}
|
||||
|
||||
from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN, JOB_TYPE_PYINT_RUN
|
||||
create_managed_run = False
|
||||
normalized_extra = dict(payload["extra"])
|
||||
|
||||
if req.engine_code == "sarscape":
|
||||
payload["mode"] = "custom" if req.profile == "custom6" else "metatask"
|
||||
job_type = JOB_TYPE_IDL_RUN_DINSAR
|
||||
max_attempts = DINSAR_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
create_managed_run = True
|
||||
elif req.engine_code in {"isce2", "pyint"}:
|
||||
if hasattr(engine, "normalize_extra"):
|
||||
try:
|
||||
payload["extra"] = engine.normalize_extra(payload["extra"])
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
normalized_extra = dict(payload["extra"])
|
||||
if req.engine_code == "isce2":
|
||||
job_type = JOB_TYPE_ISCE2_RUN
|
||||
max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
create_managed_run = True
|
||||
else:
|
||||
job_type = JOB_TYPE_PYINT_RUN
|
||||
max_attempts = PYINT_PRODUCTION_JOB_MAX_ATTEMPTS
|
||||
if validation_summary is not None:
|
||||
validated_task_count = validation_summary.get("task_count", 0)
|
||||
if pyint_preview is not None:
|
||||
validated_task_count = int(pyint_preview.get("selected_task_count", validated_task_count) or 0)
|
||||
payload["extra"].update(
|
||||
{
|
||||
"__validated_task_count": validated_task_count,
|
||||
"__validated_mode": validation_summary.get("mode", ""),
|
||||
"__rerun_mode": req.rerun_mode,
|
||||
"__discovered_task_count": int(validation_summary.get("discovered_task_count", validated_task_count) or 0),
|
||||
"__skipped_completed_count": int(validation_summary.get("skipped_completed_count", 0) or 0),
|
||||
}
|
||||
)
|
||||
else:
|
||||
@@ -255,15 +286,16 @@ async def submit_run(
|
||||
)
|
||||
|
||||
try:
|
||||
if req.engine_code == "sarscape":
|
||||
if create_managed_run:
|
||||
async with _new_session() as db:
|
||||
result = await dinsar_production_service.create_run(
|
||||
engine_code=req.engine_code,
|
||||
profile_code=req.profile,
|
||||
root_dir=req.root_dir,
|
||||
num_to_process=req.num_to_process,
|
||||
rerun_mode=req.rerun_mode,
|
||||
timeout_seconds=req.timeout_seconds,
|
||||
extra=req.extra,
|
||||
extra=normalized_extra,
|
||||
created_by=getattr(current_user, "username", None),
|
||||
db=db,
|
||||
)
|
||||
@@ -276,6 +308,9 @@ async def submit_run(
|
||||
"engine_code": req.engine_code,
|
||||
"profile": req.profile,
|
||||
"selected_task_count": result.get("selected_task_count", 0),
|
||||
"discovered_task_count": result.get("discovered_task_count", result.get("selected_task_count", 0)),
|
||||
"skipped_completed_count": result.get("skipped_completed_count", 0),
|
||||
"rerun_mode": result.get("rerun_mode", req.rerun_mode),
|
||||
"message": "Task queued.",
|
||||
}
|
||||
|
||||
@@ -303,6 +338,13 @@ async def submit_run(
|
||||
"engine_code": req.engine_code,
|
||||
"profile": req.profile,
|
||||
"selected_task_count": validation_summary.get("task_count", 1) if validation_summary else 1,
|
||||
"discovered_task_count": (
|
||||
validation_summary.get("discovered_task_count", validation_summary.get("task_count", 1))
|
||||
if validation_summary
|
||||
else 1
|
||||
),
|
||||
"skipped_completed_count": validation_summary.get("skipped_completed_count", 0) if validation_summary else 0,
|
||||
"rerun_mode": req.rerun_mode,
|
||||
"message": "Task queued.",
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,517 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from ..config import settings
|
||||
from .dinsar_naming import RUN_META_FILENAME
|
||||
from .dinsar_result_layout_service import (
|
||||
EXECUTION_MANIFEST_FILENAME,
|
||||
PACKAGE_MANIFEST_FILENAME,
|
||||
get_run_disp_asset_paths,
|
||||
get_run_native_output_dir,
|
||||
normalize_envi_run_layout,
|
||||
)
|
||||
|
||||
|
||||
_ENVI_ENGINE_CODES = {"envi", "sarscape"}
|
||||
_LEGACY_DISP_RE = re.compile(r"^.+_rsp_disp$", re.IGNORECASE)
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
return os.path.normpath(os.path.abspath(str(path or "").strip()))
|
||||
|
||||
|
||||
def _load_json(path: str) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
return payload if isinstance(payload, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _iter_managed_run_dirs(root_dir: str) -> Iterable[str]:
|
||||
normalized_root = _normalize_path(root_dir)
|
||||
if not os.path.isdir(normalized_root):
|
||||
return
|
||||
for pair_name in sorted(os.listdir(normalized_root)):
|
||||
pair_dir = os.path.join(normalized_root, pair_name)
|
||||
if not os.path.isdir(pair_dir):
|
||||
continue
|
||||
runs_dir = os.path.join(pair_dir, "runs")
|
||||
if not os.path.isdir(runs_dir):
|
||||
continue
|
||||
for run_name in sorted(os.listdir(runs_dir)):
|
||||
run_dir = os.path.join(runs_dir, run_name)
|
||||
if os.path.isdir(run_dir):
|
||||
yield run_dir
|
||||
|
||||
|
||||
def _read_run_payloads(run_dir: str) -> Dict[str, Dict[str, Any]]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
payloads: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
execution_manifest = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME))
|
||||
if execution_manifest:
|
||||
payloads["execution"] = execution_manifest
|
||||
|
||||
run_meta = _load_json(os.path.join(normalized_run_dir, RUN_META_FILENAME))
|
||||
if run_meta:
|
||||
payloads["run_meta"] = run_meta
|
||||
|
||||
package_manifest = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME))
|
||||
if package_manifest:
|
||||
payloads["package"] = package_manifest
|
||||
|
||||
return payloads
|
||||
|
||||
|
||||
def _first_text(*values: Any) -> Optional[str]:
|
||||
for value in values:
|
||||
text = str(value or "").strip()
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_engine_code(payloads: Dict[str, Dict[str, Any]]) -> str:
|
||||
package_payload = payloads.get("package") or {}
|
||||
package_engine = package_payload.get("engine") if isinstance(package_payload.get("engine"), dict) else {}
|
||||
return (
|
||||
_first_text(
|
||||
(payloads.get("execution") or {}).get("engine_code"),
|
||||
(payloads.get("run_meta") or {}).get("engine_code"),
|
||||
package_engine.get("code"),
|
||||
package_payload.get("engine_code"),
|
||||
)
|
||||
or ""
|
||||
).strip().lower()
|
||||
|
||||
|
||||
def _resolve_pair_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]:
|
||||
package_payload = payloads.get("package") or {}
|
||||
package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {}
|
||||
return _first_text(
|
||||
(payloads.get("execution") or {}).get("pair_key"),
|
||||
(payloads.get("run_meta") or {}).get("pair_key"),
|
||||
package_identity.get("pair_key"),
|
||||
package_payload.get("pair_key"),
|
||||
os.path.basename(os.path.dirname(os.path.dirname(_normalize_path(run_dir)))),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_run_key(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[str]:
|
||||
package_payload = payloads.get("package") or {}
|
||||
package_identity = package_payload.get("identity") if isinstance(package_payload.get("identity"), dict) else {}
|
||||
return _first_text(
|
||||
(payloads.get("execution") or {}).get("run_key"),
|
||||
(payloads.get("run_meta") or {}).get("run_key"),
|
||||
package_identity.get("run_key"),
|
||||
package_payload.get("run_key"),
|
||||
os.path.basename(_normalize_path(run_dir)),
|
||||
)
|
||||
|
||||
|
||||
def _relpath_lower(base_dir: str, path: str) -> str:
|
||||
return os.path.relpath(_normalize_path(path), _normalize_path(base_dir)).replace("/", os.sep).lower()
|
||||
|
||||
|
||||
def _is_standard_disp_path(run_dir: str, path: str) -> bool:
|
||||
disp_paths = get_run_disp_asset_paths(run_dir)
|
||||
normalized_path = _normalize_path(path)
|
||||
return normalized_path in {
|
||||
_normalize_path(disp_paths["primary"]),
|
||||
_normalize_path(disp_paths["hdr"]),
|
||||
_normalize_path(disp_paths["sml"]),
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_legacy_primary(path: str) -> bool:
|
||||
return bool(_LEGACY_DISP_RE.match(os.path.basename(str(path or "").strip())))
|
||||
|
||||
|
||||
def _looks_like_envi_primary(run_dir: str, path: str) -> bool:
|
||||
normalized_path = _normalize_path(path)
|
||||
if not os.path.isfile(normalized_path):
|
||||
return False
|
||||
if _is_standard_disp_path(run_dir, normalized_path):
|
||||
return True
|
||||
return _looks_like_legacy_primary(normalized_path)
|
||||
|
||||
|
||||
def _build_source_files(primary_file: str, source_files: Optional[List[str]] = None) -> List[str]:
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
candidates: List[str] = [normalized_primary]
|
||||
for raw_path in source_files or []:
|
||||
normalized = _normalize_path(raw_path)
|
||||
if normalized and normalized not in candidates and os.path.isfile(normalized):
|
||||
candidates.append(normalized)
|
||||
|
||||
for ext in (".hdr", ".sml"):
|
||||
sidecar = normalized_primary + ext
|
||||
if os.path.isfile(sidecar) and sidecar not in candidates:
|
||||
candidates.append(sidecar)
|
||||
return candidates
|
||||
|
||||
|
||||
def _find_standardized_run_files(run_dir: str) -> Optional[Dict[str, Any]]:
|
||||
disp_paths = get_run_disp_asset_paths(run_dir)
|
||||
primary_file = disp_paths["primary"]
|
||||
if not os.path.isfile(primary_file):
|
||||
return None
|
||||
return {
|
||||
"primary_file": _normalize_path(primary_file),
|
||||
"source_files": _build_source_files(primary_file),
|
||||
"discovery": "standard_asset",
|
||||
}
|
||||
|
||||
|
||||
def _iter_metadata_candidates(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
|
||||
execution_payload = payloads.get("execution") or {}
|
||||
if execution_payload:
|
||||
primary_file = execution_payload.get("primary_file")
|
||||
source_files = execution_payload.get("source_files") if isinstance(execution_payload.get("source_files"), list) else None
|
||||
if primary_file:
|
||||
yield {
|
||||
"primary_file": primary_file,
|
||||
"source_files": source_files,
|
||||
"discovery": "execution_manifest",
|
||||
}
|
||||
|
||||
package_payload = payloads.get("package") or {}
|
||||
source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {}
|
||||
if source_payload:
|
||||
primary_file = source_payload.get("primary_path")
|
||||
if primary_file:
|
||||
yield {
|
||||
"primary_file": primary_file,
|
||||
"source_files": None,
|
||||
"discovery": "package_manifest",
|
||||
}
|
||||
|
||||
|
||||
def _find_latest_legacy_primary(search_dir: str) -> Optional[str]:
|
||||
normalized_search_dir = _normalize_path(search_dir)
|
||||
if not os.path.isdir(normalized_search_dir):
|
||||
return None
|
||||
candidates: List[str] = []
|
||||
try:
|
||||
with os.scandir(normalized_search_dir) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_file(follow_symlinks=False) and _looks_like_legacy_primary(entry.name):
|
||||
candidates.append(entry.path)
|
||||
except OSError:
|
||||
continue
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda item: os.path.getmtime(item), reverse=True)
|
||||
return _normalize_path(candidates[0])
|
||||
|
||||
|
||||
def _discover_envi_run_files(run_dir: str, payloads: Dict[str, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||
standardized = _find_standardized_run_files(run_dir)
|
||||
if standardized:
|
||||
return standardized
|
||||
|
||||
for candidate in _iter_metadata_candidates(run_dir, payloads):
|
||||
primary_file = _normalize_path(str(candidate.get("primary_file") or "").strip())
|
||||
if not _looks_like_envi_primary(run_dir, primary_file):
|
||||
continue
|
||||
return {
|
||||
"primary_file": primary_file,
|
||||
"source_files": _build_source_files(primary_file, candidate.get("source_files")),
|
||||
"discovery": candidate["discovery"],
|
||||
}
|
||||
|
||||
search_dirs = [
|
||||
_normalize_path(run_dir),
|
||||
get_run_native_output_dir(run_dir),
|
||||
]
|
||||
for search_dir in search_dirs:
|
||||
primary_file = _find_latest_legacy_primary(search_dir)
|
||||
if primary_file:
|
||||
return {
|
||||
"primary_file": primary_file,
|
||||
"source_files": _build_source_files(primary_file),
|
||||
"discovery": "filesystem_scan",
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def validate_envi_run_layout(run_dir: str) -> Dict[str, Any]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
native_output_dir = get_run_native_output_dir(normalized_run_dir)
|
||||
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
|
||||
issues: List[str] = []
|
||||
|
||||
if not os.path.isdir(normalized_run_dir):
|
||||
issues.append("run_dir_missing")
|
||||
if not os.path.isdir(native_output_dir):
|
||||
issues.append("native_dir_missing")
|
||||
if not os.path.isfile(disp_paths["primary"]):
|
||||
issues.append("standard_disp_missing")
|
||||
|
||||
execution_payload = _load_json(os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)) or {}
|
||||
if execution_payload:
|
||||
if _normalize_path(str(execution_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]):
|
||||
issues.append("execution_manifest_primary_mismatch")
|
||||
if _normalize_path(str(execution_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
|
||||
issues.append("execution_manifest_native_mismatch")
|
||||
|
||||
package_payload = _load_json(os.path.join(normalized_run_dir, PACKAGE_MANIFEST_FILENAME)) or {}
|
||||
if package_payload:
|
||||
source_payload = package_payload.get("source") if isinstance(package_payload.get("source"), dict) else {}
|
||||
if _normalize_path(str(source_payload.get("primary_path") or "")) != _normalize_path(disp_paths["primary"]):
|
||||
issues.append("package_manifest_primary_mismatch")
|
||||
if _normalize_path(str(source_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
|
||||
issues.append("package_manifest_native_mismatch")
|
||||
assets_payload = package_payload.get("assets") if isinstance(package_payload.get("assets"), list) else []
|
||||
disp_asset = next(
|
||||
(item for item in assets_payload if str((item or {}).get("role") or "").strip() == "disp"),
|
||||
None,
|
||||
)
|
||||
expected_relative = _relpath_lower(normalized_run_dir, disp_paths["primary"])
|
||||
actual_relative = _relpath_lower(
|
||||
normalized_run_dir,
|
||||
os.path.join(normalized_run_dir, str((disp_asset or {}).get("relative_path") or "")),
|
||||
) if disp_asset else ""
|
||||
if not disp_asset or actual_relative != expected_relative:
|
||||
issues.append("package_manifest_disp_asset_mismatch")
|
||||
|
||||
run_key = _resolve_run_key(normalized_run_dir, _read_run_payloads(normalized_run_dir))
|
||||
current_dir = os.path.join(os.path.dirname(os.path.dirname(normalized_run_dir)), "current")
|
||||
matched_pointer_count = 0
|
||||
if run_key and os.path.isdir(current_dir):
|
||||
for name in os.listdir(current_dir):
|
||||
if not name.lower().endswith(".json"):
|
||||
continue
|
||||
pointer_payload = _load_json(os.path.join(current_dir, name))
|
||||
if not pointer_payload:
|
||||
continue
|
||||
if str(pointer_payload.get("run_key") or "").strip() != run_key:
|
||||
continue
|
||||
matched_pointer_count += 1
|
||||
if _normalize_path(str(pointer_payload.get("primary_file") or "")) != _normalize_path(disp_paths["primary"]):
|
||||
issues.append(f"current_pointer_primary_mismatch:{name}")
|
||||
if _normalize_path(str(pointer_payload.get("native_output_dir") or "")) != _normalize_path(native_output_dir):
|
||||
issues.append(f"current_pointer_native_mismatch:{name}")
|
||||
return {
|
||||
"ok": len(issues) == 0,
|
||||
"issues": issues,
|
||||
"matched_current_pointer_count": matched_pointer_count,
|
||||
}
|
||||
|
||||
|
||||
def inspect_envi_run_layout(run_dir: str) -> Dict[str, Any]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
payloads = _read_run_payloads(normalized_run_dir)
|
||||
engine_code = _resolve_engine_code(payloads)
|
||||
files = _discover_envi_run_files(normalized_run_dir, payloads)
|
||||
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
|
||||
standardized = bool(os.path.isfile(disp_paths["primary"]))
|
||||
native_exists = bool(os.path.isdir(get_run_native_output_dir(normalized_run_dir)))
|
||||
|
||||
is_envi_run = bool(files) or engine_code in _ENVI_ENGINE_CODES
|
||||
layout_state = "unknown"
|
||||
if not is_envi_run:
|
||||
layout_state = "not_envi"
|
||||
elif standardized and native_exists:
|
||||
layout_state = "normalized"
|
||||
elif files and _looks_like_legacy_primary(files["primary_file"]):
|
||||
layout_state = "legacy"
|
||||
elif standardized:
|
||||
layout_state = "partial_normalized"
|
||||
|
||||
validation = validate_envi_run_layout(normalized_run_dir) if is_envi_run else {"ok": True, "issues": [], "matched_current_pointer_count": 0}
|
||||
needs_migration = bool(
|
||||
is_envi_run
|
||||
and (
|
||||
layout_state in {"legacy", "partial_normalized"}
|
||||
or (layout_state == "normalized" and not validation["ok"])
|
||||
)
|
||||
)
|
||||
return {
|
||||
"run_dir": normalized_run_dir,
|
||||
"pair_key": _resolve_pair_key(normalized_run_dir, payloads),
|
||||
"run_key": _resolve_run_key(normalized_run_dir, payloads),
|
||||
"engine_code": engine_code,
|
||||
"is_envi_run": is_envi_run,
|
||||
"layout_state": layout_state,
|
||||
"needs_migration": needs_migration,
|
||||
"primary_file": (files or {}).get("primary_file"),
|
||||
"source_files": (files or {}).get("source_files") or [],
|
||||
"discovery": (files or {}).get("discovery"),
|
||||
"standard_disp_exists": standardized,
|
||||
"native_dir_exists": native_exists,
|
||||
"validation": validation,
|
||||
}
|
||||
|
||||
|
||||
class DinsarLayoutMigrationService:
|
||||
def inspect_managed_envi_runs(
|
||||
self,
|
||||
*,
|
||||
root_dir: Optional[str] = None,
|
||||
pair_keys: Optional[List[str]] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_root = _normalize_path(root_dir or settings.DINSAR_PRODUCT_DIR)
|
||||
filter_pair_keys = {
|
||||
str(item or "").strip()
|
||||
for item in pair_keys or []
|
||||
if str(item or "").strip()
|
||||
}
|
||||
|
||||
details: List[Dict[str, Any]] = []
|
||||
inspected = 0
|
||||
envi_run_count = 0
|
||||
legacy_count = 0
|
||||
normalized_count = 0
|
||||
pending_count = 0
|
||||
|
||||
for run_dir in _iter_managed_run_dirs(normalized_root):
|
||||
inspection = inspect_envi_run_layout(run_dir)
|
||||
if filter_pair_keys and inspection.get("pair_key") not in filter_pair_keys:
|
||||
continue
|
||||
inspected += 1
|
||||
if inspection["is_envi_run"]:
|
||||
envi_run_count += 1
|
||||
if inspection["layout_state"] == "legacy":
|
||||
legacy_count += 1
|
||||
if inspection["layout_state"] == "normalized":
|
||||
normalized_count += 1
|
||||
if inspection["needs_migration"]:
|
||||
pending_count += 1
|
||||
details.append(inspection)
|
||||
if limit is not None and len(details) >= int(limit):
|
||||
break
|
||||
|
||||
return {
|
||||
"root_dir": normalized_root,
|
||||
"inspected_run_count": inspected,
|
||||
"envi_run_count": envi_run_count,
|
||||
"legacy_run_count": legacy_count,
|
||||
"normalized_run_count": normalized_count,
|
||||
"pending_migration_count": pending_count,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
def migrate_managed_envi_runs(
|
||||
self,
|
||||
*,
|
||||
root_dir: Optional[str] = None,
|
||||
pair_keys: Optional[List[str]] = None,
|
||||
limit: Optional[int] = None,
|
||||
dry_run: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
inspection = self.inspect_managed_envi_runs(
|
||||
root_dir=root_dir,
|
||||
pair_keys=pair_keys,
|
||||
limit=limit,
|
||||
)
|
||||
details: List[Dict[str, Any]] = []
|
||||
migrated_count = 0
|
||||
rewritten_count = 0
|
||||
skipped_count = 0
|
||||
failed_count = 0
|
||||
|
||||
for item in inspection["details"]:
|
||||
if not item["is_envi_run"]:
|
||||
skipped_count += 1
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
"status": "skipped_not_envi",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if not item["primary_file"]:
|
||||
failed_count += 1
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
"status": "failed_no_primary",
|
||||
"error": "Unable to resolve ENVI primary displacement file.",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if not item["needs_migration"]:
|
||||
skipped_count += 1
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
"status": "skipped_already_normalized",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
"status": "would_migrate",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
result = normalize_envi_run_layout(
|
||||
item["run_dir"],
|
||||
primary_file=item["primary_file"],
|
||||
source_files=item["source_files"],
|
||||
rewrite_metadata=True,
|
||||
)
|
||||
validation = validate_envi_run_layout(item["run_dir"])
|
||||
status = "migrated"
|
||||
if not result["promoted_files"] and not result["moved_entries"]:
|
||||
status = "rewritten"
|
||||
if validation["ok"]:
|
||||
if status == "rewritten":
|
||||
rewritten_count += 1
|
||||
else:
|
||||
migrated_count += 1
|
||||
else:
|
||||
status = "failed_validation"
|
||||
failed_count += 1
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
**result,
|
||||
"status": status,
|
||||
"post_validation": validation,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
failed_count += 1
|
||||
details.append(
|
||||
{
|
||||
**item,
|
||||
"status": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"root_dir": inspection["root_dir"],
|
||||
"dry_run": bool(dry_run),
|
||||
"inspected_run_count": inspection["inspected_run_count"],
|
||||
"envi_run_count": inspection["envi_run_count"],
|
||||
"pending_migration_count": inspection["pending_migration_count"],
|
||||
"migrated_count": migrated_count,
|
||||
"rewritten_count": rewritten_count,
|
||||
"skipped_count": skipped_count,
|
||||
"failed_count": failed_count,
|
||||
"details": details,
|
||||
}
|
||||
|
||||
|
||||
dinsar_layout_migration_service = DinsarLayoutMigrationService()
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
@@ -12,6 +13,7 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from .. import database
|
||||
from ..config import settings
|
||||
from ..models import (
|
||||
DinsarProductionExecutionORM,
|
||||
DinsarProductionRunItemORM,
|
||||
@@ -24,6 +26,7 @@ from .workflow_service import workflow_service
|
||||
|
||||
|
||||
TASK_TYPE_DINSAR_PRODUCTION = "IDL_RUN_DINSAR"
|
||||
TASK_TYPE_ISCE2_DINSAR_PRODUCTION = "ISCE2_RUN"
|
||||
RUN_STATUS_PENDING = "PENDING"
|
||||
RUN_STATUS_RUNNING = "RUNNING"
|
||||
RUN_STATUS_COMPLETED = "COMPLETED"
|
||||
@@ -40,8 +43,15 @@ EXECUTION_STATUS_RUNNING = "RUNNING"
|
||||
EXECUTION_STATUS_COMPLETED = "COMPLETED"
|
||||
EXECUTION_STATUS_FAILED = "FAILED"
|
||||
EXECUTION_STATUS_CANCELLED = "CANCELLED"
|
||||
RERUN_MODE_UNFINISHED_ONLY = "unfinished_only"
|
||||
RERUN_MODE_RERUN_ALL = "rerun_all"
|
||||
VALID_RERUN_MODES = {
|
||||
RERUN_MODE_UNFINISHED_ONLY,
|
||||
RERUN_MODE_RERUN_ALL,
|
||||
}
|
||||
|
||||
CURRENT_POINTER_FILENAME = "current.json"
|
||||
CURRENT_POINTER_DIRNAME = "current"
|
||||
EXECUTION_MANIFEST_FILENAME = "execution_manifest.json"
|
||||
RUNS_STEP_ID = "execute_items"
|
||||
RUNS_STEP_NAME = "Execute ENVI D-InSAR items"
|
||||
@@ -56,6 +66,34 @@ TERMINAL_ITEM_STATUSES = {
|
||||
RUN_ITEM_STATUS_SKIPPED,
|
||||
RUN_ITEM_STATUS_CANCELLED,
|
||||
}
|
||||
_SAFE_POINTER_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
|
||||
|
||||
def _task_type_for_engine(engine_code: str) -> str:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "sarscape":
|
||||
return TASK_TYPE_DINSAR_PRODUCTION
|
||||
if normalized == "isce2":
|
||||
return TASK_TYPE_ISCE2_DINSAR_PRODUCTION
|
||||
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
|
||||
|
||||
|
||||
def _workflow_name_for_engine(engine_code: str) -> str:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "sarscape":
|
||||
return "dinsar_sarscape_production"
|
||||
if normalized == "isce2":
|
||||
return "dinsar_isce2_production"
|
||||
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
|
||||
|
||||
|
||||
def _workflow_step_name_for_engine(engine_code: str) -> str:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "sarscape":
|
||||
return RUNS_STEP_NAME
|
||||
if normalized == "isce2":
|
||||
return "Execute ISCE2 D-InSAR items"
|
||||
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
|
||||
|
||||
|
||||
def _new_session() -> AsyncSession:
|
||||
@@ -100,10 +138,15 @@ def _looks_like_task_dir(path: str) -> bool:
|
||||
return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave"))
|
||||
|
||||
|
||||
def _discover_run_items(root_dir: str, num_to_process: int) -> List[Dict[str, Any]]:
|
||||
def _normalize_rerun_mode(value: Optional[str]) -> str:
|
||||
normalized = str(value or "").strip().lower()
|
||||
if normalized in VALID_RERUN_MODES:
|
||||
return normalized
|
||||
return RERUN_MODE_UNFINISHED_ONLY
|
||||
|
||||
|
||||
def _discover_run_items(root_dir: str) -> List[Dict[str, Any]]:
|
||||
task_folders = [root_dir] if _looks_like_task_dir(root_dir) else _collect_task_folders(root_dir)
|
||||
if num_to_process > 0:
|
||||
task_folders = task_folders[:num_to_process]
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
for order_index, folder in enumerate(task_folders, start=1):
|
||||
@@ -121,12 +164,119 @@ def _discover_run_items(root_dir: str, num_to_process: int) -> List[Dict[str, An
|
||||
"policy_version": pair_meta.get("policy_version"),
|
||||
"selection_strategy": pair_meta.get("selection_strategy"),
|
||||
"source_task_dir": folder,
|
||||
"results_root_dir": os.path.join(folder, "dinsar_results"),
|
||||
"results_root_dir": os.path.join(settings.DINSAR_PRODUCT_DIR, pair_key),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _current_pointer_path_for_root(
|
||||
results_root_dir: str,
|
||||
*,
|
||||
engine_code: Optional[str] = None,
|
||||
profile_code: Optional[str] = None,
|
||||
) -> str:
|
||||
pointer_dir = os.path.join(results_root_dir, CURRENT_POINTER_DIRNAME)
|
||||
if engine_code or profile_code:
|
||||
pointer_name = (
|
||||
f"{_sanitize_pointer_fragment(engine_code or 'engine', 'engine')}__"
|
||||
f"{_sanitize_pointer_fragment(profile_code or 'profile', 'profile')}.json"
|
||||
)
|
||||
return os.path.join(pointer_dir, pointer_name)
|
||||
return os.path.join(pointer_dir, CURRENT_POINTER_FILENAME)
|
||||
|
||||
|
||||
def _normalize_existing_file(path: Any) -> str:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
normalized = os.path.normpath(os.path.abspath(_to_local_path(text)))
|
||||
return normalized if os.path.isfile(normalized) else ""
|
||||
|
||||
|
||||
def _normalize_existing_dir(path: Any) -> str:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
normalized = os.path.normpath(os.path.abspath(_to_local_path(text)))
|
||||
return normalized if os.path.isdir(normalized) else ""
|
||||
|
||||
|
||||
def _has_completed_current_result(
|
||||
item_payload: Dict[str, Any],
|
||||
*,
|
||||
engine_code: str,
|
||||
profile_code: str,
|
||||
) -> bool:
|
||||
pointer_path = _current_pointer_path_for_root(
|
||||
str(item_payload.get("results_root_dir") or ""),
|
||||
engine_code=engine_code,
|
||||
profile_code=profile_code,
|
||||
)
|
||||
if not os.path.isfile(pointer_path):
|
||||
return False
|
||||
|
||||
try:
|
||||
with open(pointer_path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp) or {}
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if str(payload.get("status") or "").strip().upper() != EXECUTION_STATUS_COMPLETED:
|
||||
return False
|
||||
|
||||
manifest_path = _normalize_existing_file(payload.get("manifest_path"))
|
||||
if manifest_path:
|
||||
return True
|
||||
|
||||
output_dir = _normalize_existing_dir(payload.get("output_dir"))
|
||||
if output_dir and os.path.isfile(_execution_manifest_path(output_dir)):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _select_run_items(
|
||||
root_dir: str,
|
||||
*,
|
||||
engine_code: str,
|
||||
profile_code: str,
|
||||
num_to_process: int,
|
||||
rerun_mode: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
discovered_items = _discover_run_items(root_dir)
|
||||
normalized_mode = _normalize_rerun_mode(rerun_mode)
|
||||
|
||||
skipped_completed_count = 0
|
||||
selected_items: List[Dict[str, Any]] = []
|
||||
for item_payload in discovered_items:
|
||||
if (
|
||||
normalized_mode == RERUN_MODE_UNFINISHED_ONLY
|
||||
and _has_completed_current_result(
|
||||
item_payload,
|
||||
engine_code=engine_code,
|
||||
profile_code=profile_code,
|
||||
)
|
||||
):
|
||||
skipped_completed_count += 1
|
||||
continue
|
||||
selected_items.append(dict(item_payload))
|
||||
|
||||
if num_to_process > 0:
|
||||
selected_items = selected_items[:num_to_process]
|
||||
|
||||
for order_index, item_payload in enumerate(selected_items, start=1):
|
||||
item_payload["order_index"] = order_index
|
||||
|
||||
return {
|
||||
"items": selected_items,
|
||||
"rerun_mode": normalized_mode,
|
||||
"discovered_task_count": len(discovered_items),
|
||||
"skipped_completed_count": skipped_completed_count,
|
||||
"selected_task_count": len(selected_items),
|
||||
}
|
||||
|
||||
|
||||
def _run_log_path(run_id: str) -> str:
|
||||
return os.path.join(RUNTIME_DIR, f"{run_id}.log")
|
||||
|
||||
@@ -179,8 +329,22 @@ def _execution_dir(item: DinsarProductionRunItemORM, run_key: str) -> str:
|
||||
return os.path.join(item.results_root_dir, "runs", run_key)
|
||||
|
||||
|
||||
def _current_pointer_path(item: DinsarProductionRunItemORM) -> str:
|
||||
return os.path.join(item.results_root_dir, CURRENT_POINTER_FILENAME)
|
||||
def _sanitize_pointer_fragment(value: str, default: str) -> str:
|
||||
text = _SAFE_POINTER_RE.sub("_", str(value or "").strip()).strip("._")
|
||||
return text or default
|
||||
|
||||
|
||||
def _current_pointer_path(
|
||||
item: DinsarProductionRunItemORM,
|
||||
*,
|
||||
engine_code: Optional[str] = None,
|
||||
profile_code: Optional[str] = None,
|
||||
) -> str:
|
||||
return _current_pointer_path_for_root(
|
||||
item.results_root_dir,
|
||||
engine_code=engine_code,
|
||||
profile_code=profile_code,
|
||||
)
|
||||
|
||||
|
||||
def _execution_manifest_path(execution_dir: str) -> str:
|
||||
@@ -193,6 +357,15 @@ def _safe_epoch(value: Optional[datetime]) -> Optional[int]:
|
||||
return int(value.timestamp())
|
||||
|
||||
|
||||
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "isce2":
|
||||
return settings.ISCE2_RUNTIME_ID or None
|
||||
if normalized in {"pyint", "gamma"}:
|
||||
return settings.PYINT_RUNTIME_ID or None
|
||||
return None
|
||||
|
||||
|
||||
def _public_run_status(value: str) -> str:
|
||||
normalized = str(value or "").strip().upper()
|
||||
if normalized == RUN_STATUS_COMPLETED:
|
||||
@@ -249,6 +422,7 @@ class DinsarProductionService:
|
||||
profile_code: str,
|
||||
root_dir: str,
|
||||
num_to_process: int,
|
||||
rerun_mode: Optional[str],
|
||||
timeout_seconds: Optional[int],
|
||||
extra: Optional[Dict[str, Any]],
|
||||
created_by: Optional[str],
|
||||
@@ -256,26 +430,44 @@ class DinsarProductionService:
|
||||
) -> Dict[str, Any]:
|
||||
normalized_engine = str(engine_code or "").strip().lower()
|
||||
normalized_profile = str(profile_code or "").strip()
|
||||
if normalized_engine != "sarscape":
|
||||
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
|
||||
task_type = _task_type_for_engine(normalized_engine)
|
||||
workflow_name = _workflow_name_for_engine(normalized_engine)
|
||||
workflow_step_name = _workflow_step_name_for_engine(normalized_engine)
|
||||
|
||||
normalized_root = _normalize_dir(root_dir, "root_dir")
|
||||
item_payloads = await asyncio.to_thread(
|
||||
_discover_run_items,
|
||||
selection = await asyncio.to_thread(
|
||||
_select_run_items,
|
||||
normalized_root,
|
||||
max(0, int(num_to_process or 0)),
|
||||
engine_code=normalized_engine,
|
||||
profile_code=normalized_profile,
|
||||
num_to_process=max(0, int(num_to_process or 0)),
|
||||
rerun_mode=rerun_mode,
|
||||
)
|
||||
item_payloads = selection["items"]
|
||||
if not item_payloads:
|
||||
if (
|
||||
selection["discovered_task_count"] > 0
|
||||
and selection["skipped_completed_count"] > 0
|
||||
and selection["rerun_mode"] == RERUN_MODE_UNFINISHED_ONLY
|
||||
):
|
||||
raise ValueError(
|
||||
f"All discovered Task_* directories already have completed "
|
||||
f"{normalized_engine}/{normalized_profile} results under: {normalized_root}"
|
||||
)
|
||||
raise ValueError(f"No Task_* directories found under: {normalized_root}")
|
||||
|
||||
run_id = str(uuid.uuid4())
|
||||
mode = "custom" if normalized_profile == "custom6" else "metatask"
|
||||
if normalized_engine == "sarscape":
|
||||
mode = "custom" if normalized_profile == "custom6" else "metatask"
|
||||
else:
|
||||
mode = "managed"
|
||||
task_name = f"D-InSAR production: {normalized_engine}/{normalized_profile}"
|
||||
task_params = {
|
||||
"engine_code": normalized_engine,
|
||||
"profile": normalized_profile,
|
||||
"root_dir": normalized_root,
|
||||
"num_to_process": int(num_to_process or 0),
|
||||
"rerun_mode": selection["rerun_mode"],
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"extra": dict(extra or {}),
|
||||
"mode": mode,
|
||||
@@ -285,7 +477,7 @@ class DinsarProductionService:
|
||||
task_id: Optional[str] = None
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
task_type=TASK_TYPE_DINSAR_PRODUCTION,
|
||||
task_type=task_type,
|
||||
task_name=task_name,
|
||||
params=task_params,
|
||||
db=db,
|
||||
@@ -294,10 +486,12 @@ class DinsarProductionService:
|
||||
run = DinsarProductionRunORM(
|
||||
run_id=run_id,
|
||||
task_id=task_id,
|
||||
product_family="dinsar",
|
||||
engine_code=normalized_engine,
|
||||
profile_code=normalized_profile,
|
||||
mode=mode,
|
||||
source_root=normalized_root,
|
||||
publish_root_dir=settings.DINSAR_PRODUCT_DIR,
|
||||
status=RUN_STATUS_PENDING,
|
||||
cancel_requested=False,
|
||||
total_items=len(item_payloads),
|
||||
@@ -309,6 +503,11 @@ class DinsarProductionService:
|
||||
summary_json={
|
||||
"phase": "queued",
|
||||
"selected_task_count": len(item_payloads),
|
||||
"discovered_task_count": selection["discovered_task_count"],
|
||||
"skipped_completed_count": selection["skipped_completed_count"],
|
||||
"rerun_mode": selection["rerun_mode"],
|
||||
"product_family": "dinsar",
|
||||
"publish_root_dir": settings.DINSAR_PRODUCT_DIR,
|
||||
},
|
||||
created_by=created_by,
|
||||
)
|
||||
@@ -337,12 +536,12 @@ class DinsarProductionService:
|
||||
await db.flush()
|
||||
|
||||
workflow_run_id = await workflow_service.create_run(
|
||||
workflow_name="dinsar_sarscape_production",
|
||||
workflow_name=workflow_name,
|
||||
steps=[
|
||||
{
|
||||
"step_id": RUNS_STEP_ID,
|
||||
"step_name": RUNS_STEP_NAME,
|
||||
"job_type": TASK_TYPE_DINSAR_PRODUCTION,
|
||||
"step_name": workflow_step_name,
|
||||
"job_type": task_type,
|
||||
"payload": {"production_run_id": run_id},
|
||||
"task_id": task_id,
|
||||
"max_attempts": 1,
|
||||
@@ -381,7 +580,11 @@ class DinsarProductionService:
|
||||
await asyncio.to_thread(
|
||||
_append_run_log_sync,
|
||||
run_id,
|
||||
f"[queued] run_id={run_id} profile={normalized_profile} root={normalized_root} items={len(item_payloads)}",
|
||||
(
|
||||
f"[queued] run_id={run_id} profile={normalized_profile} root={normalized_root} "
|
||||
f"items={len(item_payloads)} rerun_mode={selection['rerun_mode']} "
|
||||
f"skipped_completed={selection['skipped_completed_count']}"
|
||||
),
|
||||
)
|
||||
return {
|
||||
"run_id": run_id,
|
||||
@@ -389,6 +592,9 @@ class DinsarProductionService:
|
||||
"workflow_run_id": run.workflow_run_id,
|
||||
"status": run.status,
|
||||
"selected_task_count": len(item_payloads),
|
||||
"discovered_task_count": selection["discovered_task_count"],
|
||||
"skipped_completed_count": selection["skipped_completed_count"],
|
||||
"rerun_mode": selection["rerun_mode"],
|
||||
}
|
||||
|
||||
async def list_runs(
|
||||
@@ -434,6 +640,7 @@ class DinsarProductionService:
|
||||
"runs": [
|
||||
{
|
||||
"run_id": run.run_id,
|
||||
"product_family": run.product_family,
|
||||
"engine": run.engine_code,
|
||||
"profile_code": run.profile_code,
|
||||
"status": _public_run_status(run.status),
|
||||
@@ -443,6 +650,7 @@ class DinsarProductionService:
|
||||
"task_id": run.task_id,
|
||||
"workflow_run_id": run.workflow_run_id,
|
||||
"root_dir": run.source_root,
|
||||
"publish_root_dir": run.publish_root_dir,
|
||||
"message": run.latest_message,
|
||||
"total_items": run.total_items,
|
||||
"completed_items": run.completed_items,
|
||||
@@ -684,15 +892,18 @@ class DinsarProductionService:
|
||||
execution: DinsarProductionExecutionORM,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
native_output_dir: Optional[str],
|
||||
metrics: Optional[Dict[str, Any]],
|
||||
) -> str:
|
||||
manifest_payload = {
|
||||
"format_version": 1,
|
||||
"run_id": run.run_id,
|
||||
"product_family": run.product_family or "dinsar",
|
||||
"run_key": execution.run_key,
|
||||
"task_id": run.task_id,
|
||||
"engine_code": run.engine_code,
|
||||
"profile_code": run.profile_code,
|
||||
"runtime_id": _runtime_id_for_engine(run.engine_code),
|
||||
"mode": run.mode,
|
||||
"task_name": item.task_name,
|
||||
"task_alias": item.task_alias,
|
||||
@@ -704,7 +915,10 @@ class DinsarProductionService:
|
||||
"selection_strategy": item.selection_strategy,
|
||||
"source_root": run.source_root,
|
||||
"source_task_dir": item.source_task_dir,
|
||||
"results_root_dir": item.results_root_dir,
|
||||
"publish_root_dir": run.publish_root_dir,
|
||||
"output_dir": execution.output_dir,
|
||||
"native_output_dir": str(native_output_dir or execution.output_dir),
|
||||
"primary_file": primary_file,
|
||||
"source_files": source_files,
|
||||
"status": EXECUTION_STATUS_COMPLETED,
|
||||
@@ -718,24 +932,36 @@ class DinsarProductionService:
|
||||
def write_current_pointer(
|
||||
self,
|
||||
*,
|
||||
run: DinsarProductionRunORM,
|
||||
item: DinsarProductionRunItemORM,
|
||||
execution: DinsarProductionExecutionORM,
|
||||
manifest_path: str,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
native_output_dir: Optional[str],
|
||||
) -> str:
|
||||
pointer_payload = {
|
||||
"format_version": 1,
|
||||
"product_family": run.product_family or "dinsar",
|
||||
"engine_code": run.engine_code,
|
||||
"profile_code": run.profile_code,
|
||||
"runtime_id": _runtime_id_for_engine(run.engine_code),
|
||||
"run_key": execution.run_key,
|
||||
"execution_id": execution.execution_id,
|
||||
"status": EXECUTION_STATUS_COMPLETED,
|
||||
"output_dir": execution.output_dir,
|
||||
"native_output_dir": str(native_output_dir or execution.output_dir),
|
||||
"manifest_path": manifest_path,
|
||||
"primary_file": primary_file,
|
||||
"source_files": source_files,
|
||||
"updated_at": _utc_text(),
|
||||
}
|
||||
return _write_json(_current_pointer_path(item), pointer_payload)
|
||||
pointer_path = _current_pointer_path(
|
||||
item,
|
||||
engine_code=run.engine_code,
|
||||
profile_code=run.profile_code,
|
||||
)
|
||||
return _write_json(pointer_path, pointer_payload)
|
||||
|
||||
async def get_active_execution_by_task_id(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .dinsar_naming import RUN_META_FILENAME
|
||||
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
|
||||
|
||||
|
||||
RUN_NATIVE_DIRNAME = "native"
|
||||
RUN_ASSETS_DIRNAME = "assets"
|
||||
RUN_PREVIEW_DIRNAME = "preview"
|
||||
RUN_CURRENT_DIRNAME = "current"
|
||||
RUN_DISP_DIRNAME = "disp"
|
||||
RUN_COH_DIRNAME = "coh"
|
||||
EXECUTION_MANIFEST_FILENAME = "execution_manifest.json"
|
||||
PACKAGE_MANIFEST_FILENAME = "manifest.json"
|
||||
STANDARD_ENVI_DISP_BASENAME = "disp"
|
||||
STANDARD_ISCE2_DISP_NAME = "disp.tif"
|
||||
STANDARD_ISCE2_COH_NAME = "coh.tif"
|
||||
|
||||
_KEEP_RUN_ROOT_NAMES = {
|
||||
RUN_NATIVE_DIRNAME,
|
||||
RUN_ASSETS_DIRNAME,
|
||||
RUN_PREVIEW_DIRNAME,
|
||||
RUN_CURRENT_DIRNAME,
|
||||
RUN_META_FILENAME,
|
||||
EXECUTION_MANIFEST_FILENAME,
|
||||
PACKAGE_MANIFEST_FILENAME,
|
||||
}
|
||||
_DISP_ASSET_ROLES = {"disp", "disp_header", "disp_sidecar"}
|
||||
_ISCE2_ASSET_ROLES = {"disp", "coh"}
|
||||
|
||||
|
||||
def _normalize_path(path: str) -> str:
|
||||
return os.path.normpath(os.path.abspath(str(path or "").strip()))
|
||||
|
||||
|
||||
def _load_json(path: str) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
return payload if isinstance(payload, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _write_json(path: str, payload: Dict[str, Any]) -> str:
|
||||
target = _normalize_path(path)
|
||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||
with open(target, "w", encoding="utf-8") as fp:
|
||||
json.dump(payload, fp, ensure_ascii=False, indent=2)
|
||||
return target
|
||||
|
||||
|
||||
def get_run_native_output_dir(run_dir: str) -> str:
|
||||
return os.path.join(_normalize_path(run_dir), RUN_NATIVE_DIRNAME)
|
||||
|
||||
|
||||
def get_run_disp_asset_base(run_dir: str) -> str:
|
||||
return os.path.join(
|
||||
_normalize_path(run_dir),
|
||||
RUN_ASSETS_DIRNAME,
|
||||
RUN_DISP_DIRNAME,
|
||||
STANDARD_ENVI_DISP_BASENAME,
|
||||
)
|
||||
|
||||
|
||||
def get_run_disp_asset_paths(run_dir: str) -> Dict[str, str]:
|
||||
base = get_run_disp_asset_base(run_dir)
|
||||
return {
|
||||
"primary": base,
|
||||
"hdr": base + ".hdr",
|
||||
"sml": base + ".sml",
|
||||
}
|
||||
|
||||
|
||||
def get_run_isce2_disp_asset_path(run_dir: str) -> str:
|
||||
return os.path.join(
|
||||
_normalize_path(run_dir),
|
||||
RUN_ASSETS_DIRNAME,
|
||||
RUN_DISP_DIRNAME,
|
||||
STANDARD_ISCE2_DISP_NAME,
|
||||
)
|
||||
|
||||
|
||||
def get_run_isce2_coh_asset_path(run_dir: str) -> str:
|
||||
return os.path.join(
|
||||
_normalize_path(run_dir),
|
||||
RUN_ASSETS_DIRNAME,
|
||||
RUN_COH_DIRNAME,
|
||||
STANDARD_ISCE2_COH_NAME,
|
||||
)
|
||||
|
||||
|
||||
def is_standard_envi_disp_file(run_root: str, file_path: str) -> bool:
|
||||
run_root = _normalize_path(run_root)
|
||||
file_path = _normalize_path(file_path)
|
||||
rel_parts = [
|
||||
part.lower()
|
||||
for part in os.path.relpath(file_path, run_root).replace("/", os.sep).split(os.sep)
|
||||
if part
|
||||
]
|
||||
if len(rel_parts) < 3:
|
||||
return False
|
||||
tail = rel_parts[-3:]
|
||||
return tuple(tail) in {
|
||||
("assets", "disp", "disp"),
|
||||
("assets", "disp", "disp.hdr"),
|
||||
("assets", "disp", "disp.sml"),
|
||||
}
|
||||
|
||||
|
||||
def is_standard_isce2_disp_file(run_root: str, file_path: str) -> bool:
|
||||
run_root = _normalize_path(run_root)
|
||||
file_path = _normalize_path(file_path)
|
||||
rel_parts = [
|
||||
part.lower()
|
||||
for part in os.path.relpath(file_path, run_root).replace("/", os.sep).split(os.sep)
|
||||
if part
|
||||
]
|
||||
if len(rel_parts) < 3:
|
||||
return False
|
||||
tail = rel_parts[-3:]
|
||||
return tuple(tail) in {
|
||||
("assets", "disp", "disp.tif"),
|
||||
("assets", "disp", "disp.tiff"),
|
||||
}
|
||||
|
||||
|
||||
def is_path_within_native_dir(run_root: str, path: str) -> bool:
|
||||
run_root = _normalize_path(run_root)
|
||||
path = _normalize_path(path)
|
||||
try:
|
||||
rel_path = os.path.relpath(path, run_root)
|
||||
except ValueError:
|
||||
return False
|
||||
parts = [part.lower() for part in rel_path.split(os.sep) if part]
|
||||
return RUN_NATIVE_DIRNAME in parts
|
||||
|
||||
|
||||
def _move_file(src: str, dst: str) -> bool:
|
||||
src_path = _normalize_path(src)
|
||||
dst_path = _normalize_path(dst)
|
||||
if src_path == dst_path or not os.path.isfile(src_path):
|
||||
return False
|
||||
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
|
||||
if os.path.exists(dst_path):
|
||||
raise FileExistsError(f"Target already exists: {dst_path}")
|
||||
os.replace(src_path, dst_path)
|
||||
return True
|
||||
|
||||
|
||||
def _move_entry(src: str, dst: str) -> bool:
|
||||
src_path = _normalize_path(src)
|
||||
dst_path = _normalize_path(dst)
|
||||
if src_path == dst_path or not os.path.exists(src_path):
|
||||
return False
|
||||
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
|
||||
if os.path.exists(dst_path):
|
||||
raise FileExistsError(f"Target already exists: {dst_path}")
|
||||
os.replace(src_path, dst_path)
|
||||
return True
|
||||
|
||||
|
||||
def _copy_file(src: str, dst: str) -> bool:
|
||||
src_path = _normalize_path(src)
|
||||
dst_path = _normalize_path(dst)
|
||||
if src_path == dst_path or not os.path.isfile(src_path):
|
||||
return False
|
||||
os.makedirs(os.path.dirname(dst_path), exist_ok=True)
|
||||
shutil.copy2(src_path, dst_path)
|
||||
return True
|
||||
|
||||
|
||||
def _rewrite_run_metadata(run_dir: str, native_output_dir: str) -> None:
|
||||
meta_path = os.path.join(_normalize_path(run_dir), RUN_META_FILENAME)
|
||||
payload = _load_json(meta_path)
|
||||
if not payload:
|
||||
return
|
||||
payload["output_dir"] = _normalize_path(run_dir)
|
||||
payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
_write_json(meta_path, payload)
|
||||
|
||||
|
||||
def _rewrite_execution_manifest(
|
||||
run_dir: str,
|
||||
*,
|
||||
native_output_dir: str,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
) -> None:
|
||||
manifest_path = os.path.join(_normalize_path(run_dir), EXECUTION_MANIFEST_FILENAME)
|
||||
payload = _load_json(manifest_path)
|
||||
if not payload:
|
||||
return
|
||||
payload["output_dir"] = _normalize_path(run_dir)
|
||||
payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
payload["primary_file"] = _normalize_path(primary_file)
|
||||
payload["source_files"] = [_normalize_path(path) for path in source_files]
|
||||
_write_json(manifest_path, payload)
|
||||
|
||||
|
||||
def _rewrite_current_pointers(
|
||||
run_dir: str,
|
||||
*,
|
||||
native_output_dir: str,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
) -> None:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
runs_dir = os.path.dirname(normalized_run_dir)
|
||||
if os.path.basename(runs_dir).lower() != "runs":
|
||||
return
|
||||
pair_root = os.path.dirname(runs_dir)
|
||||
current_dir = os.path.join(pair_root, RUN_CURRENT_DIRNAME)
|
||||
if not os.path.isdir(current_dir):
|
||||
return
|
||||
|
||||
execution_manifest_path = os.path.join(normalized_run_dir, EXECUTION_MANIFEST_FILENAME)
|
||||
execution_manifest = _load_json(execution_manifest_path) or {}
|
||||
run_key = str(execution_manifest.get("run_key") or "").strip()
|
||||
if not run_key:
|
||||
return
|
||||
|
||||
for name in os.listdir(current_dir):
|
||||
if not name.lower().endswith(".json"):
|
||||
continue
|
||||
pointer_path = os.path.join(current_dir, name)
|
||||
payload = _load_json(pointer_path)
|
||||
if not payload:
|
||||
continue
|
||||
if str(payload.get("run_key") or "").strip() != run_key:
|
||||
continue
|
||||
payload["output_dir"] = normalized_run_dir
|
||||
payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
payload["manifest_path"] = execution_manifest_path
|
||||
payload["primary_file"] = _normalize_path(primary_file)
|
||||
payload["source_files"] = [_normalize_path(path) for path in source_files]
|
||||
_write_json(pointer_path, payload)
|
||||
|
||||
|
||||
def _rewrite_package_manifest(
|
||||
run_dir: str,
|
||||
*,
|
||||
native_output_dir: str,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
) -> None:
|
||||
manifest_path = os.path.join(_normalize_path(run_dir), PACKAGE_MANIFEST_FILENAME)
|
||||
payload = _load_json(manifest_path)
|
||||
if not payload:
|
||||
return
|
||||
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized_sources = [_normalize_path(path) for path in source_files]
|
||||
primary_relative = os.path.relpath(normalized_primary, normalized_run_dir)
|
||||
|
||||
source_payload = payload.get("source") if isinstance(payload.get("source"), dict) else {}
|
||||
source_payload["output_dir"] = normalized_run_dir
|
||||
source_payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
source_payload["primary_path"] = normalized_primary
|
||||
source_payload["source_dir"] = normalized_run_dir
|
||||
source_payload["publish_dir"] = normalized_run_dir
|
||||
payload["source"] = source_payload
|
||||
|
||||
summary_payload = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
|
||||
summary_payload["primary_asset_relative"] = primary_relative
|
||||
payload["summary"] = summary_payload
|
||||
|
||||
preview_relative = str(summary_payload.get("preview_relative") or "").strip()
|
||||
|
||||
existing_assets = payload.get("assets") if isinstance(payload.get("assets"), list) else []
|
||||
preserved_assets = [
|
||||
item
|
||||
for item in existing_assets
|
||||
if str((item or {}).get("role") or "").strip() not in _DISP_ASSET_ROLES
|
||||
]
|
||||
disp_assets: List[Dict[str, Any]] = [
|
||||
{
|
||||
"role": "disp",
|
||||
"asset_name": os.path.basename(normalized_primary) or STANDARD_ENVI_DISP_BASENAME,
|
||||
"relative_path": primary_relative,
|
||||
"format": "envi",
|
||||
"media_type": "application/octet-stream",
|
||||
"is_required": True,
|
||||
"is_primary": True,
|
||||
}
|
||||
]
|
||||
if len(normalized_sources) > 1 and os.path.isfile(normalized_sources[1]):
|
||||
disp_assets.append(
|
||||
{
|
||||
"role": "disp_header",
|
||||
"asset_name": os.path.basename(normalized_sources[1]),
|
||||
"relative_path": os.path.relpath(normalized_sources[1], normalized_run_dir),
|
||||
"format": "hdr",
|
||||
"media_type": "text/plain",
|
||||
"is_required": True,
|
||||
"is_primary": False,
|
||||
}
|
||||
)
|
||||
if len(normalized_sources) > 2 and os.path.isfile(normalized_sources[2]):
|
||||
disp_assets.append(
|
||||
{
|
||||
"role": "disp_sidecar",
|
||||
"asset_name": os.path.basename(normalized_sources[2]),
|
||||
"relative_path": os.path.relpath(normalized_sources[2], normalized_run_dir),
|
||||
"format": "sml",
|
||||
"media_type": "text/plain",
|
||||
"is_required": False,
|
||||
"is_primary": False,
|
||||
}
|
||||
)
|
||||
|
||||
payload["assets"] = disp_assets + preserved_assets
|
||||
payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
|
||||
canonical = build_canonical_descriptor(
|
||||
payload["assets"],
|
||||
product_family=str(payload.get("product_family") or "dinsar"),
|
||||
)
|
||||
if preview_relative:
|
||||
canonical["preview_asset_relative"] = preview_relative
|
||||
payload["canonical"] = canonical
|
||||
normalized_payload = normalize_package_manifest(payload)
|
||||
_write_json(manifest_path, normalized_payload)
|
||||
|
||||
|
||||
def _rewrite_isce2_package_manifest(
|
||||
run_dir: str,
|
||||
*,
|
||||
native_output_dir: str,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
) -> None:
|
||||
manifest_path = os.path.join(_normalize_path(run_dir), PACKAGE_MANIFEST_FILENAME)
|
||||
payload = _load_json(manifest_path)
|
||||
if not payload:
|
||||
return
|
||||
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized_sources = [_normalize_path(path) for path in source_files]
|
||||
primary_relative = os.path.relpath(normalized_primary, normalized_run_dir)
|
||||
|
||||
source_payload = payload.get("source") if isinstance(payload.get("source"), dict) else {}
|
||||
source_payload["output_dir"] = normalized_run_dir
|
||||
source_payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
source_payload["primary_path"] = normalized_primary
|
||||
source_payload["source_dir"] = normalized_run_dir
|
||||
source_payload["publish_dir"] = normalized_run_dir
|
||||
payload["source"] = source_payload
|
||||
|
||||
summary_payload = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
|
||||
summary_payload["primary_asset_relative"] = primary_relative
|
||||
payload["summary"] = summary_payload
|
||||
|
||||
preview_relative = str(summary_payload.get("preview_relative") or "").strip()
|
||||
|
||||
existing_assets = payload.get("assets") if isinstance(payload.get("assets"), list) else []
|
||||
preserved_assets = [
|
||||
item
|
||||
for item in existing_assets
|
||||
if str((item or {}).get("role") or "").strip() not in _ISCE2_ASSET_ROLES
|
||||
]
|
||||
isce2_assets: List[Dict[str, Any]] = [
|
||||
{
|
||||
"role": "disp",
|
||||
"asset_name": os.path.basename(normalized_primary) or STANDARD_ISCE2_DISP_NAME,
|
||||
"relative_path": primary_relative,
|
||||
"format": "geotiff",
|
||||
"media_type": "image/tiff",
|
||||
"is_required": True,
|
||||
"is_primary": True,
|
||||
}
|
||||
]
|
||||
if len(normalized_sources) > 1 and os.path.isfile(normalized_sources[1]):
|
||||
isce2_assets.append(
|
||||
{
|
||||
"role": "coh",
|
||||
"asset_name": os.path.basename(normalized_sources[1]) or STANDARD_ISCE2_COH_NAME,
|
||||
"relative_path": os.path.relpath(normalized_sources[1], normalized_run_dir),
|
||||
"format": "geotiff",
|
||||
"media_type": "image/tiff",
|
||||
"is_required": False,
|
||||
"is_primary": False,
|
||||
}
|
||||
)
|
||||
|
||||
payload["assets"] = isce2_assets + preserved_assets
|
||||
payload["native_output_dir"] = _normalize_path(native_output_dir)
|
||||
|
||||
canonical = build_canonical_descriptor(
|
||||
payload["assets"],
|
||||
product_family=str(payload.get("product_family") or "dinsar"),
|
||||
)
|
||||
if preview_relative:
|
||||
canonical["preview_asset_relative"] = preview_relative
|
||||
payload["canonical"] = canonical
|
||||
normalized_payload = normalize_package_manifest(payload)
|
||||
_write_json(manifest_path, normalized_payload)
|
||||
|
||||
|
||||
def normalize_envi_run_layout(
|
||||
run_dir: str,
|
||||
*,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
rewrite_metadata: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
if not os.path.isdir(normalized_run_dir):
|
||||
raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}")
|
||||
|
||||
native_output_dir = get_run_native_output_dir(normalized_run_dir)
|
||||
disp_paths = get_run_disp_asset_paths(normalized_run_dir)
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized_sources = [_normalize_path(path) for path in source_files if str(path or "").strip()]
|
||||
if normalized_primary and normalized_primary not in normalized_sources:
|
||||
normalized_sources.insert(0, normalized_primary)
|
||||
|
||||
promoted_files: List[str] = []
|
||||
if os.path.isfile(normalized_primary) and normalized_primary != disp_paths["primary"]:
|
||||
if _move_file(normalized_primary, disp_paths["primary"]):
|
||||
promoted_files.append(disp_paths["primary"])
|
||||
|
||||
for path in normalized_sources:
|
||||
lower = path.lower()
|
||||
if lower.endswith(".hdr"):
|
||||
if _move_file(path, disp_paths["hdr"]):
|
||||
promoted_files.append(disp_paths["hdr"])
|
||||
elif lower.endswith(".sml"):
|
||||
if _move_file(path, disp_paths["sml"]):
|
||||
promoted_files.append(disp_paths["sml"])
|
||||
|
||||
os.makedirs(native_output_dir, exist_ok=True)
|
||||
moved_entries: List[str] = []
|
||||
for name in os.listdir(normalized_run_dir):
|
||||
if name in _KEEP_RUN_ROOT_NAMES:
|
||||
continue
|
||||
src_path = os.path.join(normalized_run_dir, name)
|
||||
dst_path = os.path.join(native_output_dir, name)
|
||||
if _move_entry(src_path, dst_path):
|
||||
moved_entries.append(dst_path)
|
||||
|
||||
final_sources = [disp_paths["primary"]]
|
||||
if os.path.isfile(disp_paths["hdr"]):
|
||||
final_sources.append(disp_paths["hdr"])
|
||||
if os.path.isfile(disp_paths["sml"]):
|
||||
final_sources.append(disp_paths["sml"])
|
||||
|
||||
if rewrite_metadata:
|
||||
_rewrite_run_metadata(normalized_run_dir, native_output_dir)
|
||||
_rewrite_execution_manifest(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_paths["primary"],
|
||||
source_files=final_sources,
|
||||
)
|
||||
_rewrite_current_pointers(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_paths["primary"],
|
||||
source_files=final_sources,
|
||||
)
|
||||
_rewrite_package_manifest(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_paths["primary"],
|
||||
source_files=final_sources,
|
||||
)
|
||||
|
||||
return {
|
||||
"run_dir": normalized_run_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"primary_file": disp_paths["primary"],
|
||||
"source_files": final_sources,
|
||||
"promoted_files": promoted_files,
|
||||
"moved_entries": moved_entries,
|
||||
}
|
||||
|
||||
|
||||
def normalize_isce2_run_layout(
|
||||
run_dir: str,
|
||||
*,
|
||||
primary_file: str,
|
||||
source_files: List[str],
|
||||
rewrite_metadata: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_run_dir = _normalize_path(run_dir)
|
||||
if not os.path.isdir(normalized_run_dir):
|
||||
raise FileNotFoundError(f"Run directory not found: {normalized_run_dir}")
|
||||
|
||||
native_output_dir = get_run_native_output_dir(normalized_run_dir)
|
||||
disp_asset_path = get_run_isce2_disp_asset_path(normalized_run_dir)
|
||||
coh_asset_path = get_run_isce2_coh_asset_path(normalized_run_dir)
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized_sources = [_normalize_path(path) for path in source_files if str(path or "").strip()]
|
||||
if normalized_primary and normalized_primary not in normalized_sources:
|
||||
normalized_sources.insert(0, normalized_primary)
|
||||
|
||||
copied_files: List[str] = []
|
||||
if os.path.isfile(normalized_primary) and _copy_file(normalized_primary, disp_asset_path):
|
||||
copied_files.append(disp_asset_path)
|
||||
|
||||
coh_source = ""
|
||||
for path in normalized_sources[1:]:
|
||||
if os.path.isfile(path):
|
||||
coh_source = path
|
||||
break
|
||||
if coh_source and _copy_file(coh_source, coh_asset_path):
|
||||
copied_files.append(coh_asset_path)
|
||||
|
||||
if not os.path.isfile(disp_asset_path):
|
||||
raise FileNotFoundError(f"ISCE2 displacement asset not found: {disp_asset_path}")
|
||||
|
||||
os.makedirs(native_output_dir, exist_ok=True)
|
||||
moved_entries: List[str] = []
|
||||
for name in os.listdir(normalized_run_dir):
|
||||
if name in _KEEP_RUN_ROOT_NAMES:
|
||||
continue
|
||||
src_path = os.path.join(normalized_run_dir, name)
|
||||
dst_path = os.path.join(native_output_dir, name)
|
||||
if _move_entry(src_path, dst_path):
|
||||
moved_entries.append(dst_path)
|
||||
|
||||
final_sources = [disp_asset_path]
|
||||
if os.path.isfile(coh_asset_path):
|
||||
final_sources.append(coh_asset_path)
|
||||
|
||||
if rewrite_metadata:
|
||||
_rewrite_run_metadata(normalized_run_dir, native_output_dir)
|
||||
_rewrite_execution_manifest(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_asset_path,
|
||||
source_files=final_sources,
|
||||
)
|
||||
_rewrite_current_pointers(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_asset_path,
|
||||
source_files=final_sources,
|
||||
)
|
||||
_rewrite_isce2_package_manifest(
|
||||
normalized_run_dir,
|
||||
native_output_dir=native_output_dir,
|
||||
primary_file=disp_asset_path,
|
||||
source_files=final_sources,
|
||||
)
|
||||
|
||||
return {
|
||||
"run_dir": normalized_run_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"primary_file": disp_asset_path,
|
||||
"source_files": final_sources,
|
||||
"copied_files": copied_files,
|
||||
"moved_entries": moved_entries,
|
||||
}
|
||||
@@ -9,6 +9,8 @@ from __future__ import annotations
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import defusedxml.ElementTree as ET
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||
@@ -25,6 +27,7 @@ from .dinsar_naming import (
|
||||
find_json_sidecar,
|
||||
write_run_metadata,
|
||||
)
|
||||
from .dinsar_result_layout_service import get_run_native_output_dir
|
||||
|
||||
_BACKEND_DIR = type(settings).BACKEND_DIR
|
||||
|
||||
@@ -57,6 +60,93 @@ def _to_local_path(value: Any) -> str:
|
||||
return os.path.normpath(raw.replace("/", os.sep))
|
||||
|
||||
|
||||
def get_envi_runner_python() -> str:
|
||||
configured = _to_local_path(getattr(settings, "PYTHON_PATH", "") or "")
|
||||
if configured:
|
||||
return configured
|
||||
return os.path.normpath(sys.executable)
|
||||
|
||||
|
||||
def get_envi_runner_cwd() -> str:
|
||||
return os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
|
||||
|
||||
|
||||
def get_envi_runner_env() -> Dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
project_root = get_envi_runner_cwd()
|
||||
existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()]
|
||||
ordered = [project_root, *existing]
|
||||
deduped: List[str] = []
|
||||
seen = set()
|
||||
for raw_path in ordered:
|
||||
try:
|
||||
key = os.path.normcase(os.path.normpath(os.path.abspath(str(raw_path))))
|
||||
except Exception:
|
||||
key = str(raw_path)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(str(raw_path))
|
||||
env["PYTHONPATH"] = os.pathsep.join(deduped)
|
||||
return env
|
||||
|
||||
|
||||
def build_envi_runner_command(*args: Any) -> List[str]:
|
||||
command = [
|
||||
get_envi_runner_python(),
|
||||
"-m",
|
||||
"backend.app.services.envi_runner_cli",
|
||||
]
|
||||
command.extend(str(arg) for arg in args if arg is not None)
|
||||
return command
|
||||
|
||||
|
||||
def probe_envi_runner() -> Dict[str, Any]:
|
||||
python_path = get_envi_runner_python()
|
||||
project_root = get_envi_runner_cwd()
|
||||
result: Dict[str, Any] = {
|
||||
"python_path": python_path,
|
||||
"cwd": project_root,
|
||||
"ready": False,
|
||||
"returncode": None,
|
||||
"message": "",
|
||||
}
|
||||
if not python_path:
|
||||
result["message"] = "PYTHON_PATH is empty."
|
||||
return result
|
||||
if not os.path.isfile(python_path):
|
||||
result["message"] = f"Python executable not found: {python_path}"
|
||||
return result
|
||||
if not os.path.isdir(project_root):
|
||||
result["message"] = f"Project root not found: {project_root}"
|
||||
return result
|
||||
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
build_envi_runner_command("--help"),
|
||||
cwd=project_root,
|
||||
env=get_envi_runner_env(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
result["message"] = str(exc)
|
||||
return result
|
||||
|
||||
result["returncode"] = int(completed.returncode)
|
||||
if completed.returncode == 0:
|
||||
result["ready"] = True
|
||||
result["message"] = "Runner entrypoint is available."
|
||||
return result
|
||||
|
||||
stderr_text = str(completed.stderr or "").strip()
|
||||
stdout_text = str(completed.stdout or "").strip()
|
||||
result["message"] = (stderr_text or stdout_text or f"returncode={completed.returncode}")[:1000]
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration (read once at import time)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -245,6 +335,7 @@ def _write_envi_run_sidecar(
|
||||
started_at: str,
|
||||
params: Dict[str, Any],
|
||||
metrics: Dict[str, Any],
|
||||
native_output_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
write_run_metadata(
|
||||
output_dir,
|
||||
@@ -258,6 +349,7 @@ def _write_envi_run_sidecar(
|
||||
"source_root": os.path.normpath(root_dir),
|
||||
"task_dir": os.path.normpath(task_dir),
|
||||
"output_dir": os.path.normpath(output_dir),
|
||||
"native_output_dir": os.path.normpath(native_output_dir or output_dir),
|
||||
"started_at": started_at,
|
||||
"finished_at": _utc_now_text(),
|
||||
"params": params,
|
||||
@@ -1632,17 +1724,20 @@ def run_single_task_workflow(
|
||||
if not DEM_BASE_FILE:
|
||||
raise ValueError("DEM path not configured. Set IDL_DINSAR_DEM_BASE_FILE in .env")
|
||||
|
||||
native_output_dir = get_run_native_output_dir(output_dir)
|
||||
master_dir = os.path.join(task_dir, "master")
|
||||
slave_dir = os.path.join(task_dir, "slave")
|
||||
if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir):
|
||||
raise RuntimeError(f"{task_name}: master/slave dir missing")
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
os.makedirs(native_output_dir, exist_ok=True)
|
||||
log_lines: List[str] = [
|
||||
f"[envi] single task workflow={normalized_workflow}",
|
||||
f"[envi] source_root={source_root}",
|
||||
f"[envi] task_dir={task_dir}",
|
||||
f"[envi] output_dir={output_dir}",
|
||||
f"[envi] native_output_dir={native_output_dir}",
|
||||
f"[envi] dem={DEM_BASE_FILE}",
|
||||
]
|
||||
auto_imported = 0
|
||||
@@ -1689,7 +1784,7 @@ def run_single_task_workflow(
|
||||
"REFERENCE_SARSCAPEDATA": _build_sarscapedata(master_base),
|
||||
"SECONDARY_SARSCAPEDATA": _build_sarscapedata(slave_base),
|
||||
"DEM_SARSCAPEDATA": _build_sarscapedata(DEM_BASE_FILE),
|
||||
"OUTPUT_FOLDER": _normalize_path(output_dir),
|
||||
"OUTPUT_FOLDER": _normalize_path(native_output_dir),
|
||||
},
|
||||
)
|
||||
_write_progress(job_id, 1, 1, "Completed", output_dir, 1, 1, task_name)
|
||||
@@ -1698,7 +1793,7 @@ def run_single_task_workflow(
|
||||
master_base,
|
||||
slave_base,
|
||||
DEM_BASE_FILE,
|
||||
os.path.join(output_dir, "workflow"),
|
||||
os.path.join(native_output_dir, "workflow"),
|
||||
log_lines,
|
||||
job_id=job_id,
|
||||
pair_index=1,
|
||||
@@ -1746,6 +1841,7 @@ def run_single_task_workflow(
|
||||
metrics={
|
||||
"elapsed_seconds": elapsed,
|
||||
},
|
||||
native_output_dir=native_output_dir,
|
||||
)
|
||||
return {
|
||||
"summary": {
|
||||
@@ -1764,6 +1860,7 @@ def run_single_task_workflow(
|
||||
"run_key": resolved_run_key,
|
||||
"task_dir": task_dir,
|
||||
"output_dir": output_dir,
|
||||
"native_output_dir": native_output_dir,
|
||||
"success": True,
|
||||
"status": "ok",
|
||||
"elapsed_seconds": elapsed,
|
||||
@@ -1917,6 +2014,7 @@ def get_status() -> Dict[str, Any]:
|
||||
"""Return ENVI/IDL system status and DEM configuration."""
|
||||
idl_installed = bool(IDL_EXECUTABLE and os.path.isfile(IDL_EXECUTABLE))
|
||||
is_running = is_any_process_running(["idl.exe", "idlde.exe", "taskengine.exe"])
|
||||
runner_status = probe_envi_runner()
|
||||
|
||||
dem_ok = bool(
|
||||
DEM_BASE_FILE
|
||||
@@ -1930,6 +2028,11 @@ def get_status() -> Dict[str, Any]:
|
||||
"idl_running": is_running,
|
||||
"dem_base_file": DEM_BASE_FILE or "(not configured)",
|
||||
"dem_exists": dem_ok,
|
||||
"runner_python": runner_status.get("python_path", ""),
|
||||
"runner_cwd": runner_status.get("cwd", ""),
|
||||
"runner_ready": bool(runner_status.get("ready")),
|
||||
"runner_returncode": runner_status.get("returncode"),
|
||||
"runner_message": runner_status.get("message", ""),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,9 @@ from ..models import (
|
||||
SystemWorkerHeartbeatORM,
|
||||
)
|
||||
from ..idl_service import get_idl_status
|
||||
from .product_package_schema import CANONICAL_PACKAGE_SCHEMA
|
||||
from .pairing_state_service import pairing_state_service
|
||||
from .wsl_runtime_registry import wsl_runtime_registry
|
||||
|
||||
|
||||
DEFAULT_WORKER_TIMEOUT_SECONDS = 60
|
||||
@@ -282,14 +284,18 @@ async def _check_result_catalog() -> Dict[str, Any]:
|
||||
)
|
||||
|
||||
|
||||
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
|
||||
async def _check_timeseries_result_catalog() -> Dict[str, Any]:
|
||||
return await _check_catalog(
|
||||
catalog_name="psinsar",
|
||||
storage_root=settings.PSINSAR_PRODUCT_DIR,
|
||||
storage_root=settings.TIMESERIES_PRODUCT_DIR,
|
||||
enabled=bool(settings.TIMESERIES_ENABLED),
|
||||
)
|
||||
|
||||
|
||||
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
|
||||
return await _check_timeseries_result_catalog()
|
||||
|
||||
|
||||
async def _check_nginx() -> Dict[str, Any]:
|
||||
status = {"ok": False, "error": None, "status_code": None}
|
||||
nginx_health_url = settings.NGINX_HEALTH_URL
|
||||
@@ -351,6 +357,30 @@ def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
"total_count": int(payload.get("total_count") or 0),
|
||||
"canonical_count": int(payload.get("canonical_count") or 0),
|
||||
"missing_manifest_count": int(payload.get("missing_manifest_count") or 0),
|
||||
"missing_publish_dir_count": int(payload.get("missing_publish_dir_count") or 0),
|
||||
"missing_processor_count": int(payload.get("missing_processor_count") or 0),
|
||||
"missing_runtime_count": int(payload.get("missing_runtime_count") or 0),
|
||||
"missing_native_output_count": int(payload.get("missing_native_output_count") or 0),
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_wsl_runtime_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
"broker_job_root_exists": bool(payload.get("broker_job_root_exists")),
|
||||
"required_runtime_count": int(payload.get("required_runtime_count") or 0),
|
||||
"healthy_runtime_count": int(payload.get("healthy_runtime_count") or 0),
|
||||
"shared_distro": payload.get("shared_distro"),
|
||||
"shared_conda_env_name": payload.get("shared_conda_env_name"),
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
@@ -374,18 +404,26 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
worker = payload.get("worker", {}) or {}
|
||||
result_catalog = payload.get("result_catalog", {}) or {}
|
||||
dinsar_result_catalog = payload.get("dinsar_result_catalog", {}) or result_catalog
|
||||
psinsar_result_catalog = payload.get("psinsar_result_catalog", {}) or {}
|
||||
timeseries_result_catalog = (
|
||||
payload.get("timeseries_result_catalog", {}) or payload.get("psinsar_result_catalog", {}) or {}
|
||||
)
|
||||
psinsar_result_catalog = timeseries_result_catalog
|
||||
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
product_packages = payload.get("product_packages", {}) or {}
|
||||
wsl_runtime = payload.get("wsl_runtime", {}) or {}
|
||||
pairing_system = payload.get("pairing_system", {}) or {}
|
||||
idl = payload.get("idl", {}) or {}
|
||||
idl_status = idl.get("status", {}) or {}
|
||||
ollama = payload.get("ollama", {}) or {}
|
||||
nginx = payload.get("nginx", {}) or {}
|
||||
sanitized_dinsar_catalog = _sanitize_catalog_status(dinsar_result_catalog)
|
||||
sanitized_psinsar_catalog = _sanitize_catalog_status(psinsar_result_catalog)
|
||||
sanitized_timeseries_catalog = _sanitize_catalog_status(timeseries_result_catalog)
|
||||
sanitized_psinsar_catalog = sanitized_timeseries_catalog
|
||||
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
|
||||
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
|
||||
sanitized_product_packages = _sanitize_product_package_status(product_packages)
|
||||
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
|
||||
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
|
||||
|
||||
return {
|
||||
@@ -403,13 +441,17 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
},
|
||||
"result_catalog": sanitized_dinsar_catalog,
|
||||
"dinsar_result_catalog": sanitized_dinsar_catalog,
|
||||
"timeseries_result_catalog": sanitized_timeseries_catalog,
|
||||
"psinsar_result_catalog": sanitized_psinsar_catalog,
|
||||
"catalogs": {
|
||||
"dinsar": sanitized_dinsar_catalog,
|
||||
"timeseries": sanitized_timeseries_catalog,
|
||||
"psinsar": sanitized_psinsar_catalog,
|
||||
},
|
||||
"dinsar_bridge": sanitized_dinsar_bridge,
|
||||
"source_roots": sanitized_source_roots,
|
||||
"product_packages": sanitized_product_packages,
|
||||
"wsl_runtime": sanitized_wsl_runtime,
|
||||
"pairing_system": sanitized_pairing_system,
|
||||
"idl": {
|
||||
"ok": bool(idl.get("ok")),
|
||||
@@ -787,6 +829,141 @@ async def _check_source_roots() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
async def _check_product_packages() -> Dict[str, Any]:
|
||||
status = {
|
||||
"ok": False,
|
||||
"canonical_schema": CANONICAL_PACKAGE_SCHEMA,
|
||||
"total_count": 0,
|
||||
"canonical_count": 0,
|
||||
"missing_manifest_count": 0,
|
||||
"missing_publish_dir_count": 0,
|
||||
"missing_processor_count": 0,
|
||||
"missing_runtime_count": 0,
|
||||
"missing_native_output_count": 0,
|
||||
"by_family": {},
|
||||
"by_engine": {},
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
session_factory = _get_session_factory()
|
||||
async with session_factory() as db:
|
||||
result = await db.execute(
|
||||
select(
|
||||
ResultProductORM.product_family,
|
||||
ResultProductORM.engine_code,
|
||||
ResultProductORM.package_schema,
|
||||
ResultProductORM.processor_code,
|
||||
ResultProductORM.runtime_id,
|
||||
ResultProductORM.manifest_path,
|
||||
ResultProductORM.publish_dir,
|
||||
ResultProductORM.native_output_dir,
|
||||
)
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
status["total_count"] = len(rows)
|
||||
for (
|
||||
product_family,
|
||||
engine_code,
|
||||
package_schema,
|
||||
processor_code,
|
||||
runtime_id,
|
||||
manifest_path,
|
||||
publish_dir,
|
||||
native_output_dir,
|
||||
) in rows:
|
||||
family_key = str(product_family or "unknown").strip() or "unknown"
|
||||
engine_key = str(engine_code or "unknown").strip() or "unknown"
|
||||
status["by_family"][family_key] = int(status["by_family"].get(family_key, 0)) + 1
|
||||
status["by_engine"][engine_key] = int(status["by_engine"].get(engine_key, 0)) + 1
|
||||
|
||||
if str(package_schema or "").strip() == CANONICAL_PACKAGE_SCHEMA:
|
||||
status["canonical_count"] += 1
|
||||
if not str(manifest_path or "").strip() or not os.path.isfile(str(manifest_path)):
|
||||
status["missing_manifest_count"] += 1
|
||||
if not str(publish_dir or "").strip() or not os.path.isdir(str(publish_dir)):
|
||||
status["missing_publish_dir_count"] += 1
|
||||
if not str(processor_code or "").strip():
|
||||
status["missing_processor_count"] += 1
|
||||
if engine_key in {"isce2", "pyint", "gamma"} and not str(runtime_id or "").strip():
|
||||
status["missing_runtime_count"] += 1
|
||||
if not str(native_output_dir or "").strip():
|
||||
status["missing_native_output_count"] += 1
|
||||
|
||||
status["ok"] = all(
|
||||
[
|
||||
status["missing_manifest_count"] == 0,
|
||||
status["missing_publish_dir_count"] == 0,
|
||||
status["missing_processor_count"] == 0,
|
||||
status["missing_runtime_count"] == 0,
|
||||
status["missing_native_output_count"] == 0,
|
||||
status["canonical_count"] == status["total_count"],
|
||||
]
|
||||
)
|
||||
except Exception as exc:
|
||||
status["error"] = str(exc)
|
||||
return status
|
||||
|
||||
|
||||
async def _check_wsl_runtime() -> Dict[str, Any]:
|
||||
status = {
|
||||
"ok": False,
|
||||
"shared_distro": wsl_runtime_registry.shared_distro,
|
||||
"shared_conda_env_name": wsl_runtime_registry.shared_conda_env_name,
|
||||
"shared_python_path": wsl_runtime_registry.shared_python_path,
|
||||
"broker_job_root_windows": wsl_runtime_registry.broker_job_root_windows,
|
||||
"broker_job_root_exists": os.path.isdir(wsl_runtime_registry.broker_job_root_windows),
|
||||
"required_runtime_count": 0,
|
||||
"healthy_runtime_count": 0,
|
||||
"runtimes": [],
|
||||
"error": None,
|
||||
}
|
||||
try:
|
||||
required_by_engine = {
|
||||
"isce2": bool(settings.ISCE2_ENABLED or settings.TIMESERIES_ENABLED),
|
||||
"pyint": bool(settings.PYINT_ENABLED),
|
||||
}
|
||||
for runtime in wsl_runtime_registry.runtimes.values():
|
||||
required = bool(required_by_engine.get(runtime.engine_code, False))
|
||||
runner_exists = os.path.isfile(runtime.runner_path_windows)
|
||||
env_profile_exists = None
|
||||
if str(runtime.env_profile_path_windows or "").strip():
|
||||
env_profile_exists = os.path.isfile(runtime.env_profile_path_windows)
|
||||
python_matches_shared = str(runtime.python_path or "").strip() == str(
|
||||
wsl_runtime_registry.shared_python_path or ""
|
||||
).strip()
|
||||
distro_matches_shared = str(runtime.distro or "").strip() == str(
|
||||
wsl_runtime_registry.shared_distro or ""
|
||||
).strip()
|
||||
runtime_ok = runner_exists and python_matches_shared and distro_matches_shared
|
||||
if env_profile_exists is False and required:
|
||||
runtime_ok = False
|
||||
if required:
|
||||
status["required_runtime_count"] += 1
|
||||
if runtime_ok:
|
||||
status["healthy_runtime_count"] += 1
|
||||
status["runtimes"].append(
|
||||
{
|
||||
"runtime_id": runtime.runtime_id,
|
||||
"engine_code": runtime.engine_code,
|
||||
"display_name": runtime.display_name,
|
||||
"required": required,
|
||||
"ok": runtime_ok,
|
||||
"runner_exists": runner_exists,
|
||||
"env_profile_exists": env_profile_exists,
|
||||
"python_matches_shared": python_matches_shared,
|
||||
"distro_matches_shared": distro_matches_shared,
|
||||
"allowed_operations": list(runtime.allowed_operations or ()),
|
||||
}
|
||||
)
|
||||
status["ok"] = bool(status["broker_job_root_exists"]) and (
|
||||
status["healthy_runtime_count"] >= status["required_runtime_count"]
|
||||
)
|
||||
except Exception as exc:
|
||||
status["error"] = str(exc)
|
||||
return status
|
||||
|
||||
|
||||
async def get_health_status(
|
||||
include_external: bool = True,
|
||||
include_details: bool = False,
|
||||
@@ -807,9 +984,12 @@ async def get_health_status(
|
||||
ollama_status = await _check_ollama()
|
||||
|
||||
result_catalog_status = await _check_result_catalog()
|
||||
psinsar_result_catalog_status = await _check_psinsar_result_catalog()
|
||||
timeseries_result_catalog_status = await _check_timeseries_result_catalog()
|
||||
psinsar_result_catalog_status = timeseries_result_catalog_status
|
||||
dinsar_bridge_status = await _check_dinsar_bridge()
|
||||
source_roots_status = await _check_source_roots()
|
||||
product_packages_status = await _check_product_packages()
|
||||
wsl_runtime_status = await _check_wsl_runtime()
|
||||
pairing_system_status = await pairing_state_service.get_pairing_system_status()
|
||||
engines_status = {"ok": None, "overall": None, "engines": []}
|
||||
if full or include_details:
|
||||
@@ -824,8 +1004,10 @@ async def get_health_status(
|
||||
result_catalog_status.get("ok"),
|
||||
dinsar_bridge_status.get("ok"),
|
||||
source_roots_status.get("ok"),
|
||||
product_packages_status.get("ok"),
|
||||
wsl_runtime_status.get("ok"),
|
||||
pairing_system_status.get("ok"),
|
||||
(not settings.TIMESERIES_ENABLED) or psinsar_result_catalog_status.get("ok"),
|
||||
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
|
||||
]
|
||||
)
|
||||
|
||||
@@ -836,13 +1018,17 @@ async def get_health_status(
|
||||
"worker": worker_status,
|
||||
"result_catalog": result_catalog_status,
|
||||
"dinsar_result_catalog": result_catalog_status,
|
||||
"timeseries_result_catalog": timeseries_result_catalog_status,
|
||||
"psinsar_result_catalog": psinsar_result_catalog_status,
|
||||
"catalogs": {
|
||||
"dinsar": result_catalog_status,
|
||||
"timeseries": timeseries_result_catalog_status,
|
||||
"psinsar": psinsar_result_catalog_status,
|
||||
},
|
||||
"dinsar_bridge": dinsar_bridge_status,
|
||||
"source_roots": source_roots_status,
|
||||
"product_packages": product_packages_status,
|
||||
"wsl_runtime": wsl_runtime_status,
|
||||
"pairing_system": pairing_system_status,
|
||||
"idl": {
|
||||
"ok": idl_ok,
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Sequence
|
||||
|
||||
def _normalize_path(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
return os.path.normpath(os.path.abspath(text))
|
||||
|
||||
|
||||
def _dedupe_existing_files(paths: Sequence[Any]) -> List[str]:
|
||||
normalized_paths: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
normalized = _normalize_path(path)
|
||||
if not normalized or normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
normalized_paths.append(normalized)
|
||||
return normalized_paths
|
||||
|
||||
|
||||
def _inspect_raster(path: str) -> Dict[str, Any]:
|
||||
gdal_error: Exception | None = None
|
||||
try:
|
||||
from osgeo import gdal
|
||||
|
||||
gdal.UseExceptions()
|
||||
dataset = gdal.Open(path, gdal.GA_ReadOnly)
|
||||
if dataset is None:
|
||||
raise RuntimeError("GDAL returned no dataset")
|
||||
|
||||
band_count = int(dataset.RasterCount or 0)
|
||||
width = int(dataset.RasterXSize or 0)
|
||||
height = int(dataset.RasterYSize or 0)
|
||||
if band_count <= 0 or width <= 0 or height <= 0:
|
||||
raise ValueError(
|
||||
f"invalid raster geometry bands={band_count} width={width} height={height}"
|
||||
)
|
||||
|
||||
return {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"bands": band_count,
|
||||
"driver": dataset.GetDriver().ShortName if dataset.GetDriver() is not None else "",
|
||||
"projection_present": bool(dataset.GetProjection()),
|
||||
"geo_transform_present": bool(dataset.GetGeoTransform(can_return_null=True)),
|
||||
"reader": "gdal",
|
||||
}
|
||||
except Exception as exc:
|
||||
gdal_error = exc
|
||||
|
||||
try:
|
||||
import rasterio
|
||||
|
||||
with rasterio.open(path) as dataset:
|
||||
band_count = int(dataset.count or 0)
|
||||
width = int(dataset.width or 0)
|
||||
height = int(dataset.height or 0)
|
||||
if band_count <= 0 or width <= 0 or height <= 0:
|
||||
raise ValueError(
|
||||
f"invalid raster geometry bands={band_count} width={width} height={height}"
|
||||
)
|
||||
|
||||
return {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"bands": band_count,
|
||||
"driver": str(dataset.driver or ""),
|
||||
"projection_present": bool(dataset.crs),
|
||||
"geo_transform_present": dataset.transform is not None,
|
||||
"reader": "rasterio",
|
||||
}
|
||||
except Exception as exc:
|
||||
if gdal_error is not None:
|
||||
raise RuntimeError(f"gdal={gdal_error}; rasterio={exc}") from exc
|
||||
raise
|
||||
|
||||
|
||||
def validate_isce2_result_files(
|
||||
primary_file: Any,
|
||||
source_files: Sequence[Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
normalized_primary = _normalize_path(primary_file)
|
||||
normalized_sources = _dedupe_existing_files(source_files or [])
|
||||
if normalized_primary and normalized_primary not in normalized_sources and os.path.isfile(normalized_primary):
|
||||
normalized_sources.insert(0, normalized_primary)
|
||||
|
||||
issues: List[str] = []
|
||||
accepted_sources: List[str] = []
|
||||
metrics: Dict[str, Any] = {
|
||||
"primary_exists": False,
|
||||
"primary_non_empty": False,
|
||||
"primary_readable": False,
|
||||
"primary_size_bytes": 0,
|
||||
"source_file_count": len(normalized_sources),
|
||||
"coh_present": False,
|
||||
}
|
||||
|
||||
primary_metadata: Dict[str, Any] = {}
|
||||
if not normalized_primary:
|
||||
issues.append("Primary ISCE2 displacement file path is empty.")
|
||||
elif not os.path.isfile(normalized_primary):
|
||||
issues.append(f"Primary ISCE2 displacement file not found: {normalized_primary}")
|
||||
else:
|
||||
metrics["primary_exists"] = True
|
||||
try:
|
||||
primary_size = int(os.path.getsize(normalized_primary))
|
||||
except OSError:
|
||||
primary_size = 0
|
||||
metrics["primary_size_bytes"] = primary_size
|
||||
if primary_size <= 0:
|
||||
issues.append(f"Primary ISCE2 displacement file is empty: {normalized_primary}")
|
||||
else:
|
||||
metrics["primary_non_empty"] = True
|
||||
try:
|
||||
primary_metadata = _inspect_raster(normalized_primary)
|
||||
metrics["primary_readable"] = True
|
||||
metrics["primary_raster"] = primary_metadata
|
||||
accepted_sources.append(normalized_primary)
|
||||
except Exception as exc:
|
||||
issues.append(
|
||||
f"Primary ISCE2 displacement file is not a readable GeoTIFF: {normalized_primary}: {exc}"
|
||||
)
|
||||
|
||||
coh_path = ""
|
||||
coh_metadata: Dict[str, Any] = {}
|
||||
for candidate in normalized_sources:
|
||||
if candidate == normalized_primary or not os.path.isfile(candidate):
|
||||
continue
|
||||
try:
|
||||
candidate_size = int(os.path.getsize(candidate))
|
||||
except OSError:
|
||||
candidate_size = 0
|
||||
if candidate_size <= 0:
|
||||
issues.append(f"Auxiliary ISCE2 output file is empty: {candidate}")
|
||||
continue
|
||||
try:
|
||||
coh_metadata = _inspect_raster(candidate)
|
||||
coh_path = candidate
|
||||
accepted_sources.append(candidate)
|
||||
break
|
||||
except Exception as exc:
|
||||
issues.append(
|
||||
f"Auxiliary ISCE2 output file is not a readable GeoTIFF: {candidate}: {exc}"
|
||||
)
|
||||
|
||||
if coh_path:
|
||||
metrics["coh_present"] = True
|
||||
metrics["coh_raster"] = coh_metadata
|
||||
|
||||
accepted = bool(metrics["primary_readable"])
|
||||
return {
|
||||
"accepted": accepted,
|
||||
"primary_file": normalized_primary if accepted else "",
|
||||
"source_files": accepted_sources if accepted_sources else ([normalized_primary] if accepted else []),
|
||||
"coh_file": coh_path,
|
||||
"issues": issues,
|
||||
"metrics": metrics,
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
@@ -27,8 +26,14 @@ from .dinsar_compat_service import dinsar_compat_service
|
||||
from .dinsar_naming import build_run_key
|
||||
from .dinsar_production_service import dinsar_production_service
|
||||
from .dinsar_read_service import dinsar_read_service
|
||||
from .dinsar_result_layout_service import (
|
||||
get_run_disp_asset_paths,
|
||||
get_run_native_output_dir,
|
||||
normalize_envi_run_layout,
|
||||
)
|
||||
from .dinsar_scan_service import dinsar_scan_service
|
||||
from .engine_lock_service import engine_lock_service
|
||||
from .envi_service import build_envi_runner_command, get_envi_runner_cwd, get_envi_runner_env
|
||||
from .psinsar_catalog_service import psinsar_catalog_service
|
||||
from .result_catalog_service import result_catalog_service
|
||||
from .task_service import task_service
|
||||
@@ -875,9 +880,9 @@ def _get_envi_progress_file(job_id: str) -> str:
|
||||
|
||||
|
||||
def _get_envi_runtime_cwd() -> str:
|
||||
runtime_dir = os.path.normpath(os.path.abspath(settings.IDL_WORKER_RUNTIME_DIR))
|
||||
os.makedirs(runtime_dir, exist_ok=True)
|
||||
return runtime_dir
|
||||
# The ENVI runner is launched via `python -m backend.app.services.envi_runner_cli`,
|
||||
# so its import root must be the project root rather than the runtime directory.
|
||||
return get_envi_runner_cwd()
|
||||
|
||||
|
||||
# Stale threshold: no progress file update AND no output file activity
|
||||
@@ -992,18 +997,18 @@ def _clear_envi_progress_file(job_id: Optional[str]) -> None:
|
||||
def _find_latest_envi_result(output_dir: str) -> Dict[str, Any]:
|
||||
matches: List[tuple[float, str]] = []
|
||||
try:
|
||||
for entry in os.scandir(output_dir):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
if entry.name.lower().endswith((".hdr", ".sml")):
|
||||
continue
|
||||
if not _ENVI_RESULT_NAME_RE.match(entry.name):
|
||||
continue
|
||||
try:
|
||||
stat = entry.stat()
|
||||
matches.append((max(stat.st_mtime, stat.st_ctime), entry.path))
|
||||
except OSError:
|
||||
matches.append((0.0, entry.path))
|
||||
for current_root, _dirs, files in os.walk(output_dir):
|
||||
for name in files:
|
||||
if name.lower().endswith((".hdr", ".sml")):
|
||||
continue
|
||||
if not _ENVI_RESULT_NAME_RE.match(name):
|
||||
continue
|
||||
path = os.path.join(current_root, name)
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
matches.append((max(stat.st_mtime, stat.st_ctime), path))
|
||||
except OSError:
|
||||
matches.append((0.0, path))
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f"Failed to scan ENVI output directory: {output_dir}: {exc}") from exc
|
||||
|
||||
@@ -1023,6 +1028,77 @@ def _find_latest_envi_result(output_dir: str) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _is_path_within(base_dir: str, candidate_path: str) -> bool:
|
||||
try:
|
||||
return os.path.commonpath(
|
||||
[
|
||||
os.path.normpath(os.path.abspath(str(base_dir or "").strip())),
|
||||
os.path.normpath(os.path.abspath(str(candidate_path or "").strip())),
|
||||
]
|
||||
) == os.path.normpath(os.path.abspath(str(base_dir or "").strip()))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_managed_envi_output_dir(output_dir: str) -> Dict[str, Any]:
|
||||
normalized_output_dir = os.path.normpath(os.path.abspath(str(output_dir or "").strip()))
|
||||
if not normalized_output_dir or not os.path.isdir(normalized_output_dir):
|
||||
raise FileNotFoundError(f"Managed ENVI output directory not found: {output_dir}")
|
||||
|
||||
managed_root = os.path.normpath(os.path.abspath(str(settings.DINSAR_PRODUCT_DIR or "").strip()))
|
||||
if not managed_root or not _is_path_within(managed_root, normalized_output_dir):
|
||||
result_files = _find_latest_envi_result(normalized_output_dir)
|
||||
return {
|
||||
"run_dir": normalized_output_dir,
|
||||
"native_output_dir": normalized_output_dir,
|
||||
"primary_file": result_files["primary_file"],
|
||||
"source_files": result_files["source_files"],
|
||||
"promoted_files": [],
|
||||
"moved_entries": [],
|
||||
}
|
||||
|
||||
disp_paths = get_run_disp_asset_paths(normalized_output_dir)
|
||||
if os.path.isfile(disp_paths["primary"]):
|
||||
source_files = [disp_paths["primary"]]
|
||||
for ext in (".hdr", ".sml"):
|
||||
sidecar = disp_paths["primary"] + ext
|
||||
if os.path.isfile(sidecar):
|
||||
source_files.append(sidecar)
|
||||
return {
|
||||
"run_dir": normalized_output_dir,
|
||||
"native_output_dir": get_run_native_output_dir(normalized_output_dir),
|
||||
"primary_file": disp_paths["primary"],
|
||||
"source_files": source_files,
|
||||
"promoted_files": [],
|
||||
"moved_entries": [],
|
||||
}
|
||||
|
||||
native_output_dir = get_run_native_output_dir(normalized_output_dir)
|
||||
search_dirs = []
|
||||
if os.path.isdir(native_output_dir):
|
||||
search_dirs.append(native_output_dir)
|
||||
search_dirs.append(normalized_output_dir)
|
||||
|
||||
last_error: Optional[Exception] = None
|
||||
result_files: Optional[Dict[str, Any]] = None
|
||||
for search_dir in search_dirs:
|
||||
try:
|
||||
result_files = _find_latest_envi_result(search_dir)
|
||||
break
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if not result_files:
|
||||
raise RuntimeError(
|
||||
f"Failed to locate ENVI displacement result under managed run directory: {normalized_output_dir}"
|
||||
) from last_error
|
||||
|
||||
return normalize_envi_run_layout(
|
||||
normalized_output_dir,
|
||||
primary_file=result_files["primary_file"],
|
||||
source_files=result_files["source_files"],
|
||||
)
|
||||
|
||||
|
||||
async def _run_envi_runner_command(
|
||||
job: SystemJobORM,
|
||||
runner_cmd: List[str],
|
||||
@@ -1065,6 +1141,7 @@ async def _run_envi_runner_command(
|
||||
stdout=stdout_fd,
|
||||
stderr=stderr_fd,
|
||||
cwd=_get_envi_runtime_cwd(),
|
||||
env=get_envi_runner_env(),
|
||||
)
|
||||
proc_state["pid"] = proc.pid
|
||||
loop.call_soon_threadsafe(pid_ready.set)
|
||||
@@ -1232,10 +1309,7 @@ async def _run_envi_workflow_job(
|
||||
message=f"Launching ENVI worker subprocess... (tasks={task_folder_count}, timeout={effective_absolute_timeout}s)",
|
||||
)
|
||||
|
||||
runner_cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"backend.app.services.envi_runner_cli",
|
||||
runner_cmd = build_envi_runner_command(
|
||||
"--workflow",
|
||||
workflow,
|
||||
"--root-dir",
|
||||
@@ -1244,7 +1318,7 @@ async def _run_envi_workflow_job(
|
||||
str(num_to_process),
|
||||
"--job-id",
|
||||
str(job.job_id),
|
||||
]
|
||||
)
|
||||
if timeout_seconds is not None:
|
||||
runner_cmd.extend(["--timeout-seconds", str(int(timeout_seconds))])
|
||||
|
||||
@@ -1304,6 +1378,7 @@ async def _run_envi_workflow_job(
|
||||
stdout=stdout_fd,
|
||||
stderr=stderr_fd,
|
||||
cwd=_get_envi_runtime_cwd(),
|
||||
env=get_envi_runner_env(),
|
||||
)
|
||||
# Close our copy of the fds; the child process has its own.
|
||||
os.close(stdout_fd)
|
||||
@@ -1443,6 +1518,11 @@ async def _run_envi_workflow_job(
|
||||
if workflow in {"dinsar", "dinsar_custom"}:
|
||||
output_dirs = _dedupe_existing_dirs(run_meta.get("output_dirs"))
|
||||
if output_dirs:
|
||||
normalized_output_dirs: List[str] = []
|
||||
for output_dir in output_dirs:
|
||||
layout_result = await asyncio.to_thread(_normalize_managed_envi_output_dir, output_dir)
|
||||
normalized_output_dirs.append(layout_result["run_dir"])
|
||||
output_dirs = _dedupe_existing_dirs(normalized_output_dirs)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=92,
|
||||
@@ -1575,10 +1655,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
||||
db=db,
|
||||
)
|
||||
|
||||
runner_cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"backend.app.services.envi_runner_cli",
|
||||
runner_cmd = build_envi_runner_command(
|
||||
"--workflow",
|
||||
workflow,
|
||||
"--task-dir",
|
||||
@@ -1593,7 +1670,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
||||
str(run_key),
|
||||
"--profile-code",
|
||||
str(run.profile_code),
|
||||
]
|
||||
)
|
||||
if timeout_seconds is not None:
|
||||
runner_cmd.extend(["--timeout-seconds", str(timeout_seconds)])
|
||||
|
||||
@@ -1638,7 +1715,10 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
||||
keepalive_formatter=_keepalive_formatter,
|
||||
register_pid=_register_pid,
|
||||
)
|
||||
result_files = await asyncio.to_thread(_find_latest_envi_result, execution.output_dir)
|
||||
layout_result = await asyncio.to_thread(
|
||||
_normalize_managed_envi_output_dir,
|
||||
execution.output_dir,
|
||||
)
|
||||
metrics = {
|
||||
"duration_seconds": run_meta.get("duration_seconds"),
|
||||
"summary": run_meta.get("summary") or {},
|
||||
@@ -1651,17 +1731,20 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
primary_file=result_files["primary_file"],
|
||||
source_files=result_files["source_files"],
|
||||
primary_file=layout_result["primary_file"],
|
||||
source_files=layout_result["source_files"],
|
||||
native_output_dir=layout_result["native_output_dir"],
|
||||
metrics=metrics,
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
dinsar_production_service.write_current_pointer,
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
manifest_path=manifest_path,
|
||||
primary_file=result_files["primary_file"],
|
||||
source_files=result_files["source_files"],
|
||||
primary_file=layout_result["primary_file"],
|
||||
source_files=layout_result["source_files"],
|
||||
native_output_dir=layout_result["native_output_dir"],
|
||||
)
|
||||
await dinsar_production_service.mark_item_completed(
|
||||
run=run,
|
||||
@@ -1894,6 +1977,8 @@ async def _handle_queued_engine_run(
|
||||
timeout_seconds = payload.get("timeout_seconds")
|
||||
extra = payload.get("extra", {})
|
||||
selected_task_count = max(1, int(extra.get("__validated_task_count") or 0 or 1))
|
||||
rerun_mode = str(payload.get("rerun_mode") or extra.get("__rerun_mode") or "rerun_all").strip()
|
||||
skipped_completed_count = int(extra.get("__skipped_completed_count") or 0)
|
||||
pair_timeout_seconds = int(timeout_seconds or fallback_timeout_seconds)
|
||||
|
||||
await task_service.start_task(
|
||||
@@ -1903,7 +1988,10 @@ async def _handle_queued_engine_run(
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, timeout={pair_timeout_seconds}s, extra={extra}",
|
||||
(
|
||||
f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, "
|
||||
f"rerun_mode={rerun_mode}, timeout={pair_timeout_seconds}s, extra={extra}"
|
||||
),
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
@@ -1911,6 +1999,7 @@ async def _handle_queued_engine_run(
|
||||
(
|
||||
f"{engine_title} batch contains {selected_task_count} pair task(s). "
|
||||
f"Pairs run sequentially and each pair uses timeout={pair_timeout_seconds}s."
|
||||
f"{f' Skipped completed={skipped_completed_count}.' if skipped_completed_count > 0 else ''}"
|
||||
),
|
||||
)
|
||||
from ..dinsar_engines.base import RunRequest
|
||||
@@ -2114,6 +2203,18 @@ async def _handle_queued_engine_run(
|
||||
"INFO",
|
||||
f"WSL command [{item.get('task_name')}]: {item.get('command')}",
|
||||
)
|
||||
if item.get("runtime_id"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL runtime [{item.get('task_name')}]: {item.get('runtime_id')}",
|
||||
)
|
||||
if item.get("manifest_path_windows"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL manifest [{item.get('task_name')}]: {item.get('manifest_path_windows')}",
|
||||
)
|
||||
if item.get("stdout_tail"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
@@ -2163,6 +2264,18 @@ async def _handle_queued_engine_run(
|
||||
"INFO",
|
||||
f"WSL command: {detail['command']}",
|
||||
)
|
||||
if detail.get("runtime_id"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL runtime: {detail['runtime_id']}",
|
||||
)
|
||||
if detail.get("manifest_path_windows"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL manifest: {detail['manifest_path_windows']}",
|
||||
)
|
||||
if detail.get("stdout_tail"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
@@ -2243,7 +2356,532 @@ async def _handle_queued_engine_run(
|
||||
)
|
||||
|
||||
|
||||
async def _run_wsl_dinsar_production_controller(
|
||||
job: SystemJobORM,
|
||||
*,
|
||||
engine_code: str,
|
||||
engine_title: str,
|
||||
fallback_timeout_seconds: int,
|
||||
) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError(f"{engine_title} production controller requires task_id.")
|
||||
|
||||
payload = job.payload or {}
|
||||
production_run_id = str(payload.get("production_run_id") or "").strip()
|
||||
if not production_run_id:
|
||||
raise ValueError(f"{engine_title} production controller requires production_run_id.")
|
||||
|
||||
from ..dinsar_engines import registry
|
||||
from ..dinsar_engines.base import RunRequest
|
||||
|
||||
engine = registry.get_engine(engine_code)
|
||||
if engine is None:
|
||||
raise RuntimeError(f"Engine '{engine_code}' is not registered.")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||
if run is None:
|
||||
raise ValueError(f"D-InSAR production run not found: {production_run_id}")
|
||||
|
||||
items = await dinsar_production_service.list_run_items(run.run_id, db)
|
||||
if not items:
|
||||
raise ValueError(f"D-InSAR production run has no items: {production_run_id}")
|
||||
|
||||
params = run.params_json or {}
|
||||
user_extra = dict(params.get("extra") or {})
|
||||
timeout_seconds_raw = params.get("timeout_seconds")
|
||||
if timeout_seconds_raw not in (None, ""):
|
||||
per_task_timeout = int(timeout_seconds_raw)
|
||||
else:
|
||||
per_task_timeout = int(
|
||||
getattr(engine, "default_timeout_seconds", None)
|
||||
or fallback_timeout_seconds
|
||||
or 0
|
||||
)
|
||||
total_items = len(items)
|
||||
run_log = dinsar_production_service.append_run_log
|
||||
|
||||
async def _refresh_cancel_state() -> bool:
|
||||
await db.refresh(run)
|
||||
current_task = await task_service.get_task(job.task_id)
|
||||
task_cancelled = bool(current_task and current_task.status == "CANCELLED")
|
||||
if task_cancelled and not run.cancel_requested:
|
||||
run.cancel_requested = True
|
||||
await db.commit()
|
||||
return bool(run.cancel_requested or task_cancelled)
|
||||
|
||||
await task_service.start_task(
|
||||
job.task_id,
|
||||
message=f"Starting {engine_title} D-InSAR production run {run.run_id} ({total_items} items)...",
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"{engine_title} D-InSAR production controller started. run_id={run.run_id} "
|
||||
f"profile={run.profile_code} items={total_items}"
|
||||
),
|
||||
)
|
||||
await dinsar_production_service.mark_run_started(
|
||||
run,
|
||||
db=db,
|
||||
message=f"Preparing {total_items} {engine_title} item(s)",
|
||||
)
|
||||
run_log(
|
||||
run.run_id,
|
||||
f"[start] engine={engine_code} items={total_items} source_root={run.source_root}",
|
||||
)
|
||||
|
||||
successful_output_dirs: List[str] = []
|
||||
async with engine_lock_service.acquire(f"wsl_dinsar_{engine_code}"):
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=5,
|
||||
message=f"{engine_title} engine acquired. Preparing {total_items} item(s)...",
|
||||
)
|
||||
for item_index, item in enumerate(items, start=1):
|
||||
if await _refresh_cancel_state():
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"Cancellation detected before item {item.task_alias or item.task_name}.",
|
||||
)
|
||||
break
|
||||
|
||||
await db.refresh(item)
|
||||
if str(item.status or "").upper() in {"COMPLETED", "FAILED", "SKIPPED", "CANCELLED"}:
|
||||
if item.status == "COMPLETED" and item.latest_output_dir and os.path.isdir(item.latest_output_dir):
|
||||
successful_output_dirs.append(item.latest_output_dir)
|
||||
continue
|
||||
|
||||
run_key = (
|
||||
f"{build_run_key(engine_code, run.profile_code, started_at=datetime.utcnow())}"
|
||||
f"_{item.id}_{uuid.uuid4().hex[:6]}"
|
||||
)
|
||||
execution = await dinsar_production_service.begin_item_execution(
|
||||
run=run,
|
||||
item=item,
|
||||
run_key=run_key,
|
||||
db=db,
|
||||
)
|
||||
|
||||
managed_run_dir = os.path.normpath(execution.output_dir)
|
||||
managed_native_output_dir = os.path.join(managed_run_dir, "native")
|
||||
managed_work_dir = os.path.join(managed_native_output_dir, "workflow")
|
||||
managed_export_dir = os.path.join(managed_native_output_dir, "export")
|
||||
managed_orbit_output_dir = os.path.join(managed_work_dir, "orbits")
|
||||
item_label = item.task_alias or item.task_name
|
||||
base_progress = min(95, 5 + int(((item_index - 1) / max(1, total_items)) * 90))
|
||||
progress_state: Dict[str, Any] = {
|
||||
"progress": base_progress,
|
||||
"message": f"[{engine_code}/{run.profile_code}] Running {item_index}/{total_items}: {item_label}",
|
||||
"started_monotonic": time.monotonic(),
|
||||
}
|
||||
progress_queue: asyncio.Queue[Optional[Dict[str, Any]]] = asyncio.Queue()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _emit_progress(event: Dict[str, Any]) -> None:
|
||||
if not event:
|
||||
return
|
||||
try:
|
||||
loop.call_soon_threadsafe(progress_queue.put_nowait, dict(event))
|
||||
except RuntimeError:
|
||||
return
|
||||
|
||||
async def _consume_progress() -> None:
|
||||
while True:
|
||||
event = await progress_queue.get()
|
||||
if event is None:
|
||||
return
|
||||
event_type = str(event.get("event") or "").strip().lower()
|
||||
if event_type == "pair_started":
|
||||
progress_state["message"] = (
|
||||
f"[{engine_code}/{run.profile_code}] Running "
|
||||
f"{item_index}/{total_items}: {item_label}"
|
||||
)
|
||||
progress_state["started_monotonic"] = time.monotonic()
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"[{item_index}/{total_items}] {engine_title} started {item_label}",
|
||||
)
|
||||
elif event_type == "pair_finished":
|
||||
if bool(event.get("success")):
|
||||
progress_state["progress"] = min(
|
||||
98,
|
||||
5 + int((item_index / max(1, total_items)) * 90),
|
||||
)
|
||||
progress_state["message"] = (
|
||||
f"[{engine_code}/{run.profile_code}] Finished "
|
||||
f"{item_index}/{total_items}: {item_label}"
|
||||
)
|
||||
else:
|
||||
progress_state["message"] = (
|
||||
f"[{engine_code}/{run.profile_code}] Failed "
|
||||
f"{item_index}/{total_items}: {item_label}"
|
||||
)
|
||||
|
||||
async def _task_keepalive() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
message = str(progress_state.get("message") or "")
|
||||
started_monotonic = progress_state.get("started_monotonic")
|
||||
if isinstance(started_monotonic, (int, float)):
|
||||
elapsed_seconds = max(0, int(time.monotonic() - float(started_monotonic)))
|
||||
message = f"{message} (elapsed={elapsed_seconds}s)"
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=int(progress_state.get("progress") or base_progress),
|
||||
message=message,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("keepalive update failed for %s item %s: %s", engine_title, item_label, exc)
|
||||
|
||||
progress_task = asyncio.create_task(_consume_progress())
|
||||
keepalive_task = asyncio.create_task(_task_keepalive())
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"[{item_index}/{total_items}] Launching {item_label} -> {managed_run_dir}",
|
||||
)
|
||||
run_log(
|
||||
run.run_id,
|
||||
f"[item-start] {item_index}/{total_items} {item_label} run_key={run_key} output={managed_run_dir}",
|
||||
)
|
||||
|
||||
request = RunRequest(
|
||||
engine_code=engine_code,
|
||||
profile=run.profile_code,
|
||||
root_dir=str(item.source_task_dir),
|
||||
job_id=job.job_id,
|
||||
num_to_process=1,
|
||||
timeout_seconds=per_task_timeout or None,
|
||||
extra={
|
||||
**user_extra,
|
||||
"__managed_run_dir": managed_run_dir,
|
||||
"__managed_native_output_dir": managed_native_output_dir,
|
||||
"__managed_work_dir": managed_work_dir,
|
||||
"__managed_export_dir": managed_export_dir,
|
||||
"__managed_orbit_output_dir": managed_orbit_output_dir,
|
||||
"__managed_run_key": run_key,
|
||||
"__source_root_override": run.source_root,
|
||||
"__rerun_mode": "rerun_all",
|
||||
},
|
||||
progress_callback=_emit_progress,
|
||||
)
|
||||
|
||||
result = None
|
||||
run_exception_text = ""
|
||||
try:
|
||||
result = await asyncio.to_thread(engine.run, request)
|
||||
except Exception as exc:
|
||||
run_exception_text = str(exc)
|
||||
finally:
|
||||
keepalive_task.cancel()
|
||||
try:
|
||||
await keepalive_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
await progress_queue.put(None)
|
||||
await progress_task
|
||||
|
||||
detail = result.detail or {} if result else {}
|
||||
task_result = ((detail.get("task_results") or [{}])[0]) if result else {}
|
||||
|
||||
try:
|
||||
if not result or not result.success or not bool(task_result.get("success", result.success if result else False)):
|
||||
error_message = (
|
||||
str(task_result.get("error") or "").strip()
|
||||
or str(result.error or "").strip()
|
||||
or run_exception_text
|
||||
or str(task_result.get("stderr_tail") or "").strip()
|
||||
or f"{engine_title} run failed."
|
||||
)
|
||||
raise RuntimeError(error_message)
|
||||
|
||||
run_dir = os.path.normpath(
|
||||
str(task_result.get("run_dir") or task_result.get("output_dir") or execution.output_dir)
|
||||
)
|
||||
if run_dir != managed_run_dir:
|
||||
raise RuntimeError(
|
||||
f"{engine_title} managed run dir mismatch: expected {managed_run_dir}, got {run_dir}"
|
||||
)
|
||||
|
||||
primary_file = str(task_result.get("primary_file") or "").strip()
|
||||
source_files = [
|
||||
str(path)
|
||||
for path in (task_result.get("source_files") or [])
|
||||
if str(path or "").strip()
|
||||
]
|
||||
native_output_dir = str(
|
||||
task_result.get("native_output_dir") or managed_native_output_dir
|
||||
).strip() or managed_native_output_dir
|
||||
if not primary_file or not os.path.isfile(primary_file):
|
||||
raise RuntimeError(f"{engine_title} primary output is missing: {primary_file or '<empty>'}")
|
||||
if not source_files:
|
||||
source_files = [primary_file]
|
||||
|
||||
metrics = {
|
||||
"result_detail": detail,
|
||||
"task_result": task_result,
|
||||
}
|
||||
manifest_path = await asyncio.to_thread(
|
||||
dinsar_production_service.build_execution_manifest,
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
primary_file=primary_file,
|
||||
source_files=source_files,
|
||||
native_output_dir=native_output_dir,
|
||||
metrics=metrics,
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
dinsar_production_service.write_current_pointer,
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
manifest_path=manifest_path,
|
||||
primary_file=primary_file,
|
||||
source_files=source_files,
|
||||
native_output_dir=native_output_dir,
|
||||
)
|
||||
await dinsar_production_service.mark_item_completed(
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
manifest_path=manifest_path,
|
||||
metrics=metrics,
|
||||
db=db,
|
||||
)
|
||||
successful_output_dirs.append(managed_run_dir)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"[{item_index}/{total_items}] Completed {item_label}",
|
||||
)
|
||||
run_log(run.run_id, f"[item-ok] {item_index}/{total_items} {item_label}")
|
||||
except Exception as exc:
|
||||
cancelled = await _refresh_cancel_state()
|
||||
error_message = str(exc)
|
||||
if cancelled:
|
||||
await dinsar_production_service.mark_item_cancelled(
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
error_message=f"Cancelled while processing {item_label}",
|
||||
db=db,
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"[{item_index}/{total_items}] Cancelled {item_label}: {error_message}",
|
||||
)
|
||||
run_log(
|
||||
run.run_id,
|
||||
f"[item-cancelled] {item_index}/{total_items} {item_label}: {error_message}",
|
||||
)
|
||||
break
|
||||
|
||||
await dinsar_production_service.mark_item_failed(
|
||||
run=run,
|
||||
item=item,
|
||||
execution=execution,
|
||||
error_message=error_message,
|
||||
db=db,
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"[{item_index}/{total_items}] Failed {item_label}: {error_message}",
|
||||
)
|
||||
run_log(
|
||||
run.run_id,
|
||||
f"[item-failed] {item_index}/{total_items} {item_label}: {error_message}",
|
||||
)
|
||||
if task_result.get("command"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL command [{item_label}]: {task_result.get('command')}",
|
||||
)
|
||||
if task_result.get("stdout_tail"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
f"WSL stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}",
|
||||
)
|
||||
if task_result.get("stderr_tail"):
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"WSL stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}",
|
||||
)
|
||||
|
||||
publish_result = None
|
||||
rebuild_result = None
|
||||
publish_error = None
|
||||
publish_dirs = _dedupe_existing_dirs(successful_output_dirs)
|
||||
if publish_dirs:
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
progress=99,
|
||||
message=f"Publishing {len(publish_dirs)} successful {engine_title} result package(s)...",
|
||||
)
|
||||
try:
|
||||
publish_result = await result_catalog_service.publish_from_sources(db, publish_dirs)
|
||||
processed_count = int(publish_result.get("processed", 0) or 0)
|
||||
failed_count = int(publish_result.get("failed", 0) or 0)
|
||||
expected_count = len(publish_dirs)
|
||||
if processed_count > 0:
|
||||
rebuild_result = await result_catalog_service.rebuild_catalog(
|
||||
db,
|
||||
full_rebuild=True,
|
||||
)
|
||||
if processed_count != expected_count or failed_count != 0:
|
||||
raise RuntimeError(
|
||||
f"Expected to publish {expected_count} {engine_title} result package(s), "
|
||||
f"but processed={processed_count}, failed={failed_count}"
|
||||
)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"INFO",
|
||||
(
|
||||
f"Published {publish_result.get('processed', 0)} {engine_title} result package(s). "
|
||||
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
|
||||
),
|
||||
)
|
||||
run_log(
|
||||
run.run_id,
|
||||
(
|
||||
f"[publish] processed={publish_result.get('processed', 0)} "
|
||||
f"failed={publish_result.get('failed', 0)} "
|
||||
f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}"
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
publish_error = str(exc)
|
||||
await task_service.add_log(
|
||||
job.task_id,
|
||||
"WARNING",
|
||||
f"Result catalog publish failed: {publish_error}",
|
||||
)
|
||||
run_log(run.run_id, f"[publish-failed] {publish_error}")
|
||||
|
||||
cancelled = await _refresh_cancel_state()
|
||||
await dinsar_production_service.refresh_run_counters(run, db=db)
|
||||
if publish_error:
|
||||
final_status = "FAILED"
|
||||
latest_message = f"Result catalog publish failed: {publish_error}"
|
||||
elif cancelled:
|
||||
final_status = "CANCELLED"
|
||||
latest_message = (
|
||||
f"{engine_title} D-InSAR production cancelled. completed={run.completed_items} "
|
||||
f"failed={run.failed_items} total={run.total_items}"
|
||||
)
|
||||
elif int(run.failed_items or 0) > 0:
|
||||
final_status = "FAILED"
|
||||
latest_message = (
|
||||
f"{engine_title} D-InSAR production finished with failures. completed={run.completed_items} "
|
||||
f"failed={run.failed_items} total={run.total_items}"
|
||||
)
|
||||
else:
|
||||
final_status = "COMPLETED"
|
||||
latest_message = (
|
||||
f"{engine_title} D-InSAR production completed. completed={run.completed_items} "
|
||||
f"failed={run.failed_items} total={run.total_items}"
|
||||
)
|
||||
|
||||
summary_payload = {
|
||||
"workflow": f"dinsar_{engine_code}",
|
||||
"engine_code": run.engine_code,
|
||||
"profile_code": run.profile_code,
|
||||
"mode": run.mode,
|
||||
"total_items": run.total_items,
|
||||
"completed_items": run.completed_items,
|
||||
"failed_items": run.failed_items,
|
||||
"skipped_items": run.skipped_items,
|
||||
"publish": publish_result,
|
||||
"rebuild": rebuild_result,
|
||||
"publish_error": publish_error,
|
||||
"published_output_dirs": publish_dirs,
|
||||
}
|
||||
await dinsar_production_service.finalize_run(
|
||||
run,
|
||||
db=db,
|
||||
status=final_status,
|
||||
summary_payload=summary_payload,
|
||||
latest_message=latest_message,
|
||||
)
|
||||
run_log(run.run_id, f"[finish] status={final_status} message={latest_message}")
|
||||
|
||||
if final_status == "COMPLETED":
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=latest_message,
|
||||
)
|
||||
return
|
||||
|
||||
task_status = "CANCELLED" if final_status == "CANCELLED" else "FAILED"
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status=task_status,
|
||||
progress=100,
|
||||
message=latest_message,
|
||||
)
|
||||
raise RuntimeError(latest_message)
|
||||
|
||||
|
||||
async def _handle_isce2_run(job: SystemJobORM) -> None:
|
||||
payload = job.payload or {}
|
||||
production_run_id = str(payload.get("production_run_id") or "").strip()
|
||||
if production_run_id:
|
||||
try:
|
||||
await _run_wsl_dinsar_production_controller(
|
||||
job,
|
||||
engine_code="isce2",
|
||||
engine_title="ISCE2",
|
||||
fallback_timeout_seconds=settings.ISCE2_PER_TASK_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception as exc:
|
||||
latest_message = f"ISCE2 D-InSAR production controller failed: {exc}"
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
run = await dinsar_production_service.get_run(production_run_id, db)
|
||||
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
||||
summary_payload = dict(run.summary_json or {})
|
||||
summary_payload["controller_error"] = str(exc)
|
||||
await dinsar_production_service.finalize_run(
|
||||
run,
|
||||
db=db,
|
||||
status="FAILED",
|
||||
summary_payload=summary_payload,
|
||||
latest_message=latest_message,
|
||||
)
|
||||
dinsar_production_service.append_run_log(
|
||||
run.run_id,
|
||||
f"[controller-failed] {exc}",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
current_task = await task_service.get_task(job.task_id)
|
||||
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
|
||||
await task_service.add_log(job.task_id, "ERROR", latest_message)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="FAILED",
|
||||
progress=100,
|
||||
message=latest_message,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
return
|
||||
|
||||
await _handle_queued_engine_run(
|
||||
job,
|
||||
engine_title="ISCE2",
|
||||
|
||||
@@ -59,12 +59,17 @@ def _read_manifest_metadata(path: str, *, root: ManagedRootORM) -> Dict[str, Any
|
||||
for key in (
|
||||
"schema_version",
|
||||
"catalog_name",
|
||||
"product_family",
|
||||
"product_type",
|
||||
"product_id",
|
||||
"display_name",
|
||||
"pair_key",
|
||||
"stack_key",
|
||||
"run_key",
|
||||
"run_id",
|
||||
"group_key",
|
||||
"processor_code",
|
||||
"runtime_id",
|
||||
"reference_date",
|
||||
"published_at",
|
||||
"produced_at",
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
|
||||
CANONICAL_PACKAGE_SCHEMA = "insar.product-package/v1"
|
||||
CANONICAL_PACKAGE_LAYOUT = "canonical.v1"
|
||||
LEGACY_DINSAR_PACKAGE_SCHEMA = "dinsar-product/v1"
|
||||
LEGACY_TIMESERIES_PACKAGE_SCHEMA = "psinsar.publish.v1"
|
||||
|
||||
TIMESERIES_ARTIFACT_ROLE_MAP = {
|
||||
"timeseries_cube": "timeseries_cube",
|
||||
"velocity_map": "velocity_map",
|
||||
"velocity_geotiff": "velocity_geotiff",
|
||||
"temporal_coherence": "temporal_coherence",
|
||||
"temporal_coherence_geotiff": "temporal_coherence_geotiff",
|
||||
"quality_mask": "quality_mask",
|
||||
"quality_mask_geotiff": "quality_mask_geotiff",
|
||||
"preview_png": "preview_png",
|
||||
"diagnostic_png": "diagnostic_png",
|
||||
}
|
||||
|
||||
_DINSAR_PRIMARY_ROLES = {"disp"}
|
||||
_DINSAR_PREVIEW_ROLES = {"thumb"}
|
||||
_TIMESERIES_PRIMARY_ROLES = {"velocity_geotiff", "timeseries_cube", "velocity_map"}
|
||||
_TIMESERIES_PREVIEW_ROLES = {"preview_png"}
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _copy_dict(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
return copy.deepcopy(payload) if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _copy_list(payload: Optional[Iterable[Any]]) -> List[Any]:
|
||||
return copy.deepcopy(list(payload or []))
|
||||
|
||||
|
||||
def infer_asset_format(path: str) -> Optional[str]:
|
||||
ext = os.path.splitext(str(path or "").lower())[1]
|
||||
return {
|
||||
".h5": "hdf5",
|
||||
".hdr": "hdr",
|
||||
".json": "json",
|
||||
".png": "png",
|
||||
".sml": "sml",
|
||||
".tif": "geotiff",
|
||||
".tiff": "geotiff",
|
||||
".webp": "webp",
|
||||
".xml": "xml",
|
||||
"": None,
|
||||
}.get(ext, ext.lstrip(".") or None)
|
||||
|
||||
|
||||
def infer_asset_media_type(path: str) -> Optional[str]:
|
||||
ext = os.path.splitext(str(path or "").lower())[1]
|
||||
return {
|
||||
".h5": "application/x-hdf5",
|
||||
".hdr": "text/plain",
|
||||
".json": "application/json",
|
||||
".png": "image/png",
|
||||
".sml": "text/plain",
|
||||
".tif": "image/tiff",
|
||||
".tiff": "image/tiff",
|
||||
".webp": "image/webp",
|
||||
".xml": "application/xml",
|
||||
}.get(ext)
|
||||
|
||||
|
||||
def build_asset_entry(
|
||||
*,
|
||||
role: str,
|
||||
relative_path: str,
|
||||
asset_name: Optional[str] = None,
|
||||
format: Optional[str] = None,
|
||||
media_type: Optional[str] = None,
|
||||
is_required: bool = False,
|
||||
is_primary: bool = False,
|
||||
origin_role: Optional[str] = None,
|
||||
native_path: Optional[str] = None,
|
||||
extra: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
relative = _text(relative_path)
|
||||
if not relative:
|
||||
raise ValueError("relative_path is required")
|
||||
payload: Dict[str, Any] = {
|
||||
"role": _text(role) or "asset",
|
||||
"asset_name": _text(asset_name) or os.path.basename(relative) or (_text(role) or "asset"),
|
||||
"relative_path": relative,
|
||||
"format": format or infer_asset_format(relative),
|
||||
"media_type": media_type or infer_asset_media_type(relative),
|
||||
"is_required": bool(is_required),
|
||||
"is_primary": bool(is_primary),
|
||||
}
|
||||
if _text(origin_role):
|
||||
payload["origin_role"] = _text(origin_role)
|
||||
if _text(native_path):
|
||||
payload["native_path"] = _text(native_path)
|
||||
for key, value in (extra or {}).items():
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return payload
|
||||
|
||||
|
||||
def canonicalize_timeseries_artifacts(artifacts: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
assets: List[Dict[str, Any]] = []
|
||||
for artifact in artifacts or []:
|
||||
relative_path = _text((artifact or {}).get("path"))
|
||||
if not relative_path:
|
||||
continue
|
||||
product_type = _text((artifact or {}).get("product_type")) or "asset"
|
||||
asset_role = TIMESERIES_ARTIFACT_ROLE_MAP.get(product_type, product_type or "asset")
|
||||
assets.append(
|
||||
build_asset_entry(
|
||||
role=asset_role,
|
||||
relative_path=relative_path,
|
||||
asset_name=os.path.basename(relative_path),
|
||||
format=infer_asset_format(relative_path),
|
||||
media_type=infer_asset_media_type(relative_path),
|
||||
is_required=asset_role in _TIMESERIES_PRIMARY_ROLES or asset_role in _TIMESERIES_PREVIEW_ROLES,
|
||||
is_primary=asset_role in _TIMESERIES_PRIMARY_ROLES,
|
||||
origin_role=product_type,
|
||||
)
|
||||
)
|
||||
return assets
|
||||
|
||||
|
||||
def _primary_roles_for_family(product_family: str) -> set[str]:
|
||||
family = _text(product_family).lower()
|
||||
if family == "timeseries":
|
||||
return set(_TIMESERIES_PRIMARY_ROLES)
|
||||
return set(_DINSAR_PRIMARY_ROLES)
|
||||
|
||||
|
||||
def _preview_roles_for_family(product_family: str) -> set[str]:
|
||||
family = _text(product_family).lower()
|
||||
if family == "timeseries":
|
||||
return set(_TIMESERIES_PREVIEW_ROLES)
|
||||
return set(_DINSAR_PREVIEW_ROLES)
|
||||
|
||||
|
||||
def build_canonical_descriptor(
|
||||
assets: Iterable[Dict[str, Any]],
|
||||
*,
|
||||
product_family: str,
|
||||
) -> Dict[str, Any]:
|
||||
asset_items = list(assets or [])
|
||||
available_roles = [
|
||||
_text(asset.get("role"))
|
||||
for asset in asset_items
|
||||
if _text(asset.get("role"))
|
||||
]
|
||||
primary_role = None
|
||||
preview_role = None
|
||||
preferred_primary = _primary_roles_for_family(product_family)
|
||||
preferred_preview = _preview_roles_for_family(product_family)
|
||||
|
||||
for asset in asset_items:
|
||||
role = _text(asset.get("role"))
|
||||
if not primary_role and (bool(asset.get("is_primary")) or role in preferred_primary):
|
||||
primary_role = role
|
||||
if not preview_role and role in preferred_preview:
|
||||
preview_role = role
|
||||
|
||||
primary_relative = None
|
||||
preview_relative = None
|
||||
for asset in asset_items:
|
||||
role = _text(asset.get("role"))
|
||||
if primary_relative is None and role == primary_role:
|
||||
primary_relative = _text(asset.get("relative_path"))
|
||||
if preview_relative is None and role == preview_role:
|
||||
preview_relative = _text(asset.get("relative_path"))
|
||||
|
||||
return {
|
||||
"primary_asset_role": primary_role,
|
||||
"preview_asset_role": preview_role,
|
||||
"primary_asset_relative": primary_relative,
|
||||
"preview_asset_relative": preview_relative,
|
||||
"available_asset_roles": sorted({role for role in available_roles if role}),
|
||||
}
|
||||
|
||||
|
||||
def _normalize_canonical_manifest(document: Dict[str, Any]) -> Dict[str, Any]:
|
||||
document["schema_version"] = CANONICAL_PACKAGE_SCHEMA
|
||||
document["package_layout"] = _text(document.get("package_layout")) or CANONICAL_PACKAGE_LAYOUT
|
||||
document["source_schema_version"] = (
|
||||
_text(document.get("source_schema_version")) or CANONICAL_PACKAGE_SCHEMA
|
||||
)
|
||||
|
||||
identity = _copy_dict(document.get("identity"))
|
||||
if not _text(identity.get("pair_key")) and _text(document.get("pair_key")):
|
||||
identity["pair_key"] = _text(document.get("pair_key"))
|
||||
if not _text(identity.get("stack_key")) and _text(document.get("stack_key")):
|
||||
identity["stack_key"] = _text(document.get("stack_key"))
|
||||
if not _text(identity.get("run_key")):
|
||||
identity["run_key"] = _text(document.get("run_key")) or _text(document.get("run_id"))
|
||||
document["identity"] = identity
|
||||
document["pair_key"] = _text(identity.get("pair_key")) or None
|
||||
document["stack_key"] = _text(identity.get("stack_key")) or None
|
||||
document["run_key"] = _text(identity.get("run_key")) or None
|
||||
|
||||
engine = _copy_dict(document.get("engine"))
|
||||
if not _text(engine.get("code")):
|
||||
engine["code"] = _text(document.get("engine_code")) or "unknown"
|
||||
if not _text(engine.get("version")) and _text(document.get("engine_version")):
|
||||
engine["version"] = _text(document.get("engine_version"))
|
||||
document["engine"] = engine
|
||||
|
||||
processor = _copy_dict(document.get("processor"))
|
||||
if not _text(processor.get("code")):
|
||||
processor["code"] = _text(document.get("processor_code")) or _text(engine.get("code")) or "unknown"
|
||||
if not _text(processor.get("profile_code")) and _text(document.get("profile_code")):
|
||||
processor["profile_code"] = _text(document.get("profile_code"))
|
||||
document["processor"] = processor
|
||||
document["processor_code"] = _text(processor.get("code")) or None
|
||||
|
||||
runtime = _copy_dict(document.get("runtime"))
|
||||
if not _text(runtime.get("runtime_id")) and _text(document.get("runtime_id")):
|
||||
runtime["runtime_id"] = _text(document.get("runtime_id"))
|
||||
document["runtime"] = runtime
|
||||
document["runtime_id"] = _text(runtime.get("runtime_id")) or None
|
||||
|
||||
source = _copy_dict(document.get("source"))
|
||||
if not _text(source.get("primary_path")) and _text(document.get("source_primary_path")):
|
||||
source["primary_path"] = _text(document.get("source_primary_path"))
|
||||
if not _text(source.get("publish_dir")) and _text(document.get("publish_dir")):
|
||||
source["publish_dir"] = _text(document.get("publish_dir"))
|
||||
if not _text(source.get("native_output_dir")):
|
||||
source["native_output_dir"] = (
|
||||
_text(source.get("output_dir"))
|
||||
or _text(document.get("native_output_dir"))
|
||||
or None
|
||||
)
|
||||
document["source"] = source
|
||||
document["native_output_dir"] = _text(source.get("native_output_dir")) or None
|
||||
|
||||
temporal = _copy_dict(document.get("temporal"))
|
||||
if not _text(temporal.get("reference_date")) and _text(document.get("reference_date")):
|
||||
temporal["reference_date"] = _text(document.get("reference_date"))
|
||||
if not temporal.get("stack_dates") and isinstance(document.get("stack_dates"), list):
|
||||
temporal["stack_dates"] = [str(item).strip() for item in document.get("stack_dates") or [] if str(item).strip()]
|
||||
if not _text(temporal.get("produced_at")) and _text(document.get("produced_at")):
|
||||
temporal["produced_at"] = _text(document.get("produced_at"))
|
||||
if not _text(temporal.get("published_at")) and _text(document.get("published_at")):
|
||||
temporal["published_at"] = _text(document.get("published_at"))
|
||||
document["temporal"] = temporal
|
||||
document["produced_at"] = _text(temporal.get("produced_at")) or None
|
||||
document["published_at"] = _text(temporal.get("published_at")) or None
|
||||
|
||||
spatial = _copy_dict(document.get("spatial"))
|
||||
if not spatial and any(document.get(key) is not None for key in ("min_lon", "min_lat", "max_lon", "max_lat")):
|
||||
spatial = {
|
||||
"min_lon": document.get("min_lon"),
|
||||
"min_lat": document.get("min_lat"),
|
||||
"max_lon": document.get("max_lon"),
|
||||
"max_lat": document.get("max_lat"),
|
||||
"coverage_polygon": document.get("coverage_polygon"),
|
||||
}
|
||||
document["spatial"] = spatial
|
||||
|
||||
assets = _copy_list(document.get("assets"))
|
||||
if not assets and isinstance(document.get("artifacts"), list):
|
||||
assets = canonicalize_timeseries_artifacts(document.get("artifacts") or [])
|
||||
document["assets"] = assets
|
||||
|
||||
product_family = _text(document.get("product_family")) or "dinsar"
|
||||
canonical = _copy_dict(document.get("canonical"))
|
||||
defaults = build_canonical_descriptor(assets, product_family=product_family)
|
||||
for key, value in defaults.items():
|
||||
if canonical.get(key) in (None, "", []):
|
||||
canonical[key] = value
|
||||
document["canonical"] = canonical
|
||||
document["package_schema"] = CANONICAL_PACKAGE_SCHEMA
|
||||
|
||||
if not _text(document.get("catalog_name")):
|
||||
document["catalog_name"] = "dinsar" if product_family == "dinsar" else "psinsar"
|
||||
if not _text(document.get("product_type")):
|
||||
document["product_type"] = "dinsar_interferogram" if product_family == "dinsar" else "timeseries_bundle"
|
||||
if not _text(document.get("display_name")) and _text(document.get("product_id")):
|
||||
document["display_name"] = _text(document.get("product_id"))
|
||||
return document
|
||||
|
||||
|
||||
def _normalize_legacy_dinsar_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
document = copy.deepcopy(payload)
|
||||
run_payload = _copy_dict(document.get("run"))
|
||||
source_payload = _copy_dict(document.get("source"))
|
||||
engine_payload = _copy_dict(document.get("engine"))
|
||||
document["source_schema_version"] = LEGACY_DINSAR_PACKAGE_SCHEMA
|
||||
document["product_family"] = _text(document.get("product_family")) or "dinsar"
|
||||
document["product_type"] = "dinsar_interferogram"
|
||||
document["engine_code"] = _text(engine_payload.get("code")) or _text(document.get("engine_code")) or "unknown"
|
||||
document["processor"] = {
|
||||
"code": (
|
||||
_text(document.get("processor_code"))
|
||||
or _text(run_payload.get("profile_code"))
|
||||
or _text(engine_payload.get("code"))
|
||||
or "unknown"
|
||||
),
|
||||
"profile_code": _text(run_payload.get("profile_code")) or None,
|
||||
}
|
||||
runtime = _copy_dict(document.get("runtime"))
|
||||
if not _text(runtime.get("kind")):
|
||||
runtime["kind"] = "windows" if document["engine_code"] in {"envi", "sarscape"} else None
|
||||
document["runtime"] = runtime
|
||||
document["source"] = {
|
||||
**source_payload,
|
||||
"native_output_dir": (
|
||||
_text(source_payload.get("native_output_dir"))
|
||||
or _text(source_payload.get("output_dir"))
|
||||
or None
|
||||
),
|
||||
}
|
||||
document["canonical"] = build_canonical_descriptor(
|
||||
document.get("assets") or [],
|
||||
product_family="dinsar",
|
||||
)
|
||||
return _normalize_canonical_manifest(document)
|
||||
|
||||
|
||||
def _normalize_legacy_timeseries_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
document = copy.deepcopy(payload)
|
||||
runtime_payload = _copy_dict(document.get("runtime"))
|
||||
source_summary = _copy_dict(document.get("source_summary"))
|
||||
document["source_schema_version"] = LEGACY_TIMESERIES_PACKAGE_SCHEMA
|
||||
document["product_family"] = _text(document.get("product_family")) or "timeseries"
|
||||
document["product_type"] = "timeseries_bundle"
|
||||
document["identity"] = {
|
||||
**_copy_dict(document.get("identity")),
|
||||
"stack_key": (
|
||||
_text(_copy_dict(document.get("identity")).get("stack_key"))
|
||||
or _text(document.get("stack_key"))
|
||||
or _text(document.get("group_key"))
|
||||
or None
|
||||
),
|
||||
"run_key": (
|
||||
_text(_copy_dict(document.get("identity")).get("run_key"))
|
||||
or _text(document.get("run_key"))
|
||||
or _text(document.get("run_id"))
|
||||
or None
|
||||
),
|
||||
}
|
||||
document["engine"] = {
|
||||
**_copy_dict(document.get("engine")),
|
||||
"code": _text(document.get("engine_code")) or _text(_copy_dict(document.get("engine")).get("code")) or "unknown",
|
||||
}
|
||||
document["processor"] = {
|
||||
"code": _text(document.get("processor_code")) or "unknown",
|
||||
"profile_code": _text(document.get("processor_code")) or None,
|
||||
}
|
||||
if not _text(runtime_payload.get("kind")):
|
||||
runtime_payload["kind"] = "wsl"
|
||||
document["runtime"] = runtime_payload
|
||||
document["source"] = {
|
||||
**_copy_dict(document.get("source")),
|
||||
"publish_dir": (
|
||||
_text(_copy_dict(document.get("source")).get("publish_dir"))
|
||||
or _text(source_summary.get("publish_dir_windows"))
|
||||
or None
|
||||
),
|
||||
"native_output_dir": (
|
||||
_text(_copy_dict(document.get("source")).get("native_output_dir"))
|
||||
or _text(source_summary.get("mintpy_work_dir_windows"))
|
||||
or None
|
||||
),
|
||||
"work_dir": _text(source_summary.get("generated_stack_manifest_path_windows")) or None,
|
||||
"source_root": _text(source_summary.get("selected_manifest_path_windows")) or None,
|
||||
}
|
||||
document["temporal"] = {
|
||||
**_copy_dict(document.get("temporal")),
|
||||
"reference_date": _text(document.get("reference_date")) or None,
|
||||
"stack_dates": [
|
||||
str(item).strip()
|
||||
for item in document.get("stack_dates") or []
|
||||
if str(item).strip()
|
||||
],
|
||||
"published_at": _text(document.get("published_at")) or None,
|
||||
"produced_at": _text(document.get("produced_at")) or None,
|
||||
}
|
||||
document["assets"] = canonicalize_timeseries_artifacts(document.get("artifacts") or [])
|
||||
document["canonical"] = build_canonical_descriptor(
|
||||
document.get("assets") or [],
|
||||
product_family="timeseries",
|
||||
)
|
||||
return _normalize_canonical_manifest(document)
|
||||
|
||||
|
||||
def normalize_package_manifest(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
schema_version = _text((payload or {}).get("schema_version")).lower()
|
||||
if schema_version == CANONICAL_PACKAGE_SCHEMA:
|
||||
return _normalize_canonical_manifest(copy.deepcopy(payload))
|
||||
if schema_version == LEGACY_DINSAR_PACKAGE_SCHEMA:
|
||||
return _normalize_legacy_dinsar_manifest(payload)
|
||||
if schema_version == LEGACY_TIMESERIES_PACKAGE_SCHEMA:
|
||||
return _normalize_legacy_timeseries_manifest(payload)
|
||||
raise ValueError(f"Unsupported package schema_version: {schema_version or '<empty>'}")
|
||||
@@ -0,0 +1,231 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .product_package_schema import (
|
||||
CANONICAL_PACKAGE_LAYOUT,
|
||||
CANONICAL_PACKAGE_SCHEMA,
|
||||
build_canonical_descriptor,
|
||||
normalize_package_manifest,
|
||||
)
|
||||
|
||||
|
||||
def _clean_dict(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
return dict(payload or {})
|
||||
|
||||
|
||||
def _kind_from_engine(engine_code: str) -> Optional[str]:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized in {"envi", "sarscape"}:
|
||||
return "windows"
|
||||
if normalized in {"isce2", "pyint", "gamma"}:
|
||||
return "wsl"
|
||||
return None
|
||||
|
||||
|
||||
def _stable_digest(*parts: Any, length: int = 20) -> str:
|
||||
payload = "||".join(str(part or "") for part in parts)
|
||||
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
|
||||
|
||||
|
||||
def build_dinsar_package_manifest(
|
||||
*,
|
||||
product_id: str,
|
||||
display_name: str,
|
||||
task_name: Optional[str],
|
||||
engine_code: str,
|
||||
engine_version: Optional[str],
|
||||
processor_code: Optional[str],
|
||||
profile_code: Optional[str],
|
||||
runtime_id: Optional[str],
|
||||
source_primary_path: str,
|
||||
source_dir: str,
|
||||
publish_dir: str,
|
||||
identity: Dict[str, Any],
|
||||
source: Dict[str, Any],
|
||||
run: Dict[str, Any],
|
||||
temporal: Dict[str, Any],
|
||||
spatial: Dict[str, Any],
|
||||
dinsar_profile: Dict[str, Any],
|
||||
pairing_trace: Dict[str, Any],
|
||||
labels: Dict[str, Any],
|
||||
summary: Dict[str, Any],
|
||||
assets: List[Dict[str, Any]],
|
||||
issues: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
resolved_processor_code = (
|
||||
processor_code
|
||||
or (f"{engine_code}_{profile_code}" if profile_code else None)
|
||||
or engine_code
|
||||
)
|
||||
document = {
|
||||
"schema_version": CANONICAL_PACKAGE_SCHEMA,
|
||||
"package_layout": CANONICAL_PACKAGE_LAYOUT,
|
||||
"catalog_name": "dinsar",
|
||||
"product_family": "dinsar",
|
||||
"product_type": "dinsar_interferogram",
|
||||
"product_id": product_id,
|
||||
"display_name": display_name,
|
||||
"task_name": task_name or display_name,
|
||||
"pair_key": identity.get("pair_key"),
|
||||
"run_key": identity.get("run_key"),
|
||||
"identity": {
|
||||
"pair_key": identity.get("pair_key"),
|
||||
"stack_key": identity.get("stack_key"),
|
||||
"run_key": identity.get("run_key"),
|
||||
"task_alias": identity.get("task_alias"),
|
||||
},
|
||||
"engine": {
|
||||
"code": engine_code,
|
||||
"version": engine_version,
|
||||
},
|
||||
"processor": {
|
||||
"code": resolved_processor_code,
|
||||
"profile_code": profile_code,
|
||||
},
|
||||
"runtime": {
|
||||
"runtime_id": runtime_id,
|
||||
"kind": _kind_from_engine(engine_code),
|
||||
},
|
||||
"source": {
|
||||
**_clean_dict(source),
|
||||
"primary_path": source_primary_path,
|
||||
"source_dir": source_dir,
|
||||
"publish_dir": publish_dir,
|
||||
"native_output_dir": (
|
||||
_clean_dict(source).get("native_output_dir")
|
||||
or _clean_dict(source).get("output_dir")
|
||||
),
|
||||
},
|
||||
"run": _clean_dict(run),
|
||||
"temporal": _clean_dict(temporal),
|
||||
"spatial": _clean_dict(spatial),
|
||||
"dinsar_profile": _clean_dict(dinsar_profile),
|
||||
"labels": _clean_dict(labels),
|
||||
"pairing_trace": _clean_dict(pairing_trace),
|
||||
"summary": _clean_dict(summary),
|
||||
"assets": list(assets or []),
|
||||
"canonical": build_canonical_descriptor(assets or [], product_family="dinsar"),
|
||||
"issues": list(issues or []),
|
||||
"published_at": _clean_dict(temporal).get("published_at"),
|
||||
"produced_at": _clean_dict(temporal).get("produced_at"),
|
||||
"engine_code": engine_code,
|
||||
"processor_code": resolved_processor_code,
|
||||
"runtime_id": runtime_id,
|
||||
}
|
||||
return normalize_package_manifest(document)
|
||||
|
||||
|
||||
def upgrade_timeseries_package_manifest(
|
||||
payload: Dict[str, Any],
|
||||
*,
|
||||
run_context: Dict[str, Any],
|
||||
source_summary: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
document = normalize_package_manifest(payload)
|
||||
engine_code = str(run_context.get("engine_code") or (document.get("engine") or {}).get("code") or "unknown")
|
||||
processor_code = str(
|
||||
run_context.get("processor_code")
|
||||
or document.get("processor_code")
|
||||
or ((document.get("processor") or {}).get("code"))
|
||||
or "unknown"
|
||||
)
|
||||
runtime_id = run_context.get("runtime_id") or document.get("runtime_id")
|
||||
|
||||
document.update(
|
||||
{
|
||||
"schema_version": CANONICAL_PACKAGE_SCHEMA,
|
||||
"package_layout": CANONICAL_PACKAGE_LAYOUT,
|
||||
"catalog_name": "psinsar",
|
||||
"product_family": "timeseries",
|
||||
"product_type": "timeseries_bundle",
|
||||
"run_id": run_context.get("run_id"),
|
||||
"run_name": run_context.get("run_name"),
|
||||
"batch_id": run_context.get("batch_id"),
|
||||
"task_id": run_context.get("task_id"),
|
||||
"workflow_run_id": run_context.get("workflow_run_id"),
|
||||
"mode": run_context.get("mode"),
|
||||
"engine_code": engine_code,
|
||||
"processor_code": processor_code,
|
||||
"runtime_id": runtime_id,
|
||||
"stack_key": run_context.get("stack_key") or document.get("stack_key"),
|
||||
"group_key": run_context.get("group_key") or document.get("group_key"),
|
||||
"reference_date": run_context.get("reference_date") or document.get("reference_date"),
|
||||
"stack_dates": run_context.get("stack_dates") or document.get("stack_dates") or [],
|
||||
"published_at": run_context.get("published_at") or document.get("published_at"),
|
||||
"produced_at": run_context.get("produced_at") or document.get("produced_at"),
|
||||
"source_summary": {
|
||||
**_clean_dict(document.get("source_summary")),
|
||||
**_clean_dict(source_summary),
|
||||
},
|
||||
}
|
||||
)
|
||||
if not str(document.get("product_id") or "").strip():
|
||||
document["product_id"] = "psinsar_" + _stable_digest(
|
||||
run_context.get("run_id"),
|
||||
document.get("stack_key") or document.get("group_key"),
|
||||
document.get("reference_date"),
|
||||
length=20,
|
||||
)
|
||||
|
||||
document["identity"] = {
|
||||
**_clean_dict(document.get("identity")),
|
||||
"stack_key": document.get("stack_key") or document.get("group_key"),
|
||||
"run_key": run_context.get("run_id") or _clean_dict(document.get("identity")).get("run_key"),
|
||||
}
|
||||
document["engine"] = {
|
||||
**_clean_dict(document.get("engine")),
|
||||
"code": engine_code,
|
||||
}
|
||||
document["processor"] = {
|
||||
**_clean_dict(document.get("processor")),
|
||||
"code": processor_code,
|
||||
"profile_code": processor_code,
|
||||
}
|
||||
runtime_payload = {
|
||||
**_clean_dict(document.get("runtime")),
|
||||
**_clean_dict(run_context.get("runtime")),
|
||||
}
|
||||
runtime_payload["runtime_id"] = runtime_id
|
||||
runtime_payload["kind"] = runtime_payload.get("kind") or _kind_from_engine(engine_code)
|
||||
document["runtime"] = runtime_payload
|
||||
|
||||
source_payload = {
|
||||
**_clean_dict(document.get("source")),
|
||||
"publish_dir": run_context.get("publish_dir"),
|
||||
"native_output_dir": (
|
||||
run_context.get("native_output_dir")
|
||||
or _clean_dict(source_summary).get("mintpy_work_dir_windows")
|
||||
or _clean_dict(document.get("source")).get("native_output_dir")
|
||||
),
|
||||
"work_dir": (
|
||||
run_context.get("work_dir")
|
||||
or _clean_dict(source_summary).get("generated_stack_manifest_path_windows")
|
||||
or _clean_dict(document.get("source")).get("work_dir")
|
||||
),
|
||||
"source_root": (
|
||||
run_context.get("source_root")
|
||||
or _clean_dict(source_summary).get("selected_manifest_path_windows")
|
||||
or _clean_dict(document.get("source")).get("source_root")
|
||||
),
|
||||
}
|
||||
document["source"] = source_payload
|
||||
|
||||
temporal_payload = {
|
||||
**_clean_dict(document.get("temporal")),
|
||||
"reference_date": document.get("reference_date"),
|
||||
"stack_dates": [
|
||||
str(item).strip()
|
||||
for item in document.get("stack_dates") or []
|
||||
if str(item).strip()
|
||||
],
|
||||
"published_at": document.get("published_at"),
|
||||
"produced_at": document.get("produced_at"),
|
||||
}
|
||||
document["temporal"] = temporal_payload
|
||||
document["canonical"] = build_canonical_descriptor(
|
||||
document.get("assets") or [],
|
||||
product_family="timeseries",
|
||||
)
|
||||
return normalize_package_manifest(document)
|
||||
@@ -19,26 +19,14 @@ from .manifest_snapshot_service import (
|
||||
evaluate_manifest_reconcile,
|
||||
iter_manifest_paths,
|
||||
)
|
||||
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
|
||||
|
||||
|
||||
PSINSAR_CATALOG_NAME = "psinsar"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
TASK_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
|
||||
_ARTIFACT_ROLE_MAP = {
|
||||
"timeseries_cube": "timeseries_cube",
|
||||
"velocity_map": "velocity_map",
|
||||
"velocity_geotiff": "velocity_geotiff",
|
||||
"temporal_coherence": "temporal_coherence",
|
||||
"temporal_coherence_geotiff": "temporal_coherence_geotiff",
|
||||
"quality_mask": "quality_mask",
|
||||
"quality_mask_geotiff": "quality_mask_geotiff",
|
||||
"preview_png": "preview_png",
|
||||
"diagnostic_png": "diagnostic_png",
|
||||
}
|
||||
_PRIMARY_PRODUCT_TYPES = {"timeseries_cube", "velocity_geotiff"}
|
||||
_PREFERRED_PRIMARY_PRODUCT_TYPES = ("velocity_geotiff", "timeseries_cube", "velocity_map")
|
||||
_PREVIEW_PRODUCT_TYPES = {"preview_png"}
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
@@ -162,7 +150,7 @@ def _derive_bbox_from_summary(manifest: Dict[str, Any]) -> Dict[str, Optional[fl
|
||||
|
||||
class PsinsarCatalogService:
|
||||
def get_publish_root(self, publish_root: Optional[str] = None) -> str:
|
||||
root = publish_root or settings.PSINSAR_PRODUCT_DIR
|
||||
root = publish_root or settings.TIMESERIES_PRODUCT_DIR
|
||||
normalized = _normalize_path(root)
|
||||
os.makedirs(normalized, exist_ok=True)
|
||||
return normalized
|
||||
@@ -183,6 +171,7 @@ class PsinsarCatalogService:
|
||||
if state is None:
|
||||
state = ResultCatalogStateORM(
|
||||
catalog_name=PSINSAR_CATALOG_NAME,
|
||||
product_family="timeseries",
|
||||
storage_root=root,
|
||||
status="READY",
|
||||
needs_rebuild=False,
|
||||
@@ -191,6 +180,8 @@ class PsinsarCatalogService:
|
||||
await db.flush()
|
||||
elif state.storage_root != root:
|
||||
state.storage_root = root
|
||||
if state.product_family != "timeseries":
|
||||
state.product_family = "timeseries"
|
||||
return state
|
||||
|
||||
def _iter_manifest_paths(self, publish_root: str) -> List[str]:
|
||||
@@ -199,13 +190,13 @@ class PsinsarCatalogService:
|
||||
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
|
||||
with open(manifest_path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
schema_version = str(payload.get("schema_version") or "").strip().lower()
|
||||
catalog_name = str(payload.get("catalog_name") or "").strip().lower()
|
||||
if schema_version != "psinsar.publish.v1":
|
||||
raise ValueError("manifest schema_version is not psinsar.publish.v1")
|
||||
normalized = normalize_package_manifest(payload)
|
||||
catalog_name = str(normalized.get("catalog_name") or "").strip().lower()
|
||||
if str(normalized.get("product_family") or "").strip().lower() != "timeseries":
|
||||
raise ValueError("manifest product_family is not timeseries")
|
||||
if catalog_name != PSINSAR_CATALOG_NAME:
|
||||
raise ValueError("manifest catalog_name is not psinsar")
|
||||
return payload
|
||||
return normalized
|
||||
|
||||
def _build_rows_from_manifest(
|
||||
self,
|
||||
@@ -215,11 +206,28 @@ class PsinsarCatalogService:
|
||||
package_dir = _normalize_path(os.path.dirname(manifest_path))
|
||||
basename = os.path.basename(package_dir)
|
||||
group_key = str(manifest.get("group_key") or "").strip() or None
|
||||
reference_date = str(manifest.get("reference_date") or "").strip() or None
|
||||
stack_dates = [str(item).strip() for item in (manifest.get("stack_dates") or []) if str(item).strip()]
|
||||
stack_key = str(manifest.get("stack_key") or "").strip() or group_key or basename
|
||||
temporal = manifest.get("temporal") or {}
|
||||
reference_date = str(
|
||||
temporal.get("reference_date")
|
||||
or manifest.get("reference_date")
|
||||
or ""
|
||||
).strip() or None
|
||||
stack_dates = [
|
||||
str(item).strip()
|
||||
for item in (
|
||||
temporal.get("stack_dates")
|
||||
or manifest.get("stack_dates")
|
||||
or []
|
||||
)
|
||||
if str(item).strip()
|
||||
]
|
||||
run_key = str(manifest.get("run_id") or "").strip() or basename
|
||||
display_name = group_key or basename
|
||||
product_id = "psinsar_" + _stable_digest(package_dir, run_key, reference_date, length=20)
|
||||
display_name = stack_key or group_key or basename
|
||||
product_id = (
|
||||
str(manifest.get("product_id") or "").strip()
|
||||
or "psinsar_" + _stable_digest(package_dir, run_key, reference_date, length=20)
|
||||
)
|
||||
bbox = _derive_bbox_from_summary(manifest)
|
||||
poly = _build_bbox_polygon(
|
||||
bbox.get("min_lon"),
|
||||
@@ -227,19 +235,29 @@ class PsinsarCatalogService:
|
||||
bbox.get("max_lon"),
|
||||
bbox.get("max_lat"),
|
||||
)
|
||||
processor_payload = manifest.get("processor") or {}
|
||||
runtime_payload = manifest.get("runtime") or {}
|
||||
canonical_payload = manifest.get("canonical") or build_canonical_descriptor(
|
||||
manifest.get("assets") or [],
|
||||
product_family="timeseries",
|
||||
)
|
||||
|
||||
summary_json = {
|
||||
"product_family": "timeseries",
|
||||
"stack_key": stack_key,
|
||||
"group_key": group_key,
|
||||
"reference_date": reference_date,
|
||||
"reference_point": manifest.get("reference_point"),
|
||||
"stack_dates": stack_dates,
|
||||
"stack_size": len(stack_dates),
|
||||
"mode": manifest.get("mode"),
|
||||
"processor_code": manifest.get("processor_code"),
|
||||
"processor_code": processor_payload.get("code") or manifest.get("processor_code"),
|
||||
"quality": manifest.get("quality"),
|
||||
"summaries": manifest.get("summaries"),
|
||||
"canonical": canonical_payload,
|
||||
"runtime": runtime_payload,
|
||||
}
|
||||
published_at = _parse_datetime(manifest.get("published_at"))
|
||||
published_at = _parse_datetime(temporal.get("published_at") or manifest.get("published_at"))
|
||||
if published_at is None:
|
||||
try:
|
||||
published_at = datetime.utcfromtimestamp(os.path.getmtime(manifest_path))
|
||||
@@ -249,20 +267,27 @@ class PsinsarCatalogService:
|
||||
product = ResultProductORM(
|
||||
product_id=product_id,
|
||||
catalog_name=PSINSAR_CATALOG_NAME,
|
||||
product_type="psinsar_bundle",
|
||||
product_family="timeseries",
|
||||
product_type=str(manifest.get("product_type") or "timeseries_bundle").strip() or "timeseries_bundle",
|
||||
display_name=display_name,
|
||||
task_name=display_name,
|
||||
task_alias=group_key or basename,
|
||||
task_alias=stack_key or group_key or basename,
|
||||
pair_key=None,
|
||||
stack_key=stack_key,
|
||||
run_key=run_key,
|
||||
profile_code=str(manifest.get("processor_code") or "").strip() or None,
|
||||
engine_code=str(manifest.get("engine_code") or "unknown"),
|
||||
engine_version=None,
|
||||
profile_code=str(processor_payload.get("profile_code") or manifest.get("processor_code") or "").strip() or None,
|
||||
engine_code=str(((manifest.get("engine") or {}).get("code")) or manifest.get("engine_code") or "unknown"),
|
||||
engine_version=str(((manifest.get("engine") or {}).get("version")) or "") or None,
|
||||
package_schema=str(manifest.get("schema_version") or "").strip() or None,
|
||||
package_layout=str(manifest.get("package_layout") or "").strip() or None,
|
||||
processor_code=str(processor_payload.get("code") or manifest.get("processor_code") or "").strip() or None,
|
||||
runtime_id=str(runtime_payload.get("runtime_id") or manifest.get("runtime_id") or "").strip() or None,
|
||||
status="READY",
|
||||
health_status="OK",
|
||||
publish_dir=package_dir,
|
||||
manifest_path=_normalize_path(manifest_path),
|
||||
source_primary_path=None,
|
||||
native_output_dir=((manifest.get("source") or {}).get("native_output_dir")),
|
||||
preview_path=None,
|
||||
primary_asset_path=None,
|
||||
summary_json=summary_json,
|
||||
@@ -282,25 +307,30 @@ class PsinsarCatalogService:
|
||||
[bbox["min_lon"], bbox["min_lat"]],
|
||||
]],
|
||||
} if None not in (bbox["min_lon"], bbox["min_lat"], bbox["max_lon"], bbox["max_lat"]) else None,
|
||||
produced_at=_parse_datetime(manifest.get("produced_at")) or published_at,
|
||||
produced_at=_parse_datetime(temporal.get("produced_at") or manifest.get("produced_at")) or published_at,
|
||||
published_at=published_at,
|
||||
registered_at=_utcnow(),
|
||||
)
|
||||
|
||||
has_warn = False
|
||||
has_error = False
|
||||
artifacts = manifest.get("artifacts") or []
|
||||
assets_payload = manifest.get("assets") or []
|
||||
chosen_primary_path = None
|
||||
for candidate_type in _PREFERRED_PRIMARY_PRODUCT_TYPES:
|
||||
for artifact in artifacts:
|
||||
if str(artifact.get("product_type") or "").strip() == candidate_type:
|
||||
chosen_primary_path = str(artifact.get("path") or "").strip()
|
||||
preferred_primary_roles = [
|
||||
str(canonical_payload.get("primary_asset_role") or "").strip(),
|
||||
*_PREFERRED_PRIMARY_PRODUCT_TYPES,
|
||||
]
|
||||
for candidate_type in preferred_primary_roles:
|
||||
for asset in assets_payload:
|
||||
if str(asset.get("role") or "").strip() == candidate_type:
|
||||
chosen_primary_path = str(asset.get("relative_path") or "").strip()
|
||||
break
|
||||
if chosen_primary_path:
|
||||
break
|
||||
|
||||
for artifact in artifacts:
|
||||
relative_path = str(artifact.get("path") or "").strip()
|
||||
preview_role = str(canonical_payload.get("preview_asset_role") or "").strip()
|
||||
for asset_payload in assets_payload:
|
||||
relative_path = str(asset_payload.get("relative_path") or "").strip()
|
||||
if not relative_path:
|
||||
continue
|
||||
absolute_path = _resolve_relative_path(package_dir, relative_path)
|
||||
@@ -310,28 +340,27 @@ class PsinsarCatalogService:
|
||||
except OSError:
|
||||
file_size = None
|
||||
|
||||
product_type = str(artifact.get("product_type") or "asset").strip()
|
||||
asset_role = _ARTIFACT_ROLE_MAP.get(product_type, product_type or "asset")
|
||||
is_required = product_type in _PRIMARY_PRODUCT_TYPES or product_type in _PREVIEW_PRODUCT_TYPES
|
||||
asset_role = str(asset_payload.get("role") or "asset").strip()
|
||||
is_required = bool(asset_payload.get("is_required"))
|
||||
is_primary = relative_path == chosen_primary_path
|
||||
asset = ResultAssetORM(
|
||||
asset_role=asset_role,
|
||||
asset_name=os.path.basename(relative_path) or asset_role,
|
||||
relative_path=relative_path,
|
||||
absolute_path=absolute_path,
|
||||
format=_artifact_format(relative_path),
|
||||
media_type=_artifact_media_type(relative_path),
|
||||
format=asset_payload.get("format") or _artifact_format(relative_path),
|
||||
media_type=asset_payload.get("media_type") or _artifact_media_type(relative_path),
|
||||
is_required=is_required,
|
||||
is_primary=is_primary,
|
||||
exists_flag=exists_flag,
|
||||
file_size=file_size,
|
||||
)
|
||||
product.assets.append(asset)
|
||||
if product_type == "timeseries_cube" and exists_flag:
|
||||
if asset_role == "timeseries_cube" and exists_flag:
|
||||
product.source_primary_path = absolute_path
|
||||
if is_primary and exists_flag:
|
||||
product.primary_asset_path = absolute_path
|
||||
if product_type in _PREVIEW_PRODUCT_TYPES and exists_flag:
|
||||
if preview_role and asset_role == preview_role and exists_flag:
|
||||
product.preview_path = absolute_path
|
||||
if is_required and not exists_flag:
|
||||
has_error = True
|
||||
@@ -394,12 +423,12 @@ class PsinsarCatalogService:
|
||||
root = self.get_publish_root(publish_root)
|
||||
normalized_manifest_path = _normalize_path(manifest_path)
|
||||
if not os.path.isfile(normalized_manifest_path):
|
||||
raise FileNotFoundError(f"PS-InSAR manifest not found: {normalized_manifest_path}")
|
||||
raise FileNotFoundError(f"Timeseries manifest not found: {normalized_manifest_path}")
|
||||
if not (
|
||||
normalized_manifest_path == root
|
||||
or normalized_manifest_path.startswith(root + os.sep)
|
||||
):
|
||||
raise ValueError("Manifest path is outside the configured PS-InSAR publish root.")
|
||||
raise ValueError("Manifest path is outside the configured timeseries publish root.")
|
||||
|
||||
state = await self._get_or_create_catalog_state(db, storage_root=root)
|
||||
state.status = "UPDATING"
|
||||
@@ -598,6 +627,9 @@ class PsinsarCatalogService:
|
||||
"run_key": item.run_key,
|
||||
"profile_code": item.profile_code,
|
||||
"engine_code": item.engine_code,
|
||||
"package_schema": item.package_schema,
|
||||
"processor_code": item.processor_code,
|
||||
"runtime_id": item.runtime_id,
|
||||
"status": item.status,
|
||||
"health_status": item.health_status,
|
||||
"preview_path": item.preview_path,
|
||||
@@ -649,11 +681,16 @@ class PsinsarCatalogService:
|
||||
"run_key": product.run_key,
|
||||
"profile_code": product.profile_code,
|
||||
"engine_code": product.engine_code,
|
||||
"package_schema": product.package_schema,
|
||||
"package_layout": product.package_layout,
|
||||
"processor_code": product.processor_code,
|
||||
"runtime_id": product.runtime_id,
|
||||
"status": product.status,
|
||||
"health_status": product.health_status,
|
||||
"publish_dir": product.publish_dir,
|
||||
"manifest_path": product.manifest_path,
|
||||
"source_primary_path": product.source_primary_path,
|
||||
"native_output_dir": product.native_output_dir,
|
||||
"preview_path": product.preview_path,
|
||||
"primary_asset_path": product.primary_asset_path,
|
||||
"reference_date": summary.get("reference_date"),
|
||||
@@ -717,6 +754,7 @@ class PsinsarCatalogService:
|
||||
db_count = int(db_count_result.scalar_one() or 0)
|
||||
payload = {
|
||||
"catalog_name": state.catalog_name,
|
||||
"product_family": state.product_family,
|
||||
"storage_root": state.storage_root,
|
||||
"status": state.status,
|
||||
"needs_rebuild": state.needs_rebuild,
|
||||
|
||||
@@ -48,6 +48,21 @@ def _read_bool_env(name: str, default: bool = False) -> bool:
|
||||
return read_bool_env(name, default)
|
||||
|
||||
|
||||
def default_gamma_env_script_windows() -> str:
|
||||
return os.path.normpath(str(Path(settings.PROJECT_ROOT, "deploy", "wsl", "profiles", "gamma_env.sh")))
|
||||
|
||||
|
||||
def resolve_gamma_env_script(gamma_env_script: Optional[str] = None) -> str:
|
||||
explicit = str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")).strip()
|
||||
if explicit:
|
||||
return os.path.normpath(explicit)
|
||||
|
||||
candidate = default_gamma_env_script_windows()
|
||||
if os.path.isfile(candidate):
|
||||
return candidate
|
||||
return ""
|
||||
|
||||
|
||||
def normalize_date_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
@@ -296,7 +311,7 @@ def check_pyint_environment(
|
||||
work_root_wsl = to_wsl_path(str(work_root or _read_env("PYINT_WORK_ROOT", "")))
|
||||
output_root_wsl = to_wsl_path(str(output_root or _read_env("PYINT_OUTPUT_ROOT", "")))
|
||||
dem_root_wsl = to_wsl_path(str(dem_root or _read_env("PYINT_DEM_ROOT", "")))
|
||||
gamma_env_wsl = to_wsl_path(str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", "")))
|
||||
gamma_env_wsl = to_wsl_path(resolve_gamma_env_script(gamma_env_script))
|
||||
smoke_enabled = _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) if smoke_test is None else bool(smoke_test)
|
||||
precise_orbit_enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True)
|
||||
|
||||
|
||||
@@ -38,6 +38,16 @@ from .dinsar_naming import (
|
||||
build_fallback_pair_key,
|
||||
find_json_sidecar,
|
||||
)
|
||||
from .dinsar_result_layout_service import (
|
||||
RUN_CURRENT_DIRNAME,
|
||||
RUN_NATIVE_DIRNAME,
|
||||
RUN_PREVIEW_DIRNAME,
|
||||
is_path_within_native_dir,
|
||||
is_standard_envi_disp_file,
|
||||
is_standard_isce2_disp_file,
|
||||
)
|
||||
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
|
||||
from .product_packaging import build_dinsar_package_manifest
|
||||
|
||||
|
||||
DINSAR_CATALOG_NAME = "dinsar"
|
||||
@@ -120,6 +130,15 @@ def _coerce_optional_int(value: Any) -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
|
||||
normalized = str(engine_code or "").strip().lower()
|
||||
if normalized == "isce2":
|
||||
return settings.ISCE2_RUNTIME_ID or None
|
||||
if normalized in {"pyint", "gamma"}:
|
||||
return settings.PYINT_RUNTIME_ID or None
|
||||
return None
|
||||
|
||||
|
||||
def _build_pairing_trace_payload(
|
||||
candidate_meta: Dict[str, Any],
|
||||
task_item: Optional[DinsarTaskItemORM],
|
||||
@@ -194,11 +213,18 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"task_dir": _first_text(run_meta.get("task_dir")),
|
||||
"work_dir": _first_text(run_meta.get("work_dir")),
|
||||
"output_dir": _first_text(run_meta.get("output_dir"), source_dir),
|
||||
"native_output_dir": _first_text(run_meta.get("native_output_dir")),
|
||||
"started_at": _first_text(run_meta.get("started_at")),
|
||||
"finished_at": _first_text(run_meta.get("finished_at")),
|
||||
"params": run_meta.get("params") if isinstance(run_meta.get("params"), dict) else {},
|
||||
"metrics": run_meta.get("metrics") if isinstance(run_meta.get("metrics"), dict) else {},
|
||||
}
|
||||
if not resolved["native_output_dir"]:
|
||||
native_dir = os.path.join(str(resolved["output_dir"] or source_dir), RUN_NATIVE_DIRNAME)
|
||||
if os.path.isdir(native_dir):
|
||||
resolved["native_output_dir"] = native_dir
|
||||
else:
|
||||
resolved["native_output_dir"] = resolved["output_dir"]
|
||||
|
||||
for field in (
|
||||
"master_path",
|
||||
@@ -247,7 +273,8 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
|
||||
stack = [_normalize_path(root_dir)]
|
||||
normalized_root = _normalize_path(root_dir)
|
||||
stack = [normalized_root]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
try:
|
||||
@@ -255,12 +282,64 @@ def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
rel_name = os.path.relpath(entry.path, normalized_root)
|
||||
rel_parts = [part.lower() for part in rel_name.split(os.sep) if part]
|
||||
if any(
|
||||
part in {
|
||||
RUN_NATIVE_DIRNAME,
|
||||
RUN_CURRENT_DIRNAME,
|
||||
RUN_PREVIEW_DIRNAME,
|
||||
}
|
||||
for part in rel_parts
|
||||
):
|
||||
continue
|
||||
stack.append(entry.path)
|
||||
continue
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
if is_path_within_native_dir(normalized_root, entry.path):
|
||||
continue
|
||||
|
||||
lower_name = entry.name.lower()
|
||||
if is_standard_envi_disp_file(normalized_root, entry.path):
|
||||
primary_file = os.path.join(os.path.dirname(entry.path), "disp")
|
||||
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(primary_file)))
|
||||
sidecars = []
|
||||
if os.path.isfile(primary_file + ".hdr"):
|
||||
sidecars.append(primary_file + ".hdr")
|
||||
if os.path.isfile(primary_file + ".sml"):
|
||||
sidecars.append(primary_file + ".sml")
|
||||
yield {
|
||||
"engine_code": "envi",
|
||||
"name": "disp",
|
||||
"task_name": "",
|
||||
"source_dir": source_dir,
|
||||
"primary_file": primary_file,
|
||||
"source_files": [primary_file] + sidecars,
|
||||
}
|
||||
continue
|
||||
|
||||
if is_standard_isce2_disp_file(normalized_root, entry.path):
|
||||
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(entry.path)))
|
||||
source_files = [entry.path]
|
||||
coh_candidates = (
|
||||
os.path.join(source_dir, "assets", "coh", "coh.tif"),
|
||||
os.path.join(source_dir, "assets", "coh", "coh.tiff"),
|
||||
)
|
||||
for coh_path in coh_candidates:
|
||||
if os.path.isfile(coh_path):
|
||||
source_files.append(coh_path)
|
||||
break
|
||||
yield {
|
||||
"engine_code": "isce2",
|
||||
"name": os.path.splitext(entry.name)[0],
|
||||
"task_name": "",
|
||||
"source_dir": source_dir,
|
||||
"primary_file": entry.path,
|
||||
"source_files": source_files,
|
||||
}
|
||||
continue
|
||||
|
||||
if lower_name.endswith(".hdr"):
|
||||
base_name, _ = os.path.splitext(entry.name)
|
||||
if not base_name.lower().endswith("_disp"):
|
||||
@@ -344,6 +423,15 @@ def _resolve_relative_path(base_dir: str, relative_path: str) -> str:
|
||||
return target
|
||||
|
||||
|
||||
def _is_path_within(base_dir: str, candidate_path: str) -> bool:
|
||||
base = _normalize_path(base_dir)
|
||||
candidate = _normalize_path(candidate_path)
|
||||
try:
|
||||
return os.path.commonpath([base, candidate]) == base
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _build_bbox_polygon(
|
||||
min_lon: Optional[float],
|
||||
min_lat: Optional[float],
|
||||
@@ -383,6 +471,7 @@ class ResultCatalogService:
|
||||
if state is None:
|
||||
state = ResultCatalogStateORM(
|
||||
catalog_name=DINSAR_CATALOG_NAME,
|
||||
product_family="dinsar",
|
||||
storage_root=root,
|
||||
status="READY",
|
||||
needs_rebuild=False,
|
||||
@@ -391,6 +480,8 @@ class ResultCatalogService:
|
||||
await db.flush()
|
||||
elif state.storage_root != root:
|
||||
state.storage_root = root
|
||||
if state.product_family != "dinsar":
|
||||
state.product_family = "dinsar"
|
||||
return state
|
||||
|
||||
async def _lookup_task_item(
|
||||
@@ -565,57 +656,57 @@ class ResultCatalogService:
|
||||
}
|
||||
if pairing_trace:
|
||||
summary_payload["pairing_trace"] = pairing_trace
|
||||
return {
|
||||
"schema_version": "dinsar-product/v1",
|
||||
"catalog_name": DINSAR_CATALOG_NAME,
|
||||
"product_id": product_id,
|
||||
"product_type": "dinsar",
|
||||
"display_name": display_name,
|
||||
"task_name": candidate_meta.get("task_alias") or display_name,
|
||||
"identity": {
|
||||
return build_dinsar_package_manifest(
|
||||
product_id=product_id,
|
||||
display_name=display_name,
|
||||
task_name=candidate_meta.get("task_alias") or display_name,
|
||||
engine_code=engine_code,
|
||||
engine_version=candidate_meta.get("engine_version") or "",
|
||||
processor_code=candidate_meta.get("profile_code") or engine_code,
|
||||
profile_code=candidate_meta.get("profile_code"),
|
||||
runtime_id=_runtime_id_for_engine(engine_code),
|
||||
source_primary_path=source_primary_path,
|
||||
source_dir=source_dir,
|
||||
publish_dir=package_dir,
|
||||
identity={
|
||||
"pair_key": candidate_meta.get("pair_key"),
|
||||
"task_alias": candidate_meta.get("task_alias") or display_name,
|
||||
"run_key": candidate_meta.get("run_key"),
|
||||
},
|
||||
"engine": {
|
||||
"code": engine_code,
|
||||
"version": candidate_meta.get("engine_version") or "",
|
||||
},
|
||||
"source": {
|
||||
"primary_path": source_primary_path,
|
||||
"source_dir": source_dir,
|
||||
source={
|
||||
"source_root": candidate_meta.get("source_root"),
|
||||
"task_dir": candidate_meta.get("task_dir"),
|
||||
"work_dir": candidate_meta.get("work_dir"),
|
||||
"output_dir": candidate_meta.get("output_dir"),
|
||||
"publish_dir": package_dir,
|
||||
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
|
||||
},
|
||||
"run": {
|
||||
run={
|
||||
"engine_code": engine_code,
|
||||
"profile_code": candidate_meta.get("profile_code"),
|
||||
"source_root": candidate_meta.get("source_root"),
|
||||
"task_dir": candidate_meta.get("task_dir"),
|
||||
"work_dir": candidate_meta.get("work_dir"),
|
||||
"output_dir": candidate_meta.get("output_dir"),
|
||||
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
|
||||
"started_at": candidate_meta.get("started_at"),
|
||||
"finished_at": candidate_meta.get("finished_at"),
|
||||
"params": profile_params,
|
||||
"metrics": profile_metrics,
|
||||
},
|
||||
"temporal": {
|
||||
temporal={
|
||||
"master_imaging_date": master_date,
|
||||
"slave_imaging_date": slave_date,
|
||||
"produced_at": candidate_meta.get("finished_at") or candidate_meta.get("started_at"),
|
||||
"published_at": published_at,
|
||||
},
|
||||
"spatial": {
|
||||
spatial={
|
||||
"min_lon": meta.get("min_lon"),
|
||||
"min_lat": meta.get("min_lat"),
|
||||
"max_lon": meta.get("max_lon"),
|
||||
"max_lat": meta.get("max_lat"),
|
||||
"coverage_polygon": meta.get("coverage_polygon"),
|
||||
},
|
||||
"dinsar_profile": {
|
||||
dinsar_profile={
|
||||
"master_path": getattr(task_item, "master_path", None) or candidate_meta.get("master_path"),
|
||||
"slave_path": getattr(task_item, "slave_path", None) or candidate_meta.get("slave_path"),
|
||||
"master_satellite": getattr(task_item, "master_satellite", None) or candidate_meta.get("master_satellite"),
|
||||
@@ -637,15 +728,15 @@ class ResultCatalogService:
|
||||
"params": profile_params,
|
||||
"metrics": profile_metrics,
|
||||
},
|
||||
"labels": {
|
||||
pairing_trace=pairing_trace,
|
||||
labels={
|
||||
"ai_score": None,
|
||||
"user_label": None,
|
||||
},
|
||||
"pairing_trace": pairing_trace,
|
||||
"summary": summary_payload,
|
||||
"assets": asset_rows,
|
||||
"issues": [],
|
||||
}
|
||||
summary=summary_payload,
|
||||
assets=asset_rows,
|
||||
issues=[],
|
||||
)
|
||||
|
||||
async def publish_from_sources(
|
||||
self,
|
||||
@@ -691,7 +782,9 @@ class ResultCatalogService:
|
||||
run_key,
|
||||
primary_file,
|
||||
)
|
||||
package_dir = _ensure_directory(os.path.join(target_root, pair_key, run_key))
|
||||
package_dir = _ensure_directory(os.path.join(target_root, pair_key, "runs", run_key))
|
||||
source_dir = _normalize_path(candidate["source_dir"])
|
||||
in_place_source = _is_path_within(package_dir, source_dir)
|
||||
task_item = await self._lookup_task_item(
|
||||
db,
|
||||
pair_key=pair_key,
|
||||
@@ -713,24 +806,28 @@ class ResultCatalogService:
|
||||
)
|
||||
continue
|
||||
|
||||
disp_dir = _ensure_directory(os.path.join(package_dir, "assets", "disp"))
|
||||
disp_dir = os.path.join(package_dir, "assets", "disp")
|
||||
preview_dir = _ensure_directory(os.path.join(package_dir, "preview"))
|
||||
asset_rows: List[Dict[str, Any]] = []
|
||||
|
||||
if candidate["engine_code"] == "envi":
|
||||
target_base = os.path.join(disp_dir, "disp")
|
||||
target_primary = target_base
|
||||
source_primary = candidate["source_files"][0]
|
||||
for src_path in candidate["source_files"]:
|
||||
suffix = src_path[len(source_primary):]
|
||||
dst_path = target_base + suffix
|
||||
op = _copy_file_if_needed(src_path, dst_path)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
source_primary = _normalize_path(candidate["source_files"][0])
|
||||
if in_place_source:
|
||||
target_primary = source_primary
|
||||
else:
|
||||
_ensure_directory(disp_dir)
|
||||
target_base = os.path.join(disp_dir, "disp")
|
||||
target_primary = target_base
|
||||
for src_path in candidate["source_files"]:
|
||||
suffix = src_path[len(source_primary):]
|
||||
dst_path = target_base + suffix
|
||||
op = _copy_file_if_needed(src_path, dst_path)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
asset_rows.append(
|
||||
{
|
||||
"role": "disp",
|
||||
@@ -769,14 +866,18 @@ class ResultCatalogService:
|
||||
}
|
||||
)
|
||||
else:
|
||||
target_primary = os.path.join(disp_dir, "disp.tif")
|
||||
op = _copy_file_if_needed(primary_file, target_primary)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
if in_place_source:
|
||||
target_primary = _normalize_path(primary_file)
|
||||
else:
|
||||
skipped += 1
|
||||
_ensure_directory(disp_dir)
|
||||
target_primary = os.path.join(disp_dir, "disp.tif")
|
||||
op = _copy_file_if_needed(primary_file, target_primary)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
asset_rows.append(
|
||||
{
|
||||
"role": "disp",
|
||||
@@ -789,17 +890,20 @@ class ResultCatalogService:
|
||||
}
|
||||
)
|
||||
if len(candidate["source_files"]) > 1:
|
||||
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
|
||||
source_coh = candidate["source_files"][1]
|
||||
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
|
||||
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
|
||||
op = _copy_file_if_needed(source_coh, target_coh)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
if in_place_source:
|
||||
target_coh = _normalize_path(source_coh)
|
||||
else:
|
||||
skipped += 1
|
||||
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
|
||||
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
|
||||
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
|
||||
op = _copy_file_if_needed(source_coh, target_coh)
|
||||
if op == "copied":
|
||||
copied += 1
|
||||
elif op == "overwritten":
|
||||
overwritten += 1
|
||||
else:
|
||||
skipped += 1
|
||||
asset_rows.append(
|
||||
{
|
||||
"role": "coh",
|
||||
@@ -859,6 +963,7 @@ class ResultCatalogService:
|
||||
"run_key": run_key,
|
||||
"engine_code": candidate_meta["engine_code"],
|
||||
"package_dir": package_dir,
|
||||
"in_place": in_place_source,
|
||||
"thumb_created": thumb_ok,
|
||||
"status": "ok",
|
||||
}
|
||||
@@ -881,11 +986,12 @@ class ResultCatalogService:
|
||||
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
|
||||
with open(manifest_path, "r", encoding="utf-8") as fp:
|
||||
payload = json.load(fp)
|
||||
if str(payload.get("product_type") or "").strip().lower() != "dinsar":
|
||||
raise ValueError("manifest product_type is not dinsar")
|
||||
if not str(payload.get("product_id") or "").strip():
|
||||
normalized = normalize_package_manifest(payload)
|
||||
if str(normalized.get("product_family") or "").strip().lower() != "dinsar":
|
||||
raise ValueError("manifest product_family is not dinsar")
|
||||
if not str(normalized.get("product_id") or "").strip():
|
||||
raise ValueError("manifest product_id is empty")
|
||||
return payload
|
||||
return normalized
|
||||
|
||||
def _build_rows_from_manifest(
|
||||
self,
|
||||
@@ -921,39 +1027,58 @@ class ResultCatalogService:
|
||||
profile_payload = manifest.get("dinsar_profile") or {}
|
||||
labels = manifest.get("labels") or {}
|
||||
pairing_trace = manifest.get("pairing_trace") or {}
|
||||
processor_payload = manifest.get("processor") or {}
|
||||
runtime_payload = manifest.get("runtime") or {}
|
||||
canonical_payload = manifest.get("canonical") or build_canonical_descriptor(
|
||||
assets_payload,
|
||||
product_family="dinsar",
|
||||
)
|
||||
|
||||
summary_json: Optional[Dict[str, Any]] = None
|
||||
if summary or identity or run_payload or pairing_trace:
|
||||
if summary or identity or run_payload or pairing_trace or canonical_payload:
|
||||
summary_json = {
|
||||
**summary,
|
||||
"identity": identity,
|
||||
"run": run_payload,
|
||||
}
|
||||
if processor_payload:
|
||||
summary_json["processor"] = processor_payload
|
||||
if runtime_payload:
|
||||
summary_json["runtime"] = runtime_payload
|
||||
if canonical_payload:
|
||||
summary_json["canonical"] = canonical_payload
|
||||
if pairing_trace:
|
||||
summary_json["pairing_trace"] = pairing_trace
|
||||
|
||||
product = ResultProductORM(
|
||||
product_id=str(manifest.get("product_id")).strip(),
|
||||
catalog_name=str(manifest.get("catalog_name") or DINSAR_CATALOG_NAME).strip() or DINSAR_CATALOG_NAME,
|
||||
product_type="dinsar",
|
||||
product_family=str(manifest.get("product_family") or "dinsar").strip() or "dinsar",
|
||||
product_type=str(manifest.get("product_type") or "dinsar_interferogram").strip() or "dinsar_interferogram",
|
||||
display_name=str(manifest.get("display_name") or manifest.get("task_name") or manifest.get("product_id")),
|
||||
task_name=str(manifest.get("task_name") or manifest.get("display_name") or "").strip() or None,
|
||||
task_alias=str(identity.get("task_alias") or manifest.get("task_name") or "").strip() or None,
|
||||
pair_key=str(identity.get("pair_key") or "").strip() or None,
|
||||
stack_key=str(identity.get("stack_key") or "").strip() or None,
|
||||
pair_uid=str(pairing_trace.get("pair_uid") or "").strip() or None,
|
||||
run_key=str(identity.get("run_key") or "").strip() or None,
|
||||
network_run_id=str(pairing_trace.get("network_run_id") or "").strip() or None,
|
||||
network_edge_id=_coerce_optional_int(pairing_trace.get("network_edge_id")),
|
||||
policy_version=str(pairing_trace.get("policy_version") or "").strip() or None,
|
||||
selection_strategy=str(pairing_trace.get("selection_strategy") or "").strip() or None,
|
||||
profile_code=str(run_payload.get("profile_code") or "").strip() or None,
|
||||
profile_code=str(processor_payload.get("profile_code") or run_payload.get("profile_code") or "").strip() or None,
|
||||
engine_code=str(((manifest.get("engine") or {}).get("code")) or "unknown"),
|
||||
engine_version=str(((manifest.get("engine") or {}).get("version")) or "") or None,
|
||||
package_schema=str(manifest.get("schema_version") or "").strip() or None,
|
||||
package_layout=str(manifest.get("package_layout") or "").strip() or None,
|
||||
processor_code=str(processor_payload.get("code") or manifest.get("processor_code") or "").strip() or None,
|
||||
runtime_id=str(runtime_payload.get("runtime_id") or manifest.get("runtime_id") or "").strip() or None,
|
||||
status="READY",
|
||||
health_status="OK",
|
||||
publish_dir=package_dir,
|
||||
manifest_path=_normalize_path(manifest_path),
|
||||
source_primary_path=source.get("primary_path"),
|
||||
native_output_dir=source.get("native_output_dir"),
|
||||
preview_path=None,
|
||||
primary_asset_path=None,
|
||||
summary_json=summary_json,
|
||||
@@ -1000,6 +1125,7 @@ class ResultCatalogService:
|
||||
|
||||
has_warn = False
|
||||
has_error = False
|
||||
preview_role = str(canonical_payload.get("preview_asset_role") or "").strip() or "thumb"
|
||||
for asset_payload in assets_payload:
|
||||
relative_path = str(asset_payload.get("relative_path") or "").strip()
|
||||
if not relative_path:
|
||||
@@ -1031,7 +1157,7 @@ class ResultCatalogService:
|
||||
product.assets.append(asset)
|
||||
if asset.is_primary:
|
||||
product.primary_asset_path = absolute_path
|
||||
if asset.asset_role == "thumb":
|
||||
if asset.asset_role == preview_role:
|
||||
product.preview_path = absolute_path
|
||||
if asset.is_required and not exists_flag:
|
||||
has_error = True
|
||||
@@ -1256,6 +1382,9 @@ class ResultCatalogService:
|
||||
"selection_strategy": item.selection_strategy,
|
||||
"profile_code": item.profile_code,
|
||||
"engine_code": item.engine_code,
|
||||
"package_schema": item.package_schema,
|
||||
"processor_code": item.processor_code,
|
||||
"runtime_id": item.runtime_id,
|
||||
"status": item.status,
|
||||
"health_status": item.health_status,
|
||||
"preview_path": item.preview_path,
|
||||
@@ -1325,11 +1454,16 @@ class ResultCatalogService:
|
||||
"profile_code": product.profile_code,
|
||||
"engine_code": product.engine_code,
|
||||
"engine_version": product.engine_version,
|
||||
"package_schema": product.package_schema,
|
||||
"package_layout": product.package_layout,
|
||||
"processor_code": product.processor_code,
|
||||
"runtime_id": product.runtime_id,
|
||||
"status": product.status,
|
||||
"health_status": product.health_status,
|
||||
"publish_dir": product.publish_dir,
|
||||
"manifest_path": product.manifest_path,
|
||||
"source_primary_path": product.source_primary_path,
|
||||
"native_output_dir": product.native_output_dir,
|
||||
"preview_path": product.preview_path,
|
||||
"primary_asset_path": product.primary_asset_path,
|
||||
"summary_json": product.summary_json,
|
||||
@@ -1440,6 +1574,7 @@ class ResultCatalogService:
|
||||
db_count = int(db_count_result.scalar_one() or 0)
|
||||
payload = {
|
||||
"catalog_name": state.catalog_name,
|
||||
"product_family": state.product_family,
|
||||
"storage_root": state.storage_root,
|
||||
"status": state.status,
|
||||
"needs_rebuild": state.needs_rebuild,
|
||||
|
||||
@@ -274,12 +274,12 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="PSINSAR_PRODUCT_DIR",
|
||||
path=settings.PSINSAR_PRODUCT_DIR,
|
||||
root_role="publish_root_psinsar",
|
||||
display_name="PS-InSAR Publish Root",
|
||||
env_var="TIMESERIES_PRODUCT_DIR",
|
||||
path=settings.TIMESERIES_PRODUCT_DIR,
|
||||
root_role="publish_root_timeseries",
|
||||
display_name="Timeseries Publish Root",
|
||||
scan_mode="manifest_tree",
|
||||
owner_engine="psinsar",
|
||||
owner_engine="timeseries",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
@@ -311,6 +311,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
owner_engine="timeseries",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="WSL_BROKER_JOB_ROOT",
|
||||
path=settings.WSL_BROKER_JOB_ROOT,
|
||||
root_role="work_root_wsl_broker",
|
||||
display_name="WSL Broker Root",
|
||||
scan_mode="workspace",
|
||||
owner_engine="wsl",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="IDL_WORKER_RUNTIME_DIR",
|
||||
@@ -321,6 +331,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
owner_engine="idl",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="PYINT_WORK_ROOT",
|
||||
path=settings.PYINT_WORK_ROOT,
|
||||
root_role="work_root_pyint",
|
||||
display_name="Gamma / PyINT Work Root",
|
||||
scan_mode="workspace",
|
||||
owner_engine="pyint",
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -25,6 +26,7 @@ from ..models import (
|
||||
WorkflowStepORM,
|
||||
)
|
||||
from .psinsar_catalog_service import psinsar_catalog_service
|
||||
from .product_packaging import upgrade_timeseries_package_manifest
|
||||
from .task_service import task_service
|
||||
from .workflow_service import workflow_service
|
||||
from .wsl_service import run_wsl_command
|
||||
@@ -71,6 +73,7 @@ STACK_RUN_FILE_SEQUENCE = (
|
||||
"run_07_grid_baseline",
|
||||
"run_08_igram",
|
||||
)
|
||||
_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+")
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
@@ -104,6 +107,16 @@ def _normalize_lookup_key(path: Optional[str]) -> str:
|
||||
return os.path.normcase(os.path.normpath(os.path.abspath(str(path or "").strip())))
|
||||
|
||||
|
||||
def _stable_digest(*parts: Any, length: int = 10) -> str:
|
||||
payload = "||".join(str(part or "") for part in parts)
|
||||
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
|
||||
|
||||
|
||||
def _slug_fragment(value: Optional[str], *, default: str) -> str:
|
||||
text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._").lower()
|
||||
return text or default
|
||||
|
||||
|
||||
def _read_json(path: Path) -> Dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
@@ -190,6 +203,28 @@ def _build_stack_slug(
|
||||
)
|
||||
|
||||
|
||||
def _build_stack_key(group_key: Optional[str]) -> str:
|
||||
parts = [str(item or "").strip() for item in str(group_key or "").split("|")]
|
||||
while len(parts) < 5:
|
||||
parts.append("")
|
||||
satellite, imaging_mode, polarization, orbit_direction, tile_key = parts[:5]
|
||||
return "_".join(
|
||||
[
|
||||
_slug_fragment(satellite, default="sat"),
|
||||
_slug_fragment(imaging_mode, default="mode"),
|
||||
_slug_fragment(polarization, default="pol"),
|
||||
_slug_fragment(orbit_direction, default="dir"),
|
||||
_slug_fragment(str(tile_key or "").replace(".", "p"), default="tile"),
|
||||
_stable_digest(group_key, length=10),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _compose_publish_dir(run_id: str, stack_key: Optional[str]) -> str:
|
||||
stack_fragment = _slug_fragment(stack_key, default="unsorted")
|
||||
return _normalize_path(os.path.join(settings.TIMESERIES_PRODUCT_DIR, stack_fragment, "runs", run_id))
|
||||
|
||||
|
||||
def _common_source_root(paths: List[str]) -> Optional[str]:
|
||||
normalized = [_normalize_path(path) for path in paths if str(path or "").strip()]
|
||||
if not normalized:
|
||||
@@ -207,10 +242,9 @@ def _write_step_logs(logs_dir: Path, step_name: str, stdout: str, stderr: str) -
|
||||
|
||||
|
||||
class TimeseriesService:
|
||||
def _derive_paths(self, run_id: str) -> Dict[str, str]:
|
||||
year = _utcnow().strftime("%Y")
|
||||
def _derive_paths(self, run_id: str, *, stack_key: Optional[str] = None) -> Dict[str, str]:
|
||||
work_root_windows = _normalize_path(os.path.join(settings.TIMESERIES_WORK_ROOT, run_id))
|
||||
publish_dir_windows = _normalize_path(os.path.join(settings.PSINSAR_PRODUCT_DIR, year, run_id))
|
||||
publish_dir_windows = _compose_publish_dir(run_id, stack_key)
|
||||
return {
|
||||
"work_root_windows": work_root_windows,
|
||||
"work_root_wsl": _windows_path_to_wsl_mount(work_root_windows) or "",
|
||||
@@ -380,6 +414,7 @@ class TimeseriesService:
|
||||
orbit_direction=first.get("orbit_direction"),
|
||||
tile_key=manifest_tile_key,
|
||||
)
|
||||
stack_key = _build_stack_key(manifest_group_key)
|
||||
slug = _build_stack_slug(
|
||||
satellite=first.get("satellite"),
|
||||
imaging_mode=first.get("imaging_mode"),
|
||||
@@ -394,6 +429,7 @@ class TimeseriesService:
|
||||
"source_root_windows": source_root,
|
||||
"source_root_wsl": _windows_path_to_wsl_mount(source_root) if source_root else None,
|
||||
"group_key": manifest_group_key,
|
||||
"stack_key": stack_key,
|
||||
"tile_key": manifest_tile_key,
|
||||
"scene_count": len(scene_payloads),
|
||||
"reference_strategy": REFERENCE_STRATEGY_MIDDLE_BY_DATE,
|
||||
@@ -496,7 +532,7 @@ class TimeseriesService:
|
||||
def _publish_manifest_path(self, run: PsTimeseriesRunORM) -> Path:
|
||||
publish_dir = Path(
|
||||
run.publish_dir_windows
|
||||
or _normalize_path(os.path.join(settings.PSINSAR_PRODUCT_DIR, _utcnow().strftime("%Y"), run.run_id))
|
||||
or _compose_publish_dir(run.run_id, run.stack_key)
|
||||
)
|
||||
return publish_dir / "manifest.json"
|
||||
|
||||
@@ -641,9 +677,12 @@ class TimeseriesService:
|
||||
"mintpy_config_path_wsl": _windows_path_to_wsl_mount(str(mintpy_cfg_path)),
|
||||
"mintpy_work_dir_windows": str(mintpy_work_dir),
|
||||
"mintpy_work_dir_wsl": _windows_path_to_wsl_mount(str(mintpy_work_dir)),
|
||||
"publish_dir_windows": str(run.publish_dir_windows or ""),
|
||||
"publish_dir_wsl": _windows_path_to_wsl_mount(str(run.publish_dir_windows or "")),
|
||||
}
|
||||
payload.update(
|
||||
{
|
||||
payload = upgrade_timeseries_package_manifest(
|
||||
payload,
|
||||
run_context={
|
||||
"run_id": run.run_id,
|
||||
"run_name": run.run_name,
|
||||
"batch_id": run.batch_id,
|
||||
@@ -652,6 +691,8 @@ class TimeseriesService:
|
||||
"mode": run.mode,
|
||||
"engine_code": run.engine_code,
|
||||
"processor_code": run.processor_code,
|
||||
"runtime_id": run.runtime_id,
|
||||
"stack_key": run.stack_key or payload.get("stack_key") or report.get("stack_key"),
|
||||
"group_key": payload.get("group_key") or report.get("group_key"),
|
||||
"reference_date": payload.get("reference_date") or run.reference_date,
|
||||
"stack_dates": (
|
||||
@@ -668,13 +709,16 @@ class TimeseriesService:
|
||||
),
|
||||
"published_at": payload.get("published_at") or now_text,
|
||||
"produced_at": payload.get("produced_at") or now_text,
|
||||
"publish_dir": run.publish_dir_windows,
|
||||
"native_output_dir": str(mintpy_work_dir),
|
||||
"runtime": {
|
||||
"runtime_id": run.runtime_id,
|
||||
"env_name": self._effective_env_name(run),
|
||||
"wsl_distro": self._effective_wsl_distro(run),
|
||||
"water_mask_mode": run.water_mask_mode,
|
||||
},
|
||||
"source_summary": source_summary,
|
||||
}
|
||||
},
|
||||
source_summary=source_summary,
|
||||
)
|
||||
_write_json(manifest_path, payload)
|
||||
return payload
|
||||
@@ -750,11 +794,14 @@ class TimeseriesService:
|
||||
return {
|
||||
"run_id": run.run_id,
|
||||
"batch_id": run.batch_id,
|
||||
"product_family": run.product_family,
|
||||
"run_name": run.run_name,
|
||||
"catalog_name": run.catalog_name,
|
||||
"stack_key": run.stack_key,
|
||||
"mode": run.mode,
|
||||
"engine_code": run.engine_code,
|
||||
"processor_code": run.processor_code,
|
||||
"runtime_id": run.runtime_id,
|
||||
"env_name": run.env_name,
|
||||
"wsl_distro": run.wsl_distro,
|
||||
"status": run.status,
|
||||
@@ -884,7 +931,18 @@ class TimeseriesService:
|
||||
|
||||
run_id = str(uuid.uuid4())
|
||||
run_name_text = str(run_name or "").strip() or f"SBAS_{normalized_batch_id}_{run_id[:8]}"
|
||||
paths = self._derive_paths(run_id)
|
||||
work_root_windows = _normalize_path(os.path.join(settings.TIMESERIES_WORK_ROOT, run_id))
|
||||
stack_preview = await self._resolve_stack_scene_records(
|
||||
items=items,
|
||||
batch_direction=batch.direction,
|
||||
run_id=run_id,
|
||||
work_root_windows=work_root_windows,
|
||||
db=db,
|
||||
)
|
||||
stack_key = str(stack_preview.get("stack_key") or "").strip() or _build_stack_key(
|
||||
stack_preview.get("group_key")
|
||||
)
|
||||
paths = self._derive_paths(run_id, stack_key=stack_key)
|
||||
selected_manifest_path = Path(paths["work_root_windows"]) / "input" / "selected_stack_manifest.json"
|
||||
task_name = f"SBAS timeseries run {run_name_text}"
|
||||
task_id: Optional[str] = None
|
||||
@@ -904,11 +962,14 @@ class TimeseriesService:
|
||||
run = PsTimeseriesRunORM(
|
||||
run_id=run_id,
|
||||
batch_id=normalized_batch_id,
|
||||
product_family="timeseries",
|
||||
run_name=run_name_text,
|
||||
catalog_name=CATALOG_NAME_PSINSAR,
|
||||
stack_key=stack_key,
|
||||
mode="sbas",
|
||||
engine_code="isce2",
|
||||
processor_code="isce2_stack_mintpy",
|
||||
runtime_id=settings.ISCE2_RUNTIME_ID or None,
|
||||
env_name=settings.TIMESERIES_ENV_NAME or None,
|
||||
wsl_distro=settings.TIMESERIES_WSL_DISTRO or None,
|
||||
status=STATUS_PENDING,
|
||||
@@ -934,10 +995,14 @@ class TimeseriesService:
|
||||
"water_mask_mode": normalized_water_mask_mode,
|
||||
"notes": str(notes or "").strip() or None,
|
||||
"stack_workflow": settings.TIMESERIES_STACK_WORKFLOW,
|
||||
"group_key": stack_preview.get("group_key"),
|
||||
"stack_key": stack_key,
|
||||
},
|
||||
summary_json={
|
||||
"phase": "queued",
|
||||
"workflow": settings.TIMESERIES_STACK_WORKFLOW,
|
||||
"group_key": stack_preview.get("group_key"),
|
||||
"stack_key": stack_key,
|
||||
"stack_dates": stack_dates,
|
||||
"task_name": task_name,
|
||||
},
|
||||
@@ -946,7 +1011,10 @@ class TimeseriesService:
|
||||
"batch_name": batch.name,
|
||||
"direction": batch.direction,
|
||||
"scene_count": len(items),
|
||||
"group_key": stack_preview.get("group_key"),
|
||||
"stack_key": stack_key,
|
||||
"stack_dates": stack_dates,
|
||||
"source_root_windows": stack_preview.get("source_root_windows"),
|
||||
"items": self._scene_payload(items),
|
||||
},
|
||||
orbit_summary_json={
|
||||
@@ -976,8 +1044,10 @@ class TimeseriesService:
|
||||
},
|
||||
tags={
|
||||
"catalog_name": CATALOG_NAME_PSINSAR,
|
||||
"product_family": "timeseries",
|
||||
"processor_code": "isce2_stack_mintpy",
|
||||
"batch_id": normalized_batch_id,
|
||||
"stack_key": stack_key,
|
||||
},
|
||||
created_by=created_by,
|
||||
db=db,
|
||||
@@ -1065,8 +1135,13 @@ class TimeseriesService:
|
||||
_write_json(selected_manifest_path, selected_manifest)
|
||||
|
||||
run.status = STATUS_PREPARED
|
||||
run.product_family = run.product_family or "timeseries"
|
||||
run.stack_key = str(selected_manifest.get("stack_key") or run.stack_key or "").strip() or run.stack_key
|
||||
run.reference_date = effective_reference_date
|
||||
run.stack_size = len(stack_dates)
|
||||
if run.stack_key:
|
||||
run.publish_dir_windows = _compose_publish_dir(run.run_id, run.stack_key)
|
||||
run.publish_dir_wsl = _windows_path_to_wsl_mount(run.publish_dir_windows)
|
||||
run.manifest_path_windows = str(selected_manifest_path)
|
||||
run.manifest_path_wsl = _windows_path_to_wsl_mount(str(selected_manifest_path))
|
||||
run.input_snapshot_json = {
|
||||
@@ -1075,6 +1150,7 @@ class TimeseriesService:
|
||||
"scene_count": len(stack_dates),
|
||||
"stack_dates": stack_dates,
|
||||
"group_key": selected_manifest.get("group_key"),
|
||||
"stack_key": selected_manifest.get("stack_key"),
|
||||
"tile_key": selected_manifest.get("tile_key"),
|
||||
"source_root_windows": selected_manifest.get("source_root_windows"),
|
||||
"selected_manifest_path_windows": str(selected_manifest_path),
|
||||
@@ -1097,6 +1173,7 @@ class TimeseriesService:
|
||||
"phase": "prepared",
|
||||
"workflow": settings.TIMESERIES_STACK_WORKFLOW,
|
||||
"group_key": selected_manifest.get("group_key"),
|
||||
"stack_key": selected_manifest.get("stack_key"),
|
||||
"tile_key": selected_manifest.get("tile_key"),
|
||||
"reference_date": effective_reference_date,
|
||||
"stack_dates": stack_dates,
|
||||
@@ -1223,6 +1300,10 @@ class TimeseriesService:
|
||||
|
||||
run.manifest_path_windows = str(generated_manifest_path)
|
||||
run.manifest_path_wsl = _windows_path_to_wsl_mount(str(generated_manifest_path))
|
||||
run.stack_key = str(report.get("stack_key") or run.stack_key or "").strip() or run.stack_key
|
||||
if run.stack_key:
|
||||
run.publish_dir_windows = _compose_publish_dir(run.run_id, run.stack_key)
|
||||
run.publish_dir_wsl = _windows_path_to_wsl_mount(run.publish_dir_windows)
|
||||
run.reference_date = str(report.get("reference_date") or run.reference_date or "").strip() or run.reference_date
|
||||
run.stack_size = int(report.get("scene_count") or len(stack_dates) or run.stack_size or 0)
|
||||
run.orbit_summary_json = {
|
||||
@@ -1239,6 +1320,7 @@ class TimeseriesService:
|
||||
"phase": "stack_ready" if (refresh and ready) else "stack_prepared",
|
||||
"workflow": report.get("processing_workflow") or settings.TIMESERIES_STACK_WORKFLOW,
|
||||
"group_key": report.get("group_key"),
|
||||
"stack_key": report.get("stack_key"),
|
||||
"tile_key": report.get("tile_key"),
|
||||
"reference_date": report.get("reference_date"),
|
||||
"stack_dates": stack_dates,
|
||||
@@ -1655,6 +1737,7 @@ class TimeseriesService:
|
||||
"publish_dir_wsl": publish_dir_wsl,
|
||||
"manifest_path_windows": str(publish_manifest_path),
|
||||
"manifest_path_wsl": _windows_path_to_wsl_mount(str(publish_manifest_path)),
|
||||
"stack_key": publish_manifest.get("stack_key"),
|
||||
"group_key": publish_manifest.get("group_key"),
|
||||
"export_runner": export_result,
|
||||
},
|
||||
@@ -1812,10 +1895,14 @@ class TimeseriesService:
|
||||
"product_id": product.product_id,
|
||||
"display_name": product.display_name,
|
||||
"run_key": product.run_key,
|
||||
"package_schema": product.package_schema,
|
||||
"processor_code": product.processor_code,
|
||||
"runtime_id": product.runtime_id,
|
||||
"status": product.status,
|
||||
"health_status": product.health_status,
|
||||
"publish_dir": product.publish_dir,
|
||||
"manifest_path": product.manifest_path,
|
||||
"native_output_dir": product.native_output_dir,
|
||||
"preview_path": product.preview_path,
|
||||
"primary_asset_path": product.primary_asset_path,
|
||||
"reference_date": summary.get("reference_date"),
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Mapping, Optional, Sequence
|
||||
|
||||
from .wsl_runtime_registry import WslRuntimeDefinition, get_wsl_runtime, wsl_runtime_registry
|
||||
from .wsl_service import run_wsl_exec
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-zA-Z0-9_-]+")
|
||||
|
||||
|
||||
def _utcnow_text() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
|
||||
|
||||
def _slugify(value: str, *, fallback: str = "job") -> str:
|
||||
text = _SLUG_RE.sub("_", str(value or "").strip()).strip("_")
|
||||
return text or fallback
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WslManifestRef:
|
||||
runtime_id: str
|
||||
job_id: str
|
||||
operation: str
|
||||
manifest_path_windows: str
|
||||
manifest_path_wsl: str
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"runtime_id": self.runtime_id,
|
||||
"job_id": self.job_id,
|
||||
"operation": self.operation,
|
||||
"manifest_path_windows": self.manifest_path_windows,
|
||||
"manifest_path_wsl": self.manifest_path_wsl,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WslBrokerResult:
|
||||
runtime_id: str
|
||||
distro: str
|
||||
returncode: int
|
||||
argv: Sequence[str]
|
||||
manifest: WslManifestRef
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"runtime_id": self.runtime_id,
|
||||
"distro": self.distro,
|
||||
"returncode": self.returncode,
|
||||
"argv": list(self.argv),
|
||||
"manifest": self.manifest.to_dict(),
|
||||
"stdout": self.stdout,
|
||||
"stderr": self.stderr,
|
||||
}
|
||||
|
||||
|
||||
class WslBroker:
|
||||
def __init__(self, *, job_root_windows: Optional[str] = None) -> None:
|
||||
self._job_root_windows = str(
|
||||
job_root_windows or wsl_runtime_registry.broker_job_root_windows
|
||||
).strip()
|
||||
|
||||
@property
|
||||
def job_root_windows(self) -> str:
|
||||
return self._job_root_windows
|
||||
|
||||
def stage_manifest(
|
||||
self,
|
||||
*,
|
||||
runtime_id: str,
|
||||
operation: str,
|
||||
payload: Mapping[str, Any],
|
||||
job_id: Optional[str] = None,
|
||||
) -> WslManifestRef:
|
||||
runtime = get_wsl_runtime(runtime_id)
|
||||
safe_operation = _slugify(operation, fallback="operation")
|
||||
safe_job_id = _slugify(job_id or f"{safe_operation}_{_utcnow_text()}")
|
||||
manifest_dir = Path(self.job_root_windows, runtime.runtime_id, safe_operation)
|
||||
manifest_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path = manifest_dir / f"{safe_job_id}.json"
|
||||
document = {
|
||||
"job_id": safe_job_id,
|
||||
"runtime_id": runtime.runtime_id,
|
||||
"engine_code": runtime.engine_code,
|
||||
"operation": operation,
|
||||
"created_at": _utcnow_text(),
|
||||
"payload": dict(payload or {}),
|
||||
}
|
||||
manifest_path.write_text(
|
||||
json.dumps(document, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return WslManifestRef(
|
||||
runtime_id=runtime.runtime_id,
|
||||
job_id=safe_job_id,
|
||||
operation=operation,
|
||||
manifest_path_windows=str(manifest_path),
|
||||
manifest_path_wsl=self._manifest_to_wsl(str(manifest_path)),
|
||||
)
|
||||
|
||||
def _manifest_to_wsl(self, manifest_path_windows: str) -> str:
|
||||
drive = Path(manifest_path_windows).drive.rstrip(":").lower()
|
||||
tail = Path(manifest_path_windows).as_posix().split(":", 1)[-1]
|
||||
if not drive:
|
||||
return Path(manifest_path_windows).as_posix()
|
||||
return f"/mnt/{drive}{tail}"
|
||||
|
||||
def build_runner_argv(
|
||||
self,
|
||||
*,
|
||||
runtime: WslRuntimeDefinition,
|
||||
manifest: WslManifestRef,
|
||||
extra_args: Optional[Sequence[str]] = None,
|
||||
) -> list[str]:
|
||||
argv = list(runtime.entrypoint_argv())
|
||||
argv.extend(["--manifest", manifest.manifest_path_wsl])
|
||||
argv.extend(str(item) for item in (extra_args or []) if str(item))
|
||||
return argv
|
||||
|
||||
def run_manifest(
|
||||
self,
|
||||
*,
|
||||
runtime_id: str,
|
||||
operation: str,
|
||||
payload: Mapping[str, Any],
|
||||
job_id: Optional[str] = None,
|
||||
extra_args: Optional[Sequence[str]] = None,
|
||||
timeout_seconds: int = 30,
|
||||
env: Optional[Dict[str, str]] = None,
|
||||
) -> WslBrokerResult:
|
||||
runtime = get_wsl_runtime(runtime_id)
|
||||
manifest = self.stage_manifest(
|
||||
runtime_id=runtime_id,
|
||||
operation=operation,
|
||||
payload=payload,
|
||||
job_id=job_id,
|
||||
)
|
||||
argv = self.build_runner_argv(runtime=runtime, manifest=manifest, extra_args=extra_args)
|
||||
rc, stdout, stderr = run_wsl_exec(
|
||||
argv,
|
||||
distro=runtime.distro,
|
||||
timeout=max(30, int(timeout_seconds or 30)),
|
||||
env=env,
|
||||
)
|
||||
return WslBrokerResult(
|
||||
runtime_id=runtime.runtime_id,
|
||||
distro=runtime.distro,
|
||||
returncode=rc,
|
||||
argv=argv,
|
||||
manifest=manifest,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
)
|
||||
|
||||
|
||||
wsl_broker = WslBroker()
|
||||
@@ -0,0 +1,170 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Mapping, Tuple
|
||||
|
||||
from ..config import settings
|
||||
|
||||
|
||||
def _windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
drive, tail = os.path.splitdrive(os.path.normpath(text))
|
||||
if not drive:
|
||||
return text.replace("\\", "/")
|
||||
drive_letter = drive.rstrip(":").lower()
|
||||
normalized_tail = tail.replace("\\", "/")
|
||||
return f"/mnt/{drive_letter}/{normalized_tail}"
|
||||
|
||||
|
||||
def _project_file_windows(*relative_parts: str) -> str:
|
||||
return os.path.normpath(str(Path(settings.PROJECT_ROOT, *relative_parts)))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WslRuntimeDefinition:
|
||||
runtime_id: str
|
||||
engine_code: str
|
||||
display_name: str
|
||||
distro: str
|
||||
conda_env_name: str
|
||||
python_path: str
|
||||
runner_path_windows: str
|
||||
runner_path_wsl: str
|
||||
allowed_operations: Tuple[str, ...] = ()
|
||||
env_profile_path_windows: str = ""
|
||||
env_profile_path_wsl: str = ""
|
||||
metadata_json: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def entrypoint_argv(self) -> list[str]:
|
||||
return [self.python_path, self.runner_path_wsl]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"runtime_id": self.runtime_id,
|
||||
"engine_code": self.engine_code,
|
||||
"display_name": self.display_name,
|
||||
"distro": self.distro,
|
||||
"conda_env_name": self.conda_env_name,
|
||||
"python_path": self.python_path,
|
||||
"runner_path_windows": self.runner_path_windows,
|
||||
"runner_path_wsl": self.runner_path_wsl,
|
||||
"allowed_operations": list(self.allowed_operations),
|
||||
"env_profile_path_windows": self.env_profile_path_windows,
|
||||
"env_profile_path_wsl": self.env_profile_path_wsl,
|
||||
"metadata_json": dict(self.metadata_json or {}),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WslRuntimeRegistry:
|
||||
shared_distro: str
|
||||
shared_conda_env_name: str
|
||||
shared_python_path: str
|
||||
broker_job_root_windows: str
|
||||
broker_job_root_wsl: str
|
||||
runtimes: Mapping[str, WslRuntimeDefinition]
|
||||
|
||||
def require(self, runtime_id: str) -> WslRuntimeDefinition:
|
||||
key = str(runtime_id or "").strip()
|
||||
runtime = self.runtimes.get(key)
|
||||
if runtime is None:
|
||||
known_ids = ", ".join(sorted(self.runtimes.keys()))
|
||||
raise KeyError(f"Unknown WSL runtime_id '{runtime_id}'. Known: {known_ids}")
|
||||
return runtime
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"shared_distro": self.shared_distro,
|
||||
"shared_conda_env_name": self.shared_conda_env_name,
|
||||
"shared_python_path": self.shared_python_path,
|
||||
"broker_job_root_windows": self.broker_job_root_windows,
|
||||
"broker_job_root_wsl": self.broker_job_root_wsl,
|
||||
"runtimes": {
|
||||
runtime_id: runtime.to_dict()
|
||||
for runtime_id, runtime in sorted(self.runtimes.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_wsl_runtime_registry() -> WslRuntimeRegistry:
|
||||
shared_distro = str(
|
||||
settings.WSL_DISTRO
|
||||
or settings.ISCE2_WSL_DISTRO
|
||||
or settings.PYINT_WSL_DISTRO
|
||||
or "Ubuntu-24.04"
|
||||
).strip()
|
||||
shared_conda_env = str(settings.WSL_SHARED_CONDA_ENV or "insar_wsl_v1").strip() or "insar_wsl_v1"
|
||||
shared_python_path = str(
|
||||
settings.WSL_SHARED_PYTHON
|
||||
or settings.ISCE2_PYTHON
|
||||
or settings.PYINT_WSL_PYTHON
|
||||
or f"/home/administrator/miniconda3/envs/{shared_conda_env}/bin/python"
|
||||
).strip()
|
||||
broker_job_root_windows = os.path.normpath(
|
||||
settings.WSL_BROKER_JOB_ROOT
|
||||
or os.path.join(settings.BACKEND_DIR, "runtime", "wsl_jobs")
|
||||
)
|
||||
broker_job_root_wsl = _windows_path_to_wsl_mount(broker_job_root_windows)
|
||||
|
||||
isce2_runner_windows = _project_file_windows("deploy", "wsl", "runners", "isce2_runner.py")
|
||||
gamma_runner_windows = _project_file_windows("deploy", "wsl", "runners", "gamma_pyint_runner.py")
|
||||
gamma_profile_windows = _project_file_windows("deploy", "wsl", "profiles", "gamma_env.sh")
|
||||
|
||||
runtimes = {
|
||||
settings.ISCE2_RUNTIME_ID: WslRuntimeDefinition(
|
||||
runtime_id=settings.ISCE2_RUNTIME_ID,
|
||||
engine_code="isce2",
|
||||
display_name="ISCE2 Runtime V1",
|
||||
distro=shared_distro,
|
||||
conda_env_name=shared_conda_env,
|
||||
python_path=shared_python_path,
|
||||
runner_path_windows=isce2_runner_windows,
|
||||
runner_path_wsl=_windows_path_to_wsl_mount(isce2_runner_windows),
|
||||
allowed_operations=("lt1_stripmap",),
|
||||
metadata_json={
|
||||
"shared_runtime": True,
|
||||
"legacy_distro_env_var": "ISCE2_WSL_DISTRO",
|
||||
"legacy_python_env_var": "ISCE2_PYTHON",
|
||||
"legacy_pipeline_env_var": "ISCE2_PIPELINE_SCRIPT",
|
||||
},
|
||||
),
|
||||
settings.PYINT_RUNTIME_ID: WslRuntimeDefinition(
|
||||
runtime_id=settings.PYINT_RUNTIME_ID,
|
||||
engine_code="pyint",
|
||||
display_name="Gamma / PyINT Runtime V1",
|
||||
distro=shared_distro,
|
||||
conda_env_name=shared_conda_env,
|
||||
python_path=shared_python_path,
|
||||
runner_path_windows=gamma_runner_windows,
|
||||
runner_path_wsl=_windows_path_to_wsl_mount(gamma_runner_windows),
|
||||
allowed_operations=("lt1_gamma_dinsar", "gamma_refine"),
|
||||
env_profile_path_windows=gamma_profile_windows,
|
||||
env_profile_path_wsl=_windows_path_to_wsl_mount(gamma_profile_windows),
|
||||
metadata_json={
|
||||
"shared_runtime": True,
|
||||
"legacy_distro_env_var": "PYINT_WSL_DISTRO",
|
||||
"legacy_python_env_var": "PYINT_WSL_PYTHON",
|
||||
"legacy_profile_env_var": "PYINT_GAMMA_ENV_SCRIPT",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
return WslRuntimeRegistry(
|
||||
shared_distro=shared_distro,
|
||||
shared_conda_env_name=shared_conda_env,
|
||||
shared_python_path=shared_python_path,
|
||||
broker_job_root_windows=broker_job_root_windows,
|
||||
broker_job_root_wsl=broker_job_root_wsl,
|
||||
runtimes=runtimes,
|
||||
)
|
||||
|
||||
|
||||
def get_wsl_runtime(runtime_id: str) -> WslRuntimeDefinition:
|
||||
return wsl_runtime_registry.require(runtime_id)
|
||||
|
||||
|
||||
wsl_runtime_registry = build_wsl_runtime_registry()
|
||||
@@ -12,7 +12,7 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -127,15 +127,41 @@ def run_wsl_command(
|
||||
return -3, "", str(exc)
|
||||
|
||||
|
||||
def run_wsl_exec(
|
||||
argv: Sequence[str],
|
||||
distro: Optional[str] = None,
|
||||
timeout: int = 30,
|
||||
env: Optional[Dict[str, str]] = None,
|
||||
) -> Tuple[int, str, str]:
|
||||
"""Execute a structured argv in WSL via ``wsl.exe --exec``."""
|
||||
wsl_exe = _find_wsl_executable()
|
||||
if not wsl_exe:
|
||||
return -2, "", "wsl.exe not found"
|
||||
|
||||
normalized_argv = [str(part) for part in argv if str(part)]
|
||||
if not normalized_argv:
|
||||
return -3, "", "WSL argv is empty"
|
||||
|
||||
wsl_args = [wsl_exe]
|
||||
if distro:
|
||||
wsl_args += ["-d", distro]
|
||||
wsl_args += ["--exec", *normalized_argv]
|
||||
|
||||
try:
|
||||
return _run_windows_command(wsl_args, timeout=timeout, env=env)
|
||||
except subprocess.TimeoutExpired:
|
||||
return -1, "", f"command timed out ({timeout}s)"
|
||||
except FileNotFoundError:
|
||||
return -2, "", "wsl.exe not found"
|
||||
except Exception as exc:
|
||||
return -3, "", str(exc)
|
||||
|
||||
|
||||
def windows_path_to_wsl(win_path: str, distro: Optional[str] = None) -> str:
|
||||
"""将 Windows 路径转换为 WSL 路径(调用 wslpath)。"""
|
||||
if not win_path:
|
||||
return ""
|
||||
rc, stdout, _ = run_wsl_command(
|
||||
f"wslpath -u '{win_path.replace(chr(39), '')}'",
|
||||
distro=distro,
|
||||
timeout=10,
|
||||
)
|
||||
rc, stdout, _ = run_wsl_exec(["wslpath", "-u", win_path], distro=distro, timeout=10)
|
||||
return stdout if rc == 0 else ""
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# WSL Shared Conda Runtime
|
||||
|
||||
此目录用于维护当前统一的 WSL 共享 conda 运行时文档与锁定文件。
|
||||
|
||||
## 当前策略
|
||||
|
||||
当前项目不再建议为 `isce2`、`timeseries`、`pyint` 分别维护多套相互漂移的 conda 环境,而是收口为:
|
||||
|
||||
- `base`
|
||||
- `insar_wsl_v1`
|
||||
|
||||
目标是让以下链路共享同一套 Python 运行时:
|
||||
|
||||
- ISCE2 D-InSAR
|
||||
- 时序 InSAR 当前 SBAS 流程
|
||||
- Gamma / PyINT 的 Python 胶水层
|
||||
|
||||
说明:
|
||||
|
||||
- Gamma 二进制本体仍然保持固定安装位置,不建议放入 conda。
|
||||
- Gamma 相关的 `PATH`、`GAMMA_HOME`、脚本目录注入,统一由 `deploy/wsl/profiles/gamma_env.sh` 负责。
|
||||
|
||||
## 当前约定产物
|
||||
|
||||
本目录建议维护以下文件:
|
||||
|
||||
- `insar_wsl_v1.environment.yml`
|
||||
人工可读、可维护的环境定义。
|
||||
|
||||
- `insar_wsl_v1.explicit.lock`
|
||||
由 `conda list --explicit` 导出的精确锁文件。
|
||||
|
||||
- `insar_wsl_v1.fingerprint.json`
|
||||
用于记录 Python 版本、关键包版本、生成时间等信息。
|
||||
|
||||
## 推荐工作流
|
||||
|
||||
### 1. 在 WSL 中核出现有环境
|
||||
|
||||
先查看现有环境:
|
||||
|
||||
```bash
|
||||
conda env list
|
||||
```
|
||||
|
||||
逐个导出候选环境包列表,用于做并集:
|
||||
|
||||
```bash
|
||||
conda list -n isce2
|
||||
conda list -n isce2_mintpy_v1
|
||||
conda list -n pyint
|
||||
```
|
||||
|
||||
### 2. 整理并集并创建新环境
|
||||
|
||||
推荐先人工整理 `insar_wsl_v1.environment.yml`,再创建环境:
|
||||
|
||||
```bash
|
||||
conda env create -n insar_wsl_v1 -f deploy/wsl/conda/insar_wsl_v1.environment.yml
|
||||
```
|
||||
|
||||
如果环境已存在:
|
||||
|
||||
```bash
|
||||
conda env update -n insar_wsl_v1 -f deploy/wsl/conda/insar_wsl_v1.environment.yml --prune
|
||||
```
|
||||
|
||||
### 3. 生成锁文件与指纹
|
||||
|
||||
```bash
|
||||
conda env export --from-history -n insar_wsl_v1 > deploy/wsl/conda/insar_wsl_v1.environment.yml
|
||||
conda list --explicit -n insar_wsl_v1 > deploy/wsl/conda/insar_wsl_v1.explicit.lock
|
||||
```
|
||||
|
||||
```bash
|
||||
python - <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
def sh(*argv):
|
||||
return subprocess.check_output(argv, text=True).strip()
|
||||
|
||||
payload = {
|
||||
"python_version": sh("python", "--version"),
|
||||
"conda_env": "insar_wsl_v1",
|
||||
"isce_version": sh("python", "-c", "import isce, sys; print(getattr(isce, '__file__', 'unknown'))"),
|
||||
}
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
PY
|
||||
```
|
||||
|
||||
将输出整理后保存为 `insar_wsl_v1.fingerprint.json`。
|
||||
|
||||
## .env 对齐要求
|
||||
|
||||
共享环境建好后,应确保 `.env` 中以下键值一致:
|
||||
|
||||
```env
|
||||
WSL_DISTRO=Ubuntu-24.04
|
||||
WSL_SHARED_CONDA_ENV=insar_wsl_v1
|
||||
WSL_SHARED_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
|
||||
ISCE2_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
TIMESERIES_ENV_NAME=insar_wsl_v1
|
||||
TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
PYINT_WSL_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
```
|
||||
|
||||
## 运维验证
|
||||
|
||||
系统启动后,运维自检中的 `wsl_runtime` 应满足:
|
||||
|
||||
- `shared_distro = Ubuntu-24.04`
|
||||
- `shared_conda_env_name = insar_wsl_v1`
|
||||
- `required_runtime_count == healthy_runtime_count`
|
||||
|
||||
如果健康面板显示 Python 路径不一致,说明仍然有旧环境残留在配置层或运行时注册层。
|
||||
@@ -0,0 +1,358 @@
|
||||
name: insar_wsl_v1
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- _openmp_mutex=4.5
|
||||
- _x86_64-microarch-level=4
|
||||
- aom=3.9.1
|
||||
- argcomplete=3.6.3
|
||||
- attr=2.5.2
|
||||
- attrs=26.1.0
|
||||
- aws-c-auth=0.9.3
|
||||
- aws-c-cal=0.9.13
|
||||
- aws-c-common=0.12.6
|
||||
- aws-c-compression=0.3.1
|
||||
- aws-c-event-stream=0.5.7
|
||||
- aws-c-http=0.10.7
|
||||
- aws-c-io=0.23.3
|
||||
- aws-c-mqtt=0.13.3
|
||||
- aws-c-s3=0.11.3
|
||||
- aws-c-sdkutils=0.2.4
|
||||
- aws-checksums=0.2.7
|
||||
- aws-crt-cpp=0.35.2
|
||||
- aws-sdk-cpp=1.11.606
|
||||
- azure-core-cpp=1.16.1
|
||||
- azure-identity-cpp=1.13.2
|
||||
- azure-storage-blobs-cpp=12.15.0
|
||||
- azure-storage-common-cpp=12.11.0
|
||||
- azure-storage-files-datalake-cpp=12.13.0
|
||||
- backports.zstd=1.3.0
|
||||
- blosc=1.21.6
|
||||
- bokeh=3.9.0
|
||||
- brotli=1.2.0
|
||||
- brotli-bin=1.2.0
|
||||
- brotli-python=1.2.0
|
||||
- brunsli=0.1
|
||||
- bzip2=1.0.8
|
||||
- c-ares=1.34.6
|
||||
- c-blosc2=2.23.1
|
||||
- ca-certificates=2026.4.22
|
||||
- cached-property=1.5.2
|
||||
- cached_property=1.5.2
|
||||
- cairo=1.18.4
|
||||
- capnproto=1.2.0
|
||||
- cartopy=0.25.0
|
||||
- cdsapi=0.7.7
|
||||
- certifi=2026.4.22
|
||||
- cffi=2.0.0
|
||||
- cfgv=3.5.0
|
||||
- cfitsio=4.6.3
|
||||
- charls=2.4.3
|
||||
- charset-normalizer=3.4.5
|
||||
- click=8.3.2
|
||||
- cloudpickle=3.1.2
|
||||
- configobj=5.0.9
|
||||
- contourpy=1.3.3
|
||||
- cvxopt=1.3.3
|
||||
- cycler=0.12.1
|
||||
- cyrus-sasl=2.1.28
|
||||
- cytoolz=1.1.0
|
||||
- dask=2026.3.0
|
||||
- dask-core=2026.3.0
|
||||
- dask-jobqueue=0.9.0
|
||||
- dav1d=1.2.1
|
||||
- distlib=0.4.0
|
||||
- distributed=2026.3.0
|
||||
- donfig=0.8.1.post1
|
||||
- dsdp=5.8
|
||||
- eccodes=2.46.0
|
||||
- ecmwf-datastores-client=0.5.1
|
||||
- fftw=3.3.10
|
||||
- filelock=3.29.0
|
||||
- fmt=12.0.0
|
||||
- font-ttf-dejavu-sans-mono=2.37
|
||||
- font-ttf-inconsolata=3.000
|
||||
- font-ttf-source-code-pro=2.038
|
||||
- font-ttf-ubuntu=0.83
|
||||
- fontconfig=2.17.1
|
||||
- fonts-conda-ecosystem=1
|
||||
- fonts-conda-forge=1
|
||||
- fonttools=4.62.1
|
||||
- freeglut=3.2.2
|
||||
- freetype=2.14.2
|
||||
- freexl=2.0.0
|
||||
- fsspec=2026.3.0
|
||||
- gdal=3.10.3
|
||||
- geos=3.14.1
|
||||
- geotiff=1.7.4
|
||||
- gflags=2.2.2
|
||||
- giflib=5.2.2
|
||||
- glog=0.7.1
|
||||
- glpk=5.0
|
||||
- gmp=6.3.0
|
||||
- gsl=2.7
|
||||
- h2=4.3.0
|
||||
- h5py=3.15.1
|
||||
- hdf4=4.2.15
|
||||
- hdf5=1.14.6
|
||||
- hpack=4.1.0
|
||||
- hyperframe=6.1.0
|
||||
- icu=78.2
|
||||
- identify=2.6.19
|
||||
- idna=3.11
|
||||
- imagecodecs=2026.3.6
|
||||
- imageio=2.37.0
|
||||
- importlib-metadata=8.8.0
|
||||
- isce2=2.6.4
|
||||
- jasper=4.2.9
|
||||
- jinja2=3.1.6
|
||||
- joblib=1.5.3
|
||||
- json-c=0.18
|
||||
- jxrlib=1.1
|
||||
- kealib=1.6.2
|
||||
- keyutils=1.6.3
|
||||
- kiwisolver=1.5.0
|
||||
- krb5=1.22.2
|
||||
- lazy-loader=0.5
|
||||
- lcms2=2.18
|
||||
- ld_impl_linux-64=2.45.1
|
||||
- lerc=4.1.0
|
||||
- libabseil=20250512.1
|
||||
- libacl=2.3.2
|
||||
- libaec=1.1.5
|
||||
- libamd=3.3.3
|
||||
- libarchive=3.8.5
|
||||
- libarrow=22.0.0
|
||||
- libarrow-acero=22.0.0
|
||||
- libarrow-compute=22.0.0
|
||||
- libarrow-dataset=22.0.0
|
||||
- libarrow-substrait=22.0.0
|
||||
- libavif16=1.4.0
|
||||
- libblas=3.11.0
|
||||
- libbrotlicommon=1.2.0
|
||||
- libbrotlidec=1.2.0
|
||||
- libbrotlienc=1.2.0
|
||||
- libbtf=2.3.2
|
||||
- libcamd=3.3.3
|
||||
- libcblas=3.11.0
|
||||
- libccolamd=3.3.4
|
||||
- libcholmod=5.3.1
|
||||
- libcolamd=3.3.4
|
||||
- libcrc32c=1.1.2
|
||||
- libcurl=8.18.0
|
||||
- libcxsparse=4.4.1
|
||||
- libdeflate=1.25
|
||||
- libedit=3.1.20250104
|
||||
- libev=4.33
|
||||
- libevent=2.1.12
|
||||
- libexpat=2.7.4
|
||||
- libffi=3.5.2
|
||||
- libfreetype=2.14.2
|
||||
- libfreetype6=2.14.2
|
||||
- libgcc=15.2.0
|
||||
- libgcc-ng=15.2.0
|
||||
- libgdal=3.10.3
|
||||
- libgdal-core=3.10.3
|
||||
- libgdal-fits=3.10.3
|
||||
- libgdal-grib=3.10.3
|
||||
- libgdal-hdf4=3.10.3
|
||||
- libgdal-hdf5=3.10.3
|
||||
- libgdal-jp2openjpeg=3.10.3
|
||||
- libgdal-kea=3.10.3
|
||||
- libgdal-netcdf=3.10.3
|
||||
- libgdal-pdf=3.10.3
|
||||
- libgdal-pg=3.10.3
|
||||
- libgdal-postgisraster=3.10.3
|
||||
- libgdal-tiledb=3.10.3
|
||||
- libgdal-xls=3.10.3
|
||||
- libgfortran=15.2.0
|
||||
- libgfortran5=15.2.0
|
||||
- libgl=1.7.0
|
||||
- libglib=2.86.4
|
||||
- libglu=9.0.3
|
||||
- libglvnd=1.7.0
|
||||
- libglx=1.7.0
|
||||
- libgomp=15.2.0
|
||||
- libgoogle-cloud=2.39.0
|
||||
- libgoogle-cloud-storage=2.39.0
|
||||
- libgrpc=1.73.1
|
||||
- libhwy=1.3.0
|
||||
- libiconv=1.18
|
||||
- libjpeg-turbo=3.1.2
|
||||
- libjxl=0.11.2
|
||||
- libklu=2.3.5
|
||||
- libkml=1.3.0
|
||||
- liblapack=3.11.0
|
||||
- libldl=3.3.2
|
||||
- liblzma=5.8.2
|
||||
- libnetcdf=4.9.3
|
||||
- libnghttp2=1.67.0
|
||||
- libnsl=2.0.1
|
||||
- libntlm=1.8
|
||||
- libopenblas=0.3.30
|
||||
- libopengl=1.7.0
|
||||
- libopentelemetry-cpp=1.21.0
|
||||
- libopentelemetry-cpp-headers=1.21.0
|
||||
- libparquet=22.0.0
|
||||
- libparu=1.0.0
|
||||
- libpng=1.6.55
|
||||
- libpq=18.3
|
||||
- libprotobuf=6.31.1
|
||||
- librbio=4.3.4
|
||||
- libre2-11=2025.11.05
|
||||
- librttopo=1.1.0
|
||||
- libspatialite=5.1.0
|
||||
- libspex=3.2.3
|
||||
- libspqr=4.3.4
|
||||
- libsqlite=3.52.0
|
||||
- libssh2=1.11.1
|
||||
- libstdcxx=15.2.0
|
||||
- libstdcxx-ng=15.2.0
|
||||
- libsuitesparseconfig=7.10.1
|
||||
- libthrift=0.22.0
|
||||
- libtiff=4.7.1
|
||||
- libumfpack=6.3.5
|
||||
- liburing=2.14
|
||||
- libutf8proc=2.11.3
|
||||
- libuuid=2.41.3
|
||||
- libwebp-base=1.6.0
|
||||
- libxcb=1.17.0
|
||||
- libxcrypt=4.4.36
|
||||
- libxml2=2.15.2
|
||||
- libxml2-16=2.15.2
|
||||
- libxml2-devel=2.15.2
|
||||
- libxslt=1.1.43
|
||||
- libzip=1.11.2
|
||||
- libzlib=1.3.1
|
||||
- libzopfli=1.0.3
|
||||
- locket=1.0.0
|
||||
- lxml=6.0.2
|
||||
- lz4=4.4.5
|
||||
- lz4-c=1.10.0
|
||||
- lzo=2.10
|
||||
- markdown-it-py=4.0.0
|
||||
- markupsafe=3.0.3
|
||||
- matplotlib-base=3.10.8
|
||||
- mdurl=0.1.2
|
||||
- metis=5.1.0
|
||||
- minizip=4.0.10
|
||||
- mintpy=1.6.3
|
||||
- mpfr=4.2.2
|
||||
- msgpack-python=1.1.2
|
||||
- multiurl=0.3.7
|
||||
- munkres=1.1.4
|
||||
- narwhals=2.20.0
|
||||
- ncurses=6.5
|
||||
- networkx=3.6.1
|
||||
- nlohmann_json=3.12.0
|
||||
- nodeenv=1.10.0
|
||||
- nspr=4.38
|
||||
- nss=3.118
|
||||
- numpy=1.26.4
|
||||
- openjpeg=2.5.4
|
||||
- openjph=0.26.3
|
||||
- openldap=2.6.10
|
||||
- openmotif=2.3.8
|
||||
- openssl=3.6.2
|
||||
- orc=2.2.1
|
||||
- packaging=26.0
|
||||
- pandas=3.0.2
|
||||
- partd=1.4.2
|
||||
- pcre2=10.47
|
||||
- pillow=12.1.1
|
||||
- pip=26.0.1
|
||||
- pixman=0.46.4
|
||||
- platformdirs=4.9.6
|
||||
- poppler=25.07.0
|
||||
- poppler-data=0.4.12
|
||||
- postgresql=18.3
|
||||
- pre-commit=4.6.0
|
||||
- proj=9.7.1
|
||||
- prometheus-cpp=1.3.0
|
||||
- psutil=7.2.2
|
||||
- pthread-stubs=0.4
|
||||
- pyaps3=0.3.7
|
||||
- pyarrow=22.0.0
|
||||
- pyarrow-core=22.0.0
|
||||
- pycparser=2.22
|
||||
- pygments=2.20.0
|
||||
- pygrib=2.1.8
|
||||
- pykdtree=1.4.3
|
||||
- pykml=0.2.0
|
||||
- pyparsing=3.3.2
|
||||
- pyproj=3.7.2
|
||||
- pyresample=1.35.0
|
||||
- pyshp=3.0.3
|
||||
- pysocks=1.7.1
|
||||
- pysolid=0.3.4
|
||||
- python=3.11.15
|
||||
- python-dateutil=2.9.0.post0
|
||||
- python-discovery=1.2.2
|
||||
- python_abi=3.11
|
||||
- pytz=2026.1.post1
|
||||
- pyyaml=6.0.3
|
||||
- qhull=2020.2
|
||||
- rav1e=0.8.1
|
||||
- re2=2025.11.05
|
||||
- readline=8.3
|
||||
- requests=2.32.5
|
||||
- rich=15.0.0
|
||||
- s2n=1.6.2
|
||||
- scikit-image=0.26.0
|
||||
- scipy=1.17.1
|
||||
- setuptools=82.0.1
|
||||
- shapely=2.1.2
|
||||
- six=1.17.0
|
||||
- snappy=1.2.2
|
||||
- sortedcontainers=2.4.0
|
||||
- spdlog=1.16.0
|
||||
- sqlite=3.52.0
|
||||
- suitesparse=7.10.1
|
||||
- svt-av1=4.0.1
|
||||
- tblib=3.2.2
|
||||
- tifffile=2026.3.3
|
||||
- tiledb=2.29.2
|
||||
- tk=8.6.13
|
||||
- toolz=1.1.0
|
||||
- tornado=6.5.5
|
||||
- tqdm=4.67.3
|
||||
- typing_extensions=4.15.0
|
||||
- tzcode=2026a
|
||||
- tzdata=2025c
|
||||
- ukkonen=1.1.0
|
||||
- unicodedata2=17.0.1
|
||||
- uriparser=0.9.8
|
||||
- urllib3=2.6.3
|
||||
- utm=0.7.0
|
||||
- virtualenv=21.2.4
|
||||
- wheel=0.46.3
|
||||
- xerces-c=3.3.0
|
||||
- xorg-libice=1.1.2
|
||||
- xorg-libsm=1.2.6
|
||||
- xorg-libx11=1.8.13
|
||||
- xorg-libxau=1.0.12
|
||||
- xorg-libxdmcp=1.1.5
|
||||
- xorg-libxext=1.3.7
|
||||
- xorg-libxfixes=6.0.2
|
||||
- xorg-libxft=2.3.9
|
||||
- xorg-libxi=1.8.2
|
||||
- xorg-libxmu=1.3.1
|
||||
- xorg-libxp=1.0.4
|
||||
- xorg-libxrender=0.9.12
|
||||
- xorg-libxt=1.3.1
|
||||
- xorg-libxxf86vm=1.1.7
|
||||
- xyzservices=2026.3.0
|
||||
- yaml=0.2.5
|
||||
- zfp=1.0.1
|
||||
- zict=3.0.0
|
||||
- zipp=3.23.1
|
||||
- zlib=1.3.1
|
||||
- zlib-ng=2.3.3
|
||||
- zstd=1.5.7
|
||||
- pip:
|
||||
- affine==2.4.0
|
||||
- click-plugins==1.1.1.2
|
||||
- cligj==0.7.2
|
||||
- rasterio==1.4.4
|
||||
prefix: /home/administrator/miniconda3/envs/insar_wsl_v1
|
||||
@@ -0,0 +1,352 @@
|
||||
# This file may be used to create an environment using:
|
||||
# $ conda create --name <env> --file <this file>
|
||||
# platform: linux-64
|
||||
# created-by: conda 26.1.1
|
||||
@EXPLICIT
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/_x86_64-microarch-level-4-3_sapphirerapids.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/ca-certificates-2026.4.22-hbd8a1cb_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgomp-15.2.0-he0feb66_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libopentelemetry-cpp-headers-1.21.0-ha770c72_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/nlohmann_json-3.12.0-h54a6638_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/python_abi-3.11-8_cp311.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libopengl-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgcc-15.2.0-he0feb66_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/attr-2.5.2-h39aace5_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-common-0.12.6-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/bzip2-1.0.8-hda65f42_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/c-ares-1.34.6-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/json-c-0.18-h6688a6e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlicommon-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libdeflate-1.25-h17f619e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libexpat-2.7.4-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libjpeg-turbo-3.1.2-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libutf8proc-2.11.3-hfe17d71_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lzo-2.10-h280c20c_1002.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openssl-3.6.2-h35e630c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/rav1e-0.8.1-h1fbca29_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tzcode-2026a-h280c20c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxau-1.0.12-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-cal-0.9.13-h2c9d079_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-compression-0.3.1-h8b1a151_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-sdkutils-0.2.4-h8b1a151_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-checksums-0.2.7-h8b1a151_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/capnproto-1.2.0-hfc315d8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/charls-2.4.3-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fmt-12.0.0-h2b0788b_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/geos-3.14.1-h480dda7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/gflags-2.2.2-h5888daf_1005.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/giflib-5.2.2-hd590300_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/jxrlib-1.1-hd590300_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lerc-4.1.0-hdb68285_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libabseil-20250512.1-cxx17_hba17884_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libacl-2.3.2-h0f662aa_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libaec-1.1.5-h088129d_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlidec-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlienc-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libev-4.33-hd590300_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libevent-2.1.12-hf998b51_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglu-9.0.3-h5888daf_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libhwy-1.3.0-h4c17acf_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libpng-1.6.55-h421ea60_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libssh2-1.11.1-hcf80075_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libstdcxx-ng-15.2.0-hdf11a46_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liburing-2.14-hb700be7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzip-1.11.2-h6991a6a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/metis-5.1.0-hd0bcaf9_1007.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/nspr-4.38-h29cc59b_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pcre2-10.47-haa7fec5_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/readline-8.3-h853b02a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/s2n-1.6.2-he8a4886_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/snappy-1.2.2-h03e3b7b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/svt-av1-4.0.1-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zfp-1.0.1-h909a3a2_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zlib-ng-2.3.3-hceb46e0_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-io-0.23.3-hdaf4b65_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brotli-bin-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brunsli-0.1-hd1e3526_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/c-blosc2-2.23.1-hc31b594_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fftw-3.3.10-nompi_h3b011a4_112.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/glog-0.7.1-hbabe93e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/gmp-6.3.0-hac33072_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/hdf4-4.2.15-h2a13503_7.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/krb5-1.22.2-ha1258a1_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_101.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libfreetype6-2.14.2-h73754d4_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglib-2.86.4-h6548e54_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libjxl-0.11.2-ha09017c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnghttp2-1.67.0-had1ee68_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libprotobuf-6.31.1-h49aed37_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libre2-11-2025.11.05-h7b12aa8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/librttopo-1.1.0-h46dd2a8_20.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libsqlite-3.52.0-hf4e2dac_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libsuitesparseconfig-7.10.1-h92d6892_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libthrift-0.22.0-h454ac66_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libtiff-4.7.1-h9d88235_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-16-2.15.2-hca6bf5a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/minizip-4.0.10-h05a5f5f_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/qhull-2020.2-h434a139_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/spdlog-1.16.0-hffee6e0_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xerces-c-3.3.0-hd9031aa_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libx11-1.8.13-he1eb515_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-event-stream-0.5.7-h28f887f_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-http-0.10.7-ha8fc4e3_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brotli-1.2.0-hed03a55_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cyrus-sasl-2.1.28-hac629b4_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/freexl-2.0.0-h9dce30a_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/glpk-5.0-h445213a_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lcms2-2.18-h0c24ade_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libamd-3.3.3-haaf9dc3_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libavif16-1.4.0-hcfa2d63_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbtf-2.3.2-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcamd-3.3.3-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libccolamd-3.3.4-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcolamd-3.3.4-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcurl-8.18.0-hcf29cc6_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcxsparse-4.4.1-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libfreetype-2.14.2-ha770c72_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libkml-1.3.0-haa4a5bd_1022.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libldl-3.3.2-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/librbio-4.3.4-h32481e8_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-2.15.2-he237659_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/mpfr-4.2.2-he0a73b1_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/nss-3.118-h445c969_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openjpeg-2.5.4-h55fea9a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openjph-0.26.3-h8d634f6_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/orc-2.2.1-hd747db4_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/python-3.11.15-hd63d673_0_cpython.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/re2-2025.11.05-h5301d42_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/sqlite-3.52.0-h04a0ce9_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxext-1.3.7-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxfixes-6.0.2-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxt-1.3.1-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/argcomplete-3.6.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/attrs-26.1.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-auth-0.9.3-hef928c7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-mqtt-0.13.3-hc63082f_11.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-core-cpp-1.16.1-h3a458e0_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/backports.zstd-1.3.0-py311h6b1f9c4_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brotli-python-1.2.0-py311h66f275b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/certifi-2026.4.22-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cfgv-3.5.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cfitsio-4.6.3-ha0b56bc_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/charset-normalizer-3.4.5-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/click-8.3.2-pyhc90fa1f_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cloudpickle-3.1.2-pyhcf101f3_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cycler-0.12.1-pyhcf101f3_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/distlib-0.4.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/filelock-3.29.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fontconfig-2.17.1-h27c8c51_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/freetype-2.14.2-ha770c72_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/fsspec-2026.3.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/hdf5-1.14.6-nompi_h19486de_106.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/kiwisolver-1.5.0-py311h724c32c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarchive-3.8.5-gpl_hc2c16d8_100.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgrpc-1.73.1-h3288cfb_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libspex-3.2.3-had10066_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-devel-2.15.2-he237659_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxslt-1.1.43-h711ed8c_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/locket-1.0.0-pyhd8ed1ab_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lz4-4.4.5-py311h1c460e0_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/markupsafe-3.0.3-py311h3778330_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/mdurl-0.1.2-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/msgpack-python-1.1.2-py311hdf67eae_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/munkres-1.1.4-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/narwhals-2.20.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/networkx-3.6.1-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openldap-2.6.10-hbde042b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pillow-12.1.1-py311hf88fc01_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/platformdirs-4.9.6-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/proj-9.7.1-he0df7b0_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/prometheus-cpp-1.3.0-ha5d0236_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/psutil-7.2.2-py311haee01d2_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pygments-2.20.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pyparsing-3.3.2-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pyshp-3.0.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pytz-2026.1.post1-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pyyaml-6.0.3-py311h3778330_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/setuptools-82.0.1-pyh332efcf_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/sortedcontainers-2.4.0-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tblib-3.2.2-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/toolz-1.1.0-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tornado-6.5.5-py311h49ec1c0_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tqdm-4.67.3-pyh8f84b5b_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/typing_extensions-4.15.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/unicodedata2-17.0.1-py311h49ec1c0_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/utm-0.7.0-pyhd8ed1ab_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxi-1.8.2-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxmu-1.3.1-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxp-1.0.4-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxxf86vm-1.1.7-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/xyzservices-2026.3.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/zict-3.0.0-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/zipp-3.23.1-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-s3-0.11.3-h06ab39a_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-identity-cpp-1.13.2-h3a5f585_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-common-cpp-12.11.0-h3d7a050_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cairo-1.18.4-he90730b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cffi-2.0.0-py311h03d9500_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/configobj-5.0.9-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cytoolz-1.1.0-py311h49ec1c0_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/donfig-0.8.1.post1-pyhd8ed1ab_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/dsdp-5.8-hd9d9efa_1203.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fonttools-4.62.1-py311h3778330_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/freeglut-3.2.2-h215f996_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/geotiff-1.7.4-h1000f5c_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/gsl-2.7-he838d99_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/importlib-metadata-8.8.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/jinja2-3.1.6-pyhcf101f3_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/joblib-1.5.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/kealib-1.6.2-hb2f3951_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/lazy-loader-0.5-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcholmod-5.3.1-h59ddab4_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgoogle-cloud-2.39.0-hdb79228_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnetcdf-4.9.3-nompi_hbf2fc22_104.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libopentelemetry-cpp-1.21.0-hb9b0907_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libpq-18.3-h9abb657_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libspatialite-5.1.0-gpl_h2abfd87_119.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lxml-6.0.2-py311h8840267_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/markdown-it-py-4.0.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/nodeenv-1.10.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/partd-1.4.2-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pyproj-3.7.2-py311h400b93a_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/python-discovery-1.2.2-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/wheel-0.46.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxft-2.3.9-h355ab9f_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-crt-cpp-0.35.2-h8824e59_6.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-blobs-cpp-12.15.0-h2a74896_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/contourpy-1.3.3-py311h724c32c_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/dask-core-2026.3.0-pyhc364b38_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/h5py-3.15.1-nompi_py311h0b2f468_101.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/imagecodecs-2026.3.6-py311h9837d23_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/imageio-2.37.0-pyhfb79c49_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/jasper-4.2.9-h1588d4d_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-core-3.10.3-h1f481a6_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgoogle-cloud-storage-2.39.0-hdbdcf42_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libklu-2.3.5-hf24d653_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libspqr-4.3.4-h852d39f_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libumfpack-6.3.5-heb53515_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openmotif-2.3.8-hf55c2fc_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pandas-3.0.2-py311h8032f78_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pip-26.0.1-pyh8b19718_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/poppler-25.07.0-h13eef12_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/postgresql-18.3-h9d31465_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pykdtree-1.4.3-py311h0372a8f_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pykml-0.2.0-pyhd8ed1ab_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/rich-15.0.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/scipy-1.17.1-py311hbe70eeb_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/shapely-2.1.2-py311h8a92878_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/ukkonen-1.1.0-py311hdf67eae_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/urllib3-2.6.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/virtualenv-21.2.4-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-sdk-cpp-1.11.606-hf38915e_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.13.0-hf38f1be_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/bokeh-3.9.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/distributed-2026.3.0-pyhc364b38_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/eccodes-2.46.0-h83bc92c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/gdal-3.10.3-py311h34ccccb_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/identify-2.6.19-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-fits-3.10.3-hec9d828_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-grib-3.10.3-hb20eef8_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-hdf4-3.10.3-ha810028_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-hdf5-3.10.3-h966a9c2_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-jp2openjpeg-3.10.3-hdd07572_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-pdf-3.10.3-h20efda7_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-pg-3.10.3-h55c2262_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-postgisraster-3.10.3-h55c2262_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-xls-3.10.3-hdee084c_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libparu-1.0.0-h17147ab_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/matplotlib-base-3.10.8-py311h0f3be63_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pyresample-1.35.0-py311h1ddb823_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pysolid-0.3.4-py311h9bb1bfa_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/requests-2.32.5-pyhcf101f3_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tifffile-2026.3.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cartopy-0.25.0-py311hed34c8f_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/dask-jobqueue-0.9.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarrow-22.0.0-h2937f24_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-kea-3.10.3-h2bf108d_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-netcdf-3.10.3-ha526aae_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/multiurl-0.3.7-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pre-commit-4.6.0-pyha770c72_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pygrib-2.1.8-py311he4f3390_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/scikit-image-0.26.0-np2py311h2a99c40_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/suitesparse-7.10.1-ha0f6916_7100102.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tiledb-2.29.2-h8821262_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cvxopt-1.3.3-py311h3d1f434_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/ecmwf-datastores-client-0.5.1-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarrow-compute-22.0.0-h58682fd_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-tiledb-3.10.3-h6c35068_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libparquet-22.0.0-h31208bf_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cdsapi-0.7.7-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarrow-acero-22.0.0-hb826db4_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-3.10.3-h3b705f5_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pyarrow-core-22.0.0-py311hbabfba9_2_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/isce2-2.6.4-py311h916084f_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarrow-dataset-22.0.0-hb826db4_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pyaps3-0.3.7-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarrow-substrait-22.0.0-h9d9f3f8_4_cuda.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pyarrow-22.0.0-py311h38be061_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/dask-2026.3.0-pyhc364b38_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/mintpy-1.6.3-pyhd8ed1ab_0.conda
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"python": "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python",
|
||||
"python_version": "3.11.15",
|
||||
"packages": {
|
||||
"isce": true,
|
||||
"rasterio": true,
|
||||
"osgeo": true,
|
||||
"numpy": true,
|
||||
"scipy": true,
|
||||
"mintpy": true
|
||||
},
|
||||
"versions": {
|
||||
"rasterio": "1.4.4",
|
||||
"gdal": "3100300",
|
||||
"numpy": "1.26.4",
|
||||
"scipy": "1.17.1"
|
||||
},
|
||||
"module_paths": {
|
||||
"isce": "/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/__init__.py",
|
||||
"mintpy": "/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/mintpy/__init__.py"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
# This file may be used to create an environment using:
|
||||
# $ conda create --name <env> --file <this file>
|
||||
# platform: linux-64
|
||||
# created-by: conda 26.1.1
|
||||
@EXPLICIT
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/ca-certificates-2026.2.25-hbd8a1cb_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-dejavu-sans-mono-2.37-hab24e00_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-inconsolata-3.000-h77eed37_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-source-code-pro-2.038-h77eed37_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/font-ttf-ubuntu-0.83-h77eed37_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglvnd-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgomp-15.2.0-he0feb66_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/poppler-data-0.4.12-hd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/python_abi-3.11-8_cp311.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tzdata-2025c-hc9c84f9_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/fonts-conda-forge-1-hc364b38_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/fonts-conda-ecosystem-1-0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgcc-15.2.0-he0feb66_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/attr-2.5.2-h39aace5_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-common-0.12.6-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/bzip2-1.0.8-hda65f42_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/c-ares-1.34.6-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/json-c-0.18-h6688a6e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/keyutils-1.6.3-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlicommon-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libdeflate-1.25-h17f619e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libexpat-2.7.4-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libffi-3.5.2-h3435931_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgcc-ng-15.2.0-h69a702a_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgfortran5-15.2.0-h68bc16d_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libiconv-1.18-h3b78370_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libjpeg-turbo-3.1.2-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liblzma-5.8.2-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnsl-2.0.1-hb9d3cd8_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libntlm-1.8-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libstdcxx-15.2.0-h934c35e_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libuuid-2.41.3-h5347b49_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libwebp-base-1.6.0-hd42ef1d_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lzo-2.10-h280c20c_1002.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openssl-3.6.1-h35e630c_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pthread-stubs-0.4-hb9d3cd8_1002.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/rav1e-0.8.1-h1fbca29_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tzcode-2026a-h280c20c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libice-1.1.2-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxau-1.0.12-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxdmcp-1.1.5-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-cal-0.9.13-h2c9d079_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-compression-0.3.1-h8b1a151_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-sdkutils-0.2.4-h8b1a151_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-checksums-0.2.7-h8b1a151_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/capnproto-1.2.0-hfc315d8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/charls-2.4.3-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/dav1d-1.2.1-hd590300_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fmt-12.0.0-h2b0788b_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/geos-3.14.1-h480dda7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/giflib-5.2.2-hd590300_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/icu-78.2-h33c6efd_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/jxrlib-1.1-hd590300_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lerc-4.1.0-hdb68285_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libabseil-20250512.1-cxx17_hba17884_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libacl-2.3.2-h0f662aa_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libaec-1.1.5-h088129d_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlidec-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libbrotlienc-1.2.0-hb03c661_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libedit-3.1.20250104-pl5321h7949ede_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libev-4.33-hd590300_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgfortran-15.2.0-h69a702a_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libhwy-1.3.0-h4c17acf_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libpng-1.6.55-h421ea60_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libssh2-1.11.1-hcf80075_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libstdcxx-ng-15.2.0-hdf11a46_18.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liburing-2.14-hb700be7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxcb-1.17.0-h8a09558_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzip-1.11.2-h6991a6a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lz4-c-1.10.0-h5888daf_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/nspr-4.38-h29cc59b_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pcre2-10.47-haa7fec5_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pixman-0.46.4-h54a6638_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/readline-8.3-h853b02a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/s2n-1.6.2-he8a4886_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/snappy-1.2.2-h03e3b7b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/svt-av1-4.0.1-hecca717_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tk-8.6.13-noxft_h366c992_103.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libsm-1.2.6-he73a12e_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zfp-1.0.1-h909a3a2_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zlib-ng-2.3.3-hceb46e0_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aom-3.9.1-hac33072_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-io-0.23.3-hdaf4b65_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/blosc-1.21.6-he440d0b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brunsli-0.1-hd1e3526_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/c-blosc2-2.23.1-hc31b594_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fftw-3.3.10-nompi_h3b011a4_112.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/hdf4-4.2.15-h2a13503_7.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/krb5-1.22.2-ha1258a1_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/ld_impl_linux-64-2.45.1-default_hbd61a6d_101.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcrc32c-1.1.2-h9c3ff4c_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libfreetype6-2.14.2-h73754d4_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglib-2.86.4-h6548e54_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libjxl-0.11.2-ha09017c_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnghttp2-1.67.0-had1ee68_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libopenblas-0.3.30-pthreads_h94d23a6_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libprotobuf-6.31.1-h49aed37_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libre2-11-2025.11.05-h7b12aa8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/librttopo-1.1.0-h46dd2a8_20.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libsqlite-3.52.0-hf4e2dac_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libtiff-4.7.1-h9d88235_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-16-2.15.2-hca6bf5a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libzopfli-1.0.3-h9c3ff4c_0.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/minizip-4.0.10-h05a5f5f_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/spdlog-1.16.0-hffee6e0_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/uriparser-0.9.8-hac33072_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xerces-c-3.3.0-hd9031aa_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libx11-1.8.13-he1eb515_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-event-stream-0.5.7-h28f887f_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-http-0.10.7-ha8fc4e3_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cyrus-sasl-2.1.28-hac629b4_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/freexl-2.0.0-h9dce30a_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/lcms2-2.18-h0c24ade_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libavif16-1.4.0-hcfa2d63_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libblas-3.11.0-5_h4a7cf45_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcurl-8.18.0-hcf29cc6_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libfreetype-2.14.2-ha770c72_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libglx-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libkml-1.3.0-haa4a5bd_1022.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-2.15.2-he237659_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/nss-3.118-h445c969_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openjpeg-2.5.4-h55fea9a_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openjph-0.26.3-h8d634f6_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/python-3.11.15-hd63d673_0_cpython.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/re2-2025.11.05-h5301d42_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/sqlite-3.52.0-h04a0ce9_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxext-1.3.7-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxrender-0.9.12-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxt-1.3.1-hb9d3cd8_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-auth-0.9.3-hef928c7_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-mqtt-0.13.3-hc63082f_11.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-core-cpp-1.16.1-h3a458e0_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/backports.zstd-1.3.0-py311h6b1f9c4_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/brotli-python-1.2.0-py311h66f275b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/certifi-2026.2.25-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cfitsio-4.6.3-ha0b56bc_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/charset-normalizer-3.4.5-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/fontconfig-2.17.1-h27c8c51_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/freetype-2.14.2-ha770c72_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/hdf5-1.14.6-nompi_h19486de_106.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/hpack-4.1.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/hyperframe-6.1.0-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/idna-3.11-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libarchive-3.8.5-gpl_hc2c16d8_100.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libcblas-3.11.0-5_h0358290_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgl-1.7.0-ha4b6fd6_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgrpc-1.73.1-h3288cfb_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/liblapack-3.11.0-5_h47877c9_openblas.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxml2-devel-2.15.2-he237659_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libxslt-1.1.43-h711ed8c_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/networkx-3.6.1-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openldap-2.6.10-hbde042b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/packaging-26.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/pillow-12.1.1-py311hf88fc01_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/proj-9.7.1-he0df7b0_3.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pysocks-1.7.1-pyha55dd90_7.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/setuptools-82.0.1-pyh332efcf_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxmu-1.3.1-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxp-1.0.4-hb03c661_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-c-s3-0.11.3-h06ab39a_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-identity-cpp-1.13.2-h3a5f585_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-common-cpp-12.11.0-h3d7a050_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/cairo-1.18.4-he90730b_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/geotiff-1.7.4-h1000f5c_4.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/h2-4.3.0-pyhcf101f3_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/kealib-1.6.2-hb2f3951_2.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/lazy-loader-0.5-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgoogle-cloud-2.39.0-hdb79228_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libnetcdf-4.9.3-nompi_hbf2fc22_104.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libpq-18.3-h9abb657_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libspatialite-5.1.0-gpl_h2abfd87_119.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/numpy-1.26.4-py311h64a7726_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/wheel-0.46.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/xorg-libxft-2.3.9-h355ab9f_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-crt-cpp-0.35.2-h8824e59_6.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-blobs-cpp-12.15.0-h2a74896_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/h5py-3.15.1-nompi_py311h0b2f468_101.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/imagecodecs-2026.3.6-py311h9837d23_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/imageio-2.37.0-pyhfb79c49_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-core-3.10.3-h1f481a6_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgoogle-cloud-storage-2.39.0-hdbdcf42_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/openmotif-2.3.8-hf55c2fc_5.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/pip-26.0.1-pyh8b19718_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/poppler-25.07.0-h13eef12_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/postgresql-18.3-h9d31465_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/scipy-1.17.1-py311hbe70eeb_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/urllib3-2.6.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/aws-sdk-cpp-1.11.606-hf38915e_9.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/azure-storage-files-datalake-cpp-12.13.0-hf38f1be_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/gdal-3.10.3-py311h34ccccb_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-fits-3.10.3-hec9d828_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-grib-3.10.3-hb20eef8_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-hdf4-3.10.3-ha810028_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-hdf5-3.10.3-h966a9c2_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-jp2openjpeg-3.10.3-hdd07572_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-pdf-3.10.3-h20efda7_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-pg-3.10.3-h55c2262_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-postgisraster-3.10.3-h55c2262_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-xls-3.10.3-hdee084c_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/requests-2.32.5-pyhcf101f3_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/noarch/tifffile-2026.3.3-pyhd8ed1ab_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-kea-3.10.3-h2bf108d_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-netcdf-3.10.3-ha526aae_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/scikit-image-0.26.0-np2py311h2a99c40_0.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/tiledb-2.29.2-h8821262_1.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-tiledb-3.10.3-h6c35068_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/libgdal-3.10.3-h3b705f5_27.conda
|
||||
https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/linux-64/isce2-2.6.4-py311h916084f_2.conda
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
_gamma_profile_die() {
|
||||
echo "$1" >&2
|
||||
return 1 2>/dev/null || exit 1
|
||||
}
|
||||
|
||||
_gamma_profile_prepend_path() {
|
||||
local _dir="$1"
|
||||
[ -d "${_dir}" ] || return 0
|
||||
case ":${PATH:-}:" in
|
||||
*":${_dir}:"*) ;;
|
||||
*) PATH="${_dir}:${PATH:-}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
_gamma_profile_home=""
|
||||
if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then
|
||||
_gamma_profile_home="${PYINT_GAMMA_HOME}"
|
||||
elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then
|
||||
_gamma_profile_home="${GAMMA_HOME}"
|
||||
else
|
||||
for _candidate in \
|
||||
/usr/local/GAMMA_SOFTWARE-20240627 \
|
||||
/usr/local/GAMMA_SOFTWARE-* \
|
||||
/opt/GAMMA_SOFTWARE-*; do
|
||||
[ -d "${_candidate}" ] || continue
|
||||
_gamma_profile_home="${_candidate}"
|
||||
break
|
||||
done
|
||||
fi
|
||||
|
||||
[ -n "${_gamma_profile_home}" ] || _gamma_profile_die "Gamma home not found."
|
||||
|
||||
export GAMMA_HOME="${_gamma_profile_home}"
|
||||
export MSP_HOME="${GAMMA_HOME}/MSP"
|
||||
export ISP_HOME="${GAMMA_HOME}/ISP"
|
||||
export DIFF_HOME="${GAMMA_HOME}/DIFF"
|
||||
export DISP_HOME="${GAMMA_HOME}/DISP"
|
||||
export LAT_HOME="${GAMMA_HOME}/LAT"
|
||||
export IPTA_HOME="${GAMMA_HOME}/IPTA"
|
||||
export GEO_HOME="${GAMMA_HOME}/GEO"
|
||||
|
||||
for _gamma_dir in \
|
||||
"${MSP_HOME}/bin" \
|
||||
"${ISP_HOME}/bin" \
|
||||
"${DIFF_HOME}/bin" \
|
||||
"${DISP_HOME}/bin" \
|
||||
"${LAT_HOME}/bin" \
|
||||
"${IPTA_HOME}/bin" \
|
||||
"${GEO_HOME}/bin" \
|
||||
"${MSP_HOME}/scripts" \
|
||||
"${ISP_HOME}/scripts" \
|
||||
"${DIFF_HOME}/scripts" \
|
||||
"${DISP_HOME}/scripts" \
|
||||
"${LAT_HOME}/scripts" \
|
||||
"${IPTA_HOME}/scripts" \
|
||||
"${GEO_HOME}/scripts"; do
|
||||
_gamma_profile_prepend_path "${_gamma_dir}"
|
||||
done
|
||||
|
||||
_gamma_profile_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
_gamma_profile_pyint_dir="${_gamma_profile_repo_root}/third_party/PyINT/pyint"
|
||||
_gamma_profile_prepend_path "${_gamma_profile_pyint_dir}"
|
||||
|
||||
export PATH
|
||||
export OS="linux64"
|
||||
export HDF5_DISABLE_VERSION_CHECK="1"
|
||||
export GNUTERM="${GNUTERM:-qt}"
|
||||
export GAMMA_RASTER="${GAMMA_RASTER:-BMP}"
|
||||
export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
unset _candidate
|
||||
unset _gamma_dir
|
||||
unset _gamma_profile_home
|
||||
unset _gamma_profile_pyint_dir
|
||||
unset _gamma_profile_repo_root
|
||||
unset -f _gamma_profile_die
|
||||
unset -f _gamma_profile_prepend_path
|
||||
|
||||
return 0 2>/dev/null || exit 0
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
def _load_manifest(path: str) -> Dict[str, Any]:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Gamma/PyINT runtime V1 runner scaffold.")
|
||||
parser.add_argument("--manifest", required=True, help="WSL path to the staged job manifest.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Emit manifest summary and exit.")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = _load_manifest(args.manifest)
|
||||
summary = {
|
||||
"runner": "gamma_pyint_runtime_v1",
|
||||
"manifest": args.manifest,
|
||||
"job_id": manifest.get("job_id"),
|
||||
"operation": manifest.get("operation"),
|
||||
"payload_keys": sorted((manifest.get("payload") or {}).keys()),
|
||||
"dry_run": bool(args.dry_run),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Mapping
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
return Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def _pipeline_script() -> Path:
|
||||
return _repo_root() / "backend" / "app" / "isce2_pipeline" / "run_lt1_dinsar_pipeline.py"
|
||||
|
||||
|
||||
def _load_manifest(path: str) -> Dict[str, Any]:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _require_text(container: Mapping[str, Any], key: str) -> str:
|
||||
value = str(container.get(key) or "").strip()
|
||||
if not value:
|
||||
raise ValueError(f"Manifest is missing required text field: {key}")
|
||||
return value
|
||||
|
||||
|
||||
def _append_optional(argv: List[str], flag: str, value: Any) -> None:
|
||||
if value is None:
|
||||
return
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return
|
||||
argv.extend([flag, text])
|
||||
|
||||
|
||||
def _build_pipeline_argv(payload: Mapping[str, Any], *, dry_run: bool = False) -> List[str]:
|
||||
paths = payload.get("paths") or {}
|
||||
params = payload.get("params") or {}
|
||||
pair_meta = payload.get("pair_meta") or {}
|
||||
|
||||
argv: List[str] = [
|
||||
sys.executable,
|
||||
_pipeline_script().as_posix(),
|
||||
_require_text(paths, "task_dir_wsl"),
|
||||
"--task-name",
|
||||
_require_text(payload, "task_alias"),
|
||||
"--output-prefix",
|
||||
_require_text(payload, "task_alias"),
|
||||
"--work-dir",
|
||||
_require_text(paths, "work_dir_wsl"),
|
||||
"--output-dir",
|
||||
_require_text(paths, "output_dir_wsl"),
|
||||
"--orbit-output-dir",
|
||||
_require_text(paths, "orbit_output_dir_wsl"),
|
||||
]
|
||||
|
||||
_append_optional(argv, "--orbit-root", paths.get("orbit_root_wsl"))
|
||||
_append_optional(argv, "--dem", paths.get("dem_path_wsl"))
|
||||
if bool(params.get("force")):
|
||||
argv.append("--force")
|
||||
_append_optional(argv, "--target-grid-size-m", params.get("target_grid_size_m"))
|
||||
_append_optional(argv, "--bbox", params.get("bbox"))
|
||||
_append_optional(argv, "--coh-threshold", params.get("coh_threshold"))
|
||||
_append_optional(argv, "--bbox-margin", params.get("bbox_margin"))
|
||||
_append_optional(argv, "--wavelength", params.get("wavelength"))
|
||||
_append_optional(argv, "--orbit-margin-sec", params.get("orbit_margin_sec"))
|
||||
_append_optional(argv, "--reference-satellite", pair_meta.get("master_satellite"))
|
||||
_append_optional(argv, "--secondary-satellite", pair_meta.get("slave_satellite"))
|
||||
if dry_run:
|
||||
argv.append("--dry-run")
|
||||
return argv
|
||||
|
||||
|
||||
def _build_child_env() -> Dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env_root = Path(sys.executable).resolve().parents[1]
|
||||
proj_data = env_root / "share" / "proj"
|
||||
if proj_data.exists():
|
||||
env["PROJ_DATA"] = proj_data.as_posix()
|
||||
return env
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="ISCE2 runtime V1 runner.")
|
||||
parser.add_argument("--manifest", required=True, help="WSL path to the staged job manifest.")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Pass --dry-run through to the pipeline.")
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest = _load_manifest(args.manifest)
|
||||
payload = manifest.get("payload") or {}
|
||||
runner_summary = {
|
||||
"runner": "isce2_runtime_v1",
|
||||
"manifest": args.manifest,
|
||||
"job_id": manifest.get("job_id"),
|
||||
"operation": manifest.get("operation"),
|
||||
"task_alias": payload.get("task_alias"),
|
||||
"pair_key": payload.get("pair_key"),
|
||||
"dry_run": bool(args.dry_run),
|
||||
}
|
||||
print(json.dumps(runner_summary, ensure_ascii=False))
|
||||
|
||||
pipeline_path = _pipeline_script()
|
||||
if not pipeline_path.exists():
|
||||
raise FileNotFoundError(f"ISCE2 pipeline script not found: {pipeline_path}")
|
||||
|
||||
argv = _build_pipeline_argv(payload, dry_run=bool(args.dry_run))
|
||||
print(json.dumps({"pipeline_argv": argv}, ensure_ascii=False))
|
||||
completed = subprocess.run(
|
||||
argv,
|
||||
env=_build_child_env(),
|
||||
check=False,
|
||||
)
|
||||
return int(completed.returncode or 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,151 @@
|
||||
# 当前状态快照
|
||||
|
||||
更新时间:2026-04-25
|
||||
|
||||
## 1. 项目形态
|
||||
|
||||
当前项目已经形成以下稳定形态:
|
||||
|
||||
- 主界面仍然是地图工作区。
|
||||
- 顶级生产入口已经统一为“生产管理”。
|
||||
- “生产管理”同时包含:
|
||||
- D-InSAR 运行
|
||||
- 时序 InSAR 运行
|
||||
- D-InSAR 产物
|
||||
- 时序 InSAR 产物
|
||||
- 时序入口的前端显示名已经统一为“时序 InSAR”。
|
||||
- 当前时序链路默认接入的是 SBAS 实现,而不是传统 PS-InSAR 单一路径。
|
||||
|
||||
## 2. 当前生产引擎
|
||||
|
||||
### D-InSAR
|
||||
|
||||
- `sarscape`
|
||||
当前可用,已有 19 个发布产品进入 catalog。
|
||||
|
||||
- `isce2`
|
||||
已完成托管式接入,运行在 WSL 共享运行时中。
|
||||
|
||||
- `gamma / pyint`
|
||||
运行时接口与目录已经预留,后续可继续落生产设计。
|
||||
|
||||
### 时序 InSAR
|
||||
|
||||
- 当前产品名称统一为“时序 InSAR”。
|
||||
- 当前默认执行路径是 SBAS。
|
||||
- 后续可在该顶级入口下继续扩展 `psinsar`、`sbas-insar` 等类型。
|
||||
|
||||
## 3. 结果目录现状
|
||||
|
||||
当前结果目录已经统一收口到 `D:\production_results`:
|
||||
|
||||
```text
|
||||
D:\production_results
|
||||
├─ dinsar
|
||||
├─ timeseries
|
||||
└─ _quarantine
|
||||
```
|
||||
|
||||
当前健康检查中的目录状态:
|
||||
|
||||
- `D:\production_results\dinsar`
|
||||
存在,catalog 正常。
|
||||
|
||||
- `D:\production_results\timeseries`
|
||||
存在,catalog 正常。
|
||||
|
||||
- `D:\production_results\_quarantine`
|
||||
已作为统一隔离根目录纳入配置模型。
|
||||
|
||||
说明:
|
||||
|
||||
- `backend\result_products` 旧目录已经不是当前文件系统事实来源。
|
||||
- 数据库 `result_products` 表仍然保留,是发布产品登记表,不要与旧目录混淆。
|
||||
|
||||
## 4. 当前启动自维护链路
|
||||
|
||||
后端启动时会依次执行:
|
||||
|
||||
1. 数据库自维护
|
||||
2. SQLAlchemy 会话初始化
|
||||
3. 根目录登记同步
|
||||
4. manifest inventory 同步
|
||||
5. D-InSAR catalog 自举
|
||||
6. 时序 InSAR catalog 自举
|
||||
7. pairing cache 状态自举
|
||||
8. 启动健康检查
|
||||
|
||||
这说明当前系统已经不是“纯手工维护目录 + 手工修库”的模式,而是有稳定的启动自维护链路。
|
||||
|
||||
## 5. 2026-04-25 现场健康摘要
|
||||
|
||||
本次审计现场读取到的核心状态如下:
|
||||
|
||||
- 数据库:
|
||||
- `ok = true`
|
||||
- `schema_ok = true`
|
||||
- `postgis_ok = true`
|
||||
- `required_table_count = 42`
|
||||
|
||||
- D-InSAR catalog:
|
||||
- `storage_root = D:\production_results\dinsar`
|
||||
- `manifest_count = 19`
|
||||
- `db_count = 19`
|
||||
- `needs_rebuild = false`
|
||||
|
||||
- 时序 InSAR catalog:
|
||||
- `storage_root = D:\production_results\timeseries`
|
||||
- `manifest_count = 0`
|
||||
- `db_count = 0`
|
||||
- `needs_rebuild = false`
|
||||
|
||||
- 产品包:
|
||||
- `total_count = 19`
|
||||
- `canonical_schema = insar.product-package/v1`
|
||||
- 缺失 manifest / publish dir / processor / runtime / native output 均为 0
|
||||
|
||||
- WSL 共享运行时:
|
||||
- `shared_distro = Ubuntu-24.04`
|
||||
- `shared_conda_env_name = insar_wsl_v1`
|
||||
- `shared_python_path = /home/administrator/miniconda3/envs/insar_wsl_v1/bin/python`
|
||||
- `required_runtime_count = 2`
|
||||
- `healthy_runtime_count = 2`
|
||||
|
||||
- 配对系统:
|
||||
- `status = READY`
|
||||
- `scene_count = 1500`
|
||||
- `pair_count = 91737`
|
||||
- `dirty_scene_count = 0`
|
||||
|
||||
## 6. 现场源目录状态
|
||||
|
||||
当前健康检查确认以下源目录可访问:
|
||||
|
||||
- `D:\LuTan1_Image_Pool`
|
||||
- `D:\LT1_data_lsarorbit`
|
||||
- `D:\DInSARResult`
|
||||
|
||||
即当前 `source_roots` 为全绿状态,没有不可访问目录。
|
||||
|
||||
## 7. 当前数据库设计是否匹配
|
||||
|
||||
结论:
|
||||
|
||||
- 当前 ORM 与现场数据库匹配。
|
||||
- 当前 catalog、结果包、WSL 运行时和 pairing 状态与现有架构一致。
|
||||
- 当前数据库自维护机制适合现在这批“新增字段、统一结果目录、增加运行时登记”的改动。
|
||||
|
||||
边界:
|
||||
|
||||
- 它仍然只是“增量补齐型”自维护,不是全功能 migration 框架。
|
||||
- 如果后续要做字段改名、类型调整、约束重构,仍然需要显式迁移方案。
|
||||
|
||||
## 8. 当前最需要保持一致的事实
|
||||
|
||||
以下几件事已经是当前系统事实,后续文档和代码都应围绕它们展开:
|
||||
|
||||
- 顶级生产入口是“生产管理”,不是“生产规划”里的临时子入口。
|
||||
- 时序 InSAR 是顶级产品族,当前默认接入 SBAS。
|
||||
- 结果发布根目录是 `RESULT_PUBLISH_ROOT`,不是历史散落目录。
|
||||
- WSL 运行时是共享模型,当前共享环境为 `insar_wsl_v1`。
|
||||
- 数据库自维护默认保守,不自动做破坏性重建。
|
||||
@@ -0,0 +1,203 @@
|
||||
# 数据库自维护审计
|
||||
|
||||
审计日期:2026-04-25
|
||||
|
||||
## 1. 审计目标
|
||||
|
||||
本次审计关注两件事:
|
||||
|
||||
1. 当前数据库自维护机制,是否仍然符合现在的系统设计。
|
||||
2. 当前现场数据库,是否已经与现有 ORM / catalog / 运行时设计对齐。
|
||||
|
||||
## 2. 当前自维护机制实际做什么
|
||||
|
||||
代码入口位于 `backend/app/db_maintenance.py`,系统启动时由 `backend/app/main.py` 调用。
|
||||
|
||||
当前自维护能力包括:
|
||||
|
||||
- 自动创建 `postgis` 扩展
|
||||
- 自动创建缺失表
|
||||
- 自动补齐缺失列
|
||||
- 自动执行 `backend/migrations/001` 到 `006`
|
||||
- 自动引导管理员账号
|
||||
- 自动灌入灾害点数据
|
||||
|
||||
当前启动链路中的相关步骤:
|
||||
|
||||
1. `ensure_database_ready(...)`
|
||||
2. `database.init_db()`
|
||||
3. 根目录登记同步
|
||||
4. manifest inventory 同步
|
||||
5. D-InSAR / 时序 catalog 自举
|
||||
6. pairing state 自举
|
||||
7. 健康检查
|
||||
|
||||
## 3. 当前机制不做什么
|
||||
|
||||
当前实现不是完整迁移框架,它不会自动处理:
|
||||
|
||||
- 字段改名
|
||||
- 字段类型变更
|
||||
- 可空性从宽到严
|
||||
- 索引 / 约束漂移修补
|
||||
- 旧字段 / 旧表删除
|
||||
|
||||
破坏性重建只有在以下两个开关同时为 `true` 时才允许:
|
||||
|
||||
```env
|
||||
DB_SCHEMA_RESET_ON_MISMATCH=true
|
||||
DB_SCHEMA_RESET_CONFIRM=true
|
||||
```
|
||||
|
||||
默认情况下这是关闭的,符合当前内网生产环境“保守自维护”的要求。
|
||||
|
||||
## 4. 本次现场检查方法
|
||||
|
||||
本次检查直接用当前项目代码读取现场数据库并执行:
|
||||
|
||||
- `inspect_database_structure(...)`
|
||||
- `ensure_database_ready(settings.DATABASE_URL, bootstrap_admin=False, seed_hazard=False)`
|
||||
- `get_health_status(include_external=False, include_details=True, refresh=True)`
|
||||
|
||||
使用的是项目当前 `.env` 中配置的 Python 解释器和数据库连接。
|
||||
|
||||
## 5. 现场检查结果
|
||||
|
||||
### 5.1 schema 结构检查
|
||||
|
||||
结果:
|
||||
|
||||
- `mismatch = false`
|
||||
- `reason_count = 0`
|
||||
- `required_table_count = 42`
|
||||
- `missing_tables = []`
|
||||
- `extra_tables = []`
|
||||
- `missing_columns = {}`
|
||||
- `type_mismatches = []`
|
||||
- `nullable_mismatches = []`
|
||||
|
||||
结论:
|
||||
|
||||
- 现场数据库结构与当前 ORM 一致。
|
||||
- 当前新增的结果包、运行时、catalog 相关字段已经在数据库中落稳。
|
||||
|
||||
### 5.2 自维护执行结果
|
||||
|
||||
结果:
|
||||
|
||||
- `schema_reset = false`
|
||||
- `mismatch_detected = false`
|
||||
- `added_columns = []`
|
||||
- `bootstrap_initialized = false`
|
||||
|
||||
启动时仍会执行以下 SQL 文件:
|
||||
|
||||
- `001_st_intersection_agg.sql`
|
||||
- `002_spatial_functions.sql`
|
||||
- `003_pairing_enhancement.sql`
|
||||
- `004_pairing_refactor.sql`
|
||||
- `005_pairing_task_trace.sql`
|
||||
- `006_result_pairing_trace.sql`
|
||||
|
||||
结论:
|
||||
|
||||
- 当前数据库已处于“无需修补”的稳定状态。
|
||||
- 启动自维护仍会重复执行迁移 SQL,因此这些 SQL 文件必须继续保持幂等。
|
||||
|
||||
### 5.3 健康检查结果
|
||||
|
||||
结果摘要:
|
||||
|
||||
- `health.ok = true`
|
||||
- `database.ok = true`
|
||||
- `database.schema_ok = true`
|
||||
- `database.postgis_ok = true`
|
||||
- `dinsar_result_catalog.ok = true`
|
||||
- `timeseries_result_catalog.ok = true`
|
||||
- `dinsar_bridge.ok = true`
|
||||
- `source_roots.ok = true`
|
||||
- `product_packages.ok = true`
|
||||
- `wsl_runtime.ok = true`
|
||||
- `pairing_system.ok = true`
|
||||
|
||||
关键现场值:
|
||||
|
||||
- D-InSAR catalog:
|
||||
- `storage_root = D:\production_results\dinsar`
|
||||
- `manifest_count = 19`
|
||||
- `db_count = 19`
|
||||
|
||||
- 时序 InSAR catalog:
|
||||
- `storage_root = D:\production_results\timeseries`
|
||||
- `manifest_count = 0`
|
||||
- `db_count = 0`
|
||||
|
||||
- 产品包:
|
||||
- `total_count = 19`
|
||||
- `canonical_schema = insar.product-package/v1`
|
||||
- 所有缺失项计数均为 0
|
||||
|
||||
- WSL 共享运行时:
|
||||
- `shared_distro = Ubuntu-24.04`
|
||||
- `shared_conda_env_name = insar_wsl_v1`
|
||||
- `shared_python_path = /home/administrator/miniconda3/envs/insar_wsl_v1/bin/python`
|
||||
- `required_runtime_count = 2`
|
||||
- `healthy_runtime_count = 2`
|
||||
|
||||
## 6. 结论
|
||||
|
||||
结论很明确:
|
||||
|
||||
- 当前数据库自维护机制与当前系统状态相符。
|
||||
- 对于当前这轮改造引入的新增字段、catalog、结果包、WSL runtime 信息,它是足够的。
|
||||
- 当前现场数据库已经对齐当前 ORM 和结果目录设计。
|
||||
|
||||
这意味着:
|
||||
|
||||
- 现在可以继续在当前 schema 基础上推进 D-InSAR / 时序 InSAR 生产。
|
||||
- 不需要为了“数据库跟不上代码”而先清空库或强制重建。
|
||||
|
||||
## 7. 残余风险
|
||||
|
||||
虽然当前是对齐的,但仍有三个明确边界:
|
||||
|
||||
### 7.1 它不是 migration framework
|
||||
|
||||
后续如果要做以下改动,不能只靠当前自维护:
|
||||
|
||||
- 重命名字段
|
||||
- 修改字段类型
|
||||
- 增加更严格的非空约束
|
||||
- 重建索引或唯一约束
|
||||
- 删除旧结构
|
||||
|
||||
### 7.2 SQL 文件必须幂等
|
||||
|
||||
因为启动时会重复执行 `001` 到 `006`,任何新增 SQL 文件也必须遵守同样原则。
|
||||
|
||||
### 7.3 健康面板已成为设计约束的一部分
|
||||
|
||||
当前运维自检不只是“看数据库能不能连”,而是在验证:
|
||||
|
||||
- catalog 是否正常
|
||||
- product package 是否完整
|
||||
- WSL 运行时是否齐全
|
||||
- pairing trace 是否一致
|
||||
|
||||
因此后续只要改目录模型、结果包模型、运行时模型,就必须同步维护健康检查逻辑。
|
||||
|
||||
## 8. 建议
|
||||
|
||||
当前建议如下:
|
||||
|
||||
1. 保持 `DB_SCHEMA_RESET_ON_MISMATCH=false` 和 `DB_SCHEMA_RESET_CONFIRM=false`。
|
||||
2. 后续涉及 schema 破坏性调整时,单独编写受控迁移,不要指望启动自维护自动兜底。
|
||||
3. 新增数据库字段时,优先采用“可空 + 向后兼容 + 健康面板补校验”的方式推进。
|
||||
4. 每次结果目录、运行时 registry、catalog 设计变动后,都重新跑一次:
|
||||
- schema 检查
|
||||
- `ensure_database_ready(...)`
|
||||
- `GET /api/health`
|
||||
|
||||
## 9. 审计结语
|
||||
|
||||
截至 2026-04-25,数据库自维护机制与当前系统状态是匹配的,且现场数据库处于健康状态。当前更大的风险已经不在“数据库结构漂移”,而在后续若继续做结果模型或运行时模型重构时,是否同步维护 catalog、健康检查和文档。
|
||||
+252
-913
File diff suppressed because it is too large
Load Diff
+63
-92
@@ -1,117 +1,88 @@
|
||||
# 当前文档总览
|
||||
# 文档索引
|
||||
|
||||
这是 `docs/` 目录的统一入口文档。
|
||||
本索引用来区分“当前有效文档”和“历史参考文档”。
|
||||
|
||||
从现在开始,`docs/` 根目录只保留“当前仍在使用”的文档;历史过程材料、已完成实施记录、被新文档覆盖的旧方案,统一放入 `docs/archive/`。
|
||||
原则:
|
||||
|
||||
## 1. 当前有效文档
|
||||
- 以本页列为“当前有效”的文档,才作为现网架构、部署和运维的事实依据。
|
||||
- 没有列入“当前有效”区域的文档,默认只作为设计过程记录或历史参考。
|
||||
- 旧文档即使仍保留在 `docs/` 根目录,也不代表它仍然描述当前系统。
|
||||
|
||||
### 核心使用
|
||||
## 1. 当前有效
|
||||
|
||||
- **[README.md](../README.md)**
|
||||
系统主文档。优先阅读,包含整体功能、架构、快速开始和主要配置说明。
|
||||
- **[DEPLOYMENT.md](DEPLOYMENT.md)**
|
||||
部署与运维细节文档。适合正式部署、数据库初始化、服务启动和环境核对时使用。
|
||||
- **[PROJ_CONFIGURATION.md](PROJ_CONFIGURATION.md)**
|
||||
PROJ/GDAL 相关专项配置说明。只有在坐标转换或 PROJ 冲突时才需要单独查看。
|
||||
- [../README.md](../README.md)
|
||||
项目总览、当前架构和启动入口。
|
||||
|
||||
### 当前安全状态
|
||||
- [CURRENT_STATUS_20260425.md](CURRENT_STATUS_20260425.md)
|
||||
2026-04-25 的项目状态快照,包含结果目录、运行时、健康检查摘要。
|
||||
|
||||
- **[SECURITY_AUDIT_2026-03-12.md](SECURITY_AUDIT_2026-03-12.md)**
|
||||
当前有效的安全审计结论。已经把“开发机阶段可接受暴露”和“上线前必须修复的问题”分开说明,后续安全判断以这份为准。
|
||||
- [DEPLOYMENT.md](DEPLOYMENT.md)
|
||||
当前 Windows + PostgreSQL + WSL2 部署模型、环境变量和启动链路。
|
||||
|
||||
### 仍在推进的设计与任务
|
||||
- [DATABASE_SELF_MAINTENANCE_AUDIT_20260425.md](DATABASE_SELF_MAINTENANCE_AUDIT_20260425.md)
|
||||
当前数据库自维护机制与现场数据库一致性审计结果。
|
||||
|
||||
- **[DINSAR_ENHANCEMENT_TASKS.md](DINSAR_ENHANCEMENT_TASKS.md)**
|
||||
D-InSAR 多引擎生产增强与结果管理增强的总 Task 文档。覆盖前端架构、ENVI 六步链路、ISCE2 的 WSL 校验、运维自检与结果治理。
|
||||
- **[PAIRING_ENHANCEMENT_DESIGN.md](PAIRING_ENHANCEMENT_DESIGN.md)**
|
||||
配对能力增强设计。属于未完全落地的专项设计文档。
|
||||
- **[GAMMA_WSL2_INTEGRATION_PLAN.md](GAMMA_WSL2_INTEGRATION_PLAN.md)**
|
||||
GAMMA + WSL2 双引擎方案。属于后续扩展规划,不是当前默认运行链路。
|
||||
- **[PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md)**
|
||||
PyINT 生产引擎与 Gamma 精配对总体设计。明确现有多引擎架构下的接入边界、配置管理、数据库策略、运维自检扩展与前端入口分布。
|
||||
- **[PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md](PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md)**
|
||||
PyINT 生产引擎与 Gamma 精配对实施清单。按阶段拆分后端、前端、接口、运维与可选数据库任务,作为后续落地执行顺序。
|
||||
- **[PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md](PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md)**
|
||||
PyINT 输入资产适配设计。聚焦 `Task_*` 路径如何映射到 PyINT 工作区,以及 DEM、LT-1 精密轨道、运维自检和前端入口应如何纳入系统托管治理。
|
||||
- **[PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md](PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md)**
|
||||
LT-1 在 PyINT/Gamma 中 `coreg` 失败的轨道假设验证实验设计。固定输入和 DEM,只改变 `.slc.par` 的 state vector 处理方式,对照验证问题是否集中在导入后的轨道几何链条。
|
||||
- **[PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md](PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md)**
|
||||
LT-1 在 PyINT/Gamma 中 `init_offsetm` 失败的 DEM 几何链定位实验。聚焦 `HGTSIM / lt0 / mli0 / Samp` 的中间产物,区分 DEM 本体问题、DEM 几何映射链问题和中心 patch 选取问题。
|
||||
- **[WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md](WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md)**
|
||||
基于本机 `Ubuntu-24.04` WSL2、`isce2` / `mintpy` / `isce2_mintpy_v1` 实际环境核对后的 SBAS 集成落地方案,明确推荐运行时、workflow 补全顺序和正式产品边界。
|
||||
- **[ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md)**
|
||||
ISCE2 框架下 SBAS / 时序生产设计稿,说明如何从现有 `PS stack batch` 扩展到 `workflow + psinsar catalog` 的正式生产链路。
|
||||
- **[SBAS_PROGRESS_STATUS_20260406.md](SBAS_PROGRESS_STATUS_20260406.md)**
|
||||
SBAS 系统改造当前落地状态。优先查看这份文档以了解“已经做到哪一步、验证做到哪一步、下一步该补什么”。
|
||||
- **[SBAS_SYSTEM_INTEGRATION_AUDIT_20260406.md](SBAS_SYSTEM_INTEGRATION_AUDIT_20260406.md)**
|
||||
对现有系统接入 SBAS 的审计结果,聚焦现状缺口与最小可行接入点。
|
||||
- **[SBAS_SYSTEM_EMBEDDING_DESIGN_20260406.md](SBAS_SYSTEM_EMBEDDING_DESIGN_20260406.md)**
|
||||
SBAS 嵌入现有系统的详细设计稿,覆盖运行记录、作业编排、结果注册与前端页面边界。
|
||||
- **[SBAS_TASK_PATH_MODE_EVALUATION_20260406.md](SBAS_TASK_PATH_MODE_EVALUATION_20260406.md)**
|
||||
评估是否沿用现有 D-InSAR `task 路径 -> 生产` 模式,以及 SBAS 为什么必须增加 `prepare` 阶段。
|
||||
- **[SBAS_SYSTEM_CHANGE_RISK_ASSESSMENT_20260406.md](SBAS_SYSTEM_CHANGE_RISK_ASSESSMENT_20260406.md)**
|
||||
SBAS 系统改造风险评估,重点约束“新增不伤旧链”。
|
||||
- **[SBAS_RESULT_MANAGEMENT_AND_DISPLAY_SPEC_20260406.md](SBAS_RESULT_MANAGEMENT_AND_DISPLAY_SPEC_20260406.md)**
|
||||
SBAS 结果管理、编目和展示约定,明确发布级 bundle 与系统结果目录的关系。
|
||||
- **[SBAS_FRONTEND_UX_SPEC_20260406.md](SBAS_FRONTEND_UX_SPEC_20260406.md)**
|
||||
SBAS 生产页、结果页与分析页的前端交互设计。
|
||||
- **[SBAS_IMPLEMENTATION_TODO_20260406.md](SBAS_IMPLEMENTATION_TODO_20260406.md)**
|
||||
SBAS 系统改造任务清单,适合作为后续 Phase 2 / Phase 3 推进的执行列表。
|
||||
- [PROJ_CONFIGURATION.md](PROJ_CONFIGURATION.md)
|
||||
PROJ / GDAL 相关专项配置说明。
|
||||
|
||||
## 2. 已归档文档
|
||||
## 2. 当前执行中的核心设计
|
||||
|
||||
以下文档已移入 `docs/archive/`,原因通常是“历史记录”“已完成实施”或“被后续结论覆盖”。
|
||||
- [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md)
|
||||
多引擎结果目录、发布结构和 catalog 治理设计。
|
||||
|
||||
### 本轮归档
|
||||
- [WSL_RUNTIME_REFACTOR_DESIGN_20260422.md](WSL_RUNTIME_REFACTOR_DESIGN_20260422.md)
|
||||
WSL 共享运行时和 Broker 设计。
|
||||
|
||||
- `SECURITY_AUDIT_2026-03-04.md`
|
||||
- `SECURITY_AUDIT_ANALYSIS_2026-03-04.md`
|
||||
- `SECURITY_FIX_PLAN.md`
|
||||
- `SECURITY_FIX_PROGRESS.md`
|
||||
- `SECURITY_FIX_CHECKLIST.md`
|
||||
- `SECURITY_FIX_STAGE1_SUMMARY.md`
|
||||
- `SECURITY_OVERVIEW.md`
|
||||
- `HARDCODE_AUDIT_2026-03-05.md`
|
||||
- `LOG_MANAGEMENT_IMPLEMENTATION_2026-03-05.md`
|
||||
- [ISCE2_MANAGED_DINSAR_IMPLEMENTATION_20260424.md](ISCE2_MANAGED_DINSAR_IMPLEMENTATION_20260424.md)
|
||||
ISCE2 托管式 D-InSAR 落地说明。
|
||||
|
||||
### 既有归档
|
||||
- [ISCE2_PRODUCTION_RELIABILITY_HARDENING_DESIGN_20260424.md](ISCE2_PRODUCTION_RELIABILITY_HARDENING_DESIGN_20260424.md)
|
||||
ISCE2 生产链路稳定性修补与约束。
|
||||
|
||||
`docs/archive/` 中原有的旧设计、旧 TODO、阶段性分析和历史修复记录继续保留,仅作历史参考,不再视为当前执行依据。
|
||||
- [DINSAR_ENHANCEMENT_TASKS.md](DINSAR_ENHANCEMENT_TASKS.md)
|
||||
D-InSAR 生产相关任务清单。
|
||||
|
||||
## 3. 建议阅读顺序
|
||||
## 3. 时序 InSAR / SBAS
|
||||
|
||||
### 新接手项目
|
||||
- [ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md)
|
||||
- [ISCE2_SBAS_PRODUCT_SPEC.md](ISCE2_SBAS_PRODUCT_SPEC.md)
|
||||
|
||||
1. 先读 [README.md](../README.md)
|
||||
2. 再读 [DEPLOYMENT.md](DEPLOYMENT.md)
|
||||
3. 若涉及上线前风险,再读 [SECURITY_AUDIT_2026-03-12.md](SECURITY_AUDIT_2026-03-12.md)
|
||||
说明:
|
||||
|
||||
### 正式部署
|
||||
- 当前前端顶级显示名已经统一为“时序 InSAR”。
|
||||
- 当前默认接入仍然是 SBAS 路径,因此这两份文档仍然有效。
|
||||
- `SBAS_*` 命名的一批旧文档大多是阶段性设计材料,不再作为总事实来源。
|
||||
|
||||
1. [README.md](../README.md)
|
||||
2. [DEPLOYMENT.md](DEPLOYMENT.md)
|
||||
3. [PROJ_CONFIGURATION.md](PROJ_CONFIGURATION.md)(仅在需要时)
|
||||
## 4. 配对与前端导航
|
||||
|
||||
### 后续功能规划
|
||||
- [PAIRING_ENHANCEMENT_DESIGN.md](PAIRING_ENHANCEMENT_DESIGN.md)
|
||||
- [PAIRING_SYSTEM_REFACTOR_PLAN_20260413.md](PAIRING_SYSTEM_REFACTOR_PLAN_20260413.md)
|
||||
- [FRONTEND_NAVIGATION_ARCHITECTURE.md](FRONTEND_NAVIGATION_ARCHITECTURE.md)
|
||||
|
||||
1. [DINSAR_ENHANCEMENT_TASKS.md](DINSAR_ENHANCEMENT_TASKS.md)
|
||||
2. [PAIRING_ENHANCEMENT_DESIGN.md](PAIRING_ENHANCEMENT_DESIGN.md)
|
||||
3. [PAIRING_SYSTEM_REFACTOR_PLAN_20260413.md](PAIRING_SYSTEM_REFACTOR_PLAN_20260413.md)
|
||||
4. [ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md)
|
||||
5. [WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md](WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md)
|
||||
6. [GAMMA_WSL2_INTEGRATION_PLAN.md](GAMMA_WSL2_INTEGRATION_PLAN.md)
|
||||
7. [PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md)
|
||||
8. [PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md](PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md)
|
||||
9. [PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md](PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md)
|
||||
10. [PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md](PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md)
|
||||
11. [PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md](PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md)
|
||||
## 5. Gamma / PyINT / WSL 预留
|
||||
|
||||
## 4. 后续维护规则
|
||||
- [GAMMA_WSL2_INTEGRATION_PLAN.md](GAMMA_WSL2_INTEGRATION_PLAN.md)
|
||||
- [PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md)
|
||||
- [PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md](PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md)
|
||||
- [PYINT_GAMMA_AB_FINDINGS_20260420.md](PYINT_GAMMA_AB_FINDINGS_20260420.md)
|
||||
|
||||
- `docs/` 根目录只放当前有效文档。
|
||||
- 已完成实施总结、阶段性修复过程、旧审计分析、被覆盖方案,统一放入 `docs/archive/`。
|
||||
- 新增文档前,优先判断是否可以并入现有文档,避免再次堆出多份并行说明。
|
||||
- 如果某份文档只记录一次任务过程,而不是长期参考资料,不应继续留在根目录。
|
||||
说明:
|
||||
|
||||
**最后整理**:2026-03-13
|
||||
- 这部分主要作为 Gamma / PyINT 接入的预留设计和实验记录。
|
||||
- 真正的当前运行时事实,以 `WSL_RUNTIME_REFACTOR_DESIGN_20260422.md`、`.env` 和健康面板为准。
|
||||
|
||||
## 6. 安全与审计
|
||||
|
||||
- [SECURITY_AUDIT_2026-03-12.md](SECURITY_AUDIT_2026-03-12.md)
|
||||
|
||||
## 7. 历史参考
|
||||
|
||||
以下材料默认视为历史设计记录或过程文档:
|
||||
|
||||
- `docs/archive/`
|
||||
- `SBAS_*`
|
||||
- `WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md`
|
||||
- `项目汇报.md`
|
||||
- 各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*`
|
||||
|
||||
最后更新:2026-04-25
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
# ISCE2 Managed D-InSAR Implementation 2026-04-24
|
||||
|
||||
## Scope
|
||||
|
||||
This document records the backend implementation that moves ISCE2 D-InSAR onto the same managed production lifecycle already used by ENVI.
|
||||
|
||||
The goal of this round is not to redesign the scientific core of ISCE2. The goal is to make ISCE2 production operationally healthy:
|
||||
|
||||
- submitted from the same production-management entry
|
||||
- tracked by the same run / item / execution tables
|
||||
- published into the same managed result tree
|
||||
- scanned by the same catalog rebuild path
|
||||
- compatible with database self-maintenance and health checks
|
||||
|
||||
## Implemented changes
|
||||
|
||||
### 1. Submission path
|
||||
|
||||
`/dinsar-production/run` now sends both `sarscape` and `isce2` through `dinsar_production_service.create_run(...)`.
|
||||
|
||||
This means ISCE2 no longer bypasses production management when it is launched from the normal UI entry.
|
||||
|
||||
### 2. Production run lifecycle
|
||||
|
||||
`dinsar_production_service.create_run(...)` now supports:
|
||||
|
||||
- `sarscape`
|
||||
- `isce2`
|
||||
|
||||
For ISCE2 it creates:
|
||||
|
||||
- `dinsar_production_runs`
|
||||
- `dinsar_production_run_items`
|
||||
- `dinsar_production_executions`
|
||||
- workflow steps with job type `ISCE2_RUN`
|
||||
|
||||
No schema migration was required for this round.
|
||||
|
||||
### 3. Managed ISCE2 controller
|
||||
|
||||
`JOB_TYPE_ISCE2_RUN` now has two modes:
|
||||
|
||||
- legacy queued mode: no `production_run_id`, keep the old shared queued handler
|
||||
- managed production mode: `production_run_id` present, use the new WSL production controller
|
||||
|
||||
The managed controller is responsible for:
|
||||
|
||||
- creating one execution row per selected task
|
||||
- forcing a managed run directory for each task
|
||||
- writing `execution_manifest.json`
|
||||
- writing `current/isce2__<profile>.json`
|
||||
- publishing successful run directories
|
||||
- rebuilding the result catalog after publish
|
||||
|
||||
### 4. Managed result layout
|
||||
|
||||
Each successful ISCE2 task now lands in the same managed D-InSAR tree as ENVI:
|
||||
|
||||
```text
|
||||
D:\production_results\dinsar\<pair_key>\
|
||||
current\
|
||||
isce2__lt1_stripmap.json
|
||||
runs\
|
||||
<run_key>\
|
||||
.dinsar_run.json
|
||||
execution_manifest.json
|
||||
manifest.json
|
||||
native\
|
||||
workflow\
|
||||
...
|
||||
export\
|
||||
<task>_disp.tif
|
||||
<task>_coh.tif
|
||||
assets\
|
||||
disp\
|
||||
disp.tif
|
||||
coh\
|
||||
coh.tif
|
||||
preview\
|
||||
thumb.webp
|
||||
```
|
||||
|
||||
Rules:
|
||||
|
||||
- `native/` keeps the raw ISCE2 work and export outputs
|
||||
- `assets/` holds the standardized files consumed by the system
|
||||
- `manifest.json` is still written by `publish_from_sources(...)`
|
||||
- `result_products` rows are still created by `rebuild_catalog(...)`
|
||||
|
||||
### 5. ISCE2 layout normalization
|
||||
|
||||
`backend/app/services/dinsar_result_layout_service.py` now includes `normalize_isce2_run_layout(...)`.
|
||||
|
||||
Behavior:
|
||||
|
||||
- copy raw ISCE2 `*_disp.tif` into `assets/disp/disp.tif`
|
||||
- copy raw ISCE2 `*_coh.tif` into `assets/coh/coh.tif`
|
||||
- keep the raw files under `native/export`
|
||||
- rewrite run metadata / execution manifest / current pointer / package manifest when those files already exist
|
||||
|
||||
## Completion semantics
|
||||
|
||||
`rerun_mode=unfinished_only` for ISCE2 now checks the managed pointer:
|
||||
|
||||
```text
|
||||
<DINSAR_PRODUCT_DIR>\<pair_key>\current\isce2__<profile>.json
|
||||
```
|
||||
|
||||
This is intentional.
|
||||
|
||||
Legacy raw ISCE2 folders without a managed current pointer are no longer treated as a completed production result. That keeps skip logic aligned with production management rather than with stray historical outputs.
|
||||
|
||||
## Database and self-check compatibility
|
||||
|
||||
This implementation keeps the existing operational contract:
|
||||
|
||||
- no new tables
|
||||
- no destructive migration
|
||||
- no change to `result_products` / `result_assets` / `result_issues` schema
|
||||
- no change to `dinsar_production_runs` schema
|
||||
|
||||
Operational compatibility points:
|
||||
|
||||
- catalog state is still rooted at `settings.DINSAR_PRODUCT_DIR`
|
||||
- health checks still use the same catalog-state and manifest-tree logic
|
||||
- database self-maintenance still works because manifests remain canonical and catalog rebuild remains the only registration path
|
||||
|
||||
## Validation completed in this round
|
||||
|
||||
- syntax compilation in memory for the edited Python modules
|
||||
- direct module import validation for:
|
||||
- `backend.app.dinsar_engines.isce2_engine`
|
||||
- `backend.app.services.dinsar_result_layout_service`
|
||||
- `backend.app.services.dinsar_production_service`
|
||||
- `backend.app.routers.dinsar_production`
|
||||
- `backend.app.services.job_handlers`
|
||||
|
||||
## Remaining work
|
||||
|
||||
- add hard cancellation for an in-flight WSL subprocess
|
||||
- add Gamma D-InSAR on top of the same managed controller contract
|
||||
- decide whether `lt1_stripmap` should stay as the public profile code or be renamed in a later migration round
|
||||
@@ -0,0 +1,445 @@
|
||||
# ISCE2 生产可靠性加固设计 2026-04-24
|
||||
|
||||
## 1. 背景
|
||||
|
||||
当前 ISCE2 生产链路已经具备:
|
||||
|
||||
- 统一的 WSL runtime 入口
|
||||
- 共享 conda 环境 `insar_wsl_v1`
|
||||
- 结果发布到 `D:\production_results`
|
||||
- 统一 catalog / 自检 / 运维面板
|
||||
|
||||
但这条链路仍存在一个核心问题:
|
||||
|
||||
系统现在更像是“子进程执行器”,还不是“结果可靠性交付器”。
|
||||
|
||||
也就是说,当前成功判定主要依赖:
|
||||
|
||||
- WSL runner 返回码是否为 `0`
|
||||
- 输出目录是否存在
|
||||
- 自动发布阶段是否“尝试过”
|
||||
|
||||
这会导致几类假成功:
|
||||
|
||||
1. ISCE2 子进程返回 `0`,但没有形成可发布的主产物,任务仍显示完成。
|
||||
2. 历史运行只留下目录和 `.dinsar_run.json`,但主产物已损坏或缺失,“只跑未完成” 仍会跳过。
|
||||
3. 运维自检用的是一套路径规则,真实运行用的是另一套路径规则,面板和实际行为可能不一致。
|
||||
|
||||
本设计的目标不是重写 ISCE2 算法链,而是把 ISCE2 生产从“命令执行成功”收紧到“结果可验收、可发布、可重跑、可运维”。
|
||||
|
||||
## 2. 本轮范围
|
||||
|
||||
本轮只做可靠性加固,不做算法重写。
|
||||
|
||||
### 纳入本轮
|
||||
|
||||
- ISCE2 原生产结果的验收契约
|
||||
- “只跑未完成” 判定收紧
|
||||
- 自动发布阶段收紧
|
||||
- 健康检查与真实运行入口对齐
|
||||
- 文档、自检、日志字段同步
|
||||
|
||||
### 暂不纳入本轮
|
||||
|
||||
- stripmap/SNAPHU/PICKLE 恢复链的算法级改写
|
||||
- 与 ENVI/IDL 流水线逐步逐项数值对比
|
||||
- Gamma 生产实现细节
|
||||
- MintPy / 时序生产链的算法可靠性评估
|
||||
|
||||
## 3. 当前问题拆解
|
||||
|
||||
### 3.1 成功判定过宽
|
||||
|
||||
当前 `backend/app/dinsar_engines/isce2_engine.py` 中,单对任务成功主要由 `rc == 0` 决定。
|
||||
|
||||
现状后果:
|
||||
|
||||
- 只要 runner 没报错,就写入 `.dinsar_run.json`
|
||||
- 只要 `output_dir` 被加入 `output_dirs`,后续任务层就进入自动发布
|
||||
- 自动发布即使 `processed == 0`,任务最终仍可标记为 `COMPLETED`
|
||||
|
||||
这不符合“生产完成”的系统定义。
|
||||
|
||||
### 3.2 未完成重跑判定过宽
|
||||
|
||||
当前 `_has_completed_task_result()` 只看:
|
||||
|
||||
- 历史 `run_dir`
|
||||
- `.dinsar_run.json`
|
||||
- `output_dir` 目录存在
|
||||
|
||||
它不看:
|
||||
|
||||
- 主产物是否存在
|
||||
- 主产物是否可读
|
||||
- 该结果是否满足发布器识别规则
|
||||
|
||||
因此历史半成品会被误判为“已完成”。
|
||||
|
||||
### 3.3 健康检查与真实运行入口不一致
|
||||
|
||||
当前存在两套 stripmapApp 路径逻辑:
|
||||
|
||||
- 健康检查依赖 `ISCE2_STRIPMAP_APP`
|
||||
- 真正运行时在 WSL 内通过 `import isce` 动态定位
|
||||
|
||||
这会导致:
|
||||
|
||||
- 面板报错但实际能跑
|
||||
- 面板通过但实际解释器/包位置已经漂移
|
||||
|
||||
### 3.4 手工恢复链缺少显式边界
|
||||
|
||||
当前 `run_lt1_dinsar_pipeline.py` 通过复制 `PICKLE` 中文件并改写 XML 来衔接:
|
||||
|
||||
- `filter -> filter_high_band`
|
||||
- `unwrap -> ionosphere`
|
||||
|
||||
这条链路短期可用,但它对 ISCE2 内部文件结构有隐含依赖。
|
||||
|
||||
本轮不重写它,但必须把它从“隐式成功”改成“显式验收”。
|
||||
|
||||
## 4. 目标状态
|
||||
|
||||
ISCE2 生产链路调整后,系统对“完成”的定义变为:
|
||||
|
||||
1. WSL runner 执行完成。
|
||||
2. 原生产物通过本地验收。
|
||||
3. 验收通过的产物可被统一发布器识别。
|
||||
4. 自动发布成功写入标准包。
|
||||
5. 自检面板能看到 runtime、native output、publish package 三层状态一致。
|
||||
|
||||
换句话说:
|
||||
|
||||
- `rc == 0` 只表示“执行成功”
|
||||
- “任务完成” 必须是“执行成功 + 产物验收成功 + 发布链成功”
|
||||
|
||||
## 5. 具体改造方案
|
||||
|
||||
### 5.1 新增 ISCE2 原生产物验收层
|
||||
|
||||
新增一个独立验收模块,建议位置:
|
||||
|
||||
- `backend/app/services/isce2_result_validator.py`
|
||||
|
||||
职责:
|
||||
|
||||
- 不参与计算
|
||||
- 不参与数据库写入
|
||||
- 只负责判断一个 `output_dir` 是否是“可发布、可交付”的 ISCE2 原生结果
|
||||
|
||||
#### 输入
|
||||
|
||||
- `output_dir`
|
||||
- `task_alias`
|
||||
- `engine_code`
|
||||
- `profile_code`
|
||||
- `run_key`
|
||||
|
||||
#### 核心检查项
|
||||
|
||||
必选检查:
|
||||
|
||||
- `output_dir` 目录存在
|
||||
- 至少存在一个符合发布规则的主位移产物
|
||||
- 主产物文件大小大于 0
|
||||
- 主产物能被 GDAL 打开
|
||||
- 若存在配对的相干产品,则记录为辅助资产
|
||||
|
||||
建议主产物识别规则直接复用发布器当前的规则,避免两套规则分叉:
|
||||
|
||||
- `*_disp.tif`
|
||||
- 可选 `*_coh.tif`
|
||||
|
||||
#### 输出结构
|
||||
|
||||
返回统一 `acceptance_result`:
|
||||
|
||||
```json
|
||||
{
|
||||
"accepted": true,
|
||||
"primary_file": "D:/.../Task_xxx_disp.tif",
|
||||
"asset_files": [
|
||||
"D:/.../Task_xxx_disp.tif",
|
||||
"D:/.../Task_xxx_coh.tif"
|
||||
],
|
||||
"issues": [],
|
||||
"metrics": {
|
||||
"primary_exists": true,
|
||||
"primary_readable": true,
|
||||
"asset_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### 运行时行为
|
||||
|
||||
`isce2_engine.run()` 中:
|
||||
|
||||
- `rc != 0` 直接失败
|
||||
- `rc == 0` 后必须执行验收
|
||||
- 只有 `accepted == true` 才计入 `pairs_processed`
|
||||
- 只有验收通过的 `output_dir` 才进入 `output_dirs`
|
||||
|
||||
这样“runner 成功但没产物”的情况会被收口为失败。
|
||||
|
||||
### 5.2 收紧 `.dinsar_run.json` 语义
|
||||
|
||||
当前 `.dinsar_run.json` 只是运行元数据,需要补成“运行 + 验收”的统一侧写。
|
||||
|
||||
本轮不改数据库表结构,先改 sidecar 内容。
|
||||
|
||||
新增字段建议:
|
||||
|
||||
```json
|
||||
{
|
||||
"run_key": "run_xxx",
|
||||
"engine_code": "isce2",
|
||||
"profile_code": "lt1_stripmap",
|
||||
"runtime_id": "isce2_runtime_v1",
|
||||
"output_dir": "D:/.../native",
|
||||
"acceptance": {
|
||||
"accepted": true,
|
||||
"accepted_at": "2026-04-24T15:00:00Z",
|
||||
"primary_file": "D:/.../Task_xxx_disp.tif",
|
||||
"asset_files": [
|
||||
"D:/.../Task_xxx_disp.tif",
|
||||
"D:/.../Task_xxx_coh.tif"
|
||||
],
|
||||
"issues": [],
|
||||
"metrics": {
|
||||
"primary_exists": true,
|
||||
"primary_readable": true,
|
||||
"asset_count": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
原则:
|
||||
|
||||
- `.dinsar_run.json` 不再只表示“跑过”
|
||||
- 它要能表达“验收通过 / 未通过”
|
||||
|
||||
### 5.3 重写“只跑未完成”判定
|
||||
|
||||
将 `_has_completed_task_result()` 调整为 `_has_accepted_task_result()`。
|
||||
|
||||
新的跳过条件:
|
||||
|
||||
1. 历史 run 的 `.dinsar_run.json` 存在
|
||||
2. `engine_code / profile_code` 匹配当前请求
|
||||
3. `acceptance.accepted == true`
|
||||
4. `acceptance.primary_file` 存在且可读
|
||||
|
||||
只有满足以上条件,才视为“已完成”。
|
||||
|
||||
#### 不再采用的旧判断
|
||||
|
||||
- 仅凭 `output_dir` 存在
|
||||
- 仅凭 `.dinsar_run.json` 存在
|
||||
|
||||
#### 这样做的效果
|
||||
|
||||
- 目录空壳不会再被跳过
|
||||
- 历史损坏结果会重新进入生产
|
||||
- “只跑未完成” 的含义变成“只跳过已验收成功的对”
|
||||
|
||||
### 5.4 自动发布阶段从“尽力而为”改为“交付闭环”
|
||||
|
||||
当前自动发布逻辑即使 `processed == 0` 也只记日志。
|
||||
|
||||
本轮改为:
|
||||
|
||||
- 引擎层只把“已验收通过”的 `output_dir` 交给发布器
|
||||
- 任务层以“验收通过数量”作为发布期望值
|
||||
- 发布结果必须满足:
|
||||
- `processed == accepted_output_count`
|
||||
- `failed == 0`
|
||||
|
||||
若不满足,则任务整体失败。
|
||||
|
||||
#### 原则
|
||||
|
||||
- 自动发布不再是可有可无的附属步骤
|
||||
- 对托管生产来说,发布成功是交付定义的一部分
|
||||
|
||||
#### 失败语义
|
||||
|
||||
若 native output 已验收通过,但标准包发布失败:
|
||||
|
||||
- 任务状态标记为 `FAILED`
|
||||
- 保留 native output 目录
|
||||
- 日志明确写出:
|
||||
- accepted count
|
||||
- processed count
|
||||
- failed count
|
||||
|
||||
这样后续可以修发布器或手动重建 catalog,但不会再出现“任务完成却没有结果”的误导状态。
|
||||
|
||||
### 5.5 健康检查与真实运行入口统一
|
||||
|
||||
目标:只保留一套 runtime 事实来源。
|
||||
|
||||
#### 调整方向
|
||||
|
||||
`ISCE2_STRIPMAP_APP` 不再作为运行事实来源,只保留为兼容字段。
|
||||
|
||||
健康检查改成:
|
||||
|
||||
1. 从 `wsl_runtime_registry` 读取共享 Python
|
||||
2. 在对应 Python 中执行:
|
||||
|
||||
```python
|
||||
import isce
|
||||
from pathlib import Path
|
||||
print(Path(isce.__file__).resolve().parent / "applications" / "stripmapApp.py")
|
||||
```
|
||||
|
||||
3. 检查该路径是否存在
|
||||
4. 将解析出的真实路径回显到自检面板
|
||||
|
||||
#### 自检项调整
|
||||
|
||||
保留:
|
||||
|
||||
- WSL 可用
|
||||
- 共享 Python 可执行
|
||||
- `import isce` 成功
|
||||
- stripmapApp 动态解析成功
|
||||
- runner 脚本存在
|
||||
|
||||
弱化:
|
||||
|
||||
- 直接检查 `.env` 中的 `ISCE2_STRIPMAP_APP`
|
||||
|
||||
#### 自检返回建议新增字段
|
||||
|
||||
- `resolved_stripmap_app`
|
||||
- `resolved_isce_package_path`
|
||||
- `python_matches_shared`
|
||||
- `runtime_id`
|
||||
|
||||
### 5.6 数据库与自维护策略
|
||||
|
||||
本轮尽量不动数据库 schema。
|
||||
|
||||
原因:
|
||||
|
||||
- 当前问题核心在文件系统验收语义,不在表结构
|
||||
- 用户已经明确要求兼容数据库自维护与自检
|
||||
- 现阶段没有必要为了可靠性问题引入一轮 schema 变更风险
|
||||
|
||||
#### 本轮原则
|
||||
|
||||
- 不新增非必要列
|
||||
- 先把验收信息写入 `.dinsar_run.json`
|
||||
- 数据库继续通过现有 `result_products / result_assets / result_issues` 表表达发布结果
|
||||
|
||||
#### 若后续必须加列
|
||||
|
||||
只允许:
|
||||
|
||||
- nullable
|
||||
- 可由 `_add_missing_columns()` 自动补齐
|
||||
- 不要求清库
|
||||
|
||||
但这不属于本轮必须项。
|
||||
|
||||
### 5.7 运维自检扩展
|
||||
|
||||
为避免“native 有结果但 publish 不一致”的灰区,本轮建议在健康面板新增两个聚合视角。
|
||||
|
||||
#### 1. runtime 视角
|
||||
|
||||
- 共享 distro / Python / stripmapApp 动态解析是否一致
|
||||
|
||||
#### 2. product 视角
|
||||
|
||||
- 已入库产品是否存在 `native_output_dir`
|
||||
- `native_output_dir` 是否仍存在
|
||||
- `manifest_path` 是否存在
|
||||
|
||||
#### 3. 新增建议计数项
|
||||
|
||||
- `missing_native_output_count`
|
||||
- `missing_manifest_count`
|
||||
- `missing_runtime_count`
|
||||
|
||||
其中前两项已有基础,重点是让 ISCE2 runtime 检查结果和产品检查结果在面板里串起来。
|
||||
|
||||
## 6. 文件落点
|
||||
|
||||
### 核心代码
|
||||
|
||||
- `backend/app/dinsar_engines/isce2_engine.py`
|
||||
- `backend/app/services/job_handlers.py`
|
||||
- `backend/app/services/result_catalog_service.py`
|
||||
- `backend/app/services/health_service.py`
|
||||
- `backend/app/services/wsl_service.py`
|
||||
|
||||
### 新增模块
|
||||
|
||||
- `backend/app/services/isce2_result_validator.py`
|
||||
|
||||
### 文档与运维
|
||||
|
||||
- `docs/ISCE2_PRODUCTION_RELIABILITY_HARDENING_DESIGN_20260424.md`
|
||||
|
||||
## 7. 分阶段实施
|
||||
|
||||
### Phase 1: 成功判定收口
|
||||
|
||||
- 新增 `isce2_result_validator`
|
||||
- `rc == 0` 后执行验收
|
||||
- 未验收通过不写成功结果
|
||||
- `.dinsar_run.json` 补 `acceptance`
|
||||
|
||||
### Phase 2: 重跑语义收口
|
||||
|
||||
- `_has_completed_task_result()` 改为验收驱动
|
||||
- “只跑未完成” 只跳过已验收成功的结果
|
||||
|
||||
### Phase 3: 发布闭环收口
|
||||
|
||||
- 自动发布必须与验收通过数量对齐
|
||||
- `processed == 0` 不再仅告警,改为任务失败
|
||||
|
||||
### Phase 4: 自检对齐
|
||||
|
||||
- stripmapApp 改为动态解析检查
|
||||
- 面板展示 runtime 实际解析结果
|
||||
|
||||
### Phase 5: 后续科学性加固
|
||||
|
||||
不在本轮立即落地,但建议保留后续专题:
|
||||
|
||||
- PIKCLE 恢复链版本敏感性评估
|
||||
- bbox / geoPosting 的科学性收紧
|
||||
- 与 ENVI/IDL 基线链的阶段性对比
|
||||
|
||||
## 8. 验收标准
|
||||
|
||||
改造完成后,至少满足以下场景:
|
||||
|
||||
1. WSL runner 返回 `0`,但没有 `*_disp.tif`。
|
||||
- 结果:任务失败,不得显示完成。
|
||||
|
||||
2. 历史 run 目录仍在,但 `*_disp.tif` 已被删掉。
|
||||
- 结果:`只跑未完成` 不应跳过,应重新生产。
|
||||
|
||||
3. 健康面板显示 stripmapApp 路径。
|
||||
- 结果:必须来自共享 Python 动态解析,不再依赖旧硬编码。
|
||||
|
||||
4. native output 验收通过,但发布器没识别到包。
|
||||
- 结果:任务失败,日志明确指出 publish mismatch。
|
||||
|
||||
5. 自检面板与实际 runtime 使用的 Python / runner / runtime_id 一致。
|
||||
|
||||
## 9. 本轮结论
|
||||
|
||||
本轮实现不应该再把 ISCE2 生产理解为“跑完脚本”,而应理解为:
|
||||
|
||||
“执行成功 -> 原生结果验收通过 -> 标准包发布成功 -> 系统完成交付”。
|
||||
|
||||
先把这个闭环收紧,再谈后续算法科学性优化,顺序才是健康的。
|
||||
@@ -0,0 +1,388 @@
|
||||
# 多引擎生产结果管理与路径设计
|
||||
|
||||
更新日期:2026-04-24
|
||||
|
||||
## 1. 目标
|
||||
|
||||
本设计解决三个长期问题:
|
||||
|
||||
- 让 `ENVI / SARscape`、`ISCE2`、后续 `Gamma` 的原生输出可以共存,但系统只消费统一结果包。
|
||||
- 让生产、发布、catalog、数据库自维护、运维自检都围绕同一套受管理目录工作。
|
||||
- 让同一对影像可以保留多引擎、多 profile、多次重跑结果,而不会互相覆盖。
|
||||
|
||||
本设计不是要求所有引擎输出长得一样,而是要求系统托管的结果目录、指针、manifest 和数据库语义一致。
|
||||
|
||||
## 2. 核心原则
|
||||
|
||||
- 结果管理以“标准包”为中心,不以某个引擎的原生目录命名为中心。
|
||||
- 原生输出和系统发布资产必须分层,不能混在一起。
|
||||
- `pair_key` 或 `stack_key` 是业务稳定主键,`run_key` 或 `run_id` 是一次具体生产实例。
|
||||
- catalog 只扫描受管理发布根中的 `manifest.json`,不再把“历史扫描目录”和“正式生产目录”混用。
|
||||
- `current` 指针只在“验收通过并成功发布”后更新。
|
||||
- “只跑未完成”只看当前引擎和当前 profile 对应的 `current` 指针,不看裸目录是否存在。
|
||||
|
||||
## 3. 根路径设计
|
||||
|
||||
系统只保留一套正式受管理结果根:
|
||||
|
||||
```text
|
||||
D:\production_results\
|
||||
dinsar\
|
||||
timeseries\
|
||||
_quarantine\
|
||||
```
|
||||
|
||||
环境变量语义固定为:
|
||||
|
||||
- `RESULT_PUBLISH_ROOT`
|
||||
结果发布总根,默认 `D:\production_results`
|
||||
- `DINSAR_PRODUCT_DIR`
|
||||
D-InSAR 正式发布根,默认 `D:\production_results\dinsar`
|
||||
- `TIMESERIES_PRODUCT_DIR`
|
||||
时序 InSAR 正式发布根,默认 `D:\production_results\timeseries`
|
||||
- `RESULT_QUARANTINE_ROOT`
|
||||
结果隔离区,默认 `D:\production_results\_quarantine`
|
||||
|
||||
以下路径不再视为正式结果根:
|
||||
|
||||
- `MONITOR_DINSAR_DIRS`
|
||||
只保留为历史导入或兼容扫描入口,不再作为新生产的默认落盘位置
|
||||
- `Task_xxx\dinsar_results`
|
||||
只保留迁移期兼容语义,最终退出正式生产链
|
||||
|
||||
## 4. 目录分层
|
||||
|
||||
### 4.1 D-InSAR
|
||||
|
||||
```text
|
||||
D:\production_results\dinsar\<pair_key>\
|
||||
current\
|
||||
current.json
|
||||
sarscape__custom6.json
|
||||
isce2__lt1_dinsar_production_v1.json
|
||||
gamma__lt1_gamma_dinsar_v1.json
|
||||
runs\
|
||||
<run_key>\
|
||||
execution_manifest.json
|
||||
manifest.json
|
||||
native\
|
||||
...
|
||||
assets\
|
||||
disp\
|
||||
disp.tif
|
||||
coh\
|
||||
coh.tif
|
||||
preview\
|
||||
thumb.webp
|
||||
```
|
||||
|
||||
目录语义:
|
||||
|
||||
- `<pair_key>`
|
||||
同一对影像的稳定业务目录
|
||||
- `runs/<run_key>`
|
||||
一次具体生产
|
||||
- `native/`
|
||||
引擎原生输出,用于追溯、审计、复核,不直接作为系统消费入口
|
||||
- `assets/`
|
||||
系统统一消费的标准资产层
|
||||
- `manifest.json`
|
||||
正式 catalog 入库契约
|
||||
- `execution_manifest.json`
|
||||
生产执行记录
|
||||
- `current/*.json`
|
||||
当前可用结果指针
|
||||
|
||||
### 4.2 时序 InSAR
|
||||
|
||||
```text
|
||||
D:\production_results\timeseries\<stack_key>\
|
||||
current\
|
||||
current.json
|
||||
sbas__v1.json
|
||||
psinsar__v1.json
|
||||
runs\
|
||||
<run_id>\
|
||||
manifest.json
|
||||
native\
|
||||
...
|
||||
assets\
|
||||
...
|
||||
preview\
|
||||
thumb.webp
|
||||
```
|
||||
|
||||
这里沿用同一哲学,只是主键从 `pair_key` 换成 `stack_key`。
|
||||
|
||||
## 5. 指针设计
|
||||
|
||||
### 5.1 引擎级 current 指针
|
||||
|
||||
每个结果根都允许多个指针同时存在:
|
||||
|
||||
- `sarscape__custom6.json`
|
||||
- `isce2__lt1_dinsar_production_v1.json`
|
||||
- `gamma__lt1_gamma_dinsar_v1.json`
|
||||
|
||||
用途:
|
||||
|
||||
- 支持同一对影像保留多引擎结果
|
||||
- 支持前端按引擎切换结果
|
||||
- 支持“只跑未完成”按引擎和 profile 做跳过判断
|
||||
|
||||
### 5.2 默认 current 指针
|
||||
|
||||
`current.json` 不是必须立即启用。
|
||||
|
||||
建议语义:
|
||||
|
||||
- 只有在系统明确需要“默认展示结果”时才写入
|
||||
- 默认值由人工或策略层决定,而不是由最近一次运行自动覆盖
|
||||
|
||||
## 6. 原生输出与发布资产分层
|
||||
|
||||
这是本轮设计的关键约束。
|
||||
|
||||
### 6.1 原生输出层
|
||||
|
||||
原生输出层由引擎控制,但必须落到受管理 run 目录的 `native/` 下:
|
||||
|
||||
- ENVI / SARscape
|
||||
保留其成熟原生流程,但原生落盘最终收敛到 `native/`
|
||||
- ISCE2
|
||||
`stripmapApp` 原生输出、日志、中间文件全部进入 `native/`
|
||||
- Gamma
|
||||
后续接入时也遵循相同约束
|
||||
|
||||
### 6.2 标准资产层
|
||||
|
||||
系统统一暴露资产层:
|
||||
|
||||
- `assets/disp/disp.tif`
|
||||
- `assets/coh/coh.tif`
|
||||
- `preview/thumb.webp`
|
||||
|
||||
未来允许增加:
|
||||
|
||||
- `assets/conncomp/conncomp.tif`
|
||||
- `assets/debug/...`
|
||||
|
||||
但 catalog 识别仍以标准 `manifest.json` 为准,不直接猜原生文件结构。
|
||||
|
||||
## 7. D-InSAR 生产路径设计
|
||||
|
||||
### 7.1 生产服务拥有 run 根目录
|
||||
|
||||
对任意一个 `pair_key`,生产服务先创建稳定结果根:
|
||||
|
||||
```text
|
||||
<DINSAR_PRODUCT_DIR>\<pair_key>\runs\<run_key>\
|
||||
```
|
||||
|
||||
然后再把引擎执行和打包都约束到这个 run 目录下。
|
||||
|
||||
这意味着:
|
||||
|
||||
- `results_root_dir` 指向 `<DINSAR_PRODUCT_DIR>\<pair_key>`
|
||||
- `execution.output_dir` 指向 `<DINSAR_PRODUCT_DIR>\<pair_key>\runs\<run_key>`
|
||||
- 原生输出默认进入 `<run_dir>\native`
|
||||
|
||||
### 7.2 ISCE2 的路径设计
|
||||
|
||||
对正式生产链,`ISCE2_OUTPUT_ROOT` 不再承担“正式发布根”的职责。
|
||||
|
||||
建议语义调整为:
|
||||
|
||||
- `ISCE2_WORK_ROOT`
|
||||
仅用于调试、缓存、临时工作空间,或 standalone 调试运行
|
||||
- 正式生产时,由编排层直接下发 `<run_dir>\native` 作为原生输出目录
|
||||
- `ISCE2_OUTPUT_ROOT`
|
||||
只保留 fallback 含义,不再默认等于 `DINSAR_PRODUCT_DIR`
|
||||
|
||||
### 7.3 ENVI 的路径设计
|
||||
|
||||
ENVI 当前仍可能写入 `Task_xxx\dinsar_results`。迁移完成后应统一为:
|
||||
|
||||
- 原生结果写入 `<run_dir>\native`
|
||||
- 标准资产与 manifest 写入 `<run_dir>`
|
||||
|
||||
迁移期允许 packager 从历史目录导入,但这只是兼容,不再作为正式目标结构。
|
||||
|
||||
## 8. ISCE2 D-InSAR 生产设计约束
|
||||
|
||||
### 8.1 生产 profile
|
||||
|
||||
建议正式生产 profile 收敛为:
|
||||
|
||||
- `lt1_dinsar_production_v1`
|
||||
|
||||
不再为 ISCE2 复制 ENVI 的 `metatask/custom6` 形态。ISCE2 自己已经是成熟 orchestrator,项目层不应再人为拆成外部 6 步工作流。
|
||||
|
||||
### 8.2 流程边界
|
||||
|
||||
正式流程分四层:
|
||||
|
||||
1. `preflight`
|
||||
2. `native_isce2`
|
||||
3. `postprocess`
|
||||
4. `acceptance_publish`
|
||||
|
||||
其中:
|
||||
|
||||
- `preflight`
|
||||
负责解析主从影像、DEM、精轨、bbox、多视参数
|
||||
- `native_isce2`
|
||||
必须是一趟原生 `stripmapApp` 跑通到 geocode
|
||||
- `postprocess`
|
||||
负责位移导出、质量掩膜、稳定点精化、标准资产生成
|
||||
- `acceptance_publish`
|
||||
负责写 manifest、更新 current 指针、交给 catalog
|
||||
|
||||
### 8.3 参数设计
|
||||
|
||||
ISCE2 应继承 ENVI 已经验证过的工程经验,而不是机械复制 ENVI 步骤。
|
||||
|
||||
保留或新增的正式生产参数:
|
||||
|
||||
- `force`
|
||||
- `target_grid_size_m`
|
||||
- `bbox`
|
||||
- `bbox_margin`
|
||||
- `orbit_margin_sec`
|
||||
- `coh_threshold`
|
||||
- `unwrap_coh_threshold`
|
||||
- `stable_point_coh_threshold`
|
||||
- `stable_point_count`
|
||||
- `refinement_mode`
|
||||
|
||||
不建议前端暴露的参数:
|
||||
|
||||
- `wavelength`
|
||||
LT1 固定值,只读
|
||||
- `filter_method`
|
||||
v1 固定为工程默认值,不向前端开放高级切换
|
||||
|
||||
### 8.4 ENVI 经验映射
|
||||
|
||||
ENVI 现有工程经验中,值得保留的是:
|
||||
|
||||
- `target_resolution -> looks` 换算逻辑
|
||||
- 基于高相干稳定点的精化思想
|
||||
- 导出后再做稳定性等待与验收
|
||||
|
||||
不应该照搬的是:
|
||||
|
||||
- 把 ISCE2 拆成 ENVI 式的外部 6 步调度
|
||||
- 依赖扫目录“捞”中间文件来续跑
|
||||
|
||||
## 9. Catalog、数据库与自检
|
||||
|
||||
### 9.1 Catalog
|
||||
|
||||
catalog 只扫描:
|
||||
|
||||
- `DINSAR_PRODUCT_DIR`
|
||||
- `TIMESERIES_PRODUCT_DIR`
|
||||
|
||||
并且只认 `manifest.json`。
|
||||
|
||||
`MONITOR_DINSAR_DIRS` 不再参与正式 catalog rebuild,只保留外部导入用途。
|
||||
|
||||
### 9.2 数据库
|
||||
|
||||
现有数据库设计可以继续使用,不需要推倒重建:
|
||||
|
||||
- `result_products`
|
||||
- `result_assets`
|
||||
- `result_issues`
|
||||
- `result_catalog_states`
|
||||
- `dinsar_production_runs`
|
||||
- `dinsar_production_run_items`
|
||||
- `dinsar_production_executions`
|
||||
|
||||
约束:
|
||||
|
||||
- `results_root_dir` 继续指向 `<pair_key>` 根目录
|
||||
- `publish_root_dir` 继续指向 `DINSAR_PRODUCT_DIR`
|
||||
- `native_output_dir` 由 manifest 持久化
|
||||
- 启动自维护仍采用“增量补列,不破坏现有数据”的方式
|
||||
|
||||
### 9.3 运维自检
|
||||
|
||||
运维自检继续围绕 `storage_root` 工作,不围绕任务目录工作。
|
||||
|
||||
重点检查三层:
|
||||
|
||||
- 结果发布根是否存在
|
||||
- catalog 状态是否健康
|
||||
- manifest 数、数据库记录数、needs_rebuild 是否一致
|
||||
|
||||
后续再加两项专项检查:
|
||||
|
||||
- `product_packages`
|
||||
检查 manifest 契约和资产完整性
|
||||
- `wsl_runtime`
|
||||
检查 WSL shared runtime、Python、ISCE2/Gamma 入口一致性
|
||||
|
||||
## 10. 迁移顺序
|
||||
|
||||
### Phase 1
|
||||
|
||||
- 固化本设计
|
||||
- 不打断当前正在运行的 ENVI 生产
|
||||
- 仅允许文档、审计、只读检查
|
||||
|
||||
### Phase 2
|
||||
|
||||
- D-InSAR run 目录强制收敛到 `<pair_key>\runs\<run_key>`
|
||||
- 把引擎原生输出收敛到 `native/`
|
||||
- packager 只在 run 目录内生成标准资产和 manifest
|
||||
- current 指针更新改为“验收通过后再写”
|
||||
|
||||
### Phase 3
|
||||
|
||||
- ISCE2 正式 profile 切换到 `lt1_dinsar_production_v1`
|
||||
- 停止 synthetic resume 方案
|
||||
- 增加 preflight、稳定点精化、结果验收
|
||||
|
||||
### Phase 4
|
||||
|
||||
- ENVI 生产链迁移到同一结果结构
|
||||
- legacy `Task_xxx\dinsar_results` 退出正式生产路径
|
||||
|
||||
### Phase 5
|
||||
|
||||
- catalog rebuild
|
||||
- compat 视图同步
|
||||
- 运维自检补齐结果包专项检查
|
||||
|
||||
## 11. 当前执行约束
|
||||
|
||||
当前如果 ENVI 仍在生产中,以下修改必须等待生产结束后再做:
|
||||
|
||||
- 修改 `ISCE2_OUTPUT_ROOT` 默认语义
|
||||
- 修改 D-InSAR 生产 run 目录结构
|
||||
- 修改 current 指针判定逻辑
|
||||
- 修改 packager 输入路径和 in-place 规则
|
||||
- 修改 catalog rebuild 入口和默认扫描路径
|
||||
- 修改 ENVI 正式落盘位置
|
||||
|
||||
当前可以立即做的事情:
|
||||
|
||||
- 文档固化
|
||||
- 代码审计
|
||||
- 只读自检
|
||||
- 改造计划维护
|
||||
|
||||
原因很简单:本轮改造会动到结果路径、发布契约、跳过判定和指针语义。生产进行中切这些逻辑,风险不是“某个功能出错”,而是直接污染正在生成的结果根。
|
||||
|
||||
## 12. 结论
|
||||
|
||||
健康的方案不是让所有引擎“输出一样”,而是让系统消费的“受管理结果结构”一样。
|
||||
|
||||
这样之后:
|
||||
|
||||
- ENVI 可以继续保留自己的成熟原生算法链
|
||||
- ISCE2 可以按自己的原生工作流稳定生产
|
||||
- Gamma 接入时不需要再改目录哲学
|
||||
- 数据库自维护、catalog、自检、前端结果工作台都只围绕一套标准结果结构工作
|
||||
@@ -0,0 +1,274 @@
|
||||
# WSL Runtime 改造设计 2026-04-22
|
||||
|
||||
## 1. 背景
|
||||
|
||||
当前 `ISCE2` 与 `Gamma/PyINT` 的 WSL 执行存在三个根问题:
|
||||
|
||||
1. 执行边界仍是 shell 字符串,后端直接把命令交给 `wsl.exe ... bash -lc`。
|
||||
2. `.env` 直接持有 WSL Python、pipeline、Gamma 环境脚本等执行型配置,配置权和执行权耦合。
|
||||
3. WSL 内的 conda 环境没有被定义成正式运行时资产,实际运行依赖“当前机器上碰巧可用的环境”。
|
||||
|
||||
这会带来四类风险:
|
||||
|
||||
- 参数或路径一旦逃逸 quote,直接进入 shell 解释边界。
|
||||
- 引擎服务自己拼命令,执行治理无法集中落地。
|
||||
- `ISCE2` 和 `Gamma/PyINT` 虽然物理上可以共用一套 WSL/conda,但逻辑上没有独立运行时标识,审计和迁移困难。
|
||||
- WSL 环境漂移无法被健康检查和运维流程正式识别。
|
||||
|
||||
## 2. 目标
|
||||
|
||||
本次改造采用:
|
||||
|
||||
- 单 WSL distro
|
||||
- 单共享 conda env
|
||||
- 双逻辑 runtime
|
||||
- 单一 WSL broker 出口
|
||||
|
||||
目标形态:
|
||||
|
||||
```text
|
||||
业务服务
|
||||
-> Engine Adapter
|
||||
-> WslBroker
|
||||
-> RuntimeRegistry
|
||||
-> JobManifest
|
||||
-> WSL Runner
|
||||
-> shared conda env: insar_wsl_v1
|
||||
```
|
||||
|
||||
其中:
|
||||
|
||||
- `ISCE2` 使用 `isce2_runtime_v1`
|
||||
- `Gamma/PyINT` 使用 `gamma_pyint_runtime_v1`
|
||||
|
||||
二者可以共用:
|
||||
|
||||
- `WSL_DISTRO=Ubuntu-24.04`
|
||||
- `WSL_SHARED_CONDA_ENV=insar_wsl_v1`
|
||||
|
||||
但不能共用“逻辑运行时定义”。
|
||||
|
||||
## 3. 运行时模型
|
||||
|
||||
### 3.1 共享运行时
|
||||
|
||||
共享运行时负责:
|
||||
|
||||
- distro 选择
|
||||
- 共享 conda env 名称
|
||||
- 共享 Python 路径
|
||||
- broker 作业落盘目录
|
||||
|
||||
建议配置:
|
||||
|
||||
```env
|
||||
WSL_DISTRO=Ubuntu-24.04
|
||||
WSL_SHARED_CONDA_ENV=insar_wsl_v1
|
||||
WSL_SHARED_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
WSL_BROKER_JOB_ROOT=<backend/runtime/wsl_jobs>
|
||||
```
|
||||
|
||||
### 3.2 逻辑 runtime
|
||||
|
||||
每个逻辑 runtime 固定:
|
||||
|
||||
- `runtime_id`
|
||||
- `engine_code`
|
||||
- `runner`
|
||||
- `allowed_operations`
|
||||
- 固定 profile 路径
|
||||
- 固定审计标识
|
||||
|
||||
当前规划:
|
||||
|
||||
| runtime_id | engine | runner | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `isce2_runtime_v1` | `isce2` | `deploy/wsl/runners/isce2_runner.py` | LT-1 条带 D-InSAR |
|
||||
| `gamma_pyint_runtime_v1` | `pyint` | `deploy/wsl/runners/gamma_pyint_runner.py` | Gamma / PyINT |
|
||||
|
||||
Gamma 固定 profile:
|
||||
|
||||
- `deploy/wsl/profiles/gamma_env.sh`
|
||||
|
||||
注意:Gamma profile 属于部署资产,不再建议继续由 `.env` 动态指定脚本路径。
|
||||
|
||||
## 4. 执行模型
|
||||
|
||||
### 4.1 Broker
|
||||
|
||||
Broker 负责:
|
||||
|
||||
- 解析 `runtime_id`
|
||||
- 写入结构化 `job manifest`
|
||||
- 生成固定 `argv`
|
||||
- 通过 `wsl.exe --exec` 执行 runner
|
||||
|
||||
禁止的旧模式:
|
||||
|
||||
- 服务层直接拼 `bash -lc "..."`
|
||||
- 服务层直接决定 `source` 哪个环境脚本
|
||||
- 服务层直接把 `.env` 中的 Python 绝对路径当成执行入口
|
||||
|
||||
### 4.2 Manifest
|
||||
|
||||
建议 manifest 固定包含:
|
||||
|
||||
- `job_id`
|
||||
- `runtime_id`
|
||||
- `engine_code`
|
||||
- `operation`
|
||||
- `created_at`
|
||||
- `payload`
|
||||
|
||||
payload 中再放:
|
||||
|
||||
- 输入路径
|
||||
- 工作路径
|
||||
- 输出路径
|
||||
- 参数
|
||||
- 操作人
|
||||
- 引擎内部业务上下文
|
||||
|
||||
### 4.3 Runner
|
||||
|
||||
Runner 是 WSL 内的固定可信入口。
|
||||
|
||||
阶段划分:
|
||||
|
||||
1. V1:只承担 manifest 读取、参数校验和 dry-run 输出
|
||||
2. V2:接入实际 pipeline / PyINT / Gamma
|
||||
3. V3:补齐固定 profile、版本指纹、错误分类与审计摘要
|
||||
|
||||
## 5. Conda 环境治理
|
||||
|
||||
### 5.1 基本原则
|
||||
|
||||
- 不原地继续污染旧环境
|
||||
- 先冻结现有环境
|
||||
- 再建立共享正式环境 `insar_wsl_v1`
|
||||
- 旧环境只保留回滚用途
|
||||
|
||||
### 5.2 版本资产
|
||||
|
||||
`deploy/wsl/conda/` 下至少维护:
|
||||
|
||||
- `insar_wsl_v1.environment.yml`
|
||||
- `insar_wsl_v1.explicit.lock`
|
||||
- `insar_wsl_v1.fingerprint.json`
|
||||
|
||||
说明:
|
||||
|
||||
- `environment.yml` 用于人工维护和审阅
|
||||
- `explicit.lock` 用于精确重建
|
||||
- `fingerprint.json` 用于健康检查、审计和漂移检测
|
||||
|
||||
### 5.3 Gamma
|
||||
|
||||
Gamma 通常不作为 conda 包管理。
|
||||
|
||||
因此建议:
|
||||
|
||||
- Gamma 以固定安装路径存在于 WSL
|
||||
- 由 `deploy/wsl/profiles/gamma_env.sh` 统一注入环境变量
|
||||
- 不再由 `.env` 直接提供动态 profile 路径
|
||||
|
||||
## 6. 配置迁移策略
|
||||
|
||||
### 6.1 新增配置
|
||||
|
||||
新增统一配置:
|
||||
|
||||
```env
|
||||
WSL_DISTRO=Ubuntu-24.04
|
||||
WSL_SHARED_CONDA_ENV=insar_wsl_v1
|
||||
WSL_SHARED_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
|
||||
WSL_BROKER_JOB_ROOT=...
|
||||
ISCE2_RUNTIME_ID=isce2_runtime_v1
|
||||
PYINT_RUNTIME_ID=gamma_pyint_runtime_v1
|
||||
```
|
||||
|
||||
### 6.2 旧配置处理
|
||||
|
||||
旧配置短期保留,用于兼容现有业务链路:
|
||||
|
||||
- `ISCE2_PYTHON`
|
||||
- `PYINT_WSL_PYTHON`
|
||||
- `PYINT_GAMMA_ENV_SCRIPT`
|
||||
- `ISCE2_WSL_DISTRO`
|
||||
- `PYINT_WSL_DISTRO`
|
||||
|
||||
但应逐步降级为:
|
||||
|
||||
- 历史兼容字段
|
||||
- 启动检查告警来源
|
||||
- 回滚开关
|
||||
|
||||
而不再是长期正式执行入口。
|
||||
|
||||
## 7. 目录规划
|
||||
|
||||
新增目录:
|
||||
|
||||
```text
|
||||
deploy/wsl/
|
||||
conda/
|
||||
profiles/
|
||||
runners/
|
||||
backend/app/services/
|
||||
wsl_runtime_registry.py
|
||||
wsl_broker.py
|
||||
```
|
||||
|
||||
运行时作业目录:
|
||||
|
||||
```text
|
||||
backend/runtime/wsl_jobs/<runtime_id>/<operation>/<job_id>.json
|
||||
```
|
||||
|
||||
## 8. 实施阶段
|
||||
|
||||
### Phase 1
|
||||
|
||||
- 新增 `WSL shared runtime` 配置
|
||||
- 新增 `RuntimeRegistry`
|
||||
- 新增 `WslBroker`
|
||||
- 新增 runner/profile/conda 目录落点
|
||||
|
||||
### Phase 2
|
||||
|
||||
- 迁移 `ISCE2` 到 `runtime_id + manifest + broker`
|
||||
- 去掉服务层 shell 字符串拼接
|
||||
|
||||
### Phase 3
|
||||
|
||||
- 迁移 `Gamma/PyINT`
|
||||
- 把 `PYINT_GAMMA_ENV_SCRIPT` 收敛到固定 profile
|
||||
|
||||
### Phase 4
|
||||
|
||||
- 增加环境指纹校验
|
||||
- 健康检查显示 runtime 漂移
|
||||
- 审计日志记录 runtime_id / fingerprint / runner / manifest
|
||||
|
||||
## 9. 本轮代码落点
|
||||
|
||||
本轮先完成 V1 骨架:
|
||||
|
||||
- `backend/app/services/wsl_service.py`
|
||||
- 新增 `run_wsl_exec()`,支持 `wsl.exe --exec`
|
||||
- `backend/app/services/wsl_runtime_registry.py`
|
||||
- 新增共享 runtime 与双逻辑 runtime 定义
|
||||
- `backend/app/services/wsl_broker.py`
|
||||
- 新增 manifest staging 与 broker 执行骨架
|
||||
- `deploy/wsl/runners/*`
|
||||
- 新增 runner scaffold
|
||||
- `deploy/wsl/profiles/gamma_env.sh`
|
||||
- 固定 profile 落点
|
||||
- `deploy/wsl/conda/README.md`
|
||||
- 约束 conda 环境治理落点
|
||||
|
||||
下一步迁移优先级建议:
|
||||
|
||||
1. `ISCE2`
|
||||
2. `Gamma/PyINT`
|
||||
3. 健康检查与审计
|
||||
+50
-28
@@ -304,11 +304,16 @@ function App() {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const frameId = requestAnimationFrame(() => {
|
||||
const invalidateMap = () => {
|
||||
mapRef.current?.invalidateSize(false);
|
||||
});
|
||||
};
|
||||
const frameId = requestAnimationFrame(invalidateMap);
|
||||
const timeoutId = window.setTimeout(invalidateMap, 120);
|
||||
|
||||
return () => cancelAnimationFrame(frameId);
|
||||
return () => {
|
||||
cancelAnimationFrame(frameId);
|
||||
window.clearTimeout(timeoutId);
|
||||
};
|
||||
}, [isStandaloneLeftPage, leftPanelWidth, rightPanelWidth]);
|
||||
|
||||
const getVisibleLayerRefs = useCallback(() => ({
|
||||
@@ -1624,32 +1629,49 @@ function App() {
|
||||
psPanel={psPanel}
|
||||
/>
|
||||
|
||||
{!isStandaloneLeftPage && (
|
||||
<>
|
||||
<div className="panel-resizer" onMouseDown={(event) => startResize('left', event)} />
|
||||
<div
|
||||
className="panel-resizer"
|
||||
onMouseDown={(event) => startResize('left', event)}
|
||||
style={{ display: isStandaloneLeftPage ? 'none' : undefined }}
|
||||
/>
|
||||
|
||||
<AppMapWorkspace
|
||||
language={language}
|
||||
showMapRegionLocator={showMapRegionLocator}
|
||||
toggleMapRegionLocator={toggleMapRegionLocator}
|
||||
mapRegionOptions={mapRegionOptions}
|
||||
mapRegionSelection={mapRegionSelection}
|
||||
mapRegionLoading={mapRegionLoading}
|
||||
mapRegionLocating={mapRegionLocating}
|
||||
mapRegionError={mapRegionError}
|
||||
mapRegionLocatedName={mapRegionLocatedName}
|
||||
onMapRegionProvinceChange={handleMapRegionProvinceChange}
|
||||
onMapRegionCityChange={handleMapRegionCityChange}
|
||||
onLocateSelectedRegion={locateSelectedRegionOnMap}
|
||||
onClearMapRegionHighlight={clearMapRegionHighlight}
|
||||
baseLayerKey={baseLayerKey}
|
||||
setBaseLayerKey={setBaseLayerKey}
|
||||
onOpenExportModal={mapExport.openExportModal}
|
||||
/>
|
||||
<div className="panel-resizer" onMouseDown={(event) => startResize('right', event)} />
|
||||
<AppLogPanel width={rightPanelWidth} />
|
||||
</>
|
||||
)}
|
||||
<div
|
||||
style={{
|
||||
display: isStandaloneLeftPage ? 'none' : 'flex',
|
||||
flex: '1 1 auto',
|
||||
minWidth: 0,
|
||||
minHeight: 0,
|
||||
}}
|
||||
>
|
||||
<AppMapWorkspace
|
||||
language={language}
|
||||
showMapRegionLocator={showMapRegionLocator}
|
||||
toggleMapRegionLocator={toggleMapRegionLocator}
|
||||
mapRegionOptions={mapRegionOptions}
|
||||
mapRegionSelection={mapRegionSelection}
|
||||
mapRegionLoading={mapRegionLoading}
|
||||
mapRegionLocating={mapRegionLocating}
|
||||
mapRegionError={mapRegionError}
|
||||
mapRegionLocatedName={mapRegionLocatedName}
|
||||
onMapRegionProvinceChange={handleMapRegionProvinceChange}
|
||||
onMapRegionCityChange={handleMapRegionCityChange}
|
||||
onLocateSelectedRegion={locateSelectedRegionOnMap}
|
||||
onClearMapRegionHighlight={clearMapRegionHighlight}
|
||||
baseLayerKey={baseLayerKey}
|
||||
setBaseLayerKey={setBaseLayerKey}
|
||||
onOpenExportModal={mapExport.openExportModal}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div
|
||||
className="panel-resizer"
|
||||
onMouseDown={(event) => startResize('right', event)}
|
||||
style={{ display: isStandaloneLeftPage ? 'none' : undefined }}
|
||||
/>
|
||||
|
||||
<div style={{ display: isStandaloneLeftPage ? 'none' : undefined }}>
|
||||
<AppLogPanel width={rightPanelWidth} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<AppOverlays
|
||||
|
||||
@@ -74,6 +74,24 @@ const PYINT_PRECISE_ORBIT_MODE_LABEL = {
|
||||
replace_and_validate: '替换并校验',
|
||||
};
|
||||
|
||||
const RERUN_MODE_LABEL = {
|
||||
unfinished_only: '只跑未完成',
|
||||
rerun_all: '全部重跑',
|
||||
};
|
||||
|
||||
const RERUN_MODE_OPTIONS = [
|
||||
{
|
||||
value: 'unfinished_only',
|
||||
label: '只跑未完成',
|
||||
description: '按当前引擎和当前模板检查已有结果,已完成任务会跳过。',
|
||||
},
|
||||
{
|
||||
value: 'rerun_all',
|
||||
label: '全部重跑',
|
||||
description: '忽略已有结果,对本次选中的任务全部重新执行。',
|
||||
},
|
||||
];
|
||||
|
||||
function formatEngineLabel(engineCode, engineLabel = '') {
|
||||
return engineLabel || ENGINE_LABEL[engineCode] || engineCode || '-';
|
||||
}
|
||||
@@ -312,6 +330,8 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [submitMsg, setSubmitMsg] = useState('');
|
||||
const [submitError, setSubmitError] = useState(false);
|
||||
const [submitDialogOpen, setSubmitDialogOpen] = useState(false);
|
||||
const [rerunMode, setRerunMode] = useState('unfinished_only');
|
||||
const [pyintPreview, setPyintPreview] = useState(null);
|
||||
const [pyintPreviewLoading, setPyintPreviewLoading] = useState(false);
|
||||
const [pyintPreviewFeedback, setPyintPreviewFeedback] = useState({ message: '', error: false });
|
||||
@@ -523,7 +543,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
}
|
||||
}, [numToProcess, rootDir]);
|
||||
|
||||
const handleSubmit = async () => {
|
||||
const handleOpenSubmitDialog = useCallback(() => {
|
||||
if (!rootDir.trim()) {
|
||||
setSubmitError(true);
|
||||
setSubmitMsg('请输入根目录。');
|
||||
@@ -534,23 +554,37 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
setSubmitMsg('PyINT 输入资产预检未通过,请先修复阻塞项。');
|
||||
return;
|
||||
}
|
||||
setSubmitError(false);
|
||||
setSubmitMsg('');
|
||||
setSubmitDialogOpen(true);
|
||||
}, [pyintPreviewBlocksSubmit, rootDir]);
|
||||
|
||||
const handleCloseSubmitDialog = useCallback(() => {
|
||||
if (submitting) return;
|
||||
setSubmitDialogOpen(false);
|
||||
}, [submitting]);
|
||||
|
||||
const handleSubmit = async () => {
|
||||
setSubmitting(true);
|
||||
setSubmitMsg('');
|
||||
setSubmitError(false);
|
||||
try {
|
||||
const extra = buildExtraPayload(currentParamSchema, engineExtraParams);
|
||||
setSubmitDialogOpen(false);
|
||||
const result = await submitRun({
|
||||
engine_code: selectedEngine,
|
||||
profile: selectedProfile,
|
||||
root_dir: rootDir.trim(),
|
||||
num_to_process: Number(numToProcess) || 0,
|
||||
rerun_mode: rerunMode,
|
||||
timeout_seconds: timeoutSec ? Number(timeoutSec) : null,
|
||||
extra,
|
||||
});
|
||||
const taskCount = result?.selected_task_count ? `,选中 ${result.selected_task_count} 个任务` : '';
|
||||
const skippedCompleted = Number(result?.skipped_completed_count || 0);
|
||||
const skippedText = skippedCompleted > 0 ? `,跳过 ${skippedCompleted} 个已完成任务` : '';
|
||||
setSubmitError(false);
|
||||
setSubmitMsg(`任务已入队:${result.task_id}${taskCount}`);
|
||||
setSubmitMsg(`任务已入队:${result.task_id}${taskCount}${skippedText}`);
|
||||
if (onJobQueued) onJobQueued(result.task_id);
|
||||
await refreshMonitor();
|
||||
} catch (err) {
|
||||
@@ -625,6 +659,135 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
</div>
|
||||
)}
|
||||
|
||||
{submitDialogOpen && (
|
||||
<div
|
||||
style={{
|
||||
position: 'fixed',
|
||||
inset: 0,
|
||||
background: 'rgba(15,23,42,0.42)',
|
||||
zIndex: 9998,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
padding: 20,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 'min(560px, 100%)',
|
||||
background: '#fff',
|
||||
borderRadius: 14,
|
||||
border: '1px solid #cbd5e1',
|
||||
boxShadow: '0 20px 60px rgba(15, 23, 42, 0.24)',
|
||||
padding: 20,
|
||||
}}
|
||||
>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 12, marginBottom: 10 }}>
|
||||
<strong style={{ fontSize: 16, color: '#0f172a' }}>提交生产任务</strong>
|
||||
<button
|
||||
onClick={handleCloseSubmitDialog}
|
||||
disabled={submitting}
|
||||
style={{
|
||||
border: 'none',
|
||||
background: 'none',
|
||||
color: '#64748b',
|
||||
cursor: submitting ? 'not-allowed' : 'pointer',
|
||||
fontSize: 13,
|
||||
}}
|
||||
>
|
||||
关闭
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div style={{ fontSize: 12, color: '#475569', lineHeight: 1.6, marginBottom: 12 }}>
|
||||
选择本次批处理策略。任务数量限制会在“只跑未完成”过滤之后再生效,按当前引擎和当前模板判断已完成状态。
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: 10, marginBottom: 14 }}>
|
||||
{RERUN_MODE_OPTIONS.map(option => {
|
||||
const selected = rerunMode === option.value;
|
||||
return (
|
||||
<label
|
||||
key={option.value}
|
||||
style={{
|
||||
display: 'flex',
|
||||
gap: 12,
|
||||
alignItems: 'flex-start',
|
||||
padding: 12,
|
||||
borderRadius: 10,
|
||||
border: selected ? '2px solid #3b82f6' : '1px solid #dbeafe',
|
||||
background: selected ? '#eff6ff' : '#f8fafc',
|
||||
cursor: 'pointer',
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name="rerun-mode"
|
||||
checked={selected}
|
||||
onChange={() => setRerunMode(option.value)}
|
||||
/>
|
||||
<div>
|
||||
<div style={{ fontSize: 13, fontWeight: 600, color: '#0f172a', marginBottom: 4 }}>{option.label}</div>
|
||||
<div style={{ fontSize: 12, color: '#64748b', lineHeight: 1.5 }}>{option.description}</div>
|
||||
</div>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div
|
||||
style={{
|
||||
marginBottom: 16,
|
||||
padding: '10px 12px',
|
||||
borderRadius: 10,
|
||||
border: '1px solid #e2e8f0',
|
||||
background: '#f8fafc',
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(auto-fit, minmax(180px, 1fr))',
|
||||
gap: 8,
|
||||
}}
|
||||
>
|
||||
<div style={{ fontSize: 12, color: '#475569' }}>引擎:{formatEngineLabel(selectedEngine, currentEngineObj?.engine_label)}</div>
|
||||
<div style={{ fontSize: 12, color: '#475569' }}>模板:{currentProfileObj?.label || selectedProfile}</div>
|
||||
<div style={{ fontSize: 12, color: '#475569' }}>任务数量:{Number(numToProcess) > 0 ? Number(numToProcess) : '全部'}</div>
|
||||
<div style={{ fontSize: 12, color: '#475569' }}>执行策略:{RERUN_MODE_LABEL[rerunMode] || rerunMode}</div>
|
||||
</div>
|
||||
|
||||
<div style={{ display: 'flex', justifyContent: 'flex-end', gap: 10 }}>
|
||||
<button
|
||||
onClick={handleCloseSubmitDialog}
|
||||
disabled={submitting}
|
||||
style={{
|
||||
padding: '6px 14px',
|
||||
borderRadius: 6,
|
||||
border: '1px solid #cbd5e1',
|
||||
background: '#fff',
|
||||
color: '#0f172a',
|
||||
cursor: submitting ? 'not-allowed' : 'pointer',
|
||||
}}
|
||||
>
|
||||
取消
|
||||
</button>
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
disabled={submitting}
|
||||
style={{
|
||||
padding: '6px 16px',
|
||||
borderRadius: 6,
|
||||
border: 'none',
|
||||
background: '#2563eb',
|
||||
color: '#fff',
|
||||
cursor: submitting ? 'not-allowed' : 'pointer',
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
{submitting ? '提交中...' : `确认 ${RERUN_MODE_LABEL[rerunMode] || ''}`}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={card}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: 10 }}>
|
||||
<strong style={{ fontSize: 14 }}>引擎状态</strong>
|
||||
@@ -967,7 +1130,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 12 }}>
|
||||
<button
|
||||
onClick={handleSubmit}
|
||||
onClick={handleOpenSubmitDialog}
|
||||
disabled={isSubmitDisabled}
|
||||
style={{
|
||||
padding: '6px 20px',
|
||||
|
||||
@@ -335,9 +335,15 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
|
||||
const databaseStatus = status?.database || {};
|
||||
const dinsarBridge = asObject(status?.dinsar_bridge);
|
||||
const dinsarResultCatalog = asObject(status?.dinsar_result_catalog || status?.result_catalog);
|
||||
const timeseriesResultCatalog = asObject(status?.timeseries_result_catalog || status?.psinsar_result_catalog);
|
||||
const dinsarCatalogNeedsRebuild =
|
||||
typeof dinsarResultCatalog.needs_rebuild === 'boolean' ? dinsarResultCatalog.needs_rebuild : null;
|
||||
const timeseriesCatalogNeedsRebuild =
|
||||
typeof timeseriesResultCatalog.needs_rebuild === 'boolean' ? timeseriesResultCatalog.needs_rebuild : null;
|
||||
const sourceRoots = asObject(status?.source_roots);
|
||||
const productPackages = asObject(status?.product_packages);
|
||||
const wslRuntime = asObject(status?.wsl_runtime);
|
||||
const wslRuntimeItems = asArray(wslRuntime.runtimes);
|
||||
const pairingSystem = asObject(status?.pairing_system);
|
||||
const sourceRootItems = asArray(sourceRoots.items);
|
||||
const bridgeDiagnosisIssueCount =
|
||||
@@ -616,6 +622,41 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="health-card">
|
||||
<div className="health-card-title">{en ? 'Timeseries Result Catalog' : '时序 InSAR 结果目录'}</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Catalog status' : '目录状态'}</span>
|
||||
{renderBadge(
|
||||
isCatalogHealthy(timeseriesResultCatalog),
|
||||
formatCatalogStatusLabel(timeseriesResultCatalog, en)
|
||||
)}
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Needs rebuild' : '需要重建'}</span>
|
||||
{renderBadge(
|
||||
timeseriesCatalogNeedsRebuild === false,
|
||||
timeseriesCatalogNeedsRebuild === null
|
||||
? (en ? 'Unknown' : '未知')
|
||||
: (timeseriesCatalogNeedsRebuild ? (en ? 'Yes' : '是') : (en ? 'No' : '否'))
|
||||
)}
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Manifest / DB' : 'Manifest / 数据库'}</span>
|
||||
<span>
|
||||
{toNumber(timeseriesResultCatalog.manifest_count)} / {toNumber(timeseriesResultCatalog.db_count)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Issue count' : '问题数量'}</span>
|
||||
<span>{toNumber(timeseriesResultCatalog.issue_count)}</span>
|
||||
</div>
|
||||
<div className="health-card-note">
|
||||
{en
|
||||
? 'Managed timeseries products are indexed from canonical publish manifests.'
|
||||
: '时序 InSAR 产物已按标准发布包 manifest 进行索引和自检。'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="health-card">
|
||||
<div className="health-card-title">{en ? 'Pairing System' : '配对系统'}</div>
|
||||
<div className="health-card-row">
|
||||
@@ -810,6 +851,78 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="health-card">
|
||||
<div className="health-card-title">{en ? 'Product Packages' : '标准结果包'}</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Overall' : '总体状态'}</span>
|
||||
{renderBadge(
|
||||
productPackages.ok,
|
||||
`${toNumber(productPackages.canonical_count)} / ${toNumber(productPackages.total_count)}`
|
||||
)}
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Manifest / publish dir' : 'Manifest / 发布目录'}</span>
|
||||
<span>
|
||||
{toNumber(productPackages.missing_manifest_count)} / {toNumber(productPackages.missing_publish_dir_count)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Processor / runtime' : '处理器 / 运行时'}</span>
|
||||
<span>
|
||||
{toNumber(productPackages.missing_processor_count)} / {toNumber(productPackages.missing_runtime_count)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Native output trace' : '原生输出追踪'}</span>
|
||||
<span>{toNumber(productPackages.missing_native_output_count)}</span>
|
||||
</div>
|
||||
<div className="health-card-note">
|
||||
{en
|
||||
? 'Catalog registration now checks canonical package metadata instead of guessing engine-native directories.'
|
||||
: '目录登记现在直接校验标准包元数据,不再依赖猜测引擎原生目录结构。'}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="health-card">
|
||||
<div className="health-card-title">{en ? 'WSL Runtime' : 'WSL 运行时'}</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Overall' : '总体状态'}</span>
|
||||
{renderBadge(
|
||||
wslRuntime.ok,
|
||||
`${toNumber(wslRuntime.healthy_runtime_count)} / ${toNumber(wslRuntime.required_runtime_count)}`
|
||||
)}
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Broker root' : 'Broker 根目录'}</span>
|
||||
{renderBadge(wslRuntime.broker_job_root_exists)}
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Shared distro' : '共享发行版'}</span>
|
||||
<span>{wslRuntime.shared_distro || '-'}</span>
|
||||
</div>
|
||||
<div className="health-card-row">
|
||||
<span>{en ? 'Shared conda env' : '共享 conda 环境'}</span>
|
||||
<span>{wslRuntime.shared_conda_env_name || '-'}</span>
|
||||
</div>
|
||||
{wslRuntimeItems.length > 0 && (
|
||||
wslRuntimeItems.map((item) => (
|
||||
<div
|
||||
key={item.runtime_id || item.engine_code}
|
||||
className={`health-card-note ${item.ok ? 'ok' : (item.required ? 'error' : 'warn')}`}
|
||||
>
|
||||
<div style={{ fontWeight: 600 }}>
|
||||
{item.display_name || item.runtime_id || item.engine_code}
|
||||
</div>
|
||||
<div>
|
||||
{(item.required ? (en ? 'Required' : '必需') : (en ? 'Reserved' : '预留'))}
|
||||
{' · '}
|
||||
{item.runner_exists ? (en ? 'Runner OK' : 'Runner 正常') : (en ? 'Runner missing' : 'Runner 缺失')}
|
||||
</div>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="health-card">
|
||||
<div className="health-card-title">IDL/ENVI</div>
|
||||
<div className="health-card-row">
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from backend.app.config import ensure_project_env_loaded
|
||||
|
||||
ensure_project_env_loaded()
|
||||
|
||||
from backend.app import database
|
||||
from backend.app.services.dinsar_layout_migration_service import dinsar_layout_migration_service
|
||||
from backend.app.services.manifest_inventory_service import manifest_inventory_service
|
||||
from backend.app.services.result_catalog_service import result_catalog_service
|
||||
from backend.app.services.root_registry_service import root_registry_service
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Normalize managed ENVI D-InSAR run layouts under DINSAR_PRODUCT_DIR."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--root-dir",
|
||||
default=None,
|
||||
help="Managed D-InSAR publish root. Defaults to settings.DINSAR_PRODUCT_DIR.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pair-key",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Only migrate the specified pair_key. Repeatable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Stop after inspecting this many run directories.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Inspect and report only. Do not move files or rewrite metadata.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-catalog-sync",
|
||||
action="store_true",
|
||||
help="Skip manifest inventory sync and catalog rebuild after migration.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
async def _sync_catalog() -> Dict[str, Any]:
|
||||
database.init_db()
|
||||
if database.AsyncSessionLocal is None:
|
||||
raise RuntimeError("Database session factory is not initialized.")
|
||||
|
||||
root_sync = await root_registry_service.sync_from_settings()
|
||||
async with database.AsyncSessionLocal() as db:
|
||||
inventory_result = await manifest_inventory_service.sync_manifest_roots(db)
|
||||
rebuild_result = await result_catalog_service.rebuild_catalog(
|
||||
db,
|
||||
full_rebuild=True,
|
||||
)
|
||||
return {
|
||||
"root_registry": root_sync,
|
||||
"manifest_inventory": inventory_result,
|
||||
"catalog_rebuild": rebuild_result,
|
||||
}
|
||||
|
||||
|
||||
async def main_async() -> int:
|
||||
args = parse_args()
|
||||
result = dinsar_layout_migration_service.migrate_managed_envi_runs(
|
||||
root_dir=args.root_dir,
|
||||
pair_keys=args.pair_key,
|
||||
limit=args.limit,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
|
||||
should_sync_catalog = (
|
||||
not args.dry_run
|
||||
and not args.skip_catalog_sync
|
||||
and int(result.get("migrated_count", 0) or 0) + int(result.get("rewritten_count", 0) or 0) > 0
|
||||
and int(result.get("failed_count", 0) or 0) == 0
|
||||
)
|
||||
if should_sync_catalog:
|
||||
result["post_sync"] = await _sync_catalog()
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if int(result.get("failed_count", 0) or 0) == 0 else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
return asyncio.run(main_async())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user