Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint production workflow updates

Browse Source
This commit is contained in:
Harmon
2026-06-30 15:25:29 +08:00
parent 19ae3ec37f
commit 9c80b95385
66 changed files with 7639 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+20 -10
View File
@@ -88,7 +88,7 @@ GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog
GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime
GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe
GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM
GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape
GF3_SARSCAPE_POLARIZATIONS=HH,HV
GF3_SARSCAPE_KEEP_EXTRACTED=true
GF3_SARSCAPE_AUTO_STANDARDIZE=false
@@ -126,9 +126,9 @@ ORBIT_QUARANTINE_DIR=
# DEM / 其他数据
# -----------------------------------------------------------------------------
# Raw D-InSAR DEM source for SARscape/ENVI. Keep the base path without the .wgs84 suffix here.
IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape
SRTM_DEM_DIR=D:\SRTM30m
GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2
IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape
SRTM_DEM_DIR=D:\DEM
GF3_GEO_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif
WATER_RESULTS_DIR=D:\WaterResult
GF3_WATER_DEM_PATH=
GF3_WATER_DEFAULT_CARTOGRAPHIC=true
@@ -137,6 +137,14 @@ SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready
SAR_ANALYSIS_WORK_ROOT=D:\production_runtime\sar_analysis_work
SAR_ANALYSIS_NODATA_VALUE=-9999
SAR_ANALYSIS_OUTPUT_COG=true
SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0
SAR_ANALYSIS_DEM_RESOLUTION_M=30.0
SAR_ANALYSIS_RANGE_LOOKS=6
SAR_ANALYSIS_AZIMUTH_LOOKS=5
SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true
SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee
SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5
# -----------------------------------------------------------------------------
@@ -188,7 +196,7 @@ ISCE2_PROFILE=lt1_stripmap
ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/applications/stripmapApp.py
ISCE2_PIPELINE_SCRIPT=
# ISCE2 should point to the prepared WGS84 DEM after the one-time conversion.
ISCE2_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84
ISCE2_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
ISCE2_WORK_ROOT=D:\production_runtime\isce2_work
ISCE2_OUTPUT_ROOT=D:\production_results\dinsar
ISCE2_PER_TASK_TIMEOUT_SECONDS=43200
@@ -245,11 +253,11 @@ PYINT_DEM_ROOT=D:\production_runtime\pyint_dem
PYINT_DEM_MODE=local_fabdem
PYINT_FABDEM_ROOT=
# When PYINT_DEM_MODE=prepared_file, point this to the same prepared WGS84 DEM.
PYINT_PREPARED_DEM_PATH=
PYINT_PREPARED_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
# 0 means do not derive looks from a target output grid.
PYINT_DEFAULT_TARGET_GRID_SIZE_M=0
# Source DEM resolution recorded in PyINT/Gamma run metadata.
PYINT_DEM_RESOLUTION_M=30.0
PYINT_DEM_RESOLUTION_M=90.0
PYINT_OPENTOPO_DEM_TYPE=SRTMGL1
PYINT_OPENTOPO_API_KEY=
PYINT_DEM_STRICT=true
@@ -293,7 +301,7 @@ GAMMA_SBAS_TRIAL_ROOT=D:\production_runtime\gamma_ipta_trials
GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\templates\gamma_sbas
GAMMA_SBAS_SOURCE_ROOTS=D:\Task_Pool\source_materialized\lutan1
GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi
GAMMA_SBAS_DEM_PATH=D:\DEM\HeiLongJiang10M_DEM.tif
GAMMA_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
GAMMA_SBAS_DEFAULT_RLKS=8
GAMMA_SBAS_DEFAULT_AZLKS=8
GAMMA_SBAS_DEFAULT_MB_MODE=0
@@ -312,7 +320,7 @@ TIMESERIES_WSL_DISTRO=Ubuntu-24.04
TIMESERIES_ENV_NAME=insar_wsl_v1
TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python
TIMESERIES_WORK_ROOT=D:\production_runtime\timeseries_work
TIMESERIES_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84
TIMESERIES_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
TIMESERIES_ORBIT_POOL_ISCE2=
TIMESERIES_EXPERIMENT_ROOT=
TIMESERIES_STACK_PREP_SCRIPT=
@@ -350,6 +358,8 @@ CLUSTER_TRANSFER_TIMEOUT_SECONDS=3600
CLUSTER_MAIN_SERVER_URL=
# Empty means the worker can claim all job types. Remote LandSAR nodes should set:
# JOB_WORKER_ALLOWED_TYPES=LANDSAR_CLUSTER_ITEM
JOB_WORKER_POLL_INTERVAL=1.0
JOB_WORKER_CONCURRENCY=2
JOB_WORKER_ALLOWED_TYPES=
@@ -413,7 +423,7 @@ DEFAULT_VLM_MODEL=qwen3-vl:30b
VITE_TILE_SERVER_URL=
VITE_TILE_SERVER_TOKEN=change_me
VITE_APP_ORG_NAME=黑龙江省自然资源卫星应用技术中心
VITE_APP_SYSTEM_NAME=InSAR 自动化管理系统
VITE_APP_SYSTEM_NAME=雷达数据生产管理系统
VITE_APP_SYSTEM_TAGLINE=科研工程生产平台
# Optional override for deployed builds. Empty uses frontend/src/logo.jpg.
VITE_APP_LOGO_URL=
backend/app/config.py
+47 -3
View File
@@ -252,6 +252,14 @@ class Settings(BaseSettings):
SAR_ANALYSIS_WORK_ROOT: str = ""
SAR_ANALYSIS_NODATA_VALUE: float = -9999.0
SAR_ANALYSIS_OUTPUT_COG: bool = True
SAR_ANALYSIS_DEM_PATH: str = ""
SAR_ANALYSIS_TARGET_GRID_SIZE_M: float = 30.0
SAR_ANALYSIS_DEM_RESOLUTION_M: float = 30.0
SAR_ANALYSIS_RANGE_LOOKS: int = 6
SAR_ANALYSIS_AZIMUTH_LOOKS: int = 5
SAR_ANALYSIS_SPECKLE_FILTER_ENABLED: bool = True
SAR_ANALYSIS_SPECKLE_FILTER_METHOD: str = "lee"
SAR_ANALYSIS_SPECKLE_FILTER_SIZE: int = 5
SRTM_DEM_DIR: str = ""
GF3_GEO_DEM_PATH: str = ""
@@ -318,7 +326,7 @@ class Settings(BaseSettings):
ISCE2_WSL_DISTRO: str = "Ubuntu-24.04"
ISCE2_PYTHON: str = "/home/administrator/miniconda3/envs/isce2/bin/python"
ISCE2_PROFILE: str = "lt1_stripmap"
ISCE2_DEM_PATH: str = "D:\\SRTM30m\\SRTMDEM_RSP_SARscape.wgs84"
ISCE2_DEM_PATH: str = "D:\\DEM\\SRTMDEM_RSP_SARscape.wgs84"
ISCE2_WORK_ROOT: str = ""
ISCE2_OUTPUT_ROOT: str = ""
ISCE2_PER_TASK_TIMEOUT_SECONDS: int = 43200
@@ -355,7 +363,7 @@ class Settings(BaseSettings):
PYINT_DEM_MODE: str = "local_fabdem"
PYINT_FABDEM_ROOT: str = ""
PYINT_PREPARED_DEM_PATH: str = ""
PYINT_DEM_RESOLUTION_M: float = 30.0
PYINT_DEM_RESOLUTION_M: float = 90.0
PYINT_OPENTOPO_DEM_TYPE: str = "SRTMGL1"
PYINT_OPENTOPO_API_KEY: str = ""
PYINT_DEM_STRICT: bool = True
@@ -423,6 +431,7 @@ class Settings(BaseSettings):
JOB_WORKER_STALE_RECOVER_INTERVAL: float = 15.0
JOB_WORKER_STALE_RUNNING_SECONDS: int = 300
JOB_WORKER_HEARTBEAT_INTERVAL: float = 5.0
JOB_WORKER_CONCURRENCY: int = 1
JOB_WORKER_ALLOWED_TYPES: str = ""
TIMESERIES_ENABLED: bool = False
@@ -500,6 +509,39 @@ class Settings(BaseSettings):
"SAR_ANALYSIS_NODATA_VALUE",
float(self.SAR_ANALYSIS_NODATA_VALUE if self.SAR_ANALYSIS_NODATA_VALUE is not None else -9999.0),
)
if not self.SAR_ANALYSIS_DEM_PATH:
object.__setattr__(
self,
"SAR_ANALYSIS_DEM_PATH",
self.GAMMA_SBAS_DEM_PATH
or self.PYINT_PREPARED_DEM_PATH
or self.ISCE2_DEM_PATH
or self.IDL_DINSAR_DEM_BASE_FILE,
)
object.__setattr__(
self,
"SAR_ANALYSIS_TARGET_GRID_SIZE_M",
max(1.0, float(self.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)),
)
object.__setattr__(
self,
"SAR_ANALYSIS_DEM_RESOLUTION_M",
max(1.0, float(self.SAR_ANALYSIS_DEM_RESOLUTION_M or self.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)),
)
object.__setattr__(self, "SAR_ANALYSIS_RANGE_LOOKS", max(1, int(self.SAR_ANALYSIS_RANGE_LOOKS or 6)))
object.__setattr__(self, "SAR_ANALYSIS_AZIMUTH_LOOKS", max(1, int(self.SAR_ANALYSIS_AZIMUTH_LOOKS or 5)))
filter_method = str(self.SAR_ANALYSIS_SPECKLE_FILTER_METHOD or "lee").strip().lower()
if filter_method in {"", "0", "false", "none", "off", "disabled", "no"}:
filter_method = "none"
elif filter_method not in {"lee"}:
filter_method = "lee"
if not self.SAR_ANALYSIS_SPECKLE_FILTER_ENABLED:
filter_method = "none"
filter_size = max(3, min(99, int(self.SAR_ANALYSIS_SPECKLE_FILTER_SIZE or 5)))
if filter_size % 2 == 0:
filter_size += 1
object.__setattr__(self, "SAR_ANALYSIS_SPECKLE_FILTER_METHOD", filter_method)
object.__setattr__(self, "SAR_ANALYSIS_SPECKLE_FILTER_SIZE", filter_size)
if not self.SRTM_DEM_DIR:
object.__setattr__(self, "SRTM_DEM_DIR", os.path.join(backend_dir, "dem_data"))
object.__setattr__(self, "ASSET_SCAN_PARSE_WORKERS", max(1, int(self.ASSET_SCAN_PARSE_WORKERS or 1)))
@@ -514,6 +556,7 @@ class Settings(BaseSettings):
max(60, int(self.ASSET_SCAN_PARSE_TIMEOUT_SECONDS or 600)),
)
object.__setattr__(self, "ASSET_SCAN_DB_BATCH_SIZE", max(1, int(self.ASSET_SCAN_DB_BATCH_SIZE or 1)))
object.__setattr__(self, "JOB_WORKER_CONCURRENCY", max(1, int(self.JOB_WORKER_CONCURRENCY or 1)))
if not self.GF3_ARCHIVE_SOURCE_DIRS:
object.__setattr__(
self,
@@ -725,7 +768,7 @@ class Settings(BaseSettings):
if pyint_dem_mode not in {"local_fabdem", "opentopo", "prepared_file"}:
pyint_dem_mode = "local_fabdem"
object.__setattr__(self, "PYINT_DEM_MODE", pyint_dem_mode)
object.__setattr__(self, "PYINT_DEM_RESOLUTION_M", max(0.1, float(self.PYINT_DEM_RESOLUTION_M or 30.0)))
object.__setattr__(self, "PYINT_DEM_RESOLUTION_M", max(0.1, float(self.PYINT_DEM_RESOLUTION_M or 90.0)))
object.__setattr__(
self,
"PYINT_UNWRAP_COH_THRESHOLD",
@@ -1291,6 +1334,7 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_WORK_ROOT", value=settings.SAR_ANALYSIS_WORK_ROOT, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="SAR_ANALYSIS_DEM_PATH", value=settings.SAR_ANALYSIS_DEM_PATH, errors=errors, warnings=warnings, expect_file=True)
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
for label, value in (
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
backend/app/copier.py
+14
View File
@@ -305,6 +305,10 @@ def _normalize_source_bundle_archive_path(source_path: str) -> str:
def _safe_archive_member_name(member_name: str, archive_path: str) -> str:
name = str(member_name or "").replace("\\", "/").strip("/")
while name.startswith("./"):
name = name[2:]
if name in {"", "."}:
return ""
if not name or name.startswith("../") or "/../" in f"/{name}/":
raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}")
if os.path.isabs(name) or os.path.splitdrive(name)[0]:
@@ -318,6 +322,11 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int:
with zipfile.ZipFile(archive_path) as zip_obj:
for info in zip_obj.infolist():
rel_name = _safe_archive_member_name(info.filename, archive_path)
if not rel_name:
if info.is_dir():
os.makedirs(dest_dir, exist_ok=True)
continue
raise ValueError(f"Unsafe ZIP member path: {info.filename}")
dest_path = os.path.abspath(os.path.join(dest_dir, rel_name))
if not dest_path.startswith(os.path.abspath(dest_dir) + os.sep):
raise ValueError(f"Unsafe ZIP member path: {info.filename}")
@@ -335,6 +344,11 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int:
with tarfile.open(archive_path, "r:*") as tar_obj:
for member in tar_obj:
rel_name = _safe_archive_member_name(member.name, archive_path)
if not rel_name:
if member.isdir():
os.makedirs(dest_dir, exist_ok=True)
continue
raise ValueError(f"Unsafe TAR member path: {member.name}")
dest_path = os.path.abspath(os.path.join(dest_dir, rel_name))
if not dest_path.startswith(os.path.abspath(dest_dir) + os.sep):
raise ValueError(f"Unsafe TAR member path: {member.name}")
backend/app/isce2_pipeline/lt1_input_resolver.py
+4 -4
View File
@@ -31,12 +31,12 @@ PathTransform = Callable[[str | Path], Path]
DEFAULT_WINDOWS_DEM_CANDIDATES = (
r"D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84",
r"D:\SRTM30m\SRTMDEM_RSP_SARscape",
r"D:\DEM\SRTMDEM_RSP_SARscape.wgs84",
r"D:\DEM\SRTMDEM_RSP_SARscape",
)
DEFAULT_WSL_DEM_CANDIDATES = (
"/mnt/d/SRTM30m/SRTMDEM_RSP_SARscape.wgs84",
"/mnt/d/SRTM30m/SRTMDEM_RSP_SARscape",
"/mnt/d/DEM/SRTMDEM_RSP_SARscape.wgs84",
"/mnt/d/DEM/SRTMDEM_RSP_SARscape",
)
DEFAULT_WINDOWS_ORBIT_POOL_CANDIDATES = (r"D:\orbit_pools\isce2",)
DEM_SIDECAR_PROPERTY_NAMES = ("file_name", "metadata_location", "extra_file_name")
backend/app/models/schemas.py
+4
View File
@@ -231,6 +231,10 @@ class RadarData(BaseModel):
stack_selection_mode: Optional[str] = None
stack_network_edge_count: Optional[int] = None
stack_network_warnings: Optional[List[str]] = None
lt1_image_produced: bool = False
lt1_image_product: Optional[Dict[str, Any]] = None
lt1_landsar_produced: bool = False
lt1_landsar_product: Optional[Dict[str, Any]] = None
model_config = ConfigDict(from_attributes=True)
backend/app/pyint_pipeline/run_gamma_scene_preprocess.py
+455 -16
View File
@@ -2,7 +2,7 @@
"""Single-scene Gamma preprocessing to analysis-ready GeoTIFF.
The script is intentionally narrower than the full PyINT DInSAR pipeline:
LT source product -> Gamma SLC -> multilook amplitude -> geocode -> GeoTIFF.
LT source product -> Gamma SLC -> multilook amplitude -> geocode -> speckle-filtered dB GeoTIFF.
It is executed inside WSL by backend.app.services.lt_gamma_scene_service.
"""
from __future__ import annotations
@@ -27,6 +27,10 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--pyint-home", required=True)
parser.add_argument("--dem-root", required=True)
parser.add_argument("--prepared-dem-path", default="")
parser.add_argument("--dem-resolution-m", type=float, default=30.0)
parser.add_argument("--target-grid-size-m", type=float, default=30.0)
parser.add_argument("--dem-lat-ovr", type=float, default=0.0)
parser.add_argument("--dem-lon-ovr", type=float, default=0.0)
parser.add_argument("--project-name", required=True)
parser.add_argument("--date", required=True)
parser.add_argument("--satellite-family", default="LT1")
@@ -35,9 +39,175 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--geo-interp", default="1")
parser.add_argument("--nodata-value", type=float, default=-9999.0)
parser.add_argument("--to-db", action="store_true")
parser.add_argument("--speckle-filter-method", default="lee")
parser.add_argument("--speckle-filter-size", type=int, default=5)
parser.add_argument("--speckle-filter-enl", type=float, default=0.0)
return parser.parse_args()
def clamp_float(value: float, minimum: float, maximum: float) -> float:
if not math.isfinite(value):
return minimum
return min(maximum, max(minimum, float(value)))
def format_gamma_number(value: float) -> str:
text = f"{float(value):.6f}".rstrip("0").rstrip(".")
return text or "0"
def calculate_dem_oversampling(
*,
dem_resolution_m: float,
target_grid_size_m: float,
dem_lat_ovr: float,
dem_lon_ovr: float,
) -> dict[str, Any]:
dem_resolution = float(dem_resolution_m or 30.0)
target_grid = float(target_grid_size_m or 30.0)
if dem_resolution <= 0:
dem_resolution = 30.0
if target_grid <= 0:
target_grid = dem_resolution
derived = dem_resolution / target_grid
lat_factor = clamp_float(float(dem_lat_ovr or derived), 0.25, 16.0)
lon_factor = clamp_float(float(dem_lon_ovr or derived), 0.25, 16.0)
actual_grid = dem_resolution / ((lat_factor + lon_factor) / 2.0)
return {
"dem_resolution_m": dem_resolution,
"target_grid_size_m": target_grid,
"derived_oversampling": derived,
"dem_lat_ovr": lat_factor,
"dem_lon_ovr": lon_factor,
"actual_grid_size_m": actual_grid,
}
def meters_per_degree_lon(latitude_deg: float) -> float:
latitude_rad = math.radians(float(latitude_deg))
return max(1.0, 111_320.0 * math.cos(latitude_rad))
def inspect_prepared_dem_path(path_text: str) -> dict[str, str]:
text = str(path_text or "").strip()
if not text:
return {"kind": "", "direct_dem_path": "", "source_dem_path": ""}
path = Path(text)
try:
resolved = path.resolve()
except Exception:
resolved = path
if resolved.is_file() and Path(str(resolved) + ".par").is_file():
return {"kind": "gamma_ready", "direct_dem_path": str(resolved), "source_dem_path": ""}
if resolved.is_file():
return {"kind": "source_dem", "direct_dem_path": "", "source_dem_path": str(resolved)}
return {"kind": "", "direct_dem_path": "", "source_dem_path": str(resolved)}
def read_slc_bbox(
pyint_home: Path,
slc_par: Path,
env: dict[str, str],
*,
margin_deg: float = 0.1,
) -> tuple[float, float, float, float]:
result = subprocess.run(
["SLC_corners", str(slc_par)],
cwd=str(pyint_home),
env=env,
text=True,
capture_output=True,
check=False,
)
if result.returncode != 0:
detail = (result.stderr or result.stdout or "").strip()
raise RuntimeError(f"SLC_corners failed rc={result.returncode}: {detail}")
lines = result.stdout.splitlines()
if len(lines) < 10:
raise RuntimeError(f"Unexpected SLC_corners output for {slc_par}")
lat_line = lines[8].rstrip()
lon_line = lines[9].rstrip()
min_lat = float(lat_line.split(":")[1].split(" max. ")[0])
max_lat = float(lat_line.split(":")[2])
min_lon = float(lon_line.split(":")[1].split(" max. ")[0])
max_lon = float(lon_line.split(":")[2])
margin = max(0.0, float(margin_deg or 0.0))
return min_lon - margin, min_lat - margin, max_lon + margin, max_lat + margin
def build_gamma_dem_from_source(
*,
source_dem: Path,
target_base: Path,
slc_par: Path,
pyint_home: Path,
log_dir: Path,
env: dict[str, str],
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
west, south, east, north = read_slc_bbox(pyint_home, slc_par, env)
log_dir.mkdir(parents=True, exist_ok=True)
source_open = Path(str(source_dem) + ".vrt") if Path(str(source_dem) + ".vrt").is_file() else source_dem
clipped_tif = target_base.with_suffix(".prepared_source_clip.tif")
clipped_aux = Path(str(clipped_tif) + ".aux.xml")
commands: list[dict[str, Any]] = []
commands.append(run_logged(
[
"gdal_translate",
"-projwin",
str(west),
str(north),
str(east),
str(south),
"-of",
"GTiff",
str(source_open),
str(clipped_tif),
],
cwd=target_base.parent,
env=env,
log_dir=log_dir,
stage="clip_prepared_dem",
))
commands.append(run_logged(
[
"makedem.py",
"-d",
str(clipped_tif),
"-p",
"gamma",
"-o",
str(target_base),
],
cwd=target_base.parent,
env=env,
log_dir=log_dir,
stage="convert_prepared_dem",
))
for path in (clipped_tif, clipped_aux):
try:
if path.exists():
path.unlink()
except OSError:
pass
dem_path = Path(str(target_base) + ".dem")
dem_par_path = Path(str(target_base) + ".dem.par")
if not dem_path.is_file() or not dem_par_path.is_file():
raise RuntimeError(f"Prepared source DEM conversion did not create Gamma DEM: {dem_path}")
return (
{
"kind": "source_dem_converted",
"source_dem_path": str(source_dem),
"source_open_path": str(source_open),
"gamma_dem_path": str(dem_path),
"bbox": {"west": west, "south": south, "east": east, "north": north},
},
commands,
)
def run_logged(command: list[str], *, cwd: Path, env: dict[str, str], log_dir: Path, stage: str) -> dict[str, Any]:
log_dir.mkdir(parents=True, exist_ok=True)
stdout_path = log_dir / f"{stage}.stdout.log"
@@ -74,6 +244,48 @@ def read_gamma_par(path: Path, key: str) -> str:
raise KeyError(f"Cannot read {key} from {path}")
def calculate_dem_oversampling_from_gamma_dem(
*,
dem_par_path: Path,
target_grid_size_m: float,
explicit_dem_lat_ovr: float,
explicit_dem_lon_ovr: float,
) -> dict[str, Any]:
target_grid = max(1.0, float(target_grid_size_m or 30.0))
post_lat_deg = abs(float(read_gamma_par(dem_par_path, "post_lat")))
post_lon_deg = abs(float(read_gamma_par(dem_par_path, "post_lon")))
corner_lat = float(read_gamma_par(dem_par_path, "corner_lat"))
nlines = int(float(read_gamma_par(dem_par_path, "nlines")))
center_lat = corner_lat - (post_lat_deg * max(0, nlines - 1) / 2.0)
lat_spacing_m = post_lat_deg * 111_320.0
lon_spacing_m = post_lon_deg * meters_per_degree_lon(center_lat)
derived_lat = lat_spacing_m / target_grid
derived_lon = lon_spacing_m / target_grid
lat_factor = clamp_float(float(explicit_dem_lat_ovr or derived_lat), 0.25, 16.0)
lon_factor = clamp_float(float(explicit_dem_lon_ovr or derived_lon), 0.25, 16.0)
actual_lat_m = lat_spacing_m / lat_factor
actual_lon_m = lon_spacing_m / lon_factor
return {
"dem_resolution_m": (lat_spacing_m + lon_spacing_m) / 2.0,
"target_grid_size_m": target_grid,
"derived_oversampling": (derived_lat + derived_lon) / 2.0,
"derived_dem_lat_ovr": derived_lat,
"derived_dem_lon_ovr": derived_lon,
"dem_lat_ovr": lat_factor,
"dem_lon_ovr": lon_factor,
"actual_grid_size_m": (actual_lat_m + actual_lon_m) / 2.0,
"actual_lat_grid_size_m": actual_lat_m,
"actual_lon_grid_size_m": actual_lon_m,
"source_dem_post_lat_deg": post_lat_deg,
"source_dem_post_lon_deg": post_lon_deg,
"source_dem_lat_spacing_m": lat_spacing_m,
"source_dem_lon_spacing_m": lon_spacing_m,
"source_dem_center_lat": center_lat,
"source_dem_par_path": str(dem_par_path),
}
def discover_lt_inputs(source_path: Path, date: str) -> list[Path]:
patterns = [f"LT1*{date}*.tar.gz", f"LT1*{date}*.tiff", f"LT1*{date}*.tif"]
if source_path.is_file():
@@ -119,15 +331,18 @@ def write_template(
range_looks: int,
azimuth_looks: int,
geo_interp: str,
prepared_dem_path: str,
dem_path: str,
prepared_dem_source: str,
dem_oversampling: dict[str, Any],
) -> None:
lines = [
"satelite = LT",
f"masterDate = {date}",
f"range_looks = {range_looks}",
f"azimuth_looks = {azimuth_looks}",
"dem_lat_ovr = 0.5",
"dem_lon_ovr = 0.5",
f"target_grid_size_m = {format_gamma_number(float(dem_oversampling.get('target_grid_size_m') or 0.0))}",
f"dem_lat_ovr = {format_gamma_number(float(dem_oversampling.get('dem_lat_ovr') or 1.0))}",
f"dem_lon_ovr = {format_gamma_number(float(dem_oversampling.get('dem_lon_ovr') or 1.0))}",
"Simphase_rpos = -",
"Simphase_azpos = -",
"Simphase_rwin = 256",
@@ -135,24 +350,153 @@ def write_template(
"Simphase_thresh = -",
f"geo_interp = {geo_interp}",
]
dem = str(prepared_dem_path or "").strip()
dem = str(dem_path or "").strip()
if dem and Path(dem).is_file() and Path(dem + ".par").is_file():
lines.append(f"DEM = {dem}")
source = str(prepared_dem_source or "").strip()
if source:
lines.append(f"prepared_dem_source = {source}")
template_path.parent.mkdir(parents=True, exist_ok=True)
template_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def convert_to_db_geotiff(source_tif: Path, target_tif: Path, nodata_value: float) -> dict[str, Any]:
def normalize_speckle_filter_method(method: str) -> str:
text = str(method or "").strip().lower()
if text in {"", "0", "false", "none", "off", "disabled", "no"}:
return "none"
if text in {"lee", "lee_filter"}:
return "lee"
raise ValueError(f"Unsupported speckle filter method: {method}")
def normalize_speckle_filter_size(size: int | float | str) -> int:
try:
value = int(float(size or 5))
except Exception:
value = 5
value = max(3, min(99, value))
if value % 2 == 0:
value += 1
return value
def moving_sum_axis(values: Any, size: int, axis: int) -> Any:
import numpy as np
radius = size // 2
pad_width = [(0, 0)] * values.ndim
pad_width[axis] = (radius, size - 1 - radius)
padded = np.pad(values, pad_width, mode="edge")
cumulative = np.cumsum(padded, axis=axis, dtype="float64")
zero_shape = list(cumulative.shape)
zero_shape[axis] = 1
cumulative = np.concatenate([np.zeros(zero_shape, dtype="float64"), cumulative], axis=axis)
length = values.shape[axis]
start = np.arange(0, length)
end = np.arange(size, size + length)
return np.take(cumulative, end, axis=axis) - np.take(cumulative, start, axis=axis)
def box_sum(values: Any, size: int) -> Any:
return moving_sum_axis(moving_sum_axis(values, size, axis=0), size, axis=1)
def local_power_stats(data: Any, valid: Any, window_size: int) -> tuple[Any, Any, Any]:
import numpy as np
values = np.where(valid, data, 0.0).astype("float64", copy=False)
weights = valid.astype("float64", copy=False)
count = box_sum(weights, window_size)
power_sum = box_sum(values, window_size)
power_sq_sum = box_sum(values * values, window_size)
mean = np.divide(power_sum, count, out=np.zeros_like(power_sum), where=count > 0)
mean_sq = np.divide(power_sq_sum, count, out=np.zeros_like(power_sq_sum), where=count > 0)
variance = np.maximum(mean_sq - mean * mean, 0.0)
valid_fraction = count / float(window_size * window_size)
return mean, variance, valid_fraction
def apply_speckle_filter_power(
data: Any,
invalid: Any,
*,
method: str,
window_size: int,
equivalent_number_of_looks: float = 0.0,
) -> tuple[Any, dict[str, Any]]:
import numpy as np
normalized_method = normalize_speckle_filter_method(method)
normalized_size = normalize_speckle_filter_size(window_size)
record: dict[str, Any] = {
"enabled": normalized_method != "none",
"method": normalized_method,
"window_size": normalized_size,
"equivalent_number_of_looks": float(equivalent_number_of_looks or 0.0),
"domain": "linear_power",
}
if normalized_method == "none":
return data, record
valid = ~invalid
valid_count = int(np.count_nonzero(valid))
record["valid_pixels"] = valid_count
if valid_count == 0:
record["enabled"] = False
record["warning"] = "no valid positive pixels to filter"
return data, record
local_mean, local_variance, valid_fraction = local_power_stats(data, valid, normalized_size)
stats_mask = valid & np.isfinite(local_variance) & np.isfinite(local_mean) & (valid_fraction > 0.0)
enl = float(equivalent_number_of_looks or 0.0)
if math.isfinite(enl) and enl > 0:
noise_variance = np.maximum((local_mean * local_mean) / enl, 0.0)
weight = np.divide(
np.maximum(local_variance - noise_variance, 0.0),
local_variance,
out=np.zeros_like(local_variance),
where=local_variance > 0,
)
record["noise_variance_model"] = "local_mean_squared_over_enl"
else:
noise_samples = local_variance[stats_mask]
global_noise_variance = float(np.nanmedian(noise_samples)) if noise_samples.size else 0.0
if not math.isfinite(global_noise_variance) or global_noise_variance <= 0:
record["enabled"] = False
record["warning"] = "local variance estimate is zero; kept unfiltered power values"
return data, record
noise_variance = global_noise_variance
weight = np.divide(
local_variance,
local_variance + noise_variance,
out=np.zeros_like(local_variance),
where=(local_variance + noise_variance) > 0,
)
record["noise_variance"] = global_noise_variance
record["noise_variance_model"] = "global_median_local_variance"
filtered = local_mean + weight * (data.astype("float64", copy=False) - local_mean)
filtered = np.where(np.isfinite(filtered) & (filtered > 0), filtered, data)
output = data.astype("float32", copy=True)
output[stats_mask] = filtered[stats_mask].astype("float32")
return output, record
def convert_to_db_geotiff(
source_tif: Path,
target_tif: Path,
nodata_value: float,
*,
speckle_filter_method: str = "none",
speckle_filter_size: int = 5,
speckle_filter_enl: float = 0.0,
) -> dict[str, Any]:
filter_method = normalize_speckle_filter_method(speckle_filter_method)
filter_size = normalize_speckle_filter_size(speckle_filter_size)
try:
import numpy as np
import rasterio
except Exception as exc:
shutil.copy2(source_tif, target_tif)
return {
"target": str(target_tif),
"backscatter_unit": "gamma_mli_power",
"warning": f"rasterio/numpy unavailable; kept power values: {exc}",
}
raise RuntimeError(f"rasterio/numpy unavailable; cannot create filtered dB GeoTIFF: {exc}") from exc
with rasterio.open(source_tif) as src:
data = src.read(1).astype("float32")
@@ -163,14 +507,23 @@ def convert_to_db_geotiff(source_tif: Path, target_tif: Path, nodata_value: floa
if src_nodata is not None:
invalid |= data == src_nodata
invalid |= data <= 0
filtered_data, speckle_filter = apply_speckle_filter_power(
data,
invalid,
method=filter_method,
window_size=filter_size,
equivalent_number_of_looks=float(speckle_filter_enl or 0.0),
)
invalid |= ~np.isfinite(filtered_data)
invalid |= filtered_data <= 0
db_data = np.full(data.shape, nodata_value, dtype="float32")
db_data[~invalid] = (10.0 * np.log10(data[~invalid])).astype("float32")
db_data[~invalid] = (10.0 * np.log10(filtered_data[~invalid])).astype("float32")
profile.update(dtype="float32", count=1, nodata=nodata_value, compress="deflate")
target_tif.parent.mkdir(parents=True, exist_ok=True)
with rasterio.open(target_tif, "w", **profile) as dst:
dst.write(db_data, 1)
return {"target": str(target_tif), "backscatter_unit": "gamma_mli_db"}
return {"target": str(target_tif), "backscatter_unit": "gamma_mli_db", "speckle_filter": speckle_filter}
def main() -> int:
@@ -201,6 +554,18 @@ def main() -> int:
env["PATH"] = f"{pyint_home / 'pyint'}:{env.get('PATH', '')}"
staged_inputs = stage_lt_inputs(source_path, download_dir, date)
dem_oversampling = calculate_dem_oversampling(
dem_resolution_m=float(args.dem_resolution_m or 30.0),
target_grid_size_m=float(args.target_grid_size_m or 30.0),
dem_lat_ovr=float(args.dem_lat_ovr or 0.0),
dem_lon_ovr=float(args.dem_lon_ovr or 0.0),
)
prepared_dem = inspect_prepared_dem_path(args.prepared_dem_path)
if not prepared_dem.get("kind"):
raise RuntimeError(f"A prepared DEM is required for LT analysis GeoTIFF production: {args.prepared_dem_path}")
dem_path = prepared_dem.get("direct_dem_path") or ""
prepared_dem_conversion: dict[str, Any] | None = None
template_path = template_dir / f"{project_name}.template"
write_template(
template_path=template_path,
@@ -208,7 +573,9 @@ def main() -> int:
range_looks=max(1, int(args.range_looks)),
azimuth_looks=max(1, int(args.azimuth_looks)),
geo_interp=str(args.geo_interp or "1"),
prepared_dem_path=args.prepared_dem_path,
dem_path=dem_path,
prepared_dem_source=str(prepared_dem.get("source_dem_path") or ""),
dem_oversampling=dem_oversampling,
)
commands: list[dict[str, Any]] = []
@@ -221,6 +588,44 @@ def main() -> int:
stage="down2slc_lt1",
)
)
if prepared_dem.get("kind") == "source_dem":
slc_par = project_dir / "SLC" / date / f"{date}.slc.par"
if not slc_par.is_file():
raise FileNotFoundError(f"Gamma SLC parameter file missing before DEM conversion: {slc_par}")
dem_target_base = dem_root / project_name / project_name
dem_target_base.parent.mkdir(parents=True, exist_ok=True)
prepared_dem_conversion, dem_commands = build_gamma_dem_from_source(
source_dem=Path(str(prepared_dem.get("source_dem_path"))),
target_base=dem_target_base,
slc_par=slc_par,
pyint_home=pyint_home,
log_dir=log_dir,
env=env,
)
commands.extend(dem_commands)
dem_path = str(Path(str(dem_target_base) + ".dem"))
dem_par_path = Path(str(dem_path) + ".par") if dem_path else Path()
if dem_path and dem_par_path.is_file():
dem_oversampling = calculate_dem_oversampling_from_gamma_dem(
dem_par_path=dem_par_path,
target_grid_size_m=float(args.target_grid_size_m or 30.0),
explicit_dem_lat_ovr=float(args.dem_lat_ovr or 0.0),
explicit_dem_lon_ovr=float(args.dem_lon_ovr or 0.0),
)
write_template(
template_path=template_path,
date=date,
range_looks=max(1, int(args.range_looks)),
azimuth_looks=max(1, int(args.azimuth_looks)),
geo_interp=str(args.geo_interp or "1"),
dem_path=dem_path,
prepared_dem_source=str(prepared_dem.get("source_dem_path") or ""),
dem_oversampling=dem_oversampling,
)
commands.append(
run_logged(
[sys.executable, str(pyint_home / "pyint" / "generate_rdc_dem.py"), project_name],
@@ -286,9 +691,25 @@ def main() -> int:
raise RuntimeError(f"data2geotiff did not create output: {power_tif}")
final_tif = output_dir / "analysis_ready.tif"
conversion = convert_to_db_geotiff(power_tif, final_tif, float(args.nodata_value)) if args.to_db else {
speckle_filter_config = {
"method": normalize_speckle_filter_method(args.speckle_filter_method),
"window_size": normalize_speckle_filter_size(args.speckle_filter_size),
}
conversion = convert_to_db_geotiff(
power_tif,
final_tif,
float(args.nodata_value),
speckle_filter_method=args.speckle_filter_method,
speckle_filter_size=args.speckle_filter_size,
speckle_filter_enl=float(args.speckle_filter_enl or (range_looks * max(1, int(args.azimuth_looks)))),
) if args.to_db else {
"target": str(final_tif),
"backscatter_unit": "gamma_mli_power",
"speckle_filter": {
"enabled": False,
**speckle_filter_config,
"warning": "not applied because --to-db was disabled",
},
}
if not args.to_db:
shutil.copy2(power_tif, final_tif)
@@ -313,6 +734,24 @@ def main() -> int:
"geo_amp": str(geo_amp),
},
"looks": {"range": range_looks, "azimuth": max(1, int(args.azimuth_looks))},
"speckle_filter": conversion.get("speckle_filter"),
"processing_steps": {
"multilook": {
"enabled": True,
"range_looks": range_looks,
"azimuth_looks": max(1, int(args.azimuth_looks)),
},
"geocode": {"enabled": True, "interpolation": str(args.geo_interp or "1")},
"speckle_filter": conversion.get("speckle_filter"),
"db_conversion": {"enabled": bool(args.to_db), "unit": conversion.get("backscatter_unit")},
},
"dem": {
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
"prepared_dem_kind": prepared_dem.get("kind"),
"gamma_dem_path": dem_path,
"conversion": prepared_dem_conversion,
"oversampling": dem_oversampling,
},
"commands": commands,
"conversion": conversion,
}
backend/app/routers/__init__.py
+2
View File
@@ -14,6 +14,7 @@ from . import (
hazard,
health,
idl,
landsar_lt1_production,
license,
logs,
monitor,
@@ -55,6 +56,7 @@ def include_all_routers(router: APIRouter) -> None:
router.include_router(dinsar.router)
router.include_router(dinsar_products.router)
router.include_router(dinsar_production.router)
router.include_router(landsar_lt1_production.router)
router.include_router(sbas_insar_production.router)
router.include_router(sbas_insar_products.router)
router.include_router(timeseries_production.router)
backend/app/routers/idl.py
+23 -79
View File
@@ -16,7 +16,6 @@ from .dependencies import _require_admin, _get_current_user, _validate_export_pa
from ..models import AuthUserORM
from ..services import envi_service
from ..services.job_queue_service import job_queue_service
from ..services.result_catalog_service import result_catalog_service
from ..services.task_service import task_service
router = APIRouter()
@@ -61,38 +60,6 @@ class SarscapeSbasInspectRequest(BaseModel):
timeout_seconds: Optional[int] = Field(default=120, ge=10, le=600)
def _normalize_existing_dir(path: Optional[str]) -> Optional[str]:
text = str(path or "").strip()
if not text:
return None
normalized = os.path.normpath(os.path.abspath(text))
if not os.path.isdir(normalized):
return None
return normalized
def _dedupe_publish_roots(*paths: Optional[str]) -> list[str]:
ordered: list[str] = []
for raw_path in paths:
normalized = _normalize_existing_dir(raw_path)
if not normalized:
continue
if any(
normalized == existing or normalized.startswith(existing + os.sep)
for existing in ordered
):
continue
ordered = [
existing
for existing in ordered
if not existing.startswith(normalized + os.sep)
]
ordered.append(normalized)
return ordered
# ---------------------------------------------------------------------------
# Job queue helper
# ---------------------------------------------------------------------------
@@ -292,7 +259,7 @@ async def get_task_overview_endpoint(
return result
@router.post("/idl/extract-disp")
@router.post("/idl/extract-disp", status_code=202)
async def extract_disp_endpoint(
request: ExtractDispRequest,
admin_user: AuthUserORM = Depends(_require_admin),
@@ -303,52 +270,29 @@ async def extract_disp_endpoint(
if request.dest_dir:
_validate_export_path(request.dest_dir, "dest_dir")
try:
result = await asyncio.to_thread(
envi_service.extract_disp_results, request.root_dir, request.dest_dir
payload = {
"root_dir": request.root_dir,
"dest_dir": request.dest_dir,
}
task_id = await task_service.create_task(
"EXTRACT_DINSAR_PRODUCTS",
"D-InSAR 结果提取与登记",
params=payload,
db=db,
)
job_id = await job_queue_service.create_job(
"EXTRACT_DINSAR_PRODUCTS",
payload=payload,
task_id=task_id,
db=db,
)
await db.commit()
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
raise HTTPException(status_code=409, detail=str(exc)) from exc
publish_roots = _dedupe_publish_roots(result.get("target_dir"))
catalog_status: Dict[str, Any] = {
"attempted": False,
"status": "skipped",
"source_directories": publish_roots,
"message": "catalog publish skipped",
return {
"queued": True,
"task_id": task_id,
"job_id": job_id,
"message": "D-InSAR 结果提取与登记任务已入队",
}
if publish_roots:
try:
catalog_status["attempted"] = True
publish_result = await result_catalog_service.publish_from_sources(
db,
publish_roots,
)
rebuild_result = None
if int(publish_result.get("processed", 0) or 0) > 0:
rebuild_result = await result_catalog_service.rebuild_catalog(
db,
full_rebuild=True,
)
catalog_status = {
"attempted": True,
"status": "ok",
"source_directories": publish_roots,
"publish": publish_result,
"rebuild": rebuild_result,
"message": (
"catalog published and rebuilt"
if rebuild_result is not None
else "catalog publish finished with no rebuild needed"
),
}
except Exception as exc:
await db.rollback()
catalog_status = {
"attempted": True,
"status": "error",
"source_directories": publish_roots,
"message": str(exc),
}
result["catalog"] = catalog_status
return result
backend/app/routers/landsar_lt1_production.py
+485
View File
@@ -0,0 +1,485 @@
from __future__ import annotations
import os
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field, field_validator
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..database import get_db
from ..models import AuthUserORM, RadarDataORM, SARSceneGeoORM
from ..services.job_handlers import JOB_TYPE_SAR_SCENE_PREPROCESS
from ..services.job_queue_service import job_queue_service
from ..services.landsar_lt1_production_service import landsar_lt1_production_service
from ..services.task_service import task_service
from ..utils import normalize_satellite_family
from .dependencies import _add_operation_audit_log, _get_current_user, _require_admin
router = APIRouter()
STATIC_ASSET_CACHE_HEADERS = {"Cache-Control": "public, max-age=31536000, immutable"}
class LandsarLt1ImageProductionRequest(BaseModel):
source_asset_ids: List[int] = Field(default_factory=list)
radar_data_ids: List[int] = Field(default_factory=list)
mode: str = "scene"
task_name: Optional[str] = None
@field_validator("mode")
@classmethod
def _validate_mode(cls, value):
mode = str(value or "scene").strip().lower()
if mode == "stack":
mode = "batch"
if mode not in {"scene", "batch"}:
raise ValueError("mode must be scene or batch")
return mode
def _dedupe_positive_ids(values: List[int]) -> List[int]:
result: List[int] = []
for value in values or []:
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0 and parsed not in result:
result.append(parsed)
return result
def _scene_product_marker(scene: SARSceneGeoORM) -> Dict[str, Any]:
return {
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"product_id": f"sar_scene_geo:{scene.id}",
"product_family": "lt1_analysis_ready_geotiff",
"engine_code": scene.analysis_engine,
"profile_code": scene.analysis_profile,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"status": scene.status,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
}
def _scene_asset_items(scene: SARSceneGeoORM) -> List[Dict[str, Any]]:
candidates = [
(1, "analysis_tif", "analysis_ready.tif", scene.analysis_tif_path, "image/tiff", True),
(2, "preview", "preview.png", scene.analysis_preview_path, "image/png", False),
]
metadata = scene.analysis_metadata_json if isinstance(scene.analysis_metadata_json, dict) else {}
manifest_path = str(metadata.get("manifest_path") or "").strip()
if manifest_path:
candidates.append((3, "manifest", "manifest.json", manifest_path, "application/json", False))
if scene.analysis_dir:
quality_path = os.path.join(scene.analysis_dir, "quality.json")
candidates.append((4, "quality", "quality.json", quality_path, "application/json", False))
assets: List[Dict[str, Any]] = []
for asset_id, role, name, path, media_type, primary in candidates:
if not path:
continue
assets.append(
{
"id": asset_id,
"role": role,
"name": name,
"relative_path": os.path.basename(path),
"absolute_path": path,
"format": os.path.splitext(path)[1].lower().lstrip(".") or None,
"media_type": media_type,
"is_required": primary,
"is_primary": primary,
"exists": os.path.isfile(path),
"file_size": os.path.getsize(path) if os.path.isfile(path) else None,
}
)
return assets
async def _resolve_lt1_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> List[RadarDataORM]:
source_asset_ids = _dedupe_positive_ids(request.source_asset_ids)
radar_data_ids = _dedupe_positive_ids(request.radar_data_ids)
filters = []
if radar_data_ids:
filters.append(RadarDataORM.id.in_(radar_data_ids))
if source_asset_ids:
filters.append(RadarDataORM.source_product_ref_id.in_(source_asset_ids))
if not filters:
return []
result = await db.execute(select(RadarDataORM).where(*([filters[0]] if len(filters) == 1 else [filters[0] | filters[1]])))
radars = list(result.scalars().all())
unique: Dict[int, RadarDataORM] = {}
for radar in radars:
if not radar.id:
continue
family = normalize_satellite_family(radar.satellite_family or radar.satellite)
if str(family or "").upper() != "LT1":
continue
unique[int(radar.id)] = radar
return [unique[key] for key in sorted(unique.keys())]
async def _produced_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> Dict[int, dict]:
radars = await _resolve_lt1_radars_for_request(db, request)
radar_ids = [int(item.id) for item in radars if item.id]
if not radar_ids:
return {}
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()}
async def _active_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> Dict[int, dict]:
radars = await _resolve_lt1_radars_for_request(db, request)
radar_ids = [int(item.id) for item in radars if item.id]
if not radar_ids:
return {}
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status.in_(["PENDING", "RUNNING"]),
)
)
return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()}
def _already_produced_blocker(produced: Dict[int, dict]) -> str:
first_id = sorted(produced.keys())[0]
marker = produced[first_id] or {}
product_id = marker.get("product_id") or "unknown"
return f"Radar data {first_id} already has an analysis-ready GeoTIFF: {product_id}"
def _active_blocker(active: Dict[int, dict]) -> str:
first_id = sorted(active.keys())[0]
marker = active[first_id] or {}
return f"Radar data {first_id} already has an active GeoTIFF production task (scene_id={marker.get('scene_id')})."
@router.get("/landsar-lt1-production/capabilities")
async def get_landsar_lt1_capabilities(
current_user: AuthUserORM = Depends(_get_current_user),
):
_ = current_user
legacy = landsar_lt1_production_service.check_capabilities()
return {
"catalog_name": "sar_scene_geo",
"supported_profiles": ["lt1_gamma_geocoded_mli"],
"engine": "lt_gamma",
"available": True,
"status": "configured",
"message": "LT-1 image production uses the existing Gamma single-scene pipeline: multilook, geocode, and analysis-ready GeoTIFF registration.",
"legacy_landsar_import": legacy,
}
@router.post("/landsar-lt1-production/preview")
async def preview_landsar_lt1_production(
request: LandsarLt1ImageProductionRequest,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
blockers: List[str] = []
warnings: List[str] = []
radars = await _resolve_lt1_radars_for_request(db, request)
if not radars:
blockers.append("No LT-1 radar records were resolved from the selected source assets.")
if request.mode == "scene" and len(radars) != 1:
blockers.append("Scene mode requires exactly one LT-1 source asset.")
if request.mode == "batch" and len(radars) < 1:
blockers.append("Batch mode requires at least one LT-1 source asset.")
produced = await _produced_radars_for_request(db, request)
if produced:
blockers.append(_already_produced_blocker(produced))
active = await _active_radars_for_request(db, request)
if active:
blockers.append(_active_blocker(active))
if request.mode == "batch":
warnings.append("Batch mode submits one independent geocoded GeoTIFF task per scene; it does not build a D-InSAR stack.")
preview = {
"allow_submit": not blockers,
"blockers": blockers,
"warnings": warnings,
"mode": request.mode,
"profile_code": "lt1_gamma_geocoded_mli",
"engine": "lt_gamma",
"scene_count": len(radars),
"source_asset_count": len(_dedupe_positive_ids(request.source_asset_ids)),
"radar_data_count": len(radars),
"produced_radars": produced,
"active_radars": active,
"scenes": [
{
"radar_data_id": radar.id,
"source_asset_id": radar.source_product_ref_id,
"satellite": radar.satellite,
"imaging_date": radar.imaging_date,
"imaging_mode": radar.imaging_mode,
"polarization": radar.polarization,
"file_path": radar.file_path,
}
for radar in radars
],
}
return preview
@router.post("/landsar-lt1-production/run", status_code=202)
async def queue_landsar_lt1_production(
request: LandsarLt1ImageProductionRequest,
http_request: Request,
db: AsyncSession = Depends(get_db),
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
preview = await preview_landsar_lt1_production(request, current_user=admin_user, db=db)
if preview.get("blockers"):
raise HTTPException(status_code=400, detail={"blockers": preview.get("blockers")})
queued: List[Dict[str, Any]] = []
radars = await _resolve_lt1_radars_for_request(db, request)
for radar in radars:
result = await db.execute(
select(SARSceneGeoORM)
.where(SARSceneGeoORM.radar_data_id == int(radar.id))
.with_for_update(skip_locked=True)
)
scene = result.scalar_one_or_none()
if scene and scene.status in ("PENDING", "RUNNING"):
raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an active GeoTIFF production task.")
if scene and scene.status == "DONE" and scene.analysis_tif_path:
raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an analysis-ready GeoTIFF.")
if not scene:
scene = SARSceneGeoORM(radar_data_id=int(radar.id), status="PENDING")
db.add(scene)
await db.flush()
else:
scene.status = "PENDING"
scene.error_msg = None
await db.flush()
scene_id = int(scene.id)
await db.commit()
payload = {
"scene_id": scene_id,
"radar_data_id": int(radar.id),
"engine": "lt_gamma",
"source_asset_id": radar.source_product_ref_id,
"requested_from": "landsar_lt1_production",
}
task_label = request.task_name or radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}"
task_type = f"LT1_SCENE_GEOTIFF_{scene_id}"
try:
task_id = await task_service.create_task(
task_type,
f"LT-1 geocoded GeoTIFF: {task_label}",
params=payload,
)
job_id = await job_queue_service.create_job(
JOB_TYPE_SAR_SCENE_PREPROCESS,
payload=payload,
task_id=task_id,
max_attempts=3,
)
except Exception as exc:
failed_scene = await db.get(SARSceneGeoORM, scene_id)
if failed_scene and failed_scene.status == "PENDING":
failed_scene.status = "FAILED"
failed_scene.error_msg = "Job queue failed"
await db.commit()
raise HTTPException(status_code=409 if "conflict" in str(exc).lower() else 400, detail=str(exc)) from exc
queued.append(
{
"task_id": task_id,
"job_id": job_id,
"scene_id": scene_id,
"radar_data_id": int(radar.id),
"source_asset_id": radar.source_product_ref_id,
}
)
await _add_operation_audit_log(
db,
request=http_request,
action="lt1_geotiff_production_queued",
resource="landsar-lt1-production/run",
detail={
"queued": queued,
"mode": request.mode,
"scene_count": len(queued),
},
)
await db.commit()
return {
"message": "LT-1 geocoded GeoTIFF production job queued.",
"task_id": queued[0]["task_id"] if len(queued) == 1 else None,
"job_id": queued[0]["job_id"] if len(queued) == 1 else None,
"queued": queued,
"preview": preview,
}
@router.get("/landsar-lt1-production/products")
async def list_landsar_lt1_products(
limit: int = 100,
offset: int = 0,
status: Optional[str] = None,
query: Optional[str] = None,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
safe_limit = max(1, min(500, int(limit or 100)))
safe_offset = max(0, int(offset or 0))
filters = [
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
]
if status:
filters.append(SARSceneGeoORM.status == str(status).strip().upper())
if query:
like = f"%{str(query).strip()}%"
filters.append(RadarDataORM.product_unique_id.ilike(like) | RadarDataORM.unique_id.ilike(like) | RadarDataORM.file_path.ilike(like))
total_result = await db.execute(
select(func.count(SARSceneGeoORM.id))
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*filters)
)
total = int(total_result.scalar_one() or 0)
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*filters)
.order_by(SARSceneGeoORM.updated_at.desc().nullslast(), SARSceneGeoORM.id.desc())
.limit(safe_limit)
.offset(safe_offset)
)
items = []
for scene, radar in result.all():
marker = _scene_product_marker(scene)
items.append(
{
"id": scene.id,
"product_id": marker["product_id"],
"catalog_name": "sar_scene_geo",
"product_family": "lt1_analysis_ready_geotiff",
"product_type": "analysis_ready_geotiff",
"display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}",
"task_name": "",
"profile_code": scene.analysis_profile,
"engine_code": scene.analysis_engine,
"status": scene.status,
"health_status": "OK" if scene.status == "DONE" and scene.analysis_tif_path else "PENDING",
"publish_dir": scene.analysis_dir,
"manifest_path": (scene.analysis_metadata_json or {}).get("manifest_path") if isinstance(scene.analysis_metadata_json, dict) else None,
"native_output_dir": scene.analysis_dir,
"primary_asset_path": scene.analysis_tif_path,
"summary": {
"scene_count": 1,
"radar_data_id": radar.id,
"source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [],
"imaging_date": radar.imaging_date,
"polarization": radar.polarization,
"pixel_size_m": scene.pixel_size_m,
"backscatter_unit": scene.analysis_backscatter_unit,
},
"tags": {"engine": scene.analysis_engine, "profile": scene.analysis_profile},
"produced_at": scene.updated_at.isoformat() if scene.updated_at else None,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
"registered_at": scene.created_at.isoformat() if scene.created_at else None,
}
)
return {"total": total, "limit": safe_limit, "offset": safe_offset, "items": items}
@router.get("/landsar-lt1-production/products/{product_db_id}")
async def get_landsar_lt1_product_detail(
product_db_id: int,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(
SARSceneGeoORM.id == product_db_id,
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
row = result.first()
if row is None:
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found")
scene, radar = row
marker = _scene_product_marker(scene)
detail = {
"id": scene.id,
"product_id": marker["product_id"],
"catalog_name": "sar_scene_geo",
"product_family": "lt1_analysis_ready_geotiff",
"product_type": "analysis_ready_geotiff",
"display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}",
"profile_code": scene.analysis_profile,
"engine_code": scene.analysis_engine,
"status": scene.status,
"publish_dir": scene.analysis_dir,
"primary_asset_path": scene.analysis_tif_path,
"summary": {
"scene_count": 1,
"radar_data_id": radar.id,
"source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [],
"imaging_date": radar.imaging_date,
"polarization": radar.polarization,
"pixel_size_m": scene.pixel_size_m,
"backscatter_unit": scene.analysis_backscatter_unit,
},
"assets": _scene_asset_items(scene),
}
return detail
@router.get("/landsar-lt1-production/products/{product_db_id}/assets/{asset_id}")
async def get_landsar_lt1_product_asset(
product_db_id: int,
asset_id: int,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
scene = await db.get(SARSceneGeoORM, product_db_id)
if scene is None or scene.analysis_engine != "lt_gamma" or scene.analysis_profile != "lt1_gamma_geocoded_mli":
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found")
asset = next((item for item in _scene_asset_items(scene) if int(item["id"]) == int(asset_id)), None)
if asset is None:
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF asset not found")
absolute_path = str(asset.get("absolute_path") or "")
if not absolute_path or not os.path.isfile(absolute_path):
raise HTTPException(status_code=404, detail="Asset file not found")
return FileResponse(
absolute_path,
media_type=str(asset.get("media_type") or "application/octet-stream"),
filename=str(asset.get("name") or os.path.basename(absolute_path)),
headers=STATIC_ASSET_CACHE_HEADERS,
)
backend/app/routers/pairing.py
+52 -6
View File
@@ -26,7 +26,10 @@ from ..models import (
TimeseriesStackPlanORM,
)
from ..services.pairing_cache_service import pairing_cache_service
from ..services.job_handlers import JOB_TYPE_PAIRING_CACHE_REBUILD
from ..services.job_queue_service import job_queue_service
from ..services.spatial_service import spatial_service
from ..services.task_service import task_service
from .dependencies import (
_parse_aoi_from_files,
_parse_aoi_geojson_form_value,
@@ -38,6 +41,49 @@ logger = logging.getLogger(__name__)
router = APIRouter()
async def _queue_pairing_cache_job(
*,
db: AsyncSession,
mode: str,
force_full: bool = False,
) -> Dict[str, object]:
full_rebuild = mode in {"full", "full_rebuild"} or force_full
task_name = (
"D-InSAR pairing cache full rebuild"
if full_rebuild
else "D-InSAR pairing cache dirty reconcile"
)
payload = {
"mode": "full_rebuild" if full_rebuild else "auto_reconcile",
"force_full": bool(full_rebuild),
}
try:
task_id = await task_service.create_task(
JOB_TYPE_PAIRING_CACHE_REBUILD,
task_name,
params=payload,
db=db,
)
job_id = await job_queue_service.create_job(
JOB_TYPE_PAIRING_CACHE_REBUILD,
payload=payload,
max_attempts=1,
task_id=task_id,
db=db,
)
await db.commit()
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
return {
"ok": True,
"queued": True,
"mode": payload["mode"],
"task_id": task_id,
"job_id": job_id,
"message": f"{task_name} queued",
}
def get_pairing_request_from_form(
time_baseline_min: int = Form(1),
time_baseline_max: int = Form(30),
@@ -130,26 +176,26 @@ async def get_pairing_health_endpoint(
return await pairing_cache_service.get_admin_summary(db)
@router.post("/pairing/rebuild-cache")
@router.post("/pairing/rebuild-cache", status_code=202)
async def rebuild_pairing_cache_endpoint(
db: AsyncSession = Depends(get_db),
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
return await pairing_cache_service.rebuild_metric_cache(db, commit=True)
return await _queue_pairing_cache_job(db=db, mode="full_rebuild", force_full=True)
@router.post("/pairing/reconcile-dirty")
@router.post("/pairing/reconcile-dirty", status_code=202)
async def reconcile_dirty_pairing_endpoint(
force_full: bool = Query(False),
db: AsyncSession = Depends(get_db),
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
return await pairing_cache_service.reconcile_dirty_scenes(
db,
return await _queue_pairing_cache_job(
db=db,
mode="full_rebuild" if force_full else "auto_reconcile",
force_full=force_full,
commit=True,
)
backend/app/routers/radar.py
+84 -14
View File
@@ -23,6 +23,7 @@ from ..models import (
RadarDataORM,
RadarDataPage,
RadarPreviewStatusInfo,
SARSceneGeoORM,
ScanRequest,
)
from ..services.data_service import data_service
@@ -154,6 +155,51 @@ def _normalize_list_pagination(limit: int, offset: int) -> Tuple[int, int]:
return safe_limit, safe_offset
def _lt1_image_marker(scene: SARSceneGeoORM) -> Dict[str, Any]:
return {
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"product_id": f"sar_scene_geo:{scene.id}",
"product_family": "lt1_analysis_ready_geotiff",
"engine_code": scene.analysis_engine,
"profile_code": scene.analysis_profile,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"status": scene.status,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
}
async def _decorate_lt1_landsar_status(db: AsyncSession, items: List[RadarDataORM]) -> List[RadarData]:
payloads = [RadarData.model_validate(item) for item in items]
radar_ids = [
int(item.id)
for item in items
if item.id and str(item.satellite_family or item.satellite or "").upper().replace("-", "") in {"LT1", "LT"}
]
if not radar_ids:
return payloads
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
produced = {int(scene.radar_data_id): _lt1_image_marker(scene) for scene in result.scalars().all()}
for payload in payloads:
marker = produced.get(int(payload.id or 0))
if marker:
payload.lt1_image_produced = True
payload.lt1_image_product = marker
payload.lt1_landsar_produced = True
payload.lt1_landsar_product = marker
return payloads
def _normalize_optional_text(value: Optional[str]) -> Optional[str]:
if value is None:
return None
@@ -256,6 +302,24 @@ def _build_radar_preview_status(
)
def _build_cached_radar_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo:
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
preview_cache_path = str(record.preview_cache_path or "")
has_geo_cache = (
os.path.exists(geo_cache_path)
or bool(preview_cache_path and os.path.exists(preview_cache_path))
)
has_raw_cache = os.path.exists(raw_cache_path)
cached_status = str(record.preview_cache_status or "NONE").upper()
source_found = cached_status in {"READY", "FAILED"} or has_geo_cache or has_raw_cache
return _build_radar_preview_status(
record=record,
source_found=source_found,
has_geo_cache=has_geo_cache,
has_raw_cache=has_raw_cache,
)
async def _build_radar_preview_cache(
record: RadarDataORM,
db: AsyncSession,
@@ -510,10 +574,13 @@ async def search_radar_data_endpoint(
limit: int = Form(500),
offset: int = Form(0),
satellite: Optional[str] = Form(None),
satellite_family: Optional[str] = Form(None),
source_format: Optional[str] = Form(None),
satellite_mode: Optional[str] = Form(None),
receiving_station: Optional[str] = Form(None),
imaging_mode: Optional[str] = Form(None),
orbit_circle: Optional[str] = Form(None),
relative_orbit: Optional[str] = Form(None),
acquisition_time_utc: Optional[str] = Form(None),
product_type: Optional[str] = Form(None),
polarization: Optional[str] = Form(None),
@@ -536,10 +603,13 @@ async def search_radar_data_endpoint(
n_satellite_list: Optional[List[str]] = None
if n_satellite_raw and "," in n_satellite_raw:
n_satellite_list = [s.strip() for s in n_satellite_raw.split(",") if s.strip()]
n_satellite_family = _normalize_optional_text(satellite_family)
n_source_format = _normalize_optional_text(source_format)
n_satellite_mode = _normalize_optional_text(satellite_mode)
n_receiving_station = _normalize_optional_text(receiving_station)
n_imaging_mode = _normalize_optional_text(imaging_mode)
n_orbit_circle = _normalize_optional_text(orbit_circle)
n_relative_orbit = _normalize_optional_text(relative_orbit)
n_acquisition_time = _normalize_optional_text(acquisition_time_utc)
n_product_type = _normalize_optional_text(product_type)
n_polarization = _normalize_optional_text(polarization)
@@ -584,6 +654,10 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.satellite.in_(n_satellite_list))
elif n_satellite_raw:
filters.append(RadarDataORM.satellite.ilike(f"%{n_satellite_raw}%"))
if n_satellite_family:
filters.append(func.upper(RadarDataORM.satellite_family) == n_satellite_family.upper())
if n_source_format:
filters.append(func.upper(RadarDataORM.source_format) == n_source_format.upper())
if n_satellite_mode:
filters.append(RadarDataORM.satellite_mode.ilike(f"%{n_satellite_mode}%"))
if n_receiving_station:
@@ -592,6 +666,8 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.imaging_mode.ilike(f"%{n_imaging_mode}%"))
if n_orbit_circle:
filters.append(RadarDataORM.orbit_circle.ilike(f"%{n_orbit_circle}%"))
if n_relative_orbit:
filters.append(RadarDataORM.relative_orbit.ilike(f"%{n_relative_orbit}%"))
if n_acquisition_time:
filters.append(RadarDataORM.acquisition_time_utc.ilike(f"%{n_acquisition_time}%"))
if n_product_type:
@@ -606,10 +682,11 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.orbit_direction.ilike(f"%{n_orbit_direction}%"))
if has_orbit_data is not None:
filters.append(RadarDataORM.has_orbit_data == has_orbit_data)
normalized_imaging_date = func.replace(RadarDataORM.imaging_date, "-", "")
if n_date_from:
filters.append(RadarDataORM.imaging_date >= n_date_from)
filters.append(normalized_imaging_date >= n_date_from.replace("-", ""))
if n_date_to:
filters.append(RadarDataORM.imaging_date <= n_date_to)
filters.append(normalized_imaging_date <= n_date_to.replace("-", ""))
if resolved_aoi_wkt:
aoi_geom = func.ST_GeomFromText(resolved_aoi_wkt, 4326)
filters.append(ST_Intersects(RadarDataORM.geom, aoi_geom))
@@ -630,8 +707,9 @@ async def search_radar_data_endpoint(
result = await db.execute(data_stmt)
items = result.scalars().all()
decorated_items = await _decorate_lt1_landsar_status(db, items)
return RadarDataSearchPageResponse(
items=items,
items=decorated_items,
total=total,
limit=limit,
offset=offset,
@@ -663,8 +741,9 @@ async def get_all_data_endpoint(
.limit(limit)
)
items = result.scalars().all()
decorated_items = await _decorate_lt1_landsar_status(db, items)
return RadarDataPage(
items=items,
items=decorated_items,
total=total,
limit=limit,
offset=offset,
@@ -724,16 +803,7 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
source_found = bool(await asyncio.to_thread(data_service.find_radar_preview_source, record.file_path))
return _build_radar_preview_status(
record=record,
source_found=source_found,
has_geo_cache=has_geo_cache,
has_raw_cache=has_raw_cache,
)
return _build_cached_radar_preview_status(record)
@router.post("/radar-data/{data_id}/rebuild-preview-cache", response_model=RadarPreviewStatusInfo)
backend/app/routers/tasks_runtime.py
+264 -1
View File
@@ -2,16 +2,19 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime, timedelta
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from sqlalchemy import case, func, select
from .. import database
from ..auth_service import SESSION_COOKIE_NAME, get_user_by_session_token
from ..auth_utils import verify_password
from ..models import AuthUserORM, TaskInfo
from ..config import settings
from ..models import AuthUserORM, SystemJobORM, SystemTaskORM, SystemWorkerHeartbeatORM, TaskInfo
from ..services.dinsar_production_service import dinsar_production_service
from ..services.task_service import (
TASK_ACTIVE_DEFAULT_LIMIT,
@@ -47,6 +50,73 @@ def _split_csv_param(raw: Optional[str]) -> List[str]:
return values
def _dt(value):
return value.isoformat() if value else None
def _task_payload(task: SystemTaskORM) -> dict:
return TaskInfo.model_validate(task).model_dump(mode="json")
def _worker_note(worker: SystemWorkerHeartbeatORM) -> dict:
try:
parsed = json.loads(str(worker.note or "") or "{}")
return parsed if isinstance(parsed, dict) else {}
except Exception:
return {}
def _worker_concurrency(worker: SystemWorkerHeartbeatORM) -> int:
note = _worker_note(worker)
try:
return max(1, int(note.get("concurrency") or 1))
except (TypeError, ValueError):
return 1
def _job_payload(job: SystemJobORM, task_by_id: dict[str, SystemTaskORM]) -> dict:
task = task_by_id.get(str(job.task_id or ""))
return {
"job_id": job.job_id,
"job_type": job.job_type,
"status": job.status,
"priority": int(job.priority or 0),
"attempts": int(job.attempts or 0),
"max_attempts": int(job.max_attempts or 0),
"task_id": job.task_id,
"task_type": task.task_type if task else None,
"task_name": task.task_name if task else None,
"task_status": task.status if task else None,
"task_progress": int(task.progress or 0) if task else None,
"task_message": task.message if task else None,
"workflow_run_id": job.workflow_run_id,
"workflow_step_id": job.workflow_step_id,
"locked_by": job.locked_by,
"locked_at": _dt(job.locked_at),
"heartbeat_at": _dt(job.heartbeat_at),
"next_run_at": _dt(job.next_run_at),
"created_at": _dt(job.created_at),
"started_at": _dt(job.started_at),
"finished_at": _dt(job.finished_at),
"last_error": job.last_error,
}
def _worker_payload(worker: SystemWorkerHeartbeatORM, active_job_count: int, concurrency: int) -> dict:
note = _worker_note(worker)
return {
"worker_id": worker.worker_id,
"hostname": worker.hostname,
"pid": worker.pid,
"note": worker.note,
"concurrency": concurrency,
"allowed_job_types": note.get("allowed_job_types") if isinstance(note.get("allowed_job_types"), list) else [],
"started_at": _dt(worker.started_at),
"last_seen": _dt(worker.last_seen),
"active_job_count": active_job_count,
}
@router.get("/tasks/active", response_model=List[TaskInfo])
async def get_active_tasks(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
@@ -73,6 +143,199 @@ async def get_recent_tasks(
return [TaskInfo.model_validate(task) for task in orm_tasks]
@router.get("/tasks/runtime-summary")
async def get_task_runtime_summary(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
safe_offset = min(TASK_QUERY_MAX_OFFSET, max(0, int(offset or 0)))
active_job_statuses = ["READY", "RETRY", "RUNNING"]
scan_job_types = {
"SCAN_DATA",
"SCAN_DINSAR",
"SCAN_ASSET_INVENTORY",
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"GF3_SARSCAPE_SYNC",
"GF3_QUICKLOOK_WEBP",
}
worker_timeout = max(5, int(getattr(settings, "JOB_WORKER_HEALTH_TIMEOUT", 60) or 60))
worker_threshold = datetime.utcnow() - timedelta(seconds=worker_timeout)
configured_concurrency = max(1, int(getattr(settings, "JOB_WORKER_CONCURRENCY", 1) or 1))
async with _new_session() as db:
active_tasks = await task_service.get_active_tasks(limit=safe_limit, offset=safe_offset, db=db)
workers_result = await db.execute(
select(SystemWorkerHeartbeatORM)
.where(SystemWorkerHeartbeatORM.last_seen >= worker_threshold)
.order_by(SystemWorkerHeartbeatORM.last_seen.desc())
)
active_workers = workers_result.scalars().all()
active_worker_ids = {str(worker.worker_id) for worker in active_workers}
running_by_worker_result = await db.execute(
select(SystemJobORM.locked_by, func.count(SystemJobORM.id))
.where(SystemJobORM.status == "RUNNING")
.group_by(SystemJobORM.locked_by)
)
running_by_worker = {
str(worker_id or ""): int(count or 0)
for worker_id, count in running_by_worker_result.all()
}
status_counts_result = await db.execute(
select(SystemJobORM.status, func.count(SystemJobORM.id))
.where(SystemJobORM.status.in_(active_job_statuses))
.group_by(SystemJobORM.status)
)
job_status_counts = {
"READY": 0,
"RETRY": 0,
"RUNNING": 0,
}
for status, count in status_counts_result.all():
job_status_counts[str(status or "").upper()] = int(count or 0)
status_rank = case(
(SystemJobORM.status == "RUNNING", 0),
(SystemJobORM.status == "RETRY", 1),
else_=2,
)
jobs_result = await db.execute(
select(SystemJobORM)
.where(SystemJobORM.status.in_(active_job_statuses))
.order_by(status_rank, SystemJobORM.priority.desc(), SystemJobORM.id.asc())
.offset(safe_offset)
.limit(safe_limit)
)
active_jobs = jobs_result.scalars().all()
task_ids = {
str(task.task_id)
for task in active_tasks
if task.task_id
}
task_ids.update(
str(job.task_id)
for job in active_jobs
if job.task_id
)
task_by_id: dict[str, SystemTaskORM] = {}
if task_ids:
task_result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id.in_(sorted(task_ids)))
)
task_by_id = {
str(task.task_id): task
for task in task_result.scalars().all()
if task.task_id
}
worker_concurrency_by_id = {
str(worker.worker_id): _worker_concurrency(worker)
for worker in active_workers
}
total_slots = sum(worker_concurrency_by_id.values())
busy_slots = sum(
count
for worker_id, count in running_by_worker.items()
if worker_id in active_worker_ids
)
running_count = int(job_status_counts.get("RUNNING") or 0)
stale_running_count = max(0, running_count - busy_slots)
queued_count = int(job_status_counts.get("READY") or 0) + int(job_status_counts.get("RETRY") or 0)
task_items = [_task_payload(task) for task in active_tasks]
job_items = [_job_payload(job, task_by_id) for job in active_jobs]
scan_jobs = [
item for item in job_items
if str(item.get("job_type") or "").upper() in scan_job_types
]
scan_tasks = [
item for item in task_items
if str(item.get("task_type") or "").upper() in scan_job_types
]
return {
"timestamp": datetime.utcnow().isoformat() + "Z",
"worker": {
"ok": len(active_workers) > 0,
"worker_count": len(active_workers),
"configured_concurrency": configured_concurrency,
"total_slots": total_slots,
"busy_slots": busy_slots,
"idle_slots": max(0, total_slots - busy_slots),
"timeout_seconds": worker_timeout,
"stale_running_job_count": stale_running_count,
"workers": [
_worker_payload(
worker,
running_by_worker.get(str(worker.worker_id), 0),
worker_concurrency_by_id.get(str(worker.worker_id), 1),
)
for worker in active_workers
],
},
"jobs": {
"active_count": running_count + queued_count,
"running_count": running_count,
"queued_count": queued_count,
"ready_count": int(job_status_counts.get("READY") or 0),
"retry_count": int(job_status_counts.get("RETRY") or 0),
"items": job_items,
},
"tasks": {
"active_count": len(task_items),
"running_count": sum(1 for item in task_items if item.get("status") == "RUNNING"),
"pending_count": sum(1 for item in task_items if item.get("status") == "PENDING"),
"items": task_items,
},
"scan": {
"active_task_count": len(scan_tasks),
"active_job_count": len(scan_jobs),
"running_job_count": sum(1 for item in scan_jobs if item.get("status") == "RUNNING"),
"queued_job_count": sum(1 for item in scan_jobs if item.get("status") in {"READY", "RETRY"}),
"tasks": scan_tasks,
"jobs": scan_jobs,
},
}
@router.get("/tasks/runtime-summary/stream")
async def stream_task_runtime_summary(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)
if not token:
raise HTTPException(status_code=401, detail="Authentication required.")
async with _new_session() as db:
user = await get_user_by_session_token(db, token)
if not user:
raise HTTPException(status_code=401, detail="Authentication required.")
async def event_generator():
while True:
if await request.is_disconnected():
break
try:
summary = await get_task_runtime_summary(
limit=TASK_ACTIVE_MAX_LIMIT,
offset=0,
)
yield f"data: {json.dumps(summary)}\n\n"
except Exception:
yield "data: {}\n\n"
await asyncio.sleep(3)
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
@router.get("/tasks/active/stream")
async def stream_active_tasks(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)
backend/app/services/asset_inventory_service.py
+71 -17
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import gzip
import hashlib
import logging
import math
import multiprocessing as mp
import os
@@ -48,6 +49,8 @@ from ..utils import (
)
from .pairing_state_service import pairing_state_service
from .data_service import DataService
logger = logging.getLogger(__name__)
from .image_service import image_service
from .orbit_converter import sync_orbit_pools
from .task_service import task_service
@@ -2599,6 +2602,53 @@ def _image_data_format_for_source(row: Dict[str, Any]) -> str:
return "DIRECTORY"
PAIRING_RELEVANT_RADAR_FIELDS = {
"satellite",
"satellite_family",
"imaging_date",
"imaging_mode",
"orbit_direction",
"polarization",
"look_direction",
"relative_orbit",
"insar_source_ready",
"file_path",
"coverage_polygon",
"min_lon",
"min_lat",
"max_lon",
"max_lat",
"scene_center_lon",
"scene_center_lat",
}
def _normalize_pairing_compare_value(value: Any) -> Any:
if isinstance(value, str):
text = value.strip()
return text or None
if isinstance(value, bool):
return bool(value)
if isinstance(value, (int, float)):
return round(float(value), 9)
if isinstance(value, list):
return [_normalize_pairing_compare_value(item) for item in value]
if isinstance(value, tuple):
return [_normalize_pairing_compare_value(item) for item in value]
return value
def _radar_pairing_fields_changed(existing: RadarDataORM, new_values: Dict[str, Any]) -> bool:
for field in PAIRING_RELEVANT_RADAR_FIELDS:
if field not in new_values:
continue
before = _normalize_pairing_compare_value(getattr(existing, field, None))
after = _normalize_pairing_compare_value(new_values.get(field))
if before != after:
return True
return False
class AssetInventoryService:
async def _progress(self, task_id: Optional[str], message: str, progress: int) -> None:
if not task_id:
@@ -2784,6 +2834,13 @@ class AssetInventoryService:
raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path)
geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path)
product_name = os.path.basename(str(record.file_path or ""))
if task_id:
progress = progress_start + int(index / max(1, total) * max(1, progress_end - progress_start))
await task_service.update_task(
task_id,
message=f"Building archive preview cache ({index}/{total}): {product_name}",
progress=min(progress_end, progress),
)
preview_source = await asyncio.to_thread(DataService.find_radar_preview_source, record.file_path)
if not preview_source:
@@ -4675,6 +4732,7 @@ class AssetInventoryService:
profile_inputs.append((row, asset_id, int(scene.id)))
else:
before_orbit_id = existing.selected_orbit_asset_id
pairing_fields_changed = _radar_pairing_fields_changed(existing, radar_values)
for key, value in radar_values.items():
setattr(existing, key, value)
if not existing.orbit_binding_status:
@@ -4682,16 +4740,15 @@ class AssetInventoryService:
db.add(existing)
if existing.id is not None:
profile_inputs.append((row, asset_id, int(existing.id)))
if existing.id is not None and before_orbit_id != existing.selected_orbit_asset_id:
if existing.id is not None and (
pairing_fields_changed or before_orbit_id != existing.selected_orbit_asset_id
):
dirty_scene_ids.append(int(existing.id))
await db.flush()
if profile_inputs:
await self._upsert_geometry_profiles(db, profile_inputs)
await self._attach_radar_ids_to_metadata_documents(db, profile_inputs)
for _, _, radar_id in profile_inputs:
if radar_id not in dirty_scene_ids:
dirty_scene_ids.append(radar_id)
if dirty_scene_ids:
await pairing_state_service.mark_scenes_dirty(db, scene_ids=dirty_scene_ids, reason="asset_inventory_source_update", commit=False)
@@ -4988,7 +5045,6 @@ class AssetInventoryService:
matched = 0
missing = 0
candidate_count = 0
dirty_scene_ids: List[int] = []
for scene in scenes:
candidates = await self._find_orbit_candidates(db, scene)
if not candidates:
@@ -5018,8 +5074,6 @@ class AssetInventoryService:
},
)
)
if scene.id is not None:
dirty_scene_ids.append(int(scene.id))
continue
candidate_count += len(candidates)
@@ -5049,8 +5103,6 @@ class AssetInventoryService:
scene.orbit_binding_reason = selected[2]
db.add(scene)
matched += 1
if scene.id is not None:
dirty_scene_ids.append(int(scene.id))
if len(candidates) > 1 and abs(float(candidates[0][1]) - float(candidates[1][1])) < 0.001:
db.add(
@@ -5068,13 +5120,8 @@ class AssetInventoryService:
)
)
if dirty_scene_ids:
await pairing_state_service.mark_scenes_dirty(
db,
scene_ids=sorted(set(dirty_scene_ids)),
reason="asset_inventory_orbit_binding",
commit=False,
)
# Pairing queries join radar_data and filter has_orbit_data live; orbit rebinding
# does not change the cached geometric/time pairing metrics.
return {
"scene_count": len(scenes),
"matched_count": matched,
@@ -5279,8 +5326,15 @@ class AssetInventoryService:
.limit(safe_limit)
)
).scalars().all()
items = [self._source_asset_payload(row) for row in rows]
try:
from .landsar_lt1_production_service import landsar_lt1_production_service
await landsar_lt1_production_service.decorate_source_asset_payloads(db, items)
except Exception:
logger.debug("Failed to decorate LT-1 LandSAR production status", exc_info=True)
return {
"items": [self._source_asset_payload(row) for row in rows],
"items": items,
"total": total,
"limit": safe_limit,
"offset": safe_offset,
backend/app/services/data_service.py
+10 -6
View File
@@ -288,6 +288,8 @@ def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) ->
return None
member = str(member_name or "").replace("\\", "/").strip("/")
while member.startswith("./"):
member = member[2:].strip("/")
member_lower = member.lower()
product_lower = product_stem.lower()
expected_names = [
@@ -315,7 +317,10 @@ def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) ->
def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]:
lower_name = str(member_name or "").replace("\\", "/").lower()
normalized_name = str(member_name or "").replace("\\", "/").strip("/")
while normalized_name.startswith("./"):
normalized_name = normalized_name[2:].strip("/")
lower_name = normalized_name.lower()
base_name = os.path.basename(lower_name)
if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS):
return None
@@ -1024,7 +1029,6 @@ class DataService:
# 3. 更新缺失精轨的现有记录
update_progress("正在关联精轨数据...", 85)
updated_orbits = 0
updated_orbit_scene_ids = set()
if orbit_files_map:
stmt_select = select(RadarDataORM).where(RadarDataORM.has_orbit_data == False)
result = await db.execute(stmt_select)
@@ -1037,8 +1041,6 @@ class DataService:
record.orbit_file_path = orbit_files_map[key]
db.add(record)
updated_orbits += 1
if record.id is not None:
updated_orbit_scene_ids.add(int(record.id))
for data_type, root_path, mtime in scan_state_updates:
await DataService._upsert_scan_state(db, data_type, root_path, mtime)
@@ -1046,7 +1048,9 @@ class DataService:
await db.commit()
pairing_dirty_summary: Dict[str, Any] = {}
dirty_scene_ids = set(updated_orbit_scene_ids)
# Orbit availability is filtered live by pairing queries; it is not part of
# pairing_metric_cache, so orbit-only updates should not dirty the cache.
dirty_scene_ids = set()
processed_unique_ids = [key for key in radar_cache_candidates.keys() if key]
for chunk in _chunked(processed_unique_ids, 500):
id_result = await db.execute(
@@ -1061,7 +1065,7 @@ class DataService:
reason="radar_scan",
commit=True,
)
elif processed_scenes > 0 or updated_orbits > 0:
elif processed_scenes > 0:
pairing_dirty_summary = await pairing_state_service.mark_global_dirty(
db,
reason="radar_scan",
backend/app/services/dinsar_production_service.py
+1
View File
@@ -1007,6 +1007,7 @@ class DinsarProductionService:
"root_dir": run.source_root,
"publish_root_dir": run.publish_root_dir,
"message": run.latest_message,
"summary_json": run.summary_json if isinstance(run.summary_json, dict) else {},
"total_items": run.total_items,
"completed_items": run.completed_items,
"failed_items": run.failed_items,
backend/app/services/job_handlers.py
+533 -1
View File
@@ -41,7 +41,7 @@ from .dinsar_result_layout_service import (
)
from .dinsar_scan_service import dinsar_scan_service
from .engine_lock_service import engine_lock_service
from .envi_service import build_envi_runner_command, get_envi_runner_cwd, get_envi_runner_env
from .envi_service import build_envi_runner_command, extract_disp_results, get_envi_runner_cwd, get_envi_runner_env
from .psinsar_catalog_service import psinsar_catalog_service
from .result_catalog_service import result_catalog_service
from .sbas_insar_catalog_service import sbas_insar_catalog_service
@@ -101,10 +101,13 @@ JOB_TYPE_ISCE2_RUN = "ISCE2_RUN"
JOB_TYPE_PYINT_RUN = "PYINT_RUN"
JOB_TYPE_LANDSAR_RUN = "LANDSAR_RUN"
JOB_TYPE_LANDSAR_CLUSTER_ITEM = "LANDSAR_CLUSTER_ITEM"
JOB_TYPE_LANDSAR_LT1_IMPORT = "LANDSAR_LT1_IMPORT"
JOB_TYPE_EXTRACT_DINSAR_PRODUCTS = "EXTRACT_DINSAR_PRODUCTS"
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG"
JOB_TYPE_PAIRING_CACHE_REBUILD = "PAIRING_CACHE_REBUILD"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY"
JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION"
@@ -206,6 +209,142 @@ def _dedupe_existing_dirs(paths: Any) -> List[str]:
return ordered
def _compact_failure_text(value: Any, *, max_length: int = 700) -> str:
text = re.sub(r"\s+", " ", str(value or "")).strip()
if not text:
return "Unknown error"
if len(text) <= max_length:
return text
return text[: max(0, max_length - 3)].rstrip() + "..."
def _classify_dinsar_failure(error_message: Any) -> str:
text = str(error_message or "")
lowered = text.lower()
if "3221225477" in text or "status_access_violation" in lowered:
return "LandSAR access violation during coherence mask/phase unwrapping"
if "not enough gcps" in lowered or "space insar calibration failed" in lowered:
return "Insufficient GCPs for baseline/calibration"
if (
"no enough points" in lowered
or "not enough points" in lowered
or "geo_extract_gcp" in lowered
or "无满足snr" in lowered
or "离散采样点" in text
):
return "Insufficient tie/GCP points for DEM/geocoding"
if "dem/sub-terrain" in lowered or "subterrain" in lowered or "sub-terrain" in lowered:
return "DEM/sub-terrain processing failed"
if "相干性掩膜" in text and "相位解缠" in text:
return "Coherence mask/phase unwrapping failed"
if "timeout" in lowered or "timed out" in lowered or "超时" in text:
return "Processing timeout"
if "publish" in lowered:
return "Result catalog publish failed"
compact = _compact_failure_text(text, max_length=120)
return compact if compact != "Unknown error" else "Unclassified D-InSAR failure"
async def _build_dinsar_failure_summary(db, run) -> Dict[str, Any]:
result = await db.execute(
select(DinsarProductionRunItemORM)
.where(
DinsarProductionRunItemORM.run_id == run.run_id,
DinsarProductionRunItemORM.status == "FAILED",
)
.order_by(
DinsarProductionRunItemORM.order_index.asc().nullslast(),
DinsarProductionRunItemORM.id.asc(),
)
)
failed_items = result.scalars().all()
details: List[Dict[str, Any]] = []
grouped: Dict[str, Dict[str, Any]] = {}
for item in failed_items:
label = str(item.task_alias or item.task_name or f"item-{item.id}").strip()
reason = _classify_dinsar_failure(item.last_error)
compact_error = _compact_failure_text(item.last_error)
detail = {
"id": item.id,
"order_index": item.order_index,
"task_name": item.task_name,
"task_alias": item.task_alias,
"reason": reason,
"error": compact_error,
"source_task_dir": item.source_task_dir,
"latest_output_dir": item.latest_output_dir,
"latest_log_path": item.latest_log_path,
}
details.append(detail)
group = grouped.setdefault(reason, {"reason": reason, "count": 0, "items": []})
group["count"] += 1
if len(group["items"]) < 25:
group["items"].append(label)
groups = sorted(grouped.values(), key=lambda item: (-int(item["count"]), str(item["reason"])))
return {
"failed_count": len(details),
"groups": groups,
"items": details,
}
def _chunk_log_lines(lines: List[str], *, max_chars: int = 3500) -> List[str]:
chunks: List[str] = []
current: List[str] = []
current_len = 0
for line in lines:
line_len = len(line) + 1
if current and current_len + line_len > max_chars:
chunks.append("\n".join(current))
current = []
current_len = 0
current.append(line)
current_len += line_len
if current:
chunks.append("\n".join(current))
return chunks
async def _log_dinsar_failure_summary(
*,
task_id: str,
run_id: str,
engine_title: str,
run,
failure_summary: Dict[str, Any],
run_log,
) -> None:
if not failure_summary or int(failure_summary.get("failed_count") or 0) <= 0:
return
lines = [
(
f"{engine_title} D-InSAR failure summary: "
f"completed={run.completed_items} failed={run.failed_items} total={run.total_items}"
),
"Failure groups:",
]
for group in failure_summary.get("groups") or []:
items = ", ".join(str(item) for item in (group.get("items") or []))
omitted = int(group.get("count") or 0) - len(group.get("items") or [])
suffix = f" (+{omitted} more)" if omitted > 0 else ""
lines.append(f"- {group.get('reason')}: {group.get('count')} item(s): {items}{suffix}")
lines.append("Failed items:")
for item in failure_summary.get("items") or []:
order_index = item.get("order_index")
order_text = f"{order_index}/{run.total_items}" if order_index else f"id={item.get('id')}"
label = item.get("task_alias") or item.get("task_name") or f"item-{item.get('id')}"
lines.append(f"- [{order_text}] {label}: {item.get('reason')} | {item.get('error')}")
for chunk in _chunk_log_lines(lines):
await task_service.add_log(task_id, "WARNING", chunk)
run_log(run_id, f"[failure-summary]\n{chunk}")
async def _run_scan_data_custom(task_id: str, payload: Dict[str, Any]) -> None:
from ..database import AsyncSessionLocal
@@ -2086,6 +2225,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
f"failed={run.failed_items} total={run.total_items}"
)
failure_summary = await _build_dinsar_failure_summary(db, run)
summary_payload = {
"workflow": workflow,
"engine_code": run.engine_code,
@@ -2098,7 +2238,17 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None:
"publish": publish_result,
"publish_error": publish_error,
"published_output_dirs": publish_dirs,
"failure_summary": failure_summary,
}
if failure_summary.get("failed_count"):
await _log_dinsar_failure_summary(
task_id=job.task_id,
run_id=run.run_id,
engine_title="ENVI",
run=run,
failure_summary=failure_summary,
run_log=run_log,
)
await dinsar_production_service.finalize_run(
run,
db=db,
@@ -3061,6 +3211,7 @@ async def _run_wsl_dinsar_production_controller(
f"failed={run.failed_items} total={run.total_items}"
)
failure_summary = await _build_dinsar_failure_summary(db, run)
summary_payload = {
"workflow": f"dinsar_{engine_code}",
"engine_code": run.engine_code,
@@ -3074,7 +3225,17 @@ async def _run_wsl_dinsar_production_controller(
"rebuild": rebuild_result,
"publish_error": publish_error,
"published_output_dirs": publish_dirs,
"failure_summary": failure_summary,
}
if failure_summary.get("failed_count"):
await _log_dinsar_failure_summary(
task_id=job.task_id,
run_id=run.run_id,
engine_title=engine_title,
run=run,
failure_summary=failure_summary,
run_log=run_log,
)
await dinsar_production_service.finalize_run(
run,
db=db,
@@ -3266,6 +3427,211 @@ async def _handle_landsar_run(job: SystemJobORM) -> None:
)
async def _handle_landsar_lt1_import(job: SystemJobORM) -> None:
from .landsar_lt1_production_service import landsar_lt1_production_service
from .asset_inventory_service import asset_inventory_service
payload = dict(job.payload or {})
task_id = job.task_id
if not task_id:
raise ValueError("LANDSAR_LT1_IMPORT job missing task_id")
loop = asyncio.get_running_loop()
def _progress(event: Dict[str, Any]) -> None:
message = str(event.get("message") or event.get("event") or "").strip()
if not message:
return
progress = event.get("progress")
async def _write() -> None:
try:
await task_service.add_log(task_id, "INFO", message)
if progress is not None:
await task_service.update_task(task_id, progress=int(progress), message=message)
except Exception:
logger.debug("Failed to write LandSAR LT-1 progress", exc_info=True)
asyncio.run_coroutine_threadsafe(_write(), loop)
await task_service.start_task(task_id, message="LandSAR LT-1 import started")
await task_service.update_task(task_id, progress=5, message="Checking LandSAR LT-1 runtime")
try:
source_asset_ids = _dedupe_positive_ints(payload.get("source_asset_ids"))
radar_data_ids = _dedupe_positive_ints(payload.get("radar_data_ids"))
if source_asset_ids or radar_data_ids:
await task_service.update_task(task_id, progress=8, message="Preparing LT-1 source assets")
prepared = await _prepare_landsar_lt1_source_assets(
task_id,
payload,
source_asset_ids=source_asset_ids,
radar_data_ids=radar_data_ids,
landsar_lt1_production_service=landsar_lt1_production_service,
asset_inventory_service=asset_inventory_service,
)
payload = {
**payload,
"source_asset_ids": prepared["source_asset_ids"],
"radar_data_ids": prepared["radar_data_ids"],
"__prepared_scene_dirs": prepared["scene_dirs"],
"__materialized": prepared["materialized"],
"__materialize_task_root": prepared["task_root"],
}
async with _local_engine_lock("landsar"):
result = await asyncio.to_thread(
landsar_lt1_production_service.run_import,
payload,
progress_callback=_progress,
)
async with AsyncSessionLocal() as db:
catalog_result = await landsar_lt1_production_service.register_manifest(
db,
result["manifest_path"],
)
await task_service.add_log(
task_id,
"INFO",
(
"LandSAR LT-1 product registered: "
f"product_id={catalog_result.get('product_id')}, "
f"assets={catalog_result.get('asset_count')}"
),
)
await task_service.update_task(
task_id,
status="COMPLETED",
progress=100,
message=(
"LandSAR LT-1 import completed: "
f"product_id={catalog_result.get('product_id')}, "
f"Input_Data={result.get('input_data_dir')}"
),
)
except Exception as exc:
await task_service.add_log(task_id, "ERROR", f"LandSAR LT-1 import failed: {exc}")
await task_service.update_task(
task_id,
status="FAILED",
progress=100,
message=f"LandSAR LT-1 import failed: {exc}",
)
raise
def _dedupe_positive_ints(values: Any) -> List[int]:
result: List[int] = []
if not isinstance(values, list):
return result
for value in values:
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0 and parsed not in result:
result.append(parsed)
return result
async def _prepare_landsar_lt1_source_assets(
task_id: str,
payload: Dict[str, Any],
*,
source_asset_ids: List[int],
radar_data_ids: List[int],
landsar_lt1_production_service: Any,
asset_inventory_service: Any,
) -> Dict[str, Any]:
from ..models import SourceProductAssetORM
scene_dirs = _dedupe_existing_dirs(payload.get("scene_dirs") or [])
radar_ids: List[int] = []
async with AsyncSessionLocal() as db:
if radar_data_ids:
rows = (
await db.execute(select(RadarDataORM).where(RadarDataORM.id.in_(radar_data_ids)))
).scalars().all()
radar_by_id = {int(row.id): row for row in rows}
for radar_id in radar_data_ids:
radar = radar_by_id.get(int(radar_id))
if radar is None:
raise ValueError(f"Radar data not found: {radar_id}")
radar_ids.append(int(radar.id))
if radar.source_product_ref_id and int(radar.source_product_ref_id) not in source_asset_ids:
source_asset_ids.append(int(radar.source_product_ref_id))
elif radar.file_path and os.path.isdir(radar.file_path):
scene_dirs.append(os.path.normpath(os.path.abspath(radar.file_path)))
else:
raise ValueError(f"Radar data {radar_id} has no source asset or scene directory.")
produced = await landsar_lt1_production_service.find_produced_source_asset_map(db, source_asset_ids)
if produced:
first_id = sorted(produced.keys())[0]
product_id = (produced[first_id] or {}).get("product_id")
raise ValueError(f"Source asset {first_id} already has a LandSAR LT-1 product: {product_id}")
assets = []
if source_asset_ids:
assets = (
await db.execute(select(SourceProductAssetORM).where(SourceProductAssetORM.id.in_(source_asset_ids)))
).scalars().all()
asset_by_id = {int(asset.id): asset for asset in assets}
task_root = os.path.join(
os.path.normpath(os.path.abspath(settings.LANDSAR_WORK_ROOT)),
"lt1_import_tasks",
task_id,
"scenes",
)
os.makedirs(task_root, exist_ok=True)
materialized: List[Dict[str, Any]] = []
overwrite = bool(payload.get("materialize_overwrite", False))
for asset_id in source_asset_ids:
asset = asset_by_id.get(int(asset_id))
if asset is None:
raise ValueError(f"Source asset not found: {asset_id}")
if str(asset.satellite_family or "").upper() != "LT1":
raise ValueError(f"Source asset {asset_id} is not LT-1.")
result = await asyncio.to_thread(
asset_inventory_service.materialize_source_asset,
asset,
target_root=task_root,
overwrite=overwrite,
)
target_dir = os.path.normpath(os.path.abspath(str(result.get("safe_dir") or result.get("target_dir") or "")))
if not target_dir or not os.path.isdir(target_dir):
raise ValueError(f"Source asset {asset_id} materialize did not produce a directory.")
if target_dir not in scene_dirs:
scene_dirs.append(target_dir)
materialized.append(
{
"source_asset_id": int(asset.id),
"asset_uid": asset.asset_uid,
"logical_product_uid": asset.logical_product_uid,
"source_path": asset.file_path,
"scene_dir": target_dir,
"status": result.get("status"),
"member_count": result.get("member_count"),
}
)
await task_service.add_log(
task_id,
"INFO",
f"Prepared LT-1 source asset {asset.id}: {target_dir} ({result.get('status')})",
)
scene_dirs = list(dict.fromkeys(scene_dirs))
if not scene_dirs:
raise ValueError("No LT-1 scene directories were prepared.")
return {
"scene_dirs": scene_dirs,
"source_asset_ids": list(dict.fromkeys(source_asset_ids)),
"radar_data_ids": radar_ids,
"materialized": materialized,
"task_root": task_root,
}
async def _handle_landsar_cluster_item(job: SystemJobORM) -> None:
payload = job.payload or {}
production_run_id = str(payload.get("production_run_id") or "").strip()
@@ -4636,6 +5002,85 @@ async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None:
await task_service.update_task(job.task_id, status=status, progress=100, message=message)
async def _handle_extract_dinsar_products(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("EXTRACT_DINSAR_PRODUCTS requires task_id for progress tracking.")
payload = job.payload or {}
root_dir = str(payload.get("root_dir") or "").strip()
dest_dir = payload.get("dest_dir") or None
if not root_dir:
raise ValueError("EXTRACT_DINSAR_PRODUCTS requires root_dir payload.")
await task_service.start_task(job.task_id, message="正在提取 D-InSAR 位移结果...")
result = await asyncio.to_thread(
extract_disp_results,
root_dir,
dest_dir,
)
await task_service.update_task(
job.task_id,
progress=45,
message=(
"D-InSAR 位移结果提取完成: "
f"processed={result.get('processed', 0)}, "
f"copied={result.get('copied', 0)}, "
f"overwritten={result.get('overwritten', 0)}, "
f"failed={result.get('failed', 0)}"
),
)
target_dir = result.get("target_dir")
publish_roots = [target_dir] if target_dir and os.path.isdir(str(target_dir)) else []
publish_result = None
rebuild_result = None
if publish_roots:
async with AsyncSessionLocal() as db:
await task_service.update_task(
job.task_id,
progress=60,
message="正在发布 D-InSAR 标准结果包...",
)
publish_result = await result_catalog_service.publish_from_sources(
db,
publish_roots,
)
if int(publish_result.get("processed", 0) or 0) > 0:
await task_service.update_task(
job.task_id,
progress=80,
message="正在重建 D-InSAR 结果目录索引...",
)
rebuild_result = await result_catalog_service.rebuild_catalog(
db,
full_rebuild=True,
)
failed = int(result.get("failed", 0) or 0)
publish_failed = int((publish_result or {}).get("failed", 0) or 0)
rebuild_failed = int((rebuild_result or {}).get("failed", 0) or 0)
status = "FAILED" if (failed or publish_failed or rebuild_failed) else "COMPLETED"
message = (
"D-InSAR 结果提取与登记完成: "
f"提取 {int(result.get('processed', 0) or 0)} 项, "
f"复制 {int(result.get('copied', 0) or 0)} 个, "
f"覆盖 {int(result.get('overwritten', 0) or 0)} 个"
)
if publish_result is not None:
message += f", 发布 {int(publish_result.get('processed', 0) or 0)} 项"
if rebuild_result is not None:
message += f", 入库 {int(rebuild_result.get('registered', 0) or 0)} 项"
if status == "FAILED":
message += f", 失败 {failed + publish_failed + rebuild_failed} 项"
await task_service.update_task(
job.task_id,
status=status,
progress=100,
message=message,
)
async def _handle_publish_dinsar_products_clean(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("PUBLISH_DINSAR_PRODUCTS requires task_id for progress tracking.")
@@ -5669,13 +6114,99 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None:
)
async def _handle_pairing_cache_rebuild(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("PAIRING_CACHE_REBUILD requires task_id for progress tracking.")
from .pairing_cache_service import pairing_cache_service
payload = job.payload or {}
mode = str(payload.get("mode") or "auto_reconcile").strip().lower()
force_full = bool(payload.get("force_full", False))
full_rebuild = mode in {"full", "full_rebuild"} or force_full
action_label = "full rebuild" if full_rebuild else "dirty reconcile"
await task_service.start_task(
job.task_id,
message=f"D-InSAR pairing cache {action_label} started",
)
await task_service.update_task(
job.task_id,
progress=5,
message=f"D-InSAR pairing cache {action_label} is running",
)
progress_state: Dict[str, Any] = {
"progress": 5,
"message": f"D-InSAR pairing cache {action_label} is running",
}
async def _report_progress(message: str, progress: int) -> None:
safe_progress = max(5, min(95, int(progress)))
progress_state["progress"] = safe_progress
progress_state["message"] = message
await task_service.update_task(
job.task_id,
progress=safe_progress,
message=message,
)
async def _keepalive() -> None:
while True:
await asyncio.sleep(60)
message = str(progress_state.get("message") or f"D-InSAR pairing cache {action_label} is still running")
progress = int(progress_state.get("progress") or 5)
await task_service.update_task(
job.task_id,
progress=progress,
message=f"{message} (still running)",
)
keepalive_task = asyncio.create_task(_keepalive())
try:
async with AsyncSessionLocal() as db:
if full_rebuild:
result = await pairing_cache_service.rebuild_metric_cache(
db,
commit=True,
progress_callback=_report_progress,
)
else:
result = await pairing_cache_service.reconcile_dirty_scenes(
db,
force_full=False,
commit=True,
progress_callback=_report_progress,
)
finally:
keepalive_task.cancel()
try:
await keepalive_task
except asyncio.CancelledError:
pass
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"D-InSAR pairing cache completed: "
f"mode={result.get('mode')}, "
f"scenes={result.get('scene_count', 0)}, "
f"pairs={result.get('pair_count', 0)}, "
f"dirty={result.get('dirty_scene_count', 0)}"
),
)
_HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit,
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
JOB_TYPE_EXTRACT_DINSAR_PRODUCTS: _handle_extract_dinsar_products,
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
JOB_TYPE_PAIRING_CACHE_REBUILD: _handle_pairing_cache_rebuild,
JOB_TYPE_TIMESERIES_PREPARE: _handle_timeseries_prepare,
JOB_TYPE_TIMESERIES_STACK_PREP: _handle_timeseries_stack_prep,
JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize,
@@ -5701,6 +6232,7 @@ _HANDLERS = {
JOB_TYPE_ISCE2_RUN: _handle_isce2_run,
JOB_TYPE_PYINT_RUN: _handle_pyint_run,
JOB_TYPE_LANDSAR_RUN: _handle_landsar_run,
JOB_TYPE_LANDSAR_LT1_IMPORT: _handle_landsar_lt1_import,
JOB_TYPE_LANDSAR_CLUSTER_ITEM: _handle_landsar_cluster_item,
JOB_TYPE_WATER_GEOCODE: _handle_water_geocode,
JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess,
backend/app/services/job_queue_service.py
+5 -2
View File
@@ -299,10 +299,11 @@ class JobQueueService:
)
stale_jobs = result.scalars().all()
if not stale_jobs:
return {"recovered": 0, "failed": 0}
return {"recovered": 0, "failed": 0, "failed_task_ids": []}
recovered = 0
failed = 0
failed_task_ids = []
for job in stale_jobs:
attempts = int(job.attempts or 0) + 1
if attempts < int(job.max_attempts or 1):
@@ -315,6 +316,8 @@ class JobQueueService:
next_run_at = None
failed += 1
finished_at = now
if job.task_id:
failed_task_ids.append(str(job.task_id))
await db.execute(
update(SystemJobORM)
@@ -331,7 +334,7 @@ class JobQueueService:
)
)
await db.commit()
return {"recovered": recovered, "failed": failed}
return {"recovered": recovered, "failed": failed, "failed_task_ids": failed_task_ids}
finally:
if gen_db:
await db.close()
backend/app/services/job_worker.py
+28 -2
View File
@@ -1,4 +1,5 @@
import asyncio
import json
import os
import socket
import uuid
@@ -50,20 +51,35 @@ def _new_session():
return database.AsyncSessionLocal()
async def _touch_worker(worker_id: str) -> None:
async def _touch_worker(
worker_id: str,
*,
concurrency: int,
allowed_job_types: Optional[Set[str]],
) -> None:
host = socket.gethostname()
pid = os.getpid()
note = json.dumps(
{
"concurrency": max(1, int(concurrency or 1)),
"allowed_job_types": sorted(allowed_job_types or []),
},
ensure_ascii=False,
separators=(",", ":"),
)
async with _new_session() as db:
stmt = pg_insert(SystemWorkerHeartbeatORM).values(
worker_id=worker_id,
hostname=host,
pid=pid,
note=note,
)
stmt = stmt.on_conflict_do_update(
index_elements=["worker_id"],
set_={
"hostname": host,
"pid": pid,
"note": note,
"last_seen": func.now(),
},
)
@@ -172,7 +188,11 @@ async def run_worker_loop(
now = time.monotonic()
if now - last_heartbeat >= heartbeat_interval:
try:
await _touch_worker(worker_id)
await _touch_worker(
worker_id,
concurrency=concurrency,
allowed_job_types=allowed_job_types,
)
except Exception as exc:
print(f"[WARN] worker cleanup: {exc}")
last_heartbeat = now
@@ -184,6 +204,12 @@ async def run_worker_loop(
f"[*] Recovered stale jobs: retry={recovered.get('recovered', 0)} "
f"failed={recovered.get('failed', 0)}"
)
for task_id in recovered.get("failed_task_ids", []) or []:
await task_service.update_task(
task_id,
status="FAILED",
message="后台任务心跳超时，任务已被标记为失败",
)
except Exception as exc:
print(f"[WARN] recover_stale: {exc}")
last_recover = now
backend/app/services/landsar_lt1_production_service.py
+1080
View File
File diff suppressed because it is too large Load Diff
backend/app/services/lt_gamma_scene_service.py
+20 -3
View File
@@ -88,6 +88,15 @@ def run_lt_gamma_scene_preprocess(
if not runner.is_file():
raise FileNotFoundError(f"Gamma scene runner not found: {runner}")
analysis_dem_path = (
settings.SAR_ANALYSIS_DEM_PATH
or settings.GAMMA_SBAS_DEM_PATH
or settings.PYINT_PREPARED_DEM_PATH
or _prepared_dem_path()
)
if not analysis_dem_path:
raise RuntimeError("SAR_ANALYSIS_DEM_PATH is not configured for LT analysis GeoTIFF production")
args = [
pyint_python,
to_wsl_path(str(runner)),
@@ -102,7 +111,11 @@ def run_lt_gamma_scene_preprocess(
"--dem-root",
to_wsl_path(str(settings.PYINT_DEM_ROOT)),
"--prepared-dem-path",
to_wsl_path(_prepared_dem_path()),
to_wsl_path(str(analysis_dem_path)),
"--dem-resolution-m",
str(float(settings.SAR_ANALYSIS_DEM_RESOLUTION_M or 30.0)),
"--target-grid-size-m",
str(float(settings.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)),
"--project-name",
run_name,
"--date",
@@ -110,9 +123,13 @@ def run_lt_gamma_scene_preprocess(
"--satellite-family",
"LT1",
"--range-looks",
str(DEFAULT_RANGE_LOOKS),
str(int(settings.SAR_ANALYSIS_RANGE_LOOKS or DEFAULT_RANGE_LOOKS)),
"--azimuth-looks",
str(DEFAULT_AZIMUTH_LOOKS),
str(int(settings.SAR_ANALYSIS_AZIMUTH_LOOKS or DEFAULT_AZIMUTH_LOOKS)),
"--speckle-filter-method",
str(settings.SAR_ANALYSIS_SPECKLE_FILTER_METHOD or "none"),
"--speckle-filter-size",
str(int(settings.SAR_ANALYSIS_SPECKLE_FILTER_SIZE or 5)),
"--geo-interp",
str(settings.PYINT_GEO_INTERP or "1"),
"--nodata-value",
backend/app/services/pairing_cache_service.py
+77 -4
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, List, Optional, Sequence
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence
from sqlalchemy import delete, func, or_, select, text, update
from sqlalchemy.ext.asyncio import AsyncSession
@@ -380,6 +380,15 @@ def _incremental_insert_sql() -> str:
class PairingCacheService:
async def _notify_progress(
self,
progress_callback: Optional[Callable[[str, int], Awaitable[None]]],
message: str,
progress: int,
) -> None:
if progress_callback is not None:
await progress_callback(message, progress)
async def _get_state_row(self, db: AsyncSession) -> PairingCacheStateORM:
payload = await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
result = await db.execute(
@@ -501,11 +510,24 @@ class PairingCacheService:
db: AsyncSession,
*,
commit: bool = True,
progress_callback: Optional[Callable[[str, int], Awaitable[None]]] = None,
) -> Dict[str, Any]:
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
await self._set_state_rebuilding(db)
if commit:
await db.commit()
try:
await self._notify_progress(
progress_callback,
"Pairing cache full rebuild: clearing old metric rows",
10,
)
delete_result = await db.execute(delete(PairingMetricCacheORM))
await self._notify_progress(
progress_callback,
"Pairing cache full rebuild: computing spatial/temporal metrics",
20,
)
await db.execute(
text(_full_rebuild_insert_sql()),
{
@@ -513,7 +535,17 @@ class PairingCacheService:
"orientation_rule_version": pairing_state_service.orientation_rule_version,
},
)
await self._notify_progress(
progress_callback,
"Pairing cache full rebuild: resolving dirty scene markers",
80,
)
resolved_dirty = await self._resolve_dirty_rows(db)
await self._notify_progress(
progress_callback,
"Pairing cache full rebuild: finalizing state",
90,
)
summary = await self._finalize_state_success(db, full_rebuild=True)
if commit:
await db.commit()
@@ -539,6 +571,7 @@ class PairingCacheService:
*,
force_full: bool = False,
commit: bool = True,
progress_callback: Optional[Callable[[str, int], Awaitable[None]]] = None,
) -> Dict[str, Any]:
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
dirty_result = await db.execute(
@@ -557,12 +590,21 @@ class PairingCacheService:
pair_count=pair_count,
force_full=force_full,
):
result = await self.rebuild_metric_cache(db, commit=commit)
result = await self.rebuild_metric_cache(
db,
commit=commit,
progress_callback=progress_callback,
)
result["trigger_dirty_scene_count"] = dirty_scene_count
result["forced"] = force_full
return result
if dirty_scene_count == 0:
await self._notify_progress(
progress_callback,
"Pairing cache reconcile: no dirty scenes, refreshing state",
80,
)
summary = await self._finalize_state_success(db, full_rebuild=False)
if commit:
await db.commit()
@@ -582,13 +624,24 @@ class PairingCacheService:
pair_count=pair_count,
force_full=force_full,
):
result = await self.rebuild_metric_cache(db, commit=commit)
result = await self.rebuild_metric_cache(
db,
commit=commit,
progress_callback=progress_callback,
)
result["trigger_dirty_scene_count"] = dirty_scene_count
result["forced"] = force_full
return result
await self._set_state_rebuilding(db)
if commit:
await db.commit()
try:
await self._notify_progress(
progress_callback,
f"Pairing cache incremental reconcile: deleting stale rows for {dirty_scene_count} dirty scenes",
20,
)
delete_result = await db.execute(
delete(PairingMetricCacheORM).where(
or_(
@@ -600,7 +653,7 @@ class PairingCacheService:
insert_attempts = 0
insert_sql = text(_incremental_insert_sql())
for dirty_scene_id in dirty_scene_ids:
for index, dirty_scene_id in enumerate(dirty_scene_ids, start=1):
insert_result = await db.execute(
insert_sql,
{
@@ -610,8 +663,28 @@ class PairingCacheService:
},
)
insert_attempts += int(insert_result.rowcount or 0)
if index == 1 or index == dirty_scene_count or index % 25 == 0:
progress = 20 + int(index / max(1, dirty_scene_count) * 55)
await self._notify_progress(
progress_callback,
(
"Pairing cache incremental reconcile: "
f"processed {index}/{dirty_scene_count} dirty scenes"
),
progress,
)
await self._notify_progress(
progress_callback,
"Pairing cache incremental reconcile: resolving dirty scene markers",
80,
)
resolved_dirty = await self._resolve_dirty_rows(db, scene_ids=dirty_scene_ids)
await self._notify_progress(
progress_callback,
"Pairing cache incremental reconcile: finalizing state",
90,
)
summary = await self._finalize_state_success(db, full_rebuild=False)
if commit:
await db.commit()
backend/app/services/sar_analysis_ready_service.py
+7 -2
View File
@@ -207,15 +207,20 @@ def _raster_quality(path: Path) -> dict[str, Any]:
return quality
def _is_geographic_crs(crs_text: str) -> bool:
text = str(crs_text or "").upper()
return "4326" in text or "GEOGCS" in text or 'UNIT["DEGREE"' in text or "UNIT['DEGREE'" in text
def _pixel_size_m_from_quality(quality: dict[str, Any]) -> float | None:
try:
transform = quality.get("transform") or []
xres = abs(float(transform[0]))
yres = abs(float(transform[4]))
crs = str(quality.get("crs") or "").upper()
crs = str(quality.get("crs") or "")
if not xres or not yres:
return None
if crs and "4326" not in crs:
if crs and not _is_geographic_crs(crs):
return round((xres + yres) / 2.0, 3)
bounds = quality.get("bounds") or {}
lat = (float(bounds.get("bottom", 0.0)) + float(bounds.get("top", 0.0))) / 2.0
backend/migrations/009_raw_source_pairing_fields.sql
+18 -1
View File
@@ -174,7 +174,24 @@ SET
)
FROM radar_data m, radar_data s
WHERE pmc.master_scene_ref_id = m.id
AND pmc.slave_scene_ref_id = s.id;
AND pmc.slave_scene_ref_id = s.id
AND (
(pmc.scene_center_distance_meters IS NULL AND pmc.spatial_baseline_meters IS NOT NULL)
OR (pmc.master_satellite_family IS NULL AND m.satellite_family IS NOT NULL)
OR (pmc.slave_satellite_family IS NULL AND s.satellite_family IS NOT NULL)
OR (pmc.master_look_direction IS NULL AND m.look_direction IS NOT NULL)
OR (pmc.slave_look_direction IS NULL AND s.look_direction IS NOT NULL)
OR pmc.same_satellite_family IS DISTINCT FROM (
NULLIF(COALESCE(m.satellite_family, m.satellite), '') IS NOT NULL
AND NULLIF(COALESCE(s.satellite_family, s.satellite), '') IS NOT NULL
AND COALESCE(m.satellite_family, m.satellite) = COALESCE(s.satellite_family, s.satellite)
)
OR pmc.same_look_direction IS DISTINCT FROM (
NULLIF(m.look_direction, '') IS NULL
OR NULLIF(s.look_direction, '') IS NULL
OR m.look_direction = s.look_direction
)
);
UPDATE pairing_cache_state
SET
backend/tests/test_landsar_lt1_production_service.py
+307
View File
@@ -0,0 +1,307 @@
from pathlib import Path
import json
import re
import tempfile
import unittest
from datetime import datetime
from types import SimpleNamespace
import backend.app.services.landsar_lt1_production_service as landsar_lt1_module
from backend.app.services.landsar_lt1_production_service import (
IMPORT_PROID,
ORBIT_PROID,
generate_lt1_import_param_file,
generate_lt1_orbit_param_file,
landsar_lt1_production_service,
)
CN_PROCESS_ID = "\u5904\u7406\u7f16\u53f7"
CN_FOLDER_COUNT = "\u6587\u4ef6\u5939\u5bfc\u5165\u4e2a\u6570"
CN_FOLDER_PATH = "\u6587\u4ef6\u5939{index}\u8def\u5f84"
CN_OUTPUT_DIR = "\u8bbe\u7f6e\u8f93\u51fa\u6587\u4ef6\u76ee\u5f55"
CN_SAT_MODE = "\u8f93\u5165\u536b\u661f\u6570\u636e\u683c\u5f0f"
CN_DATA_COUNT = "\u8f93\u5165\u6570\u636e\u4e2a\u6570"
CN_DATA_XML = "\u8f93\u5165\u6570\u636e{index}\u7684xml"
CN_ORBIT_DIR = "\u8f93\u5165\u7cbe\u5bc6\u8f68\u9053\u6570\u636e\u6587\u4ef6\u5939"
def test_generate_lt1_import_param_file_single_scene(tmp_path: Path) -> None:
scene_dir = tmp_path / "LT1A_SCENE"
scene_dir.mkdir()
export_dir = tmp_path / "Input_Data"
param_file = tmp_path / "params" / "100016.txt"
generate_lt1_import_param_file(
str(param_file),
[str(scene_dir)],
str(export_dir),
sat_mode="MONO",
)
content = param_file.read_text(encoding="utf-8")
assert f"{CN_PROCESS_ID} {IMPORT_PROID}" in content
assert f"{CN_FOLDER_COUNT} 1" in content
assert f"{CN_FOLDER_PATH.format(index=1)} <{scene_dir}>" in content
assert f"{CN_OUTPUT_DIR} <{export_dir}>" in content
assert f"{CN_SAT_MODE} MONO" in content
def test_generate_lt1_import_param_file_stack(tmp_path: Path) -> None:
scene_dirs = []
for index in range(3):
scene_dir = tmp_path / f"LT1A_SCENE_{index}"
scene_dir.mkdir()
scene_dirs.append(str(scene_dir))
param_file = tmp_path / "params" / "100016_stack.txt"
generate_lt1_import_param_file(
str(param_file),
scene_dirs,
str(tmp_path / "Input_Data"),
sat_mode="BIST",
)
content = param_file.read_text(encoding="utf-8")
assert f"{CN_FOLDER_COUNT} 3" in content
assert f"{CN_SAT_MODE} BIST" in content
for index, scene_dir in enumerate(scene_dirs, 1):
assert f"{CN_FOLDER_PATH.format(index=index)} <{scene_dir}>" in content
def test_generate_lt1_orbit_param_file(tmp_path: Path) -> None:
xml_paths = []
for index in range(2):
xml_path = tmp_path / f"LT1A_{index}_SLC.xml"
xml_path.write_text("<root />", encoding="utf-8")
xml_paths.append(str(xml_path))
orbit_dir = tmp_path / "orbit"
orbit_dir.mkdir()
param_file = tmp_path / "params" / "100206.txt"
generate_lt1_orbit_param_file(
str(param_file),
xml_paths,
str(orbit_dir),
str(tmp_path / "Input_Data"),
)
content = param_file.read_text(encoding="utf-8")
assert f"{CN_PROCESS_ID} {ORBIT_PROID}" in content
assert f"{CN_DATA_COUNT} 2" in content
assert re.search(rf"{CN_ORBIT_DIR}\s+<{re.escape(str(orbit_dir))}>", content)
for index, xml_path in enumerate(xml_paths, 1):
assert f"{CN_DATA_XML.format(index=index)} <{xml_path}>" in content
def test_preview_blocks_scene_without_lt1_files(tmp_path: Path) -> None:
scene_dir = tmp_path / "EMPTY_SCENE"
scene_dir.mkdir()
preview = landsar_lt1_production_service.preview_import(
{
"scene_dirs": [str(scene_dir)],
"mode": "scene",
"sat_mode": "MONO",
"import_orbit": False,
}
)
assert preview["allow_submit"] is False
assert any("Missing LT-1 XML" in item for item in preview["blockers"])
assert any("Missing LT-1 SLC TIFF" in item for item in preview["blockers"])
def test_preview_accepts_source_asset_only_scene() -> None:
preview = landsar_lt1_production_service.preview_import(
{
"source_asset_ids": [101],
"mode": "scene",
"sat_mode": "MONO",
"import_orbit": False,
}
)
assert preview["allow_submit"] is True
assert preview["scene_count"] == 1
assert preview["directory_scene_count"] == 0
assert preview["source_asset_count"] == 1
def test_stack_preview_counts_source_assets() -> None:
preview = landsar_lt1_production_service.preview_import(
{
"source_asset_ids": [101, 102],
"mode": "stack",
"sat_mode": "MONO",
"import_orbit": False,
}
)
assert preview["allow_submit"] is True
assert preview["scene_count"] == 2
assert not preview["warnings"]
def test_scene_mode_rejects_multiple_directories(tmp_path: Path) -> None:
left = tmp_path / "LEFT"
right = tmp_path / "RIGHT"
left.mkdir()
right.mkdir()
try:
landsar_lt1_production_service.preview_import(
{
"scene_dirs": [str(left), str(right)],
"mode": "scene",
}
)
except ValueError as exc:
assert "exactly one" in str(exc)
else:
raise AssertionError("scene mode accepted multiple directories")
class _ScalarRows:
def __init__(self, rows):
self._rows = rows
def all(self):
return self._rows
class _ExecuteResult:
def __init__(self, rows):
self._rows = rows
def scalars(self):
return _ScalarRows(self._rows)
def all(self):
return self._rows
class _FakeAsyncDb:
def __init__(self, rows):
self.rows = rows
async def execute(self, _stmt):
return _ExecuteResult(self.rows)
def test_find_produced_source_asset_map_matches_summary_source_ids() -> None:
scene = SimpleNamespace(
id=7,
radar_data_id=77,
analysis_engine="lt_gamma",
analysis_profile="lt1_gamma_geocoded_mli",
analysis_tif_path="D:/ready/analysis_ready.tif",
analysis_dir="D:/ready",
analysis_preview_path="D:/ready/preview.png",
updated_at=datetime(2026, 6, 27, 1, 2, 3),
)
async def _run():
return await landsar_lt1_production_service.find_produced_source_asset_map(
_FakeAsyncDb([(12, scene)]),
[12, 13],
)
import asyncio
produced = asyncio.run(_run())
assert 12 in produced
assert produced[12]["product_id"] == "sar_scene_geo:7"
assert produced[12]["analysis_tif_path"] == "D:/ready/analysis_ready.tif"
assert 13 not in produced
def test_run_import_materialized_asset_does_not_double_count_scene(tmp_path: Path) -> None:
scene_dir = tmp_path / "LT1A_SCENE"
scene_dir.mkdir()
service = landsar_lt1_production_service
original_ensure = service._ensure_runtime_ready
original_console = service._console_path
original_home = service._landsar_home
original_publish_root = landsar_lt1_module.settings.RESULT_PUBLISH_ROOT
original_run_console = landsar_lt1_module._run_console
original_find_imported_xmls = service._find_imported_xmls
def fake_run_console(_console_path, param_file, log_path, *, cwd, timeout_seconds):
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
Path(log_path).write_text("console success\n", encoding="utf-8")
return {
"command": [_console_path, param_file],
"returncode": 0,
"started_at": "2026-06-27T00:00:00Z",
"finished_at": "2026-06-27T00:00:01Z",
"log_path": log_path,
"stdout_tail": "console success",
}
try:
landsar_lt1_module.settings.RESULT_PUBLISH_ROOT = str(tmp_path / "publish")
service._ensure_runtime_ready = lambda: None
service._console_path = lambda: "LandSARConsole.exe"
service._landsar_home = lambda: str(tmp_path)
service._find_imported_xmls = lambda input_data_dir: [str(Path(input_data_dir) / "LT1A_SCENE_SLC.xml")]
landsar_lt1_module._run_console = fake_run_console
result = service.run_import(
{
"source_asset_ids": [101],
"__prepared_scene_dirs": [str(scene_dir)],
"__materialized": [{"source_asset_id": 101, "scene_dir": str(scene_dir)}],
"__materialize_task_root": str(tmp_path / "tasks"),
"mode": "scene",
"sat_mode": "MONO",
"import_orbit": False,
}
)
manifest = json.loads(Path(result["manifest_path"]).read_text(encoding="utf-8"))
assert manifest["summary"]["scene_count"] == 1
assert manifest["summary"]["source_asset_ids"] == [101]
assert manifest["source"]["scene_dirs"] == [str(scene_dir)]
finally:
landsar_lt1_module.settings.RESULT_PUBLISH_ROOT = original_publish_root
service._ensure_runtime_ready = original_ensure
service._console_path = original_console
service._landsar_home = original_home
service._find_imported_xmls = original_find_imported_xmls
landsar_lt1_module._run_console = original_run_console
class LandsarLt1ProductionServiceTests(unittest.TestCase):
def _with_tmp_path(self, fn) -> None:
with tempfile.TemporaryDirectory() as root:
fn(Path(root))
def test_generate_lt1_import_param_file_single_scene_unittest(self) -> None:
self._with_tmp_path(test_generate_lt1_import_param_file_single_scene)
def test_generate_lt1_import_param_file_stack_unittest(self) -> None:
self._with_tmp_path(test_generate_lt1_import_param_file_stack)
def test_generate_lt1_orbit_param_file_unittest(self) -> None:
self._with_tmp_path(test_generate_lt1_orbit_param_file)
def test_preview_blocks_scene_without_lt1_files_unittest(self) -> None:
self._with_tmp_path(test_preview_blocks_scene_without_lt1_files)
def test_preview_accepts_source_asset_only_scene_unittest(self) -> None:
test_preview_accepts_source_asset_only_scene()
def test_stack_preview_counts_source_assets_unittest(self) -> None:
test_stack_preview_counts_source_assets()
def test_scene_mode_rejects_multiple_directories_unittest(self) -> None:
self._with_tmp_path(test_scene_mode_rejects_multiple_directories)
def test_find_produced_source_asset_map_matches_summary_source_ids_unittest(self) -> None:
test_find_produced_source_asset_map_matches_summary_source_ids()
def test_run_import_materialized_asset_does_not_double_count_scene_unittest(self) -> None:
self._with_tmp_path(test_run_import_materialized_asset_does_not_double_count_scene)
docs/DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md
+52
View File
@@ -0,0 +1,52 @@
# DEM Production Source Contract (2026-06-28)
## Decision
New production should use `D:\DEM\SRTMDEM_RSP_SARscape` as the common DEM source family unless a task explicitly declares another DEM in its manifest. The system should not mix SRTM, COPDEM, GMTED, and the interpolated Heilongjiang 10 m DEM silently.
This does not mean every engine reads the same physical file. The same SRTM source is maintained in several engine-compatible forms:
| Role | Path | Format | Intended consumers |
| --- | --- | --- | --- |
| SARscape/ENVI source | `D:\DEM\SRTMDEM_RSP_SARscape` | ENVI/SARscape float32 binary with `.hdr` | `IDL_DINSAR_DEM_BASE_FILE`, `GF3_SARSCAPE_DEM_PATH` |
| WGS84 prepared source | `D:\DEM\SRTMDEM_RSP_SARscape.wgs84` | ENVI float32 binary/VRT-readable raster | `ISCE2_DEM_PATH`, `PYINT_PREPARED_DEM_PATH`, `SAR_ANALYSIS_DEM_PATH`, `GAMMA_SBAS_DEM_PATH`, `TIMESERIES_DEM_PATH` |
| Int16 GeoTIFF source | `D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif` | GeoTIFF int16 | `LANDSAR_DEM_PATH`, `LANDSAR_SBAS_DEM_PATH`, GDAL/RPC-style GeoTIFF consumers such as `GF3_GEO_DEM_PATH` |
The LT-1 single-scene production profile uses a 30 m analysis grid. Product manifests must still record the selected SRTM-derived source file, the cropped/converted DEM path, the actual Gamma DEM spacing, the derived `dem_lat_ovr`/`dem_lon_ovr`, and the configured target grid so reviewers can distinguish the source DEM family from the output grid.
The LT-1 single-scene profile also performs multilook and speckle filtering before registering the final analysis GeoTIFF. The production manifest must record `SAR_ANALYSIS_RANGE_LOOKS`, `SAR_ANALYSIS_AZIMUTH_LOOKS`, and the speckle filter method/window so reviewers can reproduce the output pixel statistics.
## Current Configuration
The current server should use:
```env
IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape
GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape
ISCE2_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
PYINT_PREPARED_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
SAR_ANALYSIS_DEM_RESOLUTION_M=30.0
SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0
SAR_ANALYSIS_RANGE_LOOKS=6
SAR_ANALYSIS_AZIMUTH_LOOKS=5
SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true
SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee
SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5
GAMMA_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
TIMESERIES_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84
LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif
LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif
GF3_GEO_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif
```
## Non-Default DEMs
- `D:\DEM\HeiLongJiang10M_DEM.tif` is a regional interpolated DEM. It is not the default production DEM.
- `D:\DEM\landsar_prepared\HeiLongJiang10M_DEM_full_4326_int16.tif` is a LandSAR-compatible regional derivative of the interpolated DEM. It should only be used by an explicitly named regional/high-resolution experiment.
- `D:\DEM\COPDEM_GLO30_China_4326_DEM` remains a possible China-coverage fallback, but it is not the default after this contract.
- `D:\DEM\GMTED2010.jp2` is too coarse for production geocoding and should not be used as a production DEM default.
Any task that intentionally uses a non-default DEM must record the selected source path, derived/cropped path, DEM resolution, target output grid, and coverage decision in its manifest.
docs/DEPLOYMENT.md
+2 -2
View File
@@ -112,7 +112,7 @@ ORBIT_POOL_LANDSAR=
```env
IDL_EXECUTABLE=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idl.exe
IDL_WORKBENCH_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlde.exe
IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape
IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape
IDL_WORKER_RUNTIME_DIR=D:\production_runtime\idl_worker
ENVI_TASK_TIMEOUT_SECONDS=21600
```
@@ -131,7 +131,7 @@ GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog
GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime
GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe
GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM
GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape
GF3_SARSCAPE_POLARIZATIONS=HH,HV
GF3_SARSCAPE_AUTO_STANDARDIZE=false
GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true
docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md
+1 -1
View File
@@ -454,7 +454,7 @@ GF3_ARCHIVE_SOURCE_DIRS
```env
GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe
GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
GF3_SARSCAPE_DEM_PATH=D:\DEM\GMTED2010.jp2
GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape
GF3_SARSCAPE_POLARIZATIONS=HH,HV
GF3_SARSCAPE_KEEP_EXTRACTED=true
GF3_SARSCAPE_AUTO_STANDARDIZE=false
docs/INDEX.md
+8
View File
@@ -31,14 +31,22 @@
褰撳墠闄嗘帰涓€鍙枫€丼entinel-1銆侀珮鍒嗕笁鏈満鐢熶骇銆佹寜闇€瑙ｅ寘銆丟F3 澶栭儴鐢熶骇鐧昏銆佺粨鏋滅鐞嗗拰 UNC 閫€鍑虹害瀹氥€?
- [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md)
缁熶竴缁撴灉鐩綍銆佹爣鍑嗕骇鍝佸寘銆乧atalog 涓庡寮曟搸缁撴灉鍏卞瓨绾﹀畾銆?
- [RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md](RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md)
Result extraction and access-control audit: current D-InSAR export/registration boundaries, placeholder channels, admin/viewer limitations, and recommended exporter/operator/admin permission model.
- [DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md](DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md)
D-InSAR 淇濈暀 ENVI/SARscape銆丩andSAR銆丟amma/PyINT 涓夊紩鎿庯紝閫€鍑?ISCE2锛岀粺涓€ Task_Pool銆佺粨鏋滆仛鍚堝拰涓棿鏂囦欢娓呯悊鐨勫綋鍓嶈璁°€?
- [LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md](LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md)
LandSAR D-InSAR/SBAS 鐨勫叏鐞?DEM 涓€娆℃€?Int16 鏍囧噯鍖栥€佸尯鍩熻鍓?tif銆佺敓浜ч厤缃拰 guardrail 绾﹀畾銆?
- [DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md](DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md)
Production DEM source contract: SRTM-derived common source family, engine-specific derived formats, and non-default DEM guardrails.
- [LANDSAR_CLUSTER_WORKER_DEPLOYMENT_20260624.md](LANDSAR_CLUSTER_WORKER_DEPLOYMENT_20260624.md)
LandSAR D-InSAR 集群 worker 的队列分片设计、主服务器 IP 白名单、远端 Windows 节点 192.168.1.6 部署和运行约束。
- [LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md](LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md)
LandSAR 集群数据搬运（HTTP Task_Pool 下载 + 结果回传）、Windows 集群运维（Task Scheduler 开机自启 + 心跳监控）。
- [PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md](PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md)
D-InSAR 与 LT-1/Sentinel-1 单景影像生产的统一生产节点子系统设计，明确 LandSAR 集群 MVP、worker-only 部署、安全边界和后续本机/集群双模式路线。
- [LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md](LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md)
LandSAR 陆探一号单景/多景生产 proID 与参数链探索，设计 `100016/100206` 导入产品化、结果 catalog、本机/集群双模式和后续影像产品验证路线。
- [UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md](UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md)
LT-1/Sentinel-1 鏈湴婧愬帇缂╁寘绠＄悊銆佸寘鍐?XML/manifest 璧勪骇鍖栥€佹湰鍦?Task_Pool materialize锛屼互鍙?UNC 閫€鍑哄悗鐨勬湰鏈洪儴缃茶竟鐣屻€?
- [SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md](SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md)
docs/LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md
+642
View File
@@ -0,0 +1,642 @@
# LandSAR 陆探一号单景/多景生产链路探索与系统设计（2026-06-27）
## 1. 结论
LandSAR 不只承担 LT-1 D-InSAR。仓库内 LandSAR 工具和文档显示，LandSAR 已有一条明确的 LT-1 数据生产基础链：
```text
LT-1 源数据
-> 100016 LT-1 数据导入
-> 100206 LT-1 精密轨道导入
-> Task_*/Input_Data
```
这条链可以支撑两类非 D-InSAR 生产能力：
1. **单景生产基础产品**：把一景 LT-1 源数据导入为 LandSAR 统一 `Input_Data` 格式，形成后续处理可复用的标准输入。
2. **多景生产基础产品**：把同轨道、同模式、同极化的一组 LT-1 场景导入并注入精密轨道，形成 LandSAR 时序栈 `Task_TS_*/Input_Data`。
当前系统的问题不是 LandSAR 没能力，而是这条能力没有被后端产品化：
- 现有后端只在 `LandsarEngine.run()` 的 D-InSAR 前置阶段调用 `100016`。
- `100206` 精轨导入和多景导入逻辑主要沉淀在 `third_party/LandSAR/lt1_import_gui.py`，还没有进入后端生产服务。
- 单景/多景导入结果还没有独立 catalog、运行记录和本机/集群双模式。
- 地理编码、正射、强度图这类“业务影像产品”还需要进一步验证 LandSAR 的 `180016`、`180044`、`200016`、`200046` 等 proID 参数链，不能直接把 `100016` 导入结果命名为正射影像。
因此第一阶段建议先产品化：
```text
landsar.import.lt1.scene.v1
landsar.import.lt1.stack.v1
landsar.orbit.lt1.v1
```
第二阶段再在真实样例参数和运行验证基础上开放：
```text
landsar.image.lt1.geocode.v1
landsar.image.lt1.ortho.v1
```
## 1.1 当前实现状态
截至 2026-06-27，系统已落地第一阶段的本机生产入口：
- 后端新增 `LANDSAR_LT1_IMPORT` 任务类型。
- 后端新增 `/api/landsar-lt1-production/*` API。
- 已按 LandSAR GUI 中的真实参数格式生成 `100016.txt` 与可选 `100206.txt`。
- 运行结果以 `catalog_name=lt1_landsar` 写入 `result_products/result_assets`。
- 前端“生产管理 -> 陆探一工作台 -> 陆探一影像生产”已可提交单景或多景 LT-1 源资产/目录导入任务。
- 选择 `source_product_assets` 中的 LT-1 源资产时，任务会先 materialize 到 `LANDSAR_WORK_ROOT/lt1_import_tasks/<task_id>/scenes/`，再把解包后的 scene 目录交给 LandSAR。
- `manifest.summary.source_asset_ids` 会记录源资产 id；资产台账和检索列表根据 `result_products.summary_json.source_asset_ids` 下发 `lt1_landsar_produced` 标识。
- 已经登记为 READY 的 LT-1 LandSAR 产品会在生产面板中禁止再次选择；后端任务执行前也会再次拒绝已生产源资产。
当前实现边界：
- 输入支持资产台账中的 LT-1 源资产，或人工指定的已 materialize/解包 LT-1 scene 目录；人工目录无法可靠反查源资产去重状态。
- 输出产品语义是 LandSAR `Input_Data` 标准化生产结果，不是正射影像、地理编码强度图或 D-InSAR 结果。
- 集群执行尚未接入这一任务类型；当前先采用主服务器本机 LandSAR 执行，并通过同一 manifest/catalog 结构为后续生产节点适配保留边界。
## 2. 已确认的 LandSAR proID 与参数链
### 2.1 已有可复用参数生成器
当前仓库中 `third_party/LandSAR/lt1_import_gui.py` 已有以下参数生成器：
| 能力 | proID | 现有函数 | 状态 |
| --- | --- | --- | --- |
| LT-1 数据导入 | `100016` | `generate_param_file()` | 已有，偏 master/slave 两目录 |
| LT-1 多景导入 | `100016` | `generate_lt1_multiscene_import_param_file()` | 已有，支持 `文件夹导入个数=N` |
| LT-1 精密轨道导入 | `100206` | `generate_orbit_param_file()` | 已有 |
| D-InSAR | `200014` | `generate_dinsar_param_file()` | 已接入后端 |
| SBAS 一体化 | `280039` | `generate_sbas_param_file()` | 已有参数生成器，运行授权/能力需另行验证 |
| PS-InSAR 分步 | `280000~280032` | `generate_psinsar_step_param_file()` | 已有参数生成器，运行链路需另行验证 |
| Stacking | `300001` | `generate_stacking_param_file()` | 已有参数生成器 |
这说明 `100016 + 100206` 不是猜测能力，已经有明确参数格式、GUI 调用方式和成功判定逻辑。
### 2.2 已确认的第一阶段链路
#### `100016`: LT-1 数据导入
用途：
- 从 LT-1 源 scene 目录读取 XML/TIFF。
- 输出 LandSAR 统一格式 `Input_Data`。
- 可用于单景，也可用于多景。
关键参数形态：
```text
卫星数据导入LT-1
处理编号 100016
设置数据导入形式_0文件夹导入_1数据导入 文件夹导入
读取成像参数文件_0否_1是 1
读取SLC数据文件_0否_1是 1
文件夹导入标识 TRUE
文件夹导入个数 N
文件夹1路径 <scene_dir_1>
...
设置数据导出目标路径_0原目录_1新目录 1
设置输出文件目录 <Input_Data>
```
成功判定：
- 日志包含 `module [LT-1数据导入] success`。
- 或 `console success`。
- 输出目录存在 `LT1*_SLC.xml` 和对应 `LT1*_SLC.tif`。
注意：
- 当前后端 `backend/app/dinsar_engines/landsar_engine.py` 的 `_generate_import_param_file()` 写死 `文件夹导入个数 2`，适合 D-InSAR pair 前置导入。
- 单景/多景产品化应改用 N 景参数生成逻辑，而不是复用 pair-shaped 参数。
#### `100206`: LT-1 精密轨道导入
用途：
- 对 `Input_Data` 中的 LT-1 XML 注入或关联精密轨道。
- 为后续 D-InSAR、SBAS、PS、Stacking 或影像处理提供已校正输入。
关键参数形态：
```text
LT-1精密轨道数据导入
处理编号 100206
输入数据个数 N
输入数据1的xml <xml_path_1>
...
输入精密轨道数据文件夹 <orbit_dir>
选择XML文件保存方式 0
设置数据导出目录形式0原目录1新目录 0
输出更新处理后数据目录 <Input_Data>
```
成功判定：
- 日志包含 `module [LT-1精密轨道数据导入] success`。
- 或日志同时包含 `精密轨道` 和 `success`。
- 或 `console success`。
### 2.3 候选但未验证的影像产品链
LandSAR 文档列出了以下与单景影像产品相关的 proID：
| proID | 功能 | 当前判断 |
| --- | --- | --- |
| `180044` | 多视处理 | 可能用于单景强度/幅度产品前置，但参数格式未在后端沉淀 |
| `180016` | 地理编码 | 可能用于单景地理编码产品，但缺少样例参数 |
| `180070` | SLC 处理 | 可能用于单景 SLC 派生处理，但语义需验证 |
| `200016` | 地理编码（流程） | 可能是一体化地理编码流程，但缺少样例参数 |
| `200046` | SLC 处理（流程） | 可能是一体化 SLC 处理流程，但缺少样例参数 |
| `280032` | SLC 数据多视 | PS/MTInSAR 链路中的多视步骤，不应直接等同于通用单景影像生产 |
这些 proID 不能直接进入正式 UI。需要先从 LandSAR GUI 生成真实参数文件，拿一景样本跑通，再决定产品定义。
## 3. 产品能力分层
### 3.1 第一层：导入型生产产品
这是近期可落地层。
#### `landsar.import.lt1.scene.v1`
```text
输入：
LT-1 单景源压缩包或 materialized scene 目录
处理：
100016 LT-1 数据导入
可选 100206 LT-1 精密轨道导入
输出：
Input_Data/
LT1*_SLC.xml
LT1*_SLC.tif
*.thumb.jpg
params/
100016.txt
100206.txt
logs/
100016_console.log
100206_console.log
import_manifest.json
catalog:
catalog_name = lt1_landsar
product_family = lt1_scene_import
product_type = landsar_input_data
```
产品语义：
- 这是 LandSAR 输入标准化产品。
- 不是正射影像。
- 不是地理编码强度图。
- 可作为 D-InSAR、SBAS、PS、后续影像产品的上游缓存。
#### `landsar.import.lt1.stack.v1`
```text
输入：
同一轨道/模式/极化/方向的一组 LT-1 scene
处理：
100016 LT-1 多景导入
100206 LT-1 精密轨道导入
输出：
Task_TS_<track>_<pol>_<orbit_direction>_<start>_<end>_<count>/
Input_Data/
Output_Data/
stack_import_manifest.json
时序数据构建报告.txt
catalog:
catalog_name = lt1_landsar
product_family = lt1_stack_import
product_type = landsar_timeseries_input_data
```
产品语义：
- 这是 LandSAR 多景时序输入产品。
- 可作为 PS/SBAS/MT-InSAR 的上游准备结果。
- 不直接表示形变结果。
### 3.2 第二层：影像型生产产品
这层需要先验证 LandSAR 影像处理 proID。
候选 profile：
```text
landsar.image.lt1.multilook.v1
landsar.image.lt1.geocode.v1
landsar.image.lt1.ortho.v1
```
可能链路：
```text
landsar.import.lt1.scene.v1
-> 180044 多视处理
-> 180016 地理编码
```
或：
```text
landsar.import.lt1.scene.v1
-> 200046 SLC 处理（流程）
-> 200016 地理编码（流程）
```
当前必须标记为待验证：
- 缺少真实参数文件。
- 缺少真实输出样例。
- 缺少对输出单位、坐标系、辐射定标、nodata、分辨率的确认。
- 缺少成功判定和错误摘要规则。
## 4. 系统架构设计
### 4.1 后端模块
建议新增独立模块，不放进 `dinsar_engines`：
```text
backend/app/landsar_lt1/
contracts.py
param_files.py
runtime.py
discovery.py
scene_import_adapter.py
stack_import_adapter.py
result_package.py
```
职责：
- `contracts.py`：定义 scene/stack 输入 manifest、结果 manifest、能力描述。
- `param_files.py`：沉淀 `100016`、`100206` 参数文件生成器，不再依赖 GUI 代码。
- `runtime.py`：统一调用 `InSAR_Console.exe`、日志捕获、超时、成功判定、错误摘要。
- `discovery.py`：从资产表或 materialized 目录解析 LT-1 scene，按轨道/极化/日期分组。
- `scene_import_adapter.py`：执行 `landsar.import.lt1.scene.v1`。
- `stack_import_adapter.py`：执行 `landsar.import.lt1.stack.v1`。
- `result_package.py`：生成标准发布包、manifest、current 指针。
LandSAR D-InSAR 当前已有的 runtime 检查、授权服务启动、DLL 校验逻辑可以抽取共用，但不要把单景/多景生产继续塞进 `LandsarEngine.run()`。
### 4.2 调度模型
建议走统一生产节点协议：
```text
主服务器创建 production run
-> 每个 scene 或 stack group 生成 production item
-> 本机 worker 或远端 production node 领取
-> adapter 执行 LandSAR
-> 上传/登记标准产品包
```
执行模式：
| 模式 | 说明 |
| --- | --- |
| `local` | 主服务器本机执行 |
| `cluster` | 指定远端生产节点执行 |
| `auto` | 按节点能力、负载、数据缓存选择 |
节点能力：
```json
{
"capabilities": [
"landsar.import.lt1.scene.v1",
"landsar.import.lt1.stack.v1",
"landsar.orbit.lt1.v1"
]
}
```
### 4.3 运行记录模型
现有 `dinsar_production_runs` 虽然有 `product_family` 字段，但表名、字段和 item 语义都偏 pair。为了避免技术债，不建议把单景/多景陆探生产继续塞进 D-InSAR run 表。
推荐新增通用生产表：
```text
production_runs
run_id
product_family
processor_code
profile_code
execution_mode
source_scope
status
total_items
completed_items
failed_items
params_json
production_run_items
item_id
run_id
item_key
item_type # scene / stack_group
source_asset_ids_json
source_paths_json
target_key # scene_uid / stack_key
status
latest_run_key
latest_manifest_path
metrics_json
production_executions
execution_id
run_id
item_id
node_id
run_key
status
output_dir
manifest_path
log_path
error_message
```
也可以短期复用 `system_jobs` 做执行队列，但正式 UI、重试、批次统计、集群调度和结果追溯需要上述通用运行表。
### 4.4 结果 catalog
现有 `result_products` / `result_assets` 表可以承载单景/多景产品，因为它们已经有：
- `catalog_name`
- `product_family`
- `product_type`
- `engine_code`
- `processor_code`
- `profile_code`
- `stack_key`
- `summary_json`
- `assets`
但现有 `result_catalog_service._load_manifest()` 明确拒绝非 `dinsar` 的 manifest。需要拆出通用结果登记服务：
```text
result_package_registry
register_manifest(manifest_path)
validate_manifest(product_family)
upsert_result_product()
upsert_result_assets()
```
LandSAR LT-1 生产建议使用：
```text
catalog_name = lt1_landsar
product_family = lt1_scene_import / lt1_stack_import
product_type = landsar_input_data / landsar_timeseries_input_data
engine_code = landsar
processor_code = landsar.import.lt1.scene / landsar.import.lt1.stack
profile_code = landsar.import.lt1.scene.v1 / landsar.import.lt1.stack.v1
```
## 5. 输入与分组设计
### 5.1 单景输入
来源：
- `source_product_assets` 中的 `LT1_ARCHIVE`。
- 已 materialized 的 LT-1 scene 目录。
- 人工指定的受控服务器目录。
流程：
```text
source asset
-> materialize scene
-> run 100016
-> optional run 100206
-> publish result package
```
### 5.2 多景输入
分组键建议：
```text
satellite_family = LT1
track / orbit number
imaging_mode
polarization
orbit_direction
product_type = SLC
admin/aoi 或用户选择范围
date range
```
多景生产不应简单把用户勾选的所有 LT-1 都丢给 LandSAR。需要先做分组预检：
- 是否同一轨道或可构成同一时序栈。
- 是否同一极化。
- 是否同一成像模式。
- 日期是否可解析。
- 是否都有源包。
- 精轨是否匹配。
- scene footprint 是否满足业务区域覆盖要求。
输出 `stack_key` 示例：
```text
lt1_stack_<track>_<pol>_<orbit_direction>_<start_date>_<end_date>_<scene_count>_<hash>
```
## 6. 发布目录设计
```text
D:\production_results\lt1_landsar\
scene_import\
<scene_uid>\
current\
landsar.import.lt1.scene.v1.json
runs\
<run_key>\
manifest.json
execution_manifest.json
native\
Input_Data\
params\
logs\
assets\
input_data\
thumb\
metadata\
stack_import\
<stack_key>\
current\
landsar.import.lt1.stack.v1.json
runs\
<run_key>\
manifest.json
execution_manifest.json
native\
Task_TS_...\Input_Data\
Task_TS_...\Output_Data\
params\
logs\
assets\
input_data_manifest.json
stack_report.txt
scene_index.json
```
`native/` 保留 LandSAR 原始结构，`assets/` 放系统标准化索引文件和必要缩略图。不要把完整 `Input_Data` 再复制两份；标准资产层可以用 manifest 指向 native 内的相对路径。
## 7. API 与前端入口
### 7.1 API
建议新增：
```text
GET /api/landsar-lt1-production/capabilities
POST /api/landsar-lt1-production/preview
POST /api/landsar-lt1-production/run
GET /api/landsar-lt1-production/products
GET /api/landsar-lt1-production/products/{product_db_id}
GET /api/landsar-lt1-production/products/{product_db_id}/assets/{asset_id}
```
`preview` 必须先返回可执行性：
- 选中 scene 数。
- 分组结果。
- 缺失源包。
- 缺失精轨。
- 预计输出目录。
- 是否可本机执行。
- 是否有集群节点支持。
- 源资产是否已经存在 READY 状态的 LT-1 LandSAR 产品。
### 7.2 前端
生产管理里将 `陆探一生产占位` 改成实际工作台：
```text
陆探一生产
├─ 单景导入
├─ 多景时序输入构建
├─ 运行记录
└─ 产品结果
```
第一版按钮只开放：
- 预检。
- 提交单景导入。
- 提交多景导入。
- 查看日志。
- 打开结果目录。
- 查看 catalog 资产。
不要第一版就开放：
- 正射产品。
- 地理编码强度图。
- PS/SBAS 自动执行。
- Stacking 自动执行。
这些应建立在第一层 `Input_Data` 产品稳定之后。
## 8. 集群化设计
单景/多景导入非常适合纳入生产节点：
- 输入大但结构明确。
- 输出可通过 manifest 回传。
- 不需要主服务器承担长时间 LandSAR 进程。
- `Input_Data` 可作为远端缓存，后续 D-InSAR/SBAS 复用。
远端节点能力：
```text
landsar.import.lt1.scene.v1
landsar.import.lt1.stack.v1
landsar.orbit.lt1.v1
```
第一版并发建议：
- LandSAR 进程并发：每节点 1。
- 多 scene 导入内部由 LandSAR 控制，不在外层并发拆太碎。
- 多个单景任务可以排队，但不要同节点同时启动多个 `InSAR_Console.exe`，除非实测证明安全。
数据搬运：
- 主服务器给 input manifest。
- worker 下载源包或 materialized scene。
- worker 执行 `100016/100206`。
- worker 上传 manifest 和必要产物。
- 大体量 `Input_Data` 是否全量回传可配置：第一版建议回传完整受管产品；后续可做远端缓存引用。
## 9. 实施路线
### 阶段 1：参数链固化
- 从 `third_party/LandSAR/lt1_import_gui.py` 抽取 `100016`、`100206` 参数生成逻辑。
- 增加参数文件 golden tests。
- 不接 UI，不跑真实任务。
### 阶段 2：本机单景导入
- 实现 `landsar.import.lt1.scene.v1` adapter。
- 输入一个 LT-1 scene。
- 执行 `100016`。
- 可选执行 `100206`。
- 生成标准 manifest。
- 登记到 `result_products`，catalog 为 `lt1_landsar`。
### 阶段 3：本机多景导入
- 实现分组预检。
- 实现 `landsar.import.lt1.stack.v1` adapter。
- 复用 `generate_lt1_multiscene_import_param_file()` 语义。
- 执行 `100016 -> 100206`。
- 发布 `Task_TS_*/Input_Data` 产品。
### 阶段 4：生产节点接入
- 把 scene/stack import adapter 接入生产节点协议。
- 远端节点上报能力。
- 实现输入下载、结果回传、日志上报。
### 阶段 5：影像产品验证
- 用 LandSAR GUI 对单景生成多视、地理编码或正射产品。
- 收集真实参数文件。
- 确认 proID、输出命名、坐标系、单位和成功判定。
- 再实现 `landsar.image.lt1.*`。
## 10. 需要运行验证的问题
1. `100016` 单景导入时 `文件夹导入个数=1` 是否被当前 LandSAR runtime 接受。
2. `100016` 多景导入最大稳定 scene 数是多少。
3. `100206` 对已导入 XML 是原地改写还是复制输出。
4. `100206` 精轨注入后 XML 中可验证字段是什么。
5. `180044 -> 180016` 是否能独立从单景 `Input_Data` 生成地理编码影像。
6. `200046/200016` 是否比单步 `180xxx` 更适合作为正式影像产品链。
7. LandSAR 同一台机器是否允许多个导入任务并发。
8. 导入输出是否可跨机器复用，还是强依赖本机路径。
## 11. 近期不建议做的事
- 不建议把 `100016` 单景导入继续藏在 D-InSAR 前置步骤里。
- 不建议把 `100016` 输出命名为正射或地理编码产品。
- 不建议把单景/多景生产塞进 `dinsar_production_runs`。
- 不建议在未验证 `180016/200016` 参数前开放“LandSAR 正射生产”按钮。
- 不建议先做复杂 UI。应先做 adapter、manifest、catalog、真实样例验证。
docs/LT1_GEOCODED_GEOTIFF_PRODUCTION_20260628.md
+68
View File
@@ -0,0 +1,68 @@
# LT-1 Geocoded GeoTIFF Production Update (2026-06-28)
## Decision
The LT-1 production button must produce a usable geocoded GeoTIFF, not only prepare LandSAR `Input_Data` for later D-InSAR work.
The current implemented production path is:
```text
LT-1 source asset
-> radar_data record
-> SAR_SCENE_PREPROCESS job
-> lt_gamma single-scene pipeline
-> SARSceneGeoORM.analysis_tif_path = analysis_ready.tif
```
This is the platform's real LT-1 single-scene backscatter image product for now. It performs the Gamma single-scene preprocessing chain: LT source product to Gamma SLC, multilook amplitude, geocode, speckle filtering in the linear-power domain, dB conversion, and analysis-ready GeoTIFF registration.
The chain must not silently fall back to unrelated DEM sources. The default production contract is SRTM-derived on the current server:
- `SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84`
- `SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0`
- `SAR_ANALYSIS_DEM_RESOLUTION_M=30.0`
- `SAR_ANALYSIS_RANGE_LOOKS=6`
- `SAR_ANALYSIS_AZIMUTH_LOOKS=5`
- `SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true`
- `SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee`
- `SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5`
- `PYINT_GEO_INTERP=1`
The runtime clips the configured SRTM-derived DEM to the scene footprint with a small margin, converts that clip to Gamma DEM format, runs `generate_rdc_dem.py`, geocodes the multilooked amplitude with `geocode_back`, exports a GeoTIFF with `data2geotiff`, applies a Lee speckle filter to the linear power raster, then converts power to dB. The output `pixel_size_m` stored in `sar_scene_geo` is derived from the GeoTIFF transform; WGS84 degree grids are converted to approximate meters before registration.
Multilook is already part of the Gamma preprocessing path through `generate_rdc_dem.py`; it is controlled by `SAR_ANALYSIS_RANGE_LOOKS` and `SAR_ANALYSIS_AZIMUTH_LOOKS`. Speckle filtering is a separate post-geocode raster operation before dB conversion. The filter manifest records the method, window size, domain, and equivalent number of looks used for the Lee weighting.
For SRTM-derived DEMs, the configured 30 m analysis grid must be translated into Gamma `dem_lat_ovr` and `dem_lon_ovr` from the actual converted `.dem.par` spacing. Do not assume that `SAR_ANALYSIS_DEM_RESOLUTION_M=30.0` alone changes the output GeoTIFF grid. A 3 arc-second source DEM with `dem_lat_ovr=1` and `dem_lon_ovr=1` will still produce about 90 m latitude spacing. The runner now reads `post_lat`, `post_lon`, and scene latitude from the converted Gamma DEM parameter file, then writes the derived oversampling into the template before `generate_rdc_dem.py`.
The previous `D:\DEM\HeiLongJiang10M_DEM.tif` default is not the production default. It is an interpolated regional DEM and must not be mixed silently with SRTM-derived products. If a future production profile intentionally uses it, the selected DEM and output grid must be recorded in the product manifest.
## What Changed
- `/api/landsar-lt1-production/run` queues `SAR_SCENE_PREPROCESS` jobs with `engine=lt_gamma`.
- Each selected LT-1 source asset resolves to a `radar_data` scene and produces one independent `SARSceneGeoORM` record.
- Batch mode means batch single-scene GeoTIFF production. It does not create a D-InSAR pair/stack product.
- The product list under `/api/landsar-lt1-production/products` reads from `sar_scene_geo`, not from `result_products.catalog_name=lt1_landsar`.
- Asset inventory and radar search mark LT-1 assets as produced only when `sar_scene_geo.status = DONE` and `analysis_tif_path` exists for the LT-1 `lt_gamma` profile.
## UI And Query Contract
- The LT-1 GeoTIFF production UI does not accept arbitrary LT-1 scene directories. Operators must select scanned source assets so each output can be linked back to `radar_data` and `sar_scene_geo`.
- The UI does not expose a satellite-mode/BIST selector. Acquisition mode should come from scanned source metadata, and the current Gamma single-scene pipeline does not require the operator to choose LandSAR import mode manually.
- The production candidate list reuses `/radar-data/search` with `satellite_family=LT1` and `source_format=LT1_ARCHIVE`, so operators can plan production by acquisition date, administrative AOI, orbit, polarization, and product name.
- `/assets/sources` remains the asset-led inventory view. It is not the main LT-1 production planning surface because it lacks the full image search/AOI workflow.
- Items that already have a completed LT-1 GeoTIFF product are shown as produced and are not selectable for a new production task.
## What Is Not A Finished Image Product
LandSAR `100016` and `100206` are still useful, but they only prepare/import LT-1 data:
```text
100016 -> LandSAR Input_Data
100206 -> precise-orbit injection for imported XML
```
Those outputs must not be displayed as "image production complete" and must not block reprocessing as if they were geocoded images.
## Open LandSAR Work
LandSAR single-scene geocoded image production may still be possible through `180044`, `180016`, `200046`, or `200016`, but the repository does not yet contain a verified parameter chain for those modules. Do not expose those proIDs in the formal UI until a real parameter file and sample run are verified.
docs/PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md
+462
View File
@@ -0,0 +1,462 @@
# 生产节点子系统设计：D-InSAR 与单景影像生产（2026-06-27）
## 1. 结论
当前 `LANDSAR_CLUSTER_ITEM` 已经证明：LandSAR D-InSAR 可以从主服务器拆分 pair，并在远端 Windows 节点完成输入搬运、LandSAR 执行和结果回传。
但这仍是 LandSAR D-InSAR 的集群 MVP，不应直接扩展成长期架构。后续陆探一号和 Sentinel-1 的“只生产影像、不做 D-InSAR”能力也会进入生产管理域。陆探一号这条线应优先承认 LandSAR 已有的 `100016` LT-1 数据导入/统一格式转换能力，再决定是否继续扩展为地理编码或正射影像产品。因此远端节点不能只理解 `LANDSAR_CLUSTER_ITEM`，而应该抽象成一个受控的“生产节点子系统”。
建议把后续设计目标调整为：
1. 主服务器继续负责资产索引、任务编排、调度策略、结果 catalog 和权限边界。
2. 子服务器只部署生产节点运行包，不部署完整项目仓库、前端、管理后台和无关源码。
3. 所有生产任务按“产品类型 + 处理器能力”分发，同一任务可以选择本机执行或集群执行。
4. LandSAR、Gamma/PyINT、未来 Sentinel-1 影像生产处理器都通过 adapter 接入生产节点协议。
5. 结果以标准产品 manifest 回传，由主服务器统一入库，而不是让子服务器直接写主库或扫描任意目录。
## 2. 当前事实
### 2.1 已有设计边界
- [THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md](THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md) 明确 LT-1、Sentinel-1 当前管理对象是本机压缩包源池，生产时才按任务 materialize 到 `Task_Pool`。
- [DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md](DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md) 明确 D-InSAR 保留 `sarscape`、`landsar`、`pyint` 三条主线，其中 LandSAR 只处理 LT-1，Gamma/PyINT 同时支持 LT-1 和 Sentinel-1。
- [LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md](LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md) 已经为 LandSAR D-InSAR 定义了输入下载和结果上传链路。
- 当前 192.168.1.6 节点已能执行 LandSAR D-InSAR 集群 item，并调用与本机一致的 `LandsarEngine.run()`。
- `third_party/LandSAR/LT-1_数据导入功能说明.md` 明确 `100016` 是 LT-1 数据导入算法 ID；当前 `backend/app/dinsar_engines/landsar_engine.py` 已经能生成 `100016.txt` 并调用 `InSAR_Console.exe`，但这段能力目前只作为 D-InSAR 前置导入阶段存在。
### 2.2 仍未完成的能力
- 生产管理里的陆探一号已接入第一阶段本机生产链：`100016` LT-1 导入生成 LandSAR `Input_Data`，并支持可选 `100206` 精轨注入。它仍不是正射/地理编码影像产品。
- 生产管理里的 Sentinel-1“只生产影像”仍是占位，不是已实现链路。
- 陆探一号单景生产至少应拆成两层：`landsar.import.lt1` 表示 LandSAR `100016` 导入/统一格式转换，`landsar.image.lt1` 表示后续地理编码、正射或业务可用影像产品。前者已在本机任务/API/前端/catalog 中产品化；后者仍需要确认 LandSAR 调用链和输出规格。
- Sentinel-1 影像生产的处理器尚未最终确认，不能把 Sentinel-1 影像生产硬编码到 LandSAR 集群链路。
- 当前集群 worker 更接近“把后端生产代码部署到远端执行”，还不是一个最小权限、最小源码暴露的生产节点运行包。
## 3. 需要解决的问题
### 3.1 不要把集群等同于 LandSAR D-InSAR
如果继续按 `LANDSAR_CLUSTER_ITEM` 的方式增长，后续很容易出现：
- `LANDSAR_IMAGE_CLUSTER_ITEM`
- `S1_IMAGE_CLUSTER_ITEM`
- `PYINT_CLUSTER_ITEM`
- `SBAS_CLUSTER_ITEM`
每新增一种生产能力都复制一套领取、搬运、执行、上传、入库逻辑，技术债会快速扩大。
更稳妥的边界是：
```text
生产任务协议
├─ 输入 manifest
├─ 处理器 adapter
├─ 执行状态上报
├─ 结果 manifest
└─ 结果上传与 catalog
具体处理器
├─ landsar.dinsar.lt1
├─ landsar.import.lt1
├─ landsar.image.lt1
├─ pyint.dinsar.lt1
├─ pyint.dinsar.s1
└─ s1.image.<待定处理器>
```
### 3.2 单景影像生产也需要本机/集群双模式
LT-1 和 Sentinel-1 的影像生产虽然不是 D-InSAR，但仍可能是重计算、重 IO、长耗时任务。它们不应该只作为本机按钮实现。
推荐统一执行模式：
| 模式 | 含义 | 适用场景 |
| --- | --- | --- |
| `local` | 主服务器本机执行 adapter | 调试、小批量、没有可用节点 |
| `cluster` | 远端生产节点领取执行 | 大批量、长耗时、需要释放主服务器 |
| `auto` | 主服务器按能力、负载、数据位置选择 | 正式生产默认模式 |
前端可以先只暴露“本机执行 / 集群执行”，内部仍按统一任务协议创建任务。
### 3.3 子服务器不应长期部署完整代码仓库
当前 MVP 为了快速跑通，子服务器需要较完整的项目运行环境。这对验证是可接受的，但长期有三个问题：
1. 源码暴露面过大：子服务器不需要前端、后台管理、资产扫描、用户接口等源码。
2. 配置权限过宽：子服务器不应持有主库高权限连接信息。
3. 升级不可控：完整仓库部署容易出现主服务器和子服务器代码版本漂移。
长期应改为生产节点运行包：
```text
production-node/
worker_service.py
config.py
client.py
adapters/
landsar_dinsar.py
landsar_lt1_image.py
pyint_dinsar.py
s1_image.py
contracts/
job_manifest.py
product_manifest.py
status_event.py
scripts/
install_windows_service.ps1
requirements.lock
```
这个运行包只包含：
- 任务领取和心跳客户端。
- 输入下载和结果上传客户端。
- 必要的生产 adapter。
- 与主服务器共享的 manifest schema。
- Windows 服务安装脚本。
不包含：
- 前端源码。
- 管理后台路由。
- 数据扫描入口。
- 用户认证管理。
- 数据库迁移脚本。
- 与该节点能力无关的处理器源码。
## 4. 目标架构
```mermaid
flowchart LR
UI["生产管理前端"] --> API["主服务器 API"]
API --> Scheduler["调度器"]
Scheduler --> Queue["生产任务队列"]
API --> Catalog["结果 catalog"]
API --> Assets["源数据/精轨资产库"]
NodeA["本机生产节点"] --> Queue
NodeB["远端生产节点 1.6"] --> Queue
NodeC["远端生产节点 N"] --> Queue
NodeB --> Adapter1["landsar.dinsar.lt1"]
NodeB --> Adapter2["landsar.image.lt1"]
NodeC --> Adapter3["pyint.dinsar.s1"]
NodeC --> Adapter4["s1.image.<待定>"]
Queue --> Manifest["输入 manifest"]
Manifest --> NodeB
NodeB --> Upload["结果上传 API"]
Upload --> Catalog
```
### 4.1 主服务器职责
- 维护源数据、精轨、DEM、Task_Pool、结果 catalog。
- 根据资产状态生成生产任务。
- 决定任务执行模式：本机、指定节点、自动调度。
- 为 worker 生成输入 manifest，包含文件清单、hash、大小、产品类型、处理器 profile。
- 接收 worker 状态、日志摘要、进度事件和结果包。
- 校验结果 manifest 后入库。
- 维护节点注册、能力、版本、心跳和并发上限。
### 4.2 生产节点职责
- 启动后向主服务器注册或发送心跳。
- 上报能力，例如 `landsar.dinsar.lt1`、`landsar.import.lt1`、`landsar.image.lt1`、`pyint.dinsar.s1`。
- 按能力领取任务。
- 下载输入文件或复用本地缓存。
- 调用本机已安装的生产软件或 adapter。
- 将运行日志、状态、结果 manifest 和产品文件上传回主服务器。
- 清理本地临时目录，保留可配置缓存。
### 4.3 Adapter 职责
Adapter 是生产节点中唯一知道具体软件细节的层：
| Adapter | 输入 | 输出 | 备注 |
| --- | --- | --- | --- |
| `landsar.dinsar.lt1` | LT-1 pair Task_Pool | D-InSAR 标准产品包 | 当前集群 MVP 已覆盖核心执行 |
| `landsar.import.lt1` | LT-1 单景源包、解包 scene，或多景导入目录 | LandSAR `Input_Data` 统一格式、缩略图、导入 manifest | 基于 `100016`，当前代码已有 pair-shaped 前置调用，需要拆成一等单景 adapter |
| `landsar.image.lt1` | `landsar.import.lt1` 输出 + 可选精轨/DEM | LT-1 地理编码、正射或业务影像产品 | 需要确认 LandSAR 后续 proID/参数和产品规格 |
| `pyint.dinsar.lt1` | LT-1 pair Task_Pool | D-InSAR 标准产品包 | 可后续接入 |
| `pyint.dinsar.s1` | S1 pair Task_Pool + EOF | D-InSAR 标准产品包 | 当前只应按 Gamma/PyINT 能力开放 |
| `s1.image.<待定>` | S1 ZIP/SAFE + EOF | S1 单景影像产品 | 处理器未确定前保持占位 |
主服务器不应该把某个 adapter 的内部目录结构暴露给前端；前端只看到任务类型、执行位置、状态和结果。
## 5. 统一任务类型
### 5.1 产品族
建议把生产任务按产品族建模，而不是按按钮建模：
| 产品族 | 数据粒度 | 当前状态 | 目标执行模式 |
| --- | --- | --- | --- |
| `dinsar_pair` | 两景 pair | LandSAR LT-1 集群 MVP 已跑通 | 本机 + 集群 |
| `single_scene_import` | 单景或多景导入 | LT-1 LandSAR `100016/100206` 已作为本机 `LANDSAR_LT1_IMPORT` 任务、API、前端入口和 `lt1_landsar` catalog 产品发布；集群执行尚未接入 | 本机 + 集群 |
| `single_scene_image` | 单景 | LT-1 后续影像产品和 S1 均为占位 | 本机 + 集群 |
| `sbas_stack` | 多景 stack | 当前不纳入本轮集群化 | 后续再设计 |
| `gf3_native_register` | 外部结果登记 | 本机登记 `_geo` | 不建议进入生产节点 |
### 5.2 推荐任务字段
```json
{
"job_id": 123,
"product_family": "single_scene_image",
"sensor": "LT1",
"processor": "landsar",
"profile": "landsar.image.lt1",
"execution_mode": "cluster",
"input_manifest_url": "/api/production-node/jobs/123/input-manifest",
"result_contract": "standard_product_manifest.v1",
"priority": 50,
"retry_policy": {
"max_retries": 2,
"timeout_seconds": 7200
}
}
```
### 5.3 结果 manifest
D-InSAR 和单景影像生产都应该回传标准 manifest，差异放在 `product_family` 和 `product_type` 中：
```json
{
"manifest_version": 1,
"product_family": "single_scene_image",
"sensor": "LT1",
"processor": "landsar",
"profile": "landsar.image.lt1",
"scene_id": "LT1A_MONO_KSC_STRIP1_...",
"run_key": "run_20260627T010203Z_landsar_image_lt1_456",
"products": [
{
"role": "main_image",
"path": "products/main.tif",
"format": "GeoTIFF",
"crs": "EPSG:4326"
},
{
"role": "preview",
"path": "preview/main.webp",
"format": "WEBP"
},
{
"role": "metadata",
"path": "metadata/product.json",
"format": "JSON"
}
]
}
```
## 6. 陆探一号单景生产设计方向
陆探一号单景生产如果由 LandSAR 承担，应先把“导入/统一格式转换”和“正式影像产品”分开。
### 6.1 `landsar.import.lt1`
这是当前最清楚、风险最低的第一版能力。
LandSAR `100016` 的语义是 LT-1 数据导入：把 LT-1A/LT-1B SLC XML/TIFF 转成 LandSAR 统一内部格式，形成 `Task_*/Input_Data` 可消费的 XML/TIF 组织，并生成缩略图等辅助文件。当前代码已经在 `_ensure_imported_input_data()` 中调用这条链路，但有两个限制：
- 它被包在 D-InSAR 执行内部，只在缺少 `Input_Data` 时作为前置阶段触发。
- 当前参数生成器按 `master/slave` 两文件夹导入写死，不是正式的单景产品 adapter。
第一版应把它产品化为：
```text
landsar.import.lt1
输入：LT-1 单景源包、解包 scene，或显式 scene 目录
执行：InSAR_Console.exe + 100016.txt
输出：LandSAR Input_Data 统一格式 + import_manifest.json + 缩略图/日志
入库：单景预处理/影像生产 catalog
执行模式：local / cluster / auto
```
这个产品不应伪装成正射影像或地理编码强度图。它的价值是把陆探源数据转成 LandSAR 后续 D-InSAR、SBAS、影像处理可复用的标准输入。
### 6.2 `landsar.image.lt1`
如果“只生产影像”指的是业务可用影像，例如地理编码强度图、幅度图、正射 GeoTIFF、洪涝分析输入图，那么还需要确认 LandSAR 是否有对应单景 proID 或可复用处理链。不能把 `100016` 的导入输出直接命名为正射产品。
需要确认的产品规格：
- 输入是源压缩包、解包目录，还是现有 Task_Pool scene 目录。
- 输出是 SLC/SSC 的标准化影像、地理编码强度图、幅度图、还是系统用于浏览和洪水分析的 GeoTIFF。
- 是否需要精轨。
- 是否需要 DEM。
- 是否需要生成 WebP 预览。
- 是否进入 `radar_data`、`source_product_assets`、D-InSAR catalog，还是新的影像产品 catalog。
如果后续确认 LandSAR 能从 `Input_Data` 继续生成地理编码/正射产品，再实现第二层：
```text
LT-1 single scene image product
输入：landsar.import.lt1 输出 + 可选精轨 + 可选 DEM
执行器：LandSAR
输出：标准产品目录 + product_manifest.json + preview.webp
入库：影像产品 catalog
执行模式：local / cluster / auto
```
不要在第一版同时承诺“原始归档标准化、地理编码、洪水分析输入、全部极化派生物、可视化浏览缓存”这些目标。先把一个产品闭环做对，再扩展产品角色。
## 7. Sentinel-1 影像生产设计方向
Sentinel-1 单景影像生产目前不能直接套用 LandSAR。需要先确定处理器：
- 如果走 Gamma/PyINT，需要定义单景预处理 profile。
- 如果走 GDAL/SNAP/其他工具，需要单独 adapter。
- 如果只是生成浏览预览，应该归入资产扫描/预览缓存，不应叫正式生产任务。
建议在处理器未确认前只保留协议占位：
```text
s1.image.<processor>
状态：设计占位
不进入正式调度
不在 UI 上展示为可执行生产能力
```
这样可以避免前端提前出现“哨兵影像集群生产”按钮，但后端没有可信处理链。
## 8. 调度与效率
### 8.1 节点能力上报
生产节点心跳应包含：
```json
{
"node_id": "production-node-192-168-1-6",
"version": "2026.06.27",
"capabilities": [
"landsar.dinsar.lt1",
"landsar.import.lt1",
"landsar.image.lt1"
],
"max_concurrency": 1,
"active_jobs": 0,
"free_disk_gb": 512,
"runtime": {
"os": "windows",
"landsar_available": true,
"python_version": "3.12"
}
}
```
LandSAR 类任务的并发不能只看 CPU 核心数。需要同时考虑：
- LandSAR 是否支持多实例并发。
- 许可证或硬件锁是否允许并行。
- 工作目录是否互相隔离。
- 磁盘 IO 是否成为瓶颈。
- 单任务内部是否已经使用多线程。
因此第一版远端 LandSAR 节点建议 `max_concurrency=1`。等确认 LandSAR 多实例隔离和资源占用后，再按节点开放 2 个或更多并发。
### 8.2 缓存策略
单景影像生产和 D-InSAR 可以共享部分输入缓存：
- LT-1 源包下载缓存。
- LT-1 解包缓存。
- LandSAR 导入后的中间目录。
- DEM 裁剪缓存。
- Sentinel-1 ZIP/SAFE 和 EOF 缓存。
缓存键应基于源文件 hash、mtime、size、processor profile 和关键参数，不应只基于文件名。否则源包被替换后容易复用错误缓存。
### 8.3 数据搬运策略
优先级建议：
1. 第一阶段：沿用 HTTP manifest + file download + result upload，路径最清楚。
2. 第二阶段：增加断点续传和文件级 hash 校验。
3. 第三阶段：支持节点本地缓存命中，避免重复下载同一源包。
4. 第四阶段：在受控环境下可选共享只读源池，但不作为默认安全模型。
不要让 worker 任意访问主服务器磁盘路径。worker 应只根据主服务器签发的 manifest 下载白名单文件。
## 9. 安全边界
长期目标：
- worker 不持有主数据库账号。
- worker 只持有节点 token。
- token 按节点、能力和有效期管理。
- 所有输入下载和结果上传都走主服务器 API。
- 主服务器校验每个上传文件的相对路径，拒绝目录逃逸。
- 主服务器校验 result manifest，只有白名单产品角色进入 catalog。
- worker 运行包只包含生产节点必要代码。
- worker 版本和 adapter 版本必须上报，主服务器可以拒绝过旧节点领取任务。
当前 LandSAR 集群 MVP 可以作为过渡，但文档上应明确：完整仓库部署、DB 直接领取队列、共享 token 都不是长期安全边界。
## 10. 实施路线
### 阶段 0：保持现状可用
- 保留当前 LandSAR D-InSAR 集群能力。
- 不在未设计清楚前扩展新的集群 job type。
- 继续记录 1.6 节点运行结果、失败原因、传输耗时和 LandSAR 执行耗时。
### 阶段 1：抽取生产节点协议
- 定义 `job_manifest`、`input_manifest`、`product_manifest`、`status_event`。
- 把 LandSAR D-InSAR 当前输入下载、执行、上传流程映射到协议。
- 主服务器保留当前 API，同时新增通用 `/api/production-node/*` 命名空间。
### 阶段 2：拆出 worker-only 运行包
- 从完整仓库部署改成生产节点运行包部署。
- Windows 节点用服务方式启动。
- 节点只配置主服务器 URL、节点 token、工作根、结果根、缓存根和能力列表。
- 先支持 `landsar.dinsar.lt1`。
### 阶段 3：陆探一号单景生产
- 先实现 `landsar.import.lt1` adapter，把 LandSAR `100016` 从 D-InSAR 前置阶段拆成一等生产能力。
- 当前 `_generate_import_param_file()` 按 master/slave 两文件夹导入写死，单景 adapter 需要支持单 scene 输入、单文件夹导入或显式 file import。
- 本机模式和集群模式同时接入同一任务协议。
- 结果进入单景预处理/影像产品 catalog，而不是混入 D-InSAR 结果 catalog。
- 确认 LandSAR 后续单景地理编码或正射处理链后，再实现 `landsar.image.lt1`。
### 阶段 4：Sentinel-1 单景影像生产
- 先确认处理器和产品规格。
- 再实现 `s1.image.<processor>` adapter。
- 未确认前不开放 UI 执行入口。
### 阶段 5：节点运维和调度完善
- 节点版本管理。
- 能力矩阵管理。
- 节点禁用/启用。
- 任务重分配。
- 节点磁盘清理。
- 节点运行日志集中查看。
## 11. 近期不建议做的事
- 不建议继续复制 `LANDSAR_CLUSTER_ITEM` 形成多个专用 cluster item。
- 不建议在子服务器长期部署完整项目仓库。
- 不建议让子服务器直接扫描主服务器源数据目录。
- 不建议让子服务器直接写主数据库结果表。
- 不建议在 Sentinel-1 处理器未确认前实现“哨兵影像生产”按钮。
- 不建议把资产扫描阶段的 WebP 预览生成混同为正式影像生产。
## 12. 待确认问题
1. 陆探一号“只生产影像”的正式产品定义是什么：只做 LandSAR `100016` 导入/统一格式转换，还是继续生成地理编码强度图、幅度图、正射 GeoTIFF？
2. 陆探一号单景影像生产是否必须使用精轨和 DEM？
3. Sentinel-1 单景影像生产准备用哪个处理器承担？
4. 单景影像产品是否需要新建 catalog，还是复用现有 `radar_data` 资产表加产品 manifest？
5. 远端节点是否允许访问只读共享源池，还是严格走 HTTP 下载？
6. LandSAR 在同一台 Windows 节点上是否允许多个实例并发？
这些问题确认前，可以继续完善 LandSAR D-InSAR 集群，但不宜把新的影像生产能力直接硬接到当前 MVP worker 上。
docs/RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md
+219
View File
@@ -0,0 +1,219 @@
# 结果提取与用户权限审计（2026-06-30）
## 1. 审计范围
本次审计聚焦“结果提取”相关入口和用户权限边界，覆盖：
- 前端结果提取工作台：`frontend/src/ResultExtractionPanel.jsx`
- D-InSAR 结果管理页：`frontend/src/DinsarProductsPanel.jsx`
- D-InSAR 结果导出接口：`POST /api/dinsar-results/export`
- D-InSAR 生产结果提取与登记接口：`POST /api/idl/extract-disp`
- 用户与权限模型：`auth_users.role`、全局认证守卫、用户管理页
本次文档只记录审计结论和后续设计约束，不包含代码修复。
## 2. 当前实现事实
### 2.1 两条“提取”链路
当前系统里“结果提取”实际包含两种不同语义：
1. **生产结果入库**
- 前端入口：`DinsarProductsPanel.jsx`
- 后端入口：`POST /api/idl/extract-disp`
- 后台任务：`EXTRACT_DINSAR_PRODUCTS`
- 作用：从生产目录提取 D-InSAR 位移结果，发布成标准结果包，并重建结果 catalog。
2. **成果交付导出**
- 前端入口：`ResultExtractionPanel.jsx`
- 后端入口：`POST /api/dinsar-results/export`
- 作用：从已经登记的 D-InSAR catalog 中选择成果，复制到服务器指定交付目录。
这两条链路目前在产品文案上都叫“提取”，容易让用户混淆“入库”和“交付”。
### 2.2 当前接入状态
`ResultExtractionPanel.jsx` 中的通道状态：
| 通道 | 当前状态 | 说明 |
| --- | --- | --- |
| D-InSAR 结果 | 已接入 | 支持查询已登记结果并导出到服务器目录 |
| SBAS-InSAR 结果 | 半接入 | 可读取目录样例，但统一提取接口未实现 |
| LT-1 正射结果 | 占位 | 单景/正射结果 catalog 与导出链路未完成 |
| Sentinel-1 正射结果 | 占位 | 生产和导出链路未完成 |
| GF3 SARscape `_geo` | 占位 | 登记/标准化思路存在，统一导出接口未完成 |
## 3. 当前权限模型
系统当前只有两类角色：
- `admin`
- `viewer`
定义位置：`backend/app/auth_service.py`
全局认证守卫位于 `backend/app/routers/dependencies.py`：
- `GET / HEAD / OPTIONS` 默认视为只读操作，登录用户可访问。
- 除少数显式安全 POST 外，非只读请求要求 `admin`。
- 非管理员执行写操作会被拒绝，返回 `403 Read-only account cannot perform this operation.`
前端在 `App.jsx` 中把非管理员账号映射为 `readOnly`：
- viewer 可以浏览结果、查看任务、查看 catalog。
- viewer 不能提交生产、扫描、提取、导出、删除、修改。
- admin 拥有所有写权限，包括生产、扫描、结果入库、结果导出、用户管理和运维配置。
后端没有依赖前端按钮禁用来保护写操作。`/api/idl/extract-disp` 和 `/api/dinsar-results/export` 都显式要求 `admin`，这一点是正确的。
## 4. 审计发现
### P1：成果交付权限与系统管理员权限耦合过重
当前只有 `admin` 能执行成果导出，但 `admin` 同时拥有用户管理、系统配置、生产扫描、删除记录等高权限。
从业务职责看，成果交付导出不应天然等同于系统管理员权限。后续应拆出更细的权限，例如：
- `operator`：可提交生产任务、结果入库、目录重建。
- `exporter`：可导出已登记成果到受控交付目录。
- `viewer`：只读浏览、预览、查询。
- `admin`：用户管理、系统配置、根目录维护、许可证和高风险运维。
### P1：结果导出是同步请求，存在 504 风险
`POST /api/dinsar-results/export` 在请求线程内执行文件复制，最多允许 500 个结果 ID。成果文件较大或目标目录较慢时，容易再次触发前端或 Nginx 超时。
后续应改成后台任务：
- 接口只创建任务并返回 `task_id`。
- 文件复制由 worker 执行。
- 前端通过任务中心/结果提取工作台展示进度、成功数、失败数和目标目录。
### P1：生产结果入库缺少显式操作审计
`/api/dinsar-results/export` 已写入 `dinsar_results_exported` 审计日志。
`/api/idl/extract-disp` 当前会创建系统任务，但缺少独立的操作审计记录。它会改变结果 catalog，应记录：
- 操作用户
- 源生产目录
- 目标发布目录
- 创建的 `task_id` / `job_id`
- 完成后的 processed/copied/failed/published/registered 数量
### P2：页面命名和工作流边界不清
当前“D-InSAR 结果提取与登记”和“结果提取工作台”容易混淆。
建议命名：
- “生产结果入库”：从生产目录提取并登记为系统 catalog。
- “成果交付导出”：从已登记 catalog 选择成果并复制到交付目录。
这两个动作应该放在同一结果管理域下，但用不同分区和不同权限提示。
### P2：占位通道需要降低可执行暗示
SBAS、LT-1 正射、Sentinel-1 正射、GF3 `_geo` 目前不应被呈现成可执行导出能力。
建议 UI 明确显示：
- `已接入`
- `目录可查，导出未接入`
- `规划中`
- `不可执行`
并隐藏或禁用导出按钮，避免用户误以为功能已经上线。
### P2：导出目录策略需要产品化
当前后端已有 `_validate_export_path()` 和 `ALLOWED_EXPORT_DIRS` 约束能力，但前端仍允许用户输入服务器绝对路径。
后续建议：
- 普通业务用户不输入任意服务器路径。
- 管理员在系统配置中维护“交付目录白名单”。
- 结果导出页只让用户选择白名单目录和子任务名。
- 审计记录保存最终解析后的服务器路径。
### P3：部分前端文案存在历史编码损坏
`DinsarProductsPanel.jsx`、`ResultExtractionPanel.jsx`、`UserAdminPanel.jsx` 等文件存在局部中文乱码。功能不一定受影响，但会降低维护性和产品可信度。
建议后续单独做一次 UTF-8 文案修复，不与权限重构混在同一次提交中。
## 5. 建议目标模型
### 5.1 功能分区
结果管理应拆成三个清晰分区：
1. **产品目录**
- 查看已登记成果。
- 预览、筛选、查看详情。
- viewer 可访问。
2. **生产结果入库**
- 从生产结果根目录扫描、提取、发布、登记。
- operator/admin 可执行。
3. **成果交付导出**
- 从 catalog 选择成果，导出到受控交付目录。
- exporter/operator/admin 可执行。
### 5.2 权限矩阵建议
| 操作 | viewer | exporter | operator | admin |
| --- | --- | --- | --- | --- |
| 查看结果 catalog | yes | yes | yes | yes |
| 查看预览与详情 | yes | yes | yes | yes |
| 生产结果入库 | no | no | yes | yes |
| 目录重建/发布 | no | no | yes | yes |
| 成果交付导出 | no | yes | yes | yes |
| 生产任务提交 | no | no | yes | yes |
| 用户管理 | no | no | no | yes |
| 根目录/许可证/运维配置 | no | no | no | yes |
实现上可以先保留 `role` 字段，扩展角色枚举；长期可引入权限位表，避免角色继续膨胀。
## 6. 推荐实施顺序
### 阶段 1：修正产品语义和审计
- 页面文案区分“生产结果入库”和“成果交付导出”。
- `/api/idl/extract-disp` 增加操作审计。
- 结果提取工作台明确标注未接入通道。
- 修复相关页面乱码文案。
### 阶段 2：导出任务化
- 新增 `EXPORT_DINSAR_RESULTS` 后台任务类型。
- `/api/dinsar-results/export` 改为返回 `task_id`。
- 前端展示导出任务进度和失败明细。
- 导出结果保留 task log 和 audit log。
### 阶段 3：权限细分
- 扩展角色：`viewer/exporter/operator/admin`。
- 用户管理页支持新角色说明。
- 后端增加能力级依赖，例如 `require_capability("result.export")`。
- 所有高风险写操作按 capability 而不是只按 admin 判断。
### 阶段 4：交付目录白名单产品化
- 将 `ALLOWED_EXPORT_DIRS` 从环境变量能力升级为系统配置/受控根目录。
- 前端从白名单选择交付根目录。
- 用户只输入子目录名或交付批次名。
## 7. 验收标准
完成上述改造后，应满足：
1. viewer 能看结果，不能导出、不能入库。
2. exporter 能导出已登记成果，但不能提交生产、不能用户管理。
3. operator 能生产、入库、导出，但不能用户管理和系统配置。
4. admin 保留全部权限。
5. 所有入库和导出动作都有 task log 和 audit log。
6. 大批量导出不再产生 HTTP 504。
7. 未实现通道在 UI 上不会被误认为可执行功能。
docs/SOURCE_ORBIT_ASSET_SCAN_OPERATIONS_20260626.md
+9
View File
@@ -61,6 +61,15 @@ The binding step is currently a full re-evaluation of active LT-1/Sentinel-1 sou
If a scan must be stopped before retrying with new concurrency settings, stop the running worker processes and mark the active `system_jobs`, `system_tasks`, and `asset_inventory_states` rows as terminal failed states. This prevents the job queue from recovering and re-claiming the old job.
The main application worker is expected to support more than one queued job at a time so long-running preview generation does not block independent production jobs. Current main-server baseline:
```env
JOB_WORKER_CONCURRENCY=2
JOB_WORKER_ALLOWED_TYPES=
```
Do not start ad-hoc one-off workers during formal testing. Change the configured worker concurrency, stop stale processes, and let the operator restart the application normally.
Recommended health checks during a source scan:
- Task log should show `workers=16` and `pending=64` / `active_or_queued=64` after the parser pool fills.
frontend/index.html
+3 -3
View File
@@ -1,11 +1,11 @@
<!doctype html>
<html lang="en">
<html lang="zh-CN">
<head>
<meta charset="UTF-8" />
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://*.tile.openstreetmap.org https://*.tile.opentopomap.org; connect-src 'self'; font-src 'self' data:; worker-src blob:;" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<link rel="icon" type="image/png" href="/app-icon.png" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>frontend</title>
<title>雷达数据生产管理系统</title>
</head>
<body>
<div id="root"></div>
frontend/public/app-icon.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 93 KiB

frontend/src/App.css
+126
View File
@@ -2589,6 +2589,96 @@ input[type="checkbox"] {
text-align: left;
}
.task-runtime-summary {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 8px;
margin-bottom: 12px;
}
.task-runtime-summary > div {
border: 1px solid #e2e8f0;
border-radius: 8px;
background: #f8fafc;
padding: 8px;
min-width: 0;
}
.task-runtime-summary span {
display: block;
color: #64748b;
font-size: 11px;
margin-bottom: 3px;
}
.task-runtime-summary strong {
color: #0f172a;
font-size: 15px;
font-family: var(--font-mono);
}
.active-jobs-container {
display: flex;
flex-direction: column;
gap: 6px;
margin-bottom: 12px;
}
.job-runtime-row {
display: grid;
grid-template-columns: auto minmax(0, 1fr) minmax(92px, 0.8fr);
align-items: center;
gap: 8px;
padding: 7px 8px;
border: 1px solid #e2e8f0;
border-radius: 7px;
background: #ffffff;
font-size: 12px;
}
.job-status-chip {
display: inline-flex;
align-items: center;
justify-content: center;
min-width: 58px;
padding: 2px 6px;
border-radius: 999px;
border: 1px solid #cbd5e1;
color: #475569;
background: #f8fafc;
font-family: var(--font-mono);
font-size: 11px;
}
.job-status-chip.running {
border-color: rgba(37, 99, 235, 0.22);
color: #1d4ed8;
background: rgba(37, 99, 235, 0.08);
}
.job-status-chip.retry {
border-color: rgba(217, 119, 6, 0.24);
color: #b45309;
background: rgba(245, 158, 11, 0.1);
}
.job-runtime-title,
.job-runtime-worker {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.job-runtime-title {
color: #334155;
font-weight: 600;
}
.job-runtime-worker {
color: #64748b;
text-align: right;
}
.task-progress-item {
background: #f8fafc;
padding: 10px;
@@ -2596,6 +2686,10 @@ input[type="checkbox"] {
border: 1px solid #e2e8f0;
}
.task-progress-item--muted {
background: #ffffff;
}
.task-info-row {
display: flex;
justify-content: space-between;
@@ -3502,6 +3596,7 @@ input[type="checkbox"] {
color: var(--color-text-secondary);
padding: 4px 8px;
border-radius: 6px;
min-width: 132px;
}
.status-task.has-active-tasks {
@@ -3509,6 +3604,13 @@ input[type="checkbox"] {
color: #1890ff;
}
.status-task small {
color: var(--color-text-muted);
font-size: 0.86em;
line-height: 1.25;
white-space: nowrap;
}
.status-license {
font-size: 0.75em;
color: var(--color-text-muted);
@@ -5615,6 +5717,17 @@ input[type="checkbox"] {
box-sizing: border-box;
}
.production-workspace-shell .dinsar-production-shell,
.production-workspace-shell .dinsar-products-page {
max-width: none;
margin-left: 0;
margin-right: 0;
}
.production-workspace-shell .dinsar-production-shell {
padding: 0;
}
.dinsar-production-header {
display: grid;
grid-template-columns: minmax(280px, 1fr) minmax(420px, 0.95fr);
@@ -6485,6 +6598,18 @@ input[type="checkbox"] {
.dinsar-products-catalog-section {
display: grid;
gap: 10px;
width: 100%;
min-width: 0;
}
.dinsar-products-catalog-section .dinsar-catalog-shell {
width: 100%;
min-width: 0;
box-sizing: border-box;
}
.dinsar-products-catalog-section .dinsar-catalog-workspace {
grid-template-columns: minmax(360px, 420px) minmax(0, 1fr);
}
.sbas-products-page {
@@ -7170,6 +7295,7 @@ input[type="checkbox"] {
.dinsar-filter-layout,
.dinsar-catalog-summary,
.dinsar-catalog-workspace,
.dinsar-products-catalog-section .dinsar-catalog-workspace,
.dinsar-catalog-manage,
.dinsar-catalog-filter-bar,
.dinsar-catalog-hero,
frontend/src/App.jsx
+6 -1
View File
@@ -208,12 +208,14 @@ function App() {
setHealthError: state.setHealthError,
})));
const {
activeTasks, setActiveTasks,
activeTasks, setActiveTasks, runtimeSummary, setRuntimeSummary,
isCheckingTasks, setIsCheckingTasks,
pendingTaskIds, setPendingTaskIds,
} = useTaskStore(useShallow((state) => ({
activeTasks: state.activeTasks,
setActiveTasks: state.setActiveTasks,
runtimeSummary: state.runtimeSummary,
setRuntimeSummary: state.setRuntimeSummary,
isCheckingTasks: state.isCheckingTasks,
setIsCheckingTasks: state.setIsCheckingTasks,
pendingTaskIds: state.pendingTaskIds,
@@ -918,6 +920,7 @@ function App() {
licenseOk: !!licenseStatus?.ok,
activeTasks,
setActiveTasks,
setRuntimeSummary,
pendingTaskIds,
setPendingTaskIds,
setIsCheckingTasks,
@@ -2114,6 +2117,7 @@ function App() {
isReadOnlyUser={isReadOnlyUser}
activeTasks={activeTasks}
avgTaskProgress={avgTaskProgress}
runtimeSummary={runtimeSummary}
licenseStatus={licenseStatus}
healthStatus={healthStatus}
healthLoading={healthLoading}
@@ -2197,6 +2201,7 @@ function App() {
licenseFileName={licenseFileName}
licenseUploadStatus={licenseUploadStatus}
activeTasks={activeTasks}
runtimeSummary={runtimeSummary}
showCancelTask={showCancelTask}
cancelTaskPwd={cancelTaskPwd}
onShowCancelTask={() => setShowCancelTask(true)}
frontend/src/AssetInventoryPanel.jsx
+3 -1
View File
@@ -28,7 +28,7 @@ const fmtBytes = (value) => {
const StatusBadge = ({ value }) => {
const text = String(value || '-');
const status = text.toUpperCase();
const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED'
const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED' || text.startsWith('已生产')
? 'ok'
: status === 'WARNING' || status === 'OPEN' || status === 'MISSING'
? 'warn'
@@ -202,6 +202,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) {
<th>产品</th>
<th>轨道</th>
<th>状态</th>
<th>生产</th>
<th>完整性</th>
<th>动作</th>
<th>文件</th>
@@ -216,6 +217,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) {
<td>{item.source_format}<small>{item.imaging_mode} / {item.polarization}</small></td>
<td>{item.relative_orbit || '-'}<small>abs {item.absolute_orbit || '-'}</small></td>
<td><StatusBadge value={item.parse_status} /></td>
<td><StatusBadge value={(item.lt1_image_produced || item.lt1_landsar_produced) ? '已生产 GeoTIFF' : '未生产'} /></td>
<td title={item.archive_integrity_error || ''}>
<StatusBadge value={item.archive_integrity_status || 'NOT_CHECKED'} />
<small>{item.archive_integrity_member_count != null ? `${item.archive_integrity_member_count} files` : item.archive_integrity_method || '-'}</small>
frontend/src/DinsarProductionPanel.jsx
+192 -7
View File
@@ -63,6 +63,17 @@ const STATUS_LABEL = {
pending: '等待中',
};
const FAILURE_REASON_LABEL = {
'LandSAR access violation during coherence mask/phase unwrapping': 'LandSAR 访问冲突（相干性掩膜/相位解缠）',
'Insufficient tie/GCP points for DEM/geocoding': 'DEM / 地理编码控制点不足',
'DEM/sub-terrain processing failed': 'DEM / 去地形阶段失败',
'Insufficient GCPs for baseline/calibration': '基线精估计控制点不足',
'Coherence mask/phase unwrapping failed': '相干性掩膜 / 相位解缠失败',
'Processing timeout': '处理超时',
'Result catalog publish failed': '结果目录发布失败',
'Unclassified D-InSAR failure': '未分类失败',
};
const PYINT_DEM_MODE_LABEL = {
local_fabdem: '本地 FABDEM',
opentopo: 'OpenTopography',
@@ -267,6 +278,123 @@ function formatTaskRootUpdatedAt(value) {
}
}
function safeCount(value) {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : null;
}
function getRunCounts(run) {
const completed = safeCount(run?.completed_items);
const failed = safeCount(run?.failed_items);
const skipped = safeCount(run?.skipped_items);
const total = safeCount(run?.total_items);
return { completed, failed, skipped, total };
}
function hasRunCounts(run) {
const { completed, failed, skipped, total } = getRunCounts(run);
return [completed, failed, skipped, total].some(value => value != null);
}
function formatRunCounts(run) {
const { completed, failed, skipped, total } = getRunCounts(run);
const parts = [];
if (completed != null) parts.push(`成功 ${completed}`);
if (failed != null) parts.push(`失败 ${failed}`);
if (skipped != null && skipped > 0) parts.push(`跳过 ${skipped}`);
if (total != null) parts.push(`总数 ${total}`);
return parts.join(' / ');
}
function statusToneClass(status) {
const normalized = String(status || '').toLowerCase();
if (normalized === 'success' || normalized === 'completed') return 'tone-ready';
if (normalized === 'failed') return 'tone-error';
if (normalized === 'running') return 'tone-info';
if (normalized === 'cancelled' || normalized === 'canceled') return 'tone-neutral';
return 'tone-warn';
}
function formatFailureReason(reason) {
return FAILURE_REASON_LABEL[reason] || reason || '未分类失败';
}
function compactText(value, maxLength = 180) {
const text = String(value || '').replace(/\s+/g, ' ').trim();
if (!text) return '';
return text.length > maxLength ? `${text.slice(0, maxLength - 1).trim()}…` : text;
}
function classifyFailureReason(errorMessage) {
const text = String(errorMessage || '');
const lower = text.toLowerCase();
if (text.includes('3221225477') || lower.includes('status_access_violation')) {
return 'LandSAR access violation during coherence mask/phase unwrapping';
}
if (lower.includes('not enough gcps') || lower.includes('space insar calibration failed')) {
return 'Insufficient GCPs for baseline/calibration';
}
if (
lower.includes('no enough points')
|| lower.includes('not enough points')
|| lower.includes('geo_extract_gcp')
|| lower.includes('无满足snr')
|| text.includes('离散采样点')
) {
return 'Insufficient tie/GCP points for DEM/geocoding';
}
if (lower.includes('dem/sub-terrain') || lower.includes('subterrain') || lower.includes('sub-terrain')) {
return 'DEM/sub-terrain processing failed';
}
if (text.includes('相干性掩膜') && text.includes('相位解缠')) {
return 'Coherence mask/phase unwrapping failed';
}
if (lower.includes('timeout') || lower.includes('timed out') || text.includes('超时')) {
return 'Processing timeout';
}
if (lower.includes('publish')) {
return 'Result catalog publish failed';
}
return 'Unclassified D-InSAR failure';
}
function buildFailureSummaryFromItems(run) {
const items = Array.isArray(run?.items) ? run.items : [];
const failedItems = items.filter(item => String(item?.status || '').toUpperCase() === 'FAILED' || item?.last_error);
if (failedItems.length === 0) return null;
const groupsByReason = new Map();
const detailItems = failedItems.map(item => {
const reason = classifyFailureReason(item?.last_error);
const label = item?.task_alias || item?.task_name || item?.pair_key || '未命名任务';
const group = groupsByReason.get(reason) || { reason, count: 0, items: [] };
group.count += 1;
group.items.push(label);
groupsByReason.set(reason, group);
return {
task_alias: item?.task_alias,
task_name: item?.task_name,
reason,
error: compactText(item?.last_error, 260),
};
});
return {
failed_count: failedItems.length,
partial: true,
groups: Array.from(groupsByReason.values()),
items: detailItems,
};
}
function getRunFailureSummary(run) {
const summary = run?.summary_json?.failure_summary;
if (summary && Number(summary.failed_count || 0) > 0) {
return summary;
}
return buildFailureSummaryFromItems(run);
}
function RunPathBlock({ run }) {
const items = Array.isArray(run?.items) ? run.items : [];
const item = items.find(entry => entry?.status === 'RUNNING') || items[0] || null;
@@ -299,6 +427,67 @@ function RunPathBlock({ run }) {
);
}
function RunStatusBlock({ run }) {
return (
<div style={{ display: 'grid', gap: 4, minWidth: 120 }}>
<span className={`dinsar-status-pill ${statusToneClass(run?.status)}`}>
{formatStatus(run?.status)}
</span>
{hasRunCounts(run) && (
<span style={{ color: '#475569', fontSize: 11, lineHeight: 1.35 }}>
{formatRunCounts(run)}
</span>
)}
</div>
);
}
function RunSituationBlock({ run }) {
const failureSummary = getRunFailureSummary(run);
const countsText = hasRunCounts(run) ? formatRunCounts(run) : '';
const message = compactText(run?.message, 220);
if (!countsText && !failureSummary && !message) return null;
return (
<div
style={{
marginTop: 8,
padding: '8px 10px',
borderRadius: 6,
border: '1px solid #e2e8f0',
background: '#f8fafc',
color: '#334155',
lineHeight: 1.45,
}}
>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap', alignItems: 'center', marginBottom: failureSummary ? 6 : 0 }}>
<strong style={{ fontSize: 12, color: '#0f172a' }}>运行情况</strong>
{countsText && <span style={{ fontSize: 11, color: '#475569' }}>{countsText}</span>}
</div>
{failureSummary ? (
<div style={{ display: 'grid', gap: 5 }}>
{(failureSummary.groups || []).slice(0, 4).map((group, index) => (
<div key={`${group.reason || 'reason'}-${index}`} style={{ fontSize: 11, color: '#7f1d1d' }}>
<strong>{formatFailureReason(group.reason)}</strong>
<span>：{Number(group.count || 0)} 项</span>
{Array.isArray(group.items) && group.items.length > 0 && (
<span style={{ color: '#475569' }}>（{group.items.slice(0, 3).join('、')}{group.items.length > 3 ? ' 等' : ''}）</span>
)}
</div>
))}
{failureSummary.partial && Number(run?.failed_items || 0) > Number(failureSummary.failed_count || 0) && (
<div style={{ fontSize: 11, color: '#92400e' }}>
当前接口仅返回部分失败明细，请查看日志获取完整失败项。
</div>
)}
</div>
) : message ? (
<div style={{ marginTop: 6, fontSize: 11, color: '#475569' }}>{message}</div>
) : null}
</div>
);
}
function PreviewIssueList({ title, items, tone = 'warning' }) {
if (!Array.isArray(items) || items.length === 0) {
return null;
@@ -1957,19 +2146,15 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
<td style={{ padding: '6px 8px', fontFamily: 'monospace', fontSize: 11 }}>{run.run_id}</td>
<td style={{ padding: '6px 8px' }}>{formatEngineLabel(run.engine)}</td>
<td style={{ padding: '6px 8px' }}>{formatSatelliteFamilyLabel(inferSatelliteFamilyFromResultLike(run))}</td>
<td
style={{
padding: '6px 8px',
color: run.status === 'success' ? '#16a34a' : run.status === 'failed' ? '#ef4444' : '#64748b',
}}
>
{formatStatus(run.status)}
<td style={{ padding: '6px 8px', verticalAlign: 'top' }}>
<RunStatusBlock run={run} />
</td>
<td style={{ padding: '6px 8px', color: '#94a3b8', whiteSpace: 'nowrap' }}>
{run.started_at ? new Date(run.started_at * 1000).toLocaleString() : '-'}
</td>
<td style={{ padding: '6px 8px', maxWidth: 520, fontSize: 11 }}>
<RunPathBlock run={run} />
<RunSituationBlock run={run} />
</td>
<td style={{ padding: '6px 8px', whiteSpace: 'nowrap' }}>
<button
frontend/src/DinsarProductsPanel.jsx
+9 -2
View File
@@ -1,6 +1,5 @@
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { scanDinsarResults } from './api/dinsar';
import { listTaskRoots } from './api/dinsarProduction';
import { extractDispResults } from './api/idl';
import { clearTaskLogs, deleteTaskLog, getTaskLogs } from './api/tasks';
@@ -8,10 +7,16 @@ import DinsarCatalogPanel from './components/DinsarCatalogPanel';
import useTaskMonitor from './hooks/useTaskMonitor';
const PRODUCT_TASK_TYPES = [
'EXTRACT_DINSAR_PRODUCTS',
'SCAN_DINSAR',
'PUBLISH_DINSAR_PRODUCTS',
'REBUILD_DINSAR_CATALOG',
];
const TASK_TYPE_LABEL = {
EXTRACT_DINSAR_PRODUCTS: 'D-InSAR 结果提取与登记',
PUBLISH_DINSAR_PRODUCTS: 'D-InSAR 结果发布',
REBUILD_DINSAR_CATALOG: 'D-InSAR 结果目录重建',
SCAN_DINSAR: 'D-InSAR 结果扫描',
};
@@ -167,7 +172,7 @@ export default function DinsarProductsPanel({ readOnly = false, onJobQueued }) {
try {
const result = await extractDispResults(productionRoot.trim(), null);
setExtractResult(result);
const scanResult = await scanDinsarResults();
const scanResult = result;
setActionMessage(scanResult?.message || `D-InSAR 结果登记任务已提交：${scanResult?.task_id || '-'}`);
if (scanResult?.task_id) {
onJobQueued?.(scanResult.task_id);
@@ -262,6 +267,8 @@ export default function DinsarProductsPanel({ readOnly = false, onJobQueued }) {
<div className={`dinsar-products-result-card ${extractResult.error ? 'error' : 'success'}`}>
{extractResult.error ? (
<span>提取失败：{extractResult.error}</span>
) : extractResult.queued ? (
<span>D-InSAR 结果提取与登记任务已入队：{extractResult.task_id || '-'}</span>
) : (
<>
<div>提取完成：复制 {extractResult.copied || 0} 个文件，覆盖 {extractResult.overwritten || 0} 个文件。</div>
frontend/src/IDLAutomationPanel.jsx
+9 -13
View File
@@ -14,7 +14,6 @@ import {
getJobLog,
deleteRun,
} from './api/idl';
import { scanDinsarResults } from './api/dinsar';
import TaskStatusPanel from './components/tasks/TaskStatusPanel';
import useTaskMonitor from './hooks/useTaskMonitor';
@@ -38,7 +37,7 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) {
const [showCancelInput, setShowCancelInput] = useState(false);
const [cancelPassword, setCancelPassword] = useState('');
const idlTaskMonitor = useTaskMonitor({
taskTypes: ['IDL_IMPORT', 'IDL_DINSAR'],
taskTypes: ['IDL_RUN_IMPORT', 'IDL_RUN_DINSAR', 'EXTRACT_DINSAR_PRODUCTS'],
showRecent: true,
recentLimit: 1,
});
@@ -147,17 +146,8 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) {
runAction(async () => {
const r = await extractDispResults(root, dest);
setExtractResult(r);
// 提取完成后自动触发扫描
let scanMsg = '';
if (r.copied > 0 || r.overwritten > 0) {
try {
await scanDinsarResults({ results_directories: [r.target_dir] });
scanMsg = '，已自动触发结果扫描入库';
} catch (e) {
scanMsg = '，扫描入库触发失败: ' + (e?.response?.data?.detail || e?.message);
}
}
setMessage(`提取完成: ${r.copied} 新增, ${r.overwritten} 更新, ${r.skipped} 跳过${scanMsg}`);
setMessage(`D-InSAR 结果提取与登记任务已入队。task_id=${r?.task_id || '-'}`);
if (r?.task_id) onJobQueued?.(r.task_id);
});
};
@@ -533,6 +523,10 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) {
</div>
{extractResult && (
<div style={{ marginTop: '8px', padding: '10px', background: '#f0fdf4', borderRadius: '6px', border: '1px solid #bbf7d0', fontSize: '12px', color: '#166534' }}>
{extractResult.queued ? (
<div>D-InSAR 结果提取与登记任务已入队。task_id={extractResult.task_id || '-'}</div>
) : (
<>
<div>目标目录: <code style={{ fontSize: '11px' }}>{extractResult.target_dir}</code></div>
<div style={{ marginTop: '4px' }}>
处理 {extractResult.processed} 个 Task &nbsp;·&nbsp;
@@ -543,6 +537,8 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) {
<span style={{ color: '#dc2626' }}> &nbsp;·&nbsp; 失败 {extractResult.failed}</span>
)}
</div>
</>
)}
</div>
)}
</div>
frontend/src/LandsarLt1ProductionPanel.jsx
+816
View File
@@ -0,0 +1,816 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import {
getLandsarLt1Capabilities,
listLandsarLt1Products,
previewLandsarLt1Production,
submitLandsarLt1Production,
} from './api/landsarLt1Production';
import { searchRadarData } from './api/radar';
import { getRegionChildren } from './api/aoi';
const PAGE_SIZE_OPTIONS = [50, 100, 200, 500];
const DEFAULT_SEARCH = {
imaging_date_from: '',
imaging_date_to: '',
imaging_mode: '',
polarization: '',
product_level: '',
orbit_direction: '',
relative_orbit: '',
product_unique_id: '',
};
const shellStyle = { display: 'grid', gap: 12 };
const sectionStyle = {
background: '#ffffff',
border: '1px solid #d8dee8',
borderRadius: 8,
padding: 14,
};
const gridStyle = {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fit, minmax(220px, 1fr))',
gap: 12,
};
const labelStyle = {
display: 'grid',
gap: 5,
color: '#475569',
fontSize: 12,
fontWeight: 650,
};
const inputStyle = {
width: '100%',
boxSizing: 'border-box',
border: '1px solid #cbd5e1',
borderRadius: 6,
padding: '8px 9px',
color: '#0f172a',
background: '#ffffff',
fontSize: 13,
lineHeight: 1.35,
};
const mutedStyle = { color: '#64748b', fontSize: 12, lineHeight: 1.55 };
const buttonStyle = {
border: '1px solid #2563eb',
borderRadius: 6,
background: '#2563eb',
color: '#ffffff',
padding: '8px 12px',
fontSize: 13,
fontWeight: 700,
cursor: 'pointer',
};
const ghostButtonStyle = {
...buttonStyle,
border: '1px solid #cbd5e1',
background: '#ffffff',
color: '#334155',
};
const disabledButtonStyle = {
opacity: 0.5,
cursor: 'not-allowed',
};
const tableHeaderStyle = {
textAlign: 'left',
padding: 8,
borderBottom: '1px solid #e2e8f0',
};
const tableCellStyle = {
padding: 8,
borderBottom: '1px solid #e2e8f0',
};
function formatTime(value) {
if (!value) return '-';
const date = new Date(value);
if (Number.isNaN(date.getTime())) return value;
return date.toLocaleString();
}
function formatYmd(value) {
const text = String(value || '').trim();
const compact = text.match(/^(\d{4})(\d{2})(\d{2})$/);
if (compact) return `${compact[1]}-${compact[2]}-${compact[3]}`;
return text || '-';
}
function StatusPill({ ok, text }) {
return (
<span
style={{
display: 'inline-flex',
alignItems: 'center',
border: `1px solid ${ok ? '#86efac' : '#fecaca'}`,
borderRadius: 999,
padding: '3px 8px',
color: ok ? '#166534' : '#991b1b',
background: ok ? '#f0fdf4' : '#fef2f2',
fontSize: 12,
fontWeight: 700,
}}
>
{text}
</span>
);
}
function sceneProduced(scene) {
return Boolean(scene.lt1_image_produced || scene.lt1_landsar_produced);
}
function getSceneTitle(scene) {
return scene.product_unique_id || scene.unique_id || scene.source_product_token || `radar:${scene.id}`;
}
function getErrorMessage(error, fallback) {
const detail = error?.response?.data?.detail;
if (typeof detail === 'string') return detail;
if (detail) return JSON.stringify(detail);
return error?.message || fallback;
}
function buildRadarSearchFormData(criteria, page, regionTreeId) {
const formData = new FormData();
formData.append('limit', String(page.limit));
formData.append('offset', String(page.offset));
formData.append('satellite_family', 'LT1');
formData.append('source_format', 'LT1_ARCHIVE');
Object.entries(criteria || {}).forEach(([key, rawValue]) => {
const value = String(rawValue ?? '').trim();
if (value) formData.append(key, value);
});
if (regionTreeId) {
formData.append('region_tree_id', regionTreeId);
}
return formData;
}
export default function LandsarLt1ProductionPanel({ readOnly, onJobQueued }) {
const [capabilities, setCapabilities] = useState(null);
const [products, setProducts] = useState([]);
const [scenes, setScenes] = useState([]);
const [scenePage, setScenePage] = useState({
limit: 100,
offset: 0,
total: 0,
hasMore: false,
});
const [searchDraft, setSearchDraft] = useState(DEFAULT_SEARCH);
const [searchApplied, setSearchApplied] = useState(DEFAULT_SEARCH);
const [regionMode, setRegionMode] = useState('none');
const [regionSelection, setRegionSelection] = useState({ province: '', city: '' });
const [regionOptions, setRegionOptions] = useState({ provinces: [], cities: [] });
const [selectedRadarIds, setSelectedRadarIds] = useState(() => new Set());
const [preview, setPreview] = useState(null);
const [message, setMessage] = useState('');
const [actionLoading, setActionLoading] = useState(false);
const [searchLoading, setSearchLoading] = useState(false);
const [regionLoading, setRegionLoading] = useState(false);
const [form, setForm] = useState({ mode: 'scene', taskName: '' });
const selectedRegionTreeId = regionSelection.city || regionSelection.province || '';
const producedRadarIds = useMemo(
() => new Set(scenes.filter(sceneProduced).map(scene => Number(scene.id))),
[scenes],
);
const selectedRadarIdList = useMemo(
() => [...selectedRadarIds].filter(id => !producedRadarIds.has(Number(id))),
[selectedRadarIds, producedRadarIds],
);
const selectableCurrentScenes = useMemo(
() => scenes.filter(scene => !sceneProduced(scene) && scene.source_product_ref_id),
[scenes],
);
const allCurrentSelectableSelected = selectableCurrentScenes.length > 0
&& selectableCurrentScenes.every(scene => selectedRadarIds.has(scene.id));
const sceneStart = scenePage.total === 0 ? 0 : scenePage.offset + 1;
const sceneEnd = Math.min(scenePage.offset + scenes.length, scenePage.total || scenePage.offset + scenes.length);
const payload = useMemo(() => ({
radar_data_ids: selectedRadarIdList,
mode: form.mode,
task_name: form.taskName.trim() || undefined,
}), [form, selectedRadarIdList]);
const refreshProducts = useCallback(async () => {
const result = await listLandsarLt1Products({ limit: 20, offset: 0 });
setProducts(Array.isArray(result?.items) ? result.items : []);
}, []);
const refreshScenes = useCallback(async () => {
setSearchLoading(true);
try {
const result = await searchRadarData(
buildRadarSearchFormData(
searchApplied,
scenePage,
regionMode === 'region' ? selectedRegionTreeId : '',
),
);
const items = Array.isArray(result?.items) ? result.items : [];
const total = Number(result?.total ?? items.length);
const offset = Number(result?.offset ?? scenePage.offset);
const limit = Number(result?.limit ?? scenePage.limit);
setScenes(items);
setScenePage(current => ({
...current,
limit,
offset,
total,
hasMore: Boolean(result?.has_more ?? (offset + items.length < total)),
}));
} finally {
setSearchLoading(false);
}
}, [regionMode, scenePage.limit, scenePage.offset, searchApplied, selectedRegionTreeId]);
const refreshCapabilities = useCallback(async () => {
const result = await getLandsarLt1Capabilities();
setCapabilities(result);
}, []);
useEffect(() => {
refreshCapabilities().catch(error => setMessage(getErrorMessage(error, '读取 LT-1 生产能力失败')));
refreshProducts().catch(() => {});
}, [refreshCapabilities, refreshProducts]);
useEffect(() => {
refreshScenes().catch(error => setMessage(getErrorMessage(error, '检索 LT-1 影像失败')));
}, [refreshScenes]);
useEffect(() => {
if (!producedRadarIds.size) return;
setSelectedRadarIds(current => {
let changed = false;
const next = new Set();
current.forEach(id => {
if (producedRadarIds.has(Number(id))) changed = true;
else next.add(id);
});
return changed ? next : current;
});
}, [producedRadarIds]);
const loadProvinces = useCallback(async () => {
setRegionLoading(true);
setMessage('');
try {
const result = await getRegionChildren('1');
setRegionOptions({ provinces: Array.isArray(result?.children) ? result.children : [], cities: [] });
} catch (error) {
setMessage(getErrorMessage(error, '加载行政区失败'));
} finally {
setRegionLoading(false);
}
}, []);
const loadCities = useCallback(async provinceId => {
if (!provinceId) {
setRegionOptions(current => ({ ...current, cities: [] }));
return;
}
setRegionLoading(true);
setMessage('');
try {
const result = await getRegionChildren(provinceId);
setRegionOptions(current => ({ ...current, cities: Array.isArray(result?.children) ? result.children : [] }));
} catch (error) {
setMessage(getErrorMessage(error, '加载地市失败'));
} finally {
setRegionLoading(false);
}
}, []);
const updateField = (field, value) => {
setForm(current => ({ ...current, [field]: value }));
setPreview(null);
setMessage('');
};
const updateSearchDraft = (field, value) => {
setSearchDraft(current => ({ ...current, [field]: value }));
setPreview(null);
setMessage('');
};
const updateRegionMode = async value => {
setRegionMode(value);
setRegionSelection({ province: '', city: '' });
setPreview(null);
setMessage('');
if (value === 'region' && regionOptions.provinces.length === 0) {
await loadProvinces();
}
};
const updateProvince = async value => {
setRegionSelection({ province: value, city: '' });
setPreview(null);
if (value) await loadCities(value);
else setRegionOptions(current => ({ ...current, cities: [] }));
};
const updateCity = value => {
setRegionSelection(current => ({ ...current, city: value }));
setPreview(null);
};
const toggleScene = scene => {
if (sceneProduced(scene) || !scene.source_product_ref_id) return;
setPreview(null);
setMessage('');
setSelectedRadarIds(current => {
const next = new Set(current);
if (next.has(scene.id)) next.delete(scene.id);
else next.add(scene.id);
return next;
});
};
const toggleCurrentPageSelection = () => {
setPreview(null);
setMessage('');
setSelectedRadarIds(current => {
const next = new Set(current);
if (allCurrentSelectableSelected) {
selectableCurrentScenes.forEach(scene => next.delete(scene.id));
} else {
selectableCurrentScenes.forEach(scene => next.add(scene.id));
}
return next;
});
};
const updatePageSize = value => {
const limit = Number(value);
setScenePage(current => ({ ...current, limit, offset: 0 }));
};
const goToScenePage = offset => {
setScenePage(current => ({ ...current, offset: Math.max(0, offset) }));
};
const applySearch = () => {
if (regionMode === 'region' && !selectedRegionTreeId) {
setMessage('请选择行政区。');
return;
}
setSearchApplied(searchDraft);
setScenePage(current => ({ ...current, offset: 0 }));
setPreview(null);
setMessage('');
};
const resetSearch = () => {
setSearchDraft(DEFAULT_SEARCH);
setSearchApplied(DEFAULT_SEARCH);
setRegionMode('none');
setRegionSelection({ province: '', city: '' });
setScenePage(current => ({ ...current, offset: 0 }));
setPreview(null);
setMessage('');
};
const handleRefresh = async () => {
setMessage('');
try {
await Promise.all([refreshProducts(), refreshScenes()]);
} catch (error) {
setMessage(getErrorMessage(error, '刷新失败'));
}
};
const handlePreview = async () => {
setActionLoading(true);
setMessage('');
try {
const result = await previewLandsarLt1Production(payload);
setPreview(result);
setMessage(result.allow_submit ? '预览通过' : '预览未通过');
} catch (error) {
setPreview(null);
setMessage(getErrorMessage(error, '预览失败'));
} finally {
setActionLoading(false);
}
};
const handleSubmit = async () => {
setActionLoading(true);
setMessage('');
try {
const result = await submitLandsarLt1Production(payload);
const queued = Array.isArray(result.queued) ? result.queued : [];
setMessage(`已提交 ${queued.length || 1} 个地理编码 GeoTIFF 生产任务`);
if (result.task_id) onJobQueued?.(result.task_id);
else queued.forEach(item => item.task_id && onJobQueued?.(item.task_id));
setPreview(null);
setSelectedRadarIds(new Set());
await Promise.all([refreshProducts(), refreshScenes()]);
} catch (error) {
setMessage(getErrorMessage(error, '提交失败'));
} finally {
setActionLoading(false);
}
};
const busy = actionLoading || searchLoading || regionLoading;
const canSubmitSelection = !readOnly && selectedRadarIdList.length > 0 && !actionLoading;
return (
<div style={shellStyle}>
<section style={sectionStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}>
<div>
<h3 style={{ margin: 0, fontSize: 16 }}>LT-1 地理编码影像生产</h3>
<div style={{ ...mutedStyle, marginTop: 5 }}>
复用影像检索筛选 LT-1 场景，再提交 Gamma 单景流水线生成 analysis_ready.tif。
</div>
</div>
<StatusPill
ok={capabilities?.engine === 'lt_gamma'}
text={capabilities?.engine === 'lt_gamma' ? 'lt_gamma 已配置' : '未配置'}
/>
</div>
{capabilities?.message && <div style={{ ...mutedStyle, marginTop: 8 }}>{capabilities.message}</div>}
</section>
<section style={sectionStyle}>
<div style={gridStyle}>
<label style={labelStyle}>
生产模式
<select
style={inputStyle}
value={form.mode}
onChange={event => updateField('mode', event.target.value)}
disabled={actionLoading || readOnly}
>
<option value="scene">单景</option>
<option value="batch">批量单景</option>
</select>
</label>
<label style={labelStyle}>
任务名
<input
style={inputStyle}
value={form.taskName}
onChange={event => updateField('taskName', event.target.value)}
disabled={actionLoading || readOnly}
placeholder="可选"
/>
</label>
</div>
<div style={{ display: 'flex', gap: 8, marginTop: 14, flexWrap: 'wrap' }}>
<button
type="button"
style={{ ...ghostButtonStyle, ...((!canSubmitSelection || actionLoading) ? disabledButtonStyle : {}) }}
onClick={handlePreview}
disabled={!canSubmitSelection}
>
预览
</button>
<button
type="button"
style={{ ...buttonStyle, ...((!canSubmitSelection || preview?.allow_submit === false) ? disabledButtonStyle : {}) }}
onClick={handleSubmit}
disabled={!canSubmitSelection || preview?.allow_submit === false}
>
提交生产
</button>
<button
type="button"
style={{ ...ghostButtonStyle, ...(busy ? disabledButtonStyle : {}) }}
onClick={handleRefresh}
disabled={busy}
>
刷新
</button>
</div>
{message && <div style={{ ...mutedStyle, marginTop: 10 }}>{message}</div>}
</section>
<section style={sectionStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}>
<div>
<h3 style={{ margin: 0, fontSize: 16 }}>生产候选检索</h3>
<div style={{ ...mutedStyle, marginTop: 4 }}>
调用影像检索能力，按时间、行政区、轨道、极化等条件规划要生产的 LT-1 场景。
</div>
</div>
<div style={mutedStyle}>已选 {selectedRadarIdList.length} 景</div>
</div>
<div style={{ ...gridStyle, marginTop: 12 }}>
<label style={labelStyle}>
成像时间起
<input
type="date"
style={inputStyle}
value={searchDraft.imaging_date_from}
onChange={event => updateSearchDraft('imaging_date_from', event.target.value)}
/>
</label>
<label style={labelStyle}>
成像时间止
<input
type="date"
style={inputStyle}
value={searchDraft.imaging_date_to}
onChange={event => updateSearchDraft('imaging_date_to', event.target.value)}
/>
</label>
<label style={labelStyle}>
成像模式
<input
style={inputStyle}
value={searchDraft.imaging_mode}
onChange={event => updateSearchDraft('imaging_mode', event.target.value)}
placeholder="如 MONO / KSC"
/>
</label>
<label style={labelStyle}>
极化
<input
style={inputStyle}
value={searchDraft.polarization}
onChange={event => updateSearchDraft('polarization', event.target.value)}
placeholder="如 HH"
/>
</label>
<label style={labelStyle}>
相对轨道
<input
style={inputStyle}
value={searchDraft.relative_orbit}
onChange={event => updateSearchDraft('relative_orbit', event.target.value)}
placeholder="可选"
/>
</label>
<label style={labelStyle}>
产品名
<input
style={inputStyle}
value={searchDraft.product_unique_id}
onChange={event => updateSearchDraft('product_unique_id', event.target.value)}
placeholder="模糊匹配"
/>
</label>
</div>
<div style={{ ...gridStyle, marginTop: 12 }}>
<label style={labelStyle}>
空间范围
<select
style={inputStyle}
value={regionMode}
onChange={event => updateRegionMode(event.target.value)}
disabled={regionLoading}
>
<option value="none">不限</option>
<option value="region">行政区</option>
</select>
</label>
{regionMode === 'region' && (
<>
<label style={labelStyle}>
省份
<select
style={inputStyle}
value={regionSelection.province}
onChange={event => updateProvince(event.target.value)}
disabled={regionLoading}
>
<option value="">选择省份</option>
{regionOptions.provinces.map(item => (
<option key={item.tree_id} value={item.tree_id}>{item.name}</option>
))}
</select>
</label>
<label style={labelStyle}>
地市
<select
style={inputStyle}
value={regionSelection.city}
onChange={event => updateCity(event.target.value)}
disabled={regionLoading || !regionSelection.province}
>
<option value="">不限地市</option>
{regionOptions.cities.map(item => (
<option key={item.tree_id} value={item.tree_id}>{item.name}</option>
))}
</select>
</label>
</>
)}
<label style={labelStyle}>
每页数量
<select
style={inputStyle}
value={scenePage.limit}
onChange={event => updatePageSize(event.target.value)}
disabled={searchLoading}
>
{PAGE_SIZE_OPTIONS.map(size => (
<option key={size} value={size}>{size}</option>
))}
</select>
</label>
</div>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 10, alignItems: 'center', marginTop: 12, flexWrap: 'wrap' }}>
<div style={mutedStyle}>
{searchLoading ? '正在检索影像...' : `第 ${sceneStart}-${sceneEnd} 景 / 共 ${scenePage.total} 景`}
</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
<button
type="button"
style={{ ...buttonStyle, ...(busy ? disabledButtonStyle : {}) }}
onClick={applySearch}
disabled={busy}
>
检索
</button>
<button
type="button"
style={{ ...ghostButtonStyle, ...(busy ? disabledButtonStyle : {}) }}
onClick={resetSearch}
disabled={busy}
>
重置
</button>
<button
type="button"
style={{ ...ghostButtonStyle, ...((readOnly || searchLoading || selectableCurrentScenes.length === 0) ? disabledButtonStyle : {}) }}
onClick={toggleCurrentPageSelection}
disabled={readOnly || searchLoading || selectableCurrentScenes.length === 0}
>
{allCurrentSelectableSelected ? '取消本页选择' : '选择本页可生产'}
</button>
<button
type="button"
style={{ ...ghostButtonStyle, ...((searchLoading || scenePage.offset <= 0) ? disabledButtonStyle : {}) }}
onClick={() => goToScenePage(scenePage.offset - scenePage.limit)}
disabled={searchLoading || scenePage.offset <= 0}
>
上一页
</button>
<button
type="button"
style={{ ...ghostButtonStyle, ...((searchLoading || !scenePage.hasMore) ? disabledButtonStyle : {}) }}
onClick={() => goToScenePage(scenePage.offset + scenePage.limit)}
disabled={searchLoading || !scenePage.hasMore}
>
下一页
</button>
</div>
</div>
<div style={{ overflowX: 'auto', marginTop: 10 }}>
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 12 }}>
<thead>
<tr style={{ color: '#475569', background: '#f8fafc' }}>
<th style={tableHeaderStyle}>选择</th>
<th style={tableHeaderStyle}>产品</th>
<th style={tableHeaderStyle}>日期</th>
<th style={tableHeaderStyle}>模式</th>
<th style={tableHeaderStyle}>轨道</th>
<th style={tableHeaderStyle}>极化</th>
<th style={tableHeaderStyle}>状态</th>
<th style={tableHeaderStyle}>路径</th>
</tr>
</thead>
<tbody>
{scenes.map(scene => {
const produced = sceneProduced(scene);
const selectable = !produced && Boolean(scene.source_product_ref_id);
const selected = selectedRadarIds.has(scene.id);
return (
<tr key={scene.id} style={{ background: selected ? '#eff6ff' : '#ffffff', opacity: produced ? 0.62 : 1 }}>
<td style={tableCellStyle}>
<input
type="checkbox"
checked={selected}
disabled={readOnly || actionLoading || !selectable}
onChange={() => toggleScene(scene)}
/>
</td>
<td style={{ ...tableCellStyle, color: '#0f172a', fontWeight: 650 }}>
{getSceneTitle(scene)}
</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{formatYmd(scene.imaging_date)}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{scene.imaging_mode || '-'}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{scene.relative_orbit || scene.orbit_circle || '-'}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{scene.polarization || '-'}</td>
<td style={{ ...tableCellStyle, color: produced ? '#166534' : '#475569', fontWeight: produced ? 700 : 500 }}>
{produced ? '已生产 GeoTIFF' : (scene.source_product_ref_id ? '可生产' : '未关联源资产')}
</td>
<td
title={scene.file_path}
style={{
...tableCellStyle,
color: '#475569',
maxWidth: 360,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
}}
>
{scene.file_path}
</td>
</tr>
);
})}
{scenes.length === 0 && (
<tr>
<td colSpan="8" style={{ ...tableCellStyle, color: '#64748b' }}>
暂无符合条件的 LT-1 影像
</td>
</tr>
)}
</tbody>
</table>
</div>
</section>
{preview && (
<section style={sectionStyle}>
<h3 style={{ margin: 0, fontSize: 16 }}>预览结果</h3>
<div style={{ ...gridStyle, marginTop: 10 }}>
<div style={mutedStyle}>场景数: {preview.scene_count}</div>
<div style={mutedStyle}>engine: {preview.engine}</div>
<div style={mutedStyle}>profile: {preview.profile_code}</div>
</div>
{Array.isArray(preview.blockers) && preview.blockers.length > 0 && (
<div style={{ marginTop: 10, display: 'grid', gap: 6 }}>
{preview.blockers.map(item => (
<div key={item} style={{ color: '#991b1b', fontSize: 12 }}>{item}</div>
))}
</div>
)}
{Array.isArray(preview.warnings) && preview.warnings.length > 0 && (
<div style={{ marginTop: 10, display: 'grid', gap: 6 }}>
{preview.warnings.map(item => (
<div key={item} style={{ color: '#92400e', fontSize: 12 }}>{item}</div>
))}
</div>
)}
</section>
)}
<section style={sectionStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'center' }}>
<h3 style={{ margin: 0, fontSize: 16 }}>最近 LT-1 GeoTIFF 产品</h3>
<div style={mutedStyle}>{products.length} 项</div>
</div>
<div style={{ overflowX: 'auto', marginTop: 10 }}>
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 12 }}>
<thead>
<tr style={{ color: '#475569', background: '#f8fafc' }}>
<th style={tableHeaderStyle}>产品</th>
<th style={tableHeaderStyle}>状态</th>
<th style={tableHeaderStyle}>日期</th>
<th style={tableHeaderStyle}>单位</th>
<th style={tableHeaderStyle}>时间</th>
<th style={tableHeaderStyle}>GeoTIFF</th>
</tr>
</thead>
<tbody>
{products.map(product => (
<tr key={product.id}>
<td style={{ ...tableCellStyle, color: '#0f172a', fontWeight: 650 }}>
{product.display_name || product.product_id}
</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{product.status}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{formatYmd(product.summary?.imaging_date)}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{product.summary?.backscatter_unit || '-'}</td>
<td style={{ ...tableCellStyle, color: '#475569' }}>{formatTime(product.published_at)}</td>
<td
title={product.primary_asset_path}
style={{
...tableCellStyle,
color: '#475569',
maxWidth: 360,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
}}
>
{product.primary_asset_path || product.publish_dir || '-'}
</td>
</tr>
))}
{products.length === 0 && (
<tr>
<td colSpan="6" style={{ ...tableCellStyle, color: '#64748b' }}>
暂无产品
</td>
</tr>
)}
</tbody>
</table>
</div>
</section>
</div>
);
}
frontend/src/ProductionWorkspace.jsx
+23 -18
View File
@@ -11,6 +11,7 @@ const LazyDinsarProductionPanel = lazy(() => import('./DinsarProductionPanel'));
const LazySbasInsarProductionPanel = lazy(() => import('./SbasInsarProductionPanel'));
const LazySbasInsarProductsPanel = lazy(() => import('./SbasInsarProductsPanel'));
const LazyDinsarProductsPanel = lazy(() => import('./DinsarProductsPanel'));
const LazyLandsarLt1ProductionPanel = lazy(() => import('./LandsarLt1ProductionPanel'));
const LazyPairPlanningPanel = lazy(() => import('./panels/PairPlanningPanel'));
const LazyPairsListPanel = lazy(() => import('./panels/PairsListPanel'));
const LazyBatchPanel = lazy(() => import('./panels/BatchPanel'));
@@ -25,17 +26,6 @@ const WORKFLOW_STEPS = [
];
const SENSOR_PRODUCTION_PLACEHOLDERS = {
lt1_production: {
title: '陆探一生产占位',
note: '当前保留 LT-1 源压缩包本机登记与按需 materialize 入口。',
rows: [
['数据来源', '本机源压缩包 archive'],
['精轨策略', '按生产任务关联 orbit 资产'],
['准备方式', '按需 materialize 到 Task_Pool'],
['生产边界', 'D-InSAR/SBAS 不走 UNC'],
['结果管理', '进入统一产品 catalog'],
],
},
sentinel1_production: {
title: 'Sentinel-1 生产占位',
note: '当前主要沉淀数据与精轨管理约束，SBAS 仅保留规划能力。',
@@ -206,6 +196,13 @@ export default function ProductionWorkspace({
});
};
const handleLt1ImageQueued = taskId => {
onTaskStart?.(taskId, 'LT-1 地理编码 GeoTIFF 生产任务已入队。', {
taskType: 'SAR_SCENE_PREPROCESS',
nonBlocking: true,
});
};
const renderContent = () => {
if (activeView === 'dinsar_pairing') {
return (
@@ -215,11 +212,11 @@ export default function ProductionWorkspace({
isLoading={isLoading}
isReadOnlyUser={readOnly}
hasEnoughRadarScenesForPlanning={hasEnoughRadarScenesForPlanning}
onOpenPairingModal={pairingPanel?.openModal}
onOpenPairingModal={pairingPanel?.onOpenPairingModal}
hasRadarSearched={hasRadarSearched}
onRefreshRadarSearch={radarPanel?.refresh}
onSearchAll={radarPanel?.searchAll}
onRefreshDinsar={pairsPanel?.refreshDinsar}
onRefreshRadarSearch={pairingPanel?.onRefreshRadarSearch}
onSearchAll={radarPanel?.onSearchAll}
onRefreshDinsar={pairingPanel?.onRefreshDinsar}
language={language}
/>
);
@@ -228,8 +225,12 @@ export default function ProductionWorkspace({
if (activeView === 'dinsar_pairs') {
return (
<div style={{ display: 'grid', gridTemplateColumns: 'minmax(0, 1.1fr) minmax(360px, 0.9fr)', gap: 14 }}>
<LazyPairsListPanel pairsPanel={pairsPanel} isReadOnlyUser={readOnly} language={language} />
<LazyBatchPanel pairsPanel={pairsPanel} isReadOnlyUser={readOnly} language={language} />
<LazyPairsListPanel
onVisualizePair={pairsPanel?.onVisualizePair}
onTogglePairVisibility={pairsPanel?.onTogglePairVisibility}
onCreateDinsarBatch={pairsPanel?.onCreateDinsarBatch}
/>
<LazyBatchPanel />
</div>
);
}
@@ -272,11 +273,15 @@ export default function ProductionWorkspace({
return <LazySbasInsarProductsPanel readOnly={readOnly} onJobQueued={handleSbasProductQueued} />;
}
if (activeView === 'lt1_production') {
return <LazyLandsarLt1ProductionPanel readOnly={readOnly} onJobQueued={handleLt1ImageQueued} />;
}
return <PlaceholderView config={SENSOR_PRODUCTION_PLACEHOLDERS[activeView]} />;
};
return (
<div style={shellStyle}>
<div className="production-workspace-shell" style={shellStyle}>
<div style={headerStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 16, alignItems: 'flex-start', flexWrap: 'wrap' }}>
<div>
frontend/src/api/index.js
+1
View File
@@ -17,3 +17,4 @@ export * as unpackApi from './unpack';
export * as statsApi from './stats';
export * as timeseriesProductionApi from './timeseriesProduction';
export * as psinsarProductsApi from './psinsarProducts';
export * as landsarLt1ProductionApi from './landsarLt1Production';
frontend/src/api/landsarLt1Production.js
+22
View File
@@ -0,0 +1,22 @@
import apiClient from './client';
export const getLandsarLt1Capabilities = () =>
apiClient.get('/landsar-lt1-production/capabilities').then(r => r.data);
export const previewLandsarLt1Production = payload =>
apiClient.post('/landsar-lt1-production/preview', payload).then(r => r.data);
export const submitLandsarLt1Production = payload =>
apiClient.post('/landsar-lt1-production/run', payload).then(r => r.data);
export const previewLandsarLt1Import = previewLandsarLt1Production;
export const submitLandsarLt1Import = submitLandsarLt1Production;
export const listLandsarLt1Products = (params = {}) =>
apiClient.get('/landsar-lt1-production/products', { params }).then(r => r.data);
export const getLandsarLt1Product = productId =>
apiClient.get(`/landsar-lt1-production/products/${encodeURIComponent(productId)}`).then(r => r.data);
export const getLandsarLt1AssetUrl = (productId, assetId) =>
`/api/landsar-lt1-production/products/${encodeURIComponent(productId)}/assets/${encodeURIComponent(assetId)}`;
frontend/src/api/tasks.js
+1
View File
@@ -1,6 +1,7 @@
import apiClient from './client';
export const getActiveTasks = () => apiClient.get('/tasks/active').then(r => r.data);
export const getTaskRuntimeSummary = () => apiClient.get('/tasks/runtime-summary').then(r => r.data);
export const getRecentTasks = (taskTypes = [], statuses = [], limit = 20, offset = 0) =>
apiClient.get('/tasks/recent', {
params: {
frontend/src/components/GlobalTaskCenter.jsx
+57 -5
View File
@@ -4,6 +4,7 @@ import { getTaskTypeLabel } from '../config/taskUiPolicies';
export default function GlobalTaskCenter({
isVisible,
activeTasks,
runtimeSummary,
t,
isAdmin,
showCancelTask,
@@ -14,14 +15,27 @@ export default function GlobalTaskCenter({
onCloseCancelTask,
}) {
const [expanded, setExpanded] = useState(false);
if (!isVisible || activeTasks.length === 0) {
const jobs = runtimeSummary?.jobs || {};
const worker = runtimeSummary?.worker || {};
const scan = runtimeSummary?.scan || {};
const activeJobs = Array.isArray(jobs.items) ? jobs.items : [];
const activeCount = Math.max(
activeTasks.length,
Number(jobs.active_count) || 0,
);
const workerCount = Number(worker.worker_count) || 0;
const queuedJobs = Number(jobs.queued_count) || 0;
const runningJobs = Number(jobs.running_count) || 0;
const scanJobCount = Number(scan.active_job_count) || 0;
if (!isVisible || activeCount === 0) {
return null;
}
const activeCount = activeTasks.length;
const avgProgress = Math.round(
activeTasks.reduce((sum, task) => sum + (Number(task.progress) || 0), 0) / Math.max(1, activeCount)
);
const visibleJobs = activeJobs.slice(0, 8);
return (
<div className="global-task-overlay">
@@ -29,7 +43,7 @@ export default function GlobalTaskCenter({
<button className="task-center-button" onClick={() => setExpanded(true)}>
<span className="task-center-dot" />
<span>后台任务 {activeCount}</span>
<strong>{avgProgress}%</strong>
<strong>{runningJobs > 0 ? `执行 ${runningJobs}` : `${avgProgress}%`}</strong>
</button>
)}
{expanded && (
@@ -37,12 +51,45 @@ export default function GlobalTaskCenter({
<div className="task-center-header">
<div>
<h3>后台任务</h3>
<p>任务正在执行，你可以继续使用其他功能；同类重复提交由系统限制。</p>
<p>展示 Worker、执行中 Job、排队 Job 和任务进度；同类重复提交由后端冲突检查处理。</p>
</div>
<button className="task-center-close" onClick={() => setExpanded(false)} aria-label="关闭任务中心">
×
</button>
</div>
<div className="task-runtime-summary">
<div>
<span>Worker</span>
<strong>{workerCount}</strong>
</div>
<div>
<span>执行中</span>
<strong>{runningJobs}</strong>
</div>
<div>
<span>排队</span>
<strong>{queuedJobs}</strong>
</div>
<div>
<span>扫描</span>
<strong>{scanJobCount}</strong>
</div>
</div>
{visibleJobs.length > 0 && (
<div className="active-jobs-container">
{visibleJobs.map((job) => (
<div key={job.job_id} className="job-runtime-row">
<span className={`job-status-chip ${String(job.status || '').toLowerCase()}`}>{job.status || '-'}</span>
<span className="job-runtime-title">
{getTaskTypeLabel(job.task_type || job.job_type)}
</span>
<span className="job-runtime-worker" title={job.locked_by || ''}>
{job.locked_by ? `Worker ${job.locked_by}` : (job.status === 'RETRY' ? '等待重试' : '等待领取')}
</span>
</div>
))}
</div>
)}
<div className="active-tasks-container">
{(() => {
const waterTasks = activeTasks.filter(task =>
@@ -73,11 +120,16 @@ export default function GlobalTaskCenter({
</div>
</div>
)}
{otherTasks.length === 0 && waterTasks.length === 0 && visibleJobs.length > 0 && (
<div className="task-progress-item task-progress-item--muted">
<p className="task-status-msg">当前只有 Job 运行态，任务进度尚未写入 system_tasks。</p>
</div>
)}
</>
);
})()}
</div>
<p className="overlay-footer-hint">任务中心只展示状态，不再锁定整个界面。需要互斥的操作由功能页按钮和后端任务冲突检查处理。</p>
<p className="overlay-footer-hint">取消按钮只作用于可跟踪的 Task；纯 Job 取消需要在对应功能页或运维接口处理。</p>
{isAdmin && (
<div style={{ marginTop: '16px', textAlign: 'center' }}>
{!showCancelTask ? (
frontend/src/components/app/AppOverlays.jsx
+4 -2
View File
@@ -26,6 +26,7 @@ export default function AppOverlays({
licenseFileName,
licenseUploadStatus,
activeTasks,
runtimeSummary,
showCancelTask,
cancelTaskPwd,
onShowCancelTask,
@@ -97,11 +98,12 @@ export default function AppOverlays({
</Suspense>
)}
{activeTasks.length > 0 && (
{(activeTasks.length > 0 || Number(runtimeSummary?.jobs?.active_count || 0) > 0) && (
<Suspense fallback={<ModalLoadingFallback message="正在加载任务中心..." />}>
<LazyGlobalTaskCenter
isVisible={activeTasks.length > 0}
isVisible={activeTasks.length > 0 || Number(runtimeSummary?.jobs?.active_count || 0) > 0}
activeTasks={activeTasks}
runtimeSummary={runtimeSummary}
t={t}
isAdmin={isAdmin}
showCancelTask={showCancelTask}
frontend/src/components/app/AppStatusHeader.jsx
+36 -6
View File
@@ -3,7 +3,7 @@ import defaultLogoUrl from '../../logo.jpg';
import { formatUtc } from '../../utils/appUiHelpers';
const ORGANIZATION_NAME = import.meta.env.VITE_APP_ORG_NAME || '黑龙江省自然资源卫星应用技术中心';
const SYSTEM_NAME = import.meta.env.VITE_APP_SYSTEM_NAME || 'InSAR 自动化管理系统';
const SYSTEM_NAME = import.meta.env.VITE_APP_SYSTEM_NAME || '雷达数据生产管理系统';
const SYSTEM_TAGLINE = import.meta.env.VITE_APP_SYSTEM_TAGLINE || '科研工程生产平台';
const LOGO_URL = import.meta.env.VITE_APP_LOGO_URL || defaultLogoUrl;
@@ -14,11 +14,40 @@ function AppStatusHeader({
isReadOnlyUser,
activeTasks,
avgTaskProgress,
runtimeSummary,
licenseStatus,
onLogout,
}) {
const licenseOk = !!licenseStatus?.ok;
const hasActiveTasks = activeTasks.length > 0;
const worker = runtimeSummary?.worker || {};
const jobs = runtimeSummary?.jobs || {};
const scan = runtimeSummary?.scan || {};
const workerCount = Number(worker.worker_count) || 0;
const runningJobs = Number(jobs.running_count) || 0;
const queuedJobs = Number(jobs.queued_count) || 0;
const scanJobs = Number(scan.active_job_count) || 0;
const scanRunningJobs = Number(scan.running_job_count) || 0;
const staleJobs = Number(worker.stale_running_job_count) || 0;
const hasRuntimeActivity = activeTasks.length > 0 || runningJobs > 0 || queuedJobs > 0;
const taskProgress = hasRuntimeActivity ? avgTaskProgress : 0;
let runtimeLabel = 'Worker 未连接';
if (!runtimeSummary && activeTasks.length > 0) {
runtimeLabel = `运行中 ${activeTasks.length}`;
} else if (runningJobs > 0 && staleJobs > 0) {
runtimeLabel = `运行态待恢复 ${staleJobs}`;
} else if (runningJobs > 0) {
runtimeLabel = `执行中 ${runningJobs}`;
} else if (queuedJobs > 0) {
runtimeLabel = `排队 ${queuedJobs}`;
} else if (workerCount > 0) {
runtimeLabel = `Worker ${workerCount} 空闲`;
}
const runtimeDetail = !runtimeSummary && activeTasks.length > 0
? '任务状态来自兼容接口'
: workerCount > 0
? `Worker ${workerCount}`
: '无在线 worker';
const staleDetail = staleJobs > 0 ? ` · 待恢复 ${staleJobs}` : '';
return (
<>
@@ -50,11 +79,12 @@ function AppStatusHeader({
</div>
<div className="status-actions">
<div className={`status-task ${hasActiveTasks ? 'has-active-tasks' : ''}`}>
<span>{hasActiveTasks ? `运行中 ${activeTasks.length}` : '任务空闲'}</span>
{hasActiveTasks && (
<div className={`status-task ${hasRuntimeActivity ? 'has-active-tasks' : ''}`}>
<span>{runtimeLabel}</span>
<small>{runtimeDetail}{staleDetail}{scanJobs > 0 ? ` · 扫描 ${scanRunningJobs}/${scanJobs}` : ''}</small>
{hasRuntimeActivity && (
<div className="status-task-bar" aria-hidden="true">
<div className="status-task-fill" style={{ width: `${avgTaskProgress}%` }} />
<div className="status-task-fill" style={{ width: `${taskProgress}%` }} />
</div>
)}
</div>
frontend/src/components/panels/RadarDataRow.jsx
+6
View File
@@ -12,10 +12,16 @@ function RadarDataRow({
onRebuildPreview,
onToggleLayer,
}) {
const isProduced = Boolean(item.lt1_image_produced || item.lt1_landsar_produced);
return (
<li className="data-item radar-data-item" onClick={() => onFlyTo(item)}>
<span className="data-item-name" title={item.displayName}>
{item.displayName}
{isProduced && (
<small style={{ marginLeft: 8, color: '#166534', fontWeight: 700 }}>
已生产 GeoTIFF
</small>
)}
</span>
<div className="data-item-controls">
<span
frontend/src/config/appConstants.js
+16 -5
View File
@@ -123,6 +123,14 @@ export const PRODUCTION_WORKSPACE_SBAS_VIEWS = [
},
];
export const PRODUCTION_WORKSPACE_LT1_VIEWS = [
{
key: 'lt1_production',
label: '陆探一 GeoTIFF 生产',
description: '运行 LT-1 Gamma 单景流水线，输出多视、地理编码后的 analysis_ready.tif。',
},
];
export const PRODUCTION_WORKSPACE_WORKBENCHES = [
{
key: 'dinsar_workbench',
@@ -138,16 +146,19 @@ export const PRODUCTION_WORKSPACE_WORKBENCHES = [
defaultView: 'sbas_insar_planning',
views: PRODUCTION_WORKSPACE_SBAS_VIEWS,
},
{
key: 'lt1_workbench',
label: '陆探一工作台',
description: '面向 LT-1 非 D-InSAR 影像生产，组织源资产选择、单景地理编码和 GeoTIFF 产品登记。',
defaultView: 'lt1_production',
views: PRODUCTION_WORKSPACE_LT1_VIEWS,
},
];
export const PRODUCTION_WORKSPACE_VIEWS = [
...PRODUCTION_WORKSPACE_DINSAR_VIEWS,
...PRODUCTION_WORKSPACE_SBAS_VIEWS,
{
key: 'lt1_production',
label: '陆探一生产占位',
description: 'LT-1 源压缩包本机登记，按需 materialize 到 Task_Pool；D-InSAR/SBAS 生产不走 UNC。',
},
...PRODUCTION_WORKSPACE_LT1_VIEWS,
{
key: 'sentinel1_production',
label: 'Sentinel-1 生产占位',
frontend/src/hooks/useGlobalTaskControl.js
+36 -6
View File
@@ -9,6 +9,7 @@ export default function useGlobalTaskControl({
licenseOk,
activeTasks,
setActiveTasks,
setRuntimeSummary,
pendingTaskIds,
setPendingTaskIds,
setIsCheckingTasks,
@@ -21,8 +22,11 @@ export default function useGlobalTaskControl({
const pendingTaskIdsRef = useRef(pendingTaskIds);
useEffect(() => { pendingTaskIdsRef.current = pendingTaskIds; }, [pendingTaskIds]);
const handleTasksUpdate = useCallback(async (tasks) => {
const handleTasksUpdate = useCallback(async (tasks, runtimeSummary = null) => {
setActiveTasks(tasks);
if (setRuntimeSummary) {
setRuntimeSummary(runtimeSummary);
}
const hasRunningTasks = tasks.length > 0;
// 首次检查完成，清除检查状态
@@ -93,21 +97,45 @@ export default function useGlobalTaskControl({
}, [
setActiveTasks,
setRuntimeSummary,
setIsCheckingTasks,
handleTaskCompletionRef,
setPendingTaskIds,
]);
const normalizeRuntimeSummary = useCallback((payload) => {
if (!payload || typeof payload !== 'object') return null;
const items = payload.tasks?.items;
return {
...payload,
tasks: {
...(payload.tasks || {}),
items: Array.isArray(items) ? items : [],
},
};
}, []);
// Fallback polling (used when SSE is unavailable)
const syncActiveTasks = useCallback(async () => {
try {
const response = await apiClient.get('/tasks/runtime-summary');
const summary = normalizeRuntimeSummary(response.data);
if (summary) {
await handleTasksUpdate(summary.tasks.items, summary);
return;
}
} catch (error) {
console.error('同步任务运行概览失败:', error);
}
try {
const response = await apiClient.get('/tasks/active');
const tasks = Array.isArray(response.data) ? response.data : [];
await handleTasksUpdate(tasks);
await handleTasksUpdate(tasks, null);
} catch (error) {
console.error('同步任务状态失败:', error);
}
}, [handleTasksUpdate]);
}, [handleTasksUpdate, normalizeRuntimeSummary]);
useEffect(() => {
if (!currentUser || !licenseOk) return;
@@ -121,12 +149,14 @@ export default function useGlobalTaskControl({
const startSSE = () => {
const baseURL = apiClient.defaults.baseURL || '';
es = new EventSource(`${baseURL}/tasks/active/stream`);
es = new EventSource(`${baseURL}/tasks/runtime-summary/stream`);
es.onmessage = (event) => {
try {
const tasks = JSON.parse(event.data);
handleTasksUpdate(Array.isArray(tasks) ? tasks : []);
const summary = normalizeRuntimeSummary(JSON.parse(event.data));
if (summary) {
handleTasksUpdate(summary.tasks.items, summary);
}
} catch (e) {
console.error('SSE parse error:', e);
}
frontend/src/hooks/usePairingLogic.js
+28 -6
View File
@@ -216,24 +216,46 @@ export default function usePairingLogic({
}
};
const createDinsarBatch = async () => {
const createDinsarBatch = async (options = {}) => {
if (!ensureCanOperate()) return;
const chunkSize = Number(options?.chunkSize || 0);
const foundPairs = usePairingStore.getState().foundPairs;
const selectedPairs = foundPairs.filter(p => p.isSelected);
if (selectedPairs.length === 0) {
addLog('warn', '没有选中的配对可保存。');
return;
}
if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
addLog('warn', '每批条数必须是大于 0 的整数。');
return;
}
try {
const batchPairs = selectedPairs.map(compactDinsarBatchPair);
const createdBatchIds = [];
const createdAt = new Date().toISOString().slice(0, 10);
const totalChunks = Math.ceil(selectedPairs.length / chunkSize);
for (let offset = 0; offset < selectedPairs.length; offset += chunkSize) {
const chunkIndex = Math.floor(offset / chunkSize) + 1;
const chunkPairs = selectedPairs
.slice(offset, offset + chunkSize)
.map(compactDinsarBatchPair);
const response = await apiClient.post('/task-batches/dinsar', {
name: `DINSAR_${new Date().toISOString().slice(0, 10)}`,
pairs: batchPairs,
name: totalChunks > 1
? `DINSAR_${createdAt}_${String(chunkIndex).padStart(3, '0')}_of_${String(totalChunks).padStart(3, '0')}`
: `DINSAR_${createdAt}`,
pairs: chunkPairs,
});
const batchId = response.data?.batch_id || '';
addLog('success', `已创建 D-InSAR 批次: ${batchId || 'OK'}`);
if (batchId) {
await focusBatchAfterCreate('dinsar', batchId);
createdBatchIds.push(batchId);
}
addLog(
'success',
`已创建 D-InSAR 批次 ${chunkIndex}/${totalChunks}: ${batchId || 'OK'} (${chunkPairs.length} 条)`
);
}
if (createdBatchIds.length > 0) {
addLog('info', `已按每批 ${chunkSize} 条拆分为 ${createdBatchIds.length} 个 D-InSAR 批次。`);
await focusBatchAfterCreate('dinsar', createdBatchIds[0]);
}
} catch (error) {
const errorMessage = error.response?.data?.detail || error.message || '未知错误';
frontend/src/i18n/translations.js
+1 -1
View File
@@ -1,7 +1,7 @@
const TRANSLATION_PAIRS = [
{ zh: '正在检查登录状态...', en: 'Checking login status...' },
{ zh: '请稍候，系统正在验证会话。', en: 'Please wait, verifying your session.' },
{ zh: 'InSAR 自动化管理系统', en: 'InSAR Automation Management System' },
{ zh: '雷达数据生产管理系统', en: 'Radar Data Production Management System' },
{ zh: '科研工程模式', en: 'Research Engineering Mode' },
{ zh: '已授权', en: 'Licensed' },
{ zh: '未授权', en: 'Unlicensed' },
frontend/src/panels/PairPlanningPanel.jsx
+18
View File
@@ -21,6 +21,8 @@ const formatActionMode = (mode, en = false) => {
return en ? 'Full rebuild' : '全量重建';
case 'incremental_reconcile':
return en ? 'Incremental reconcile' : '增量修复';
case 'auto_reconcile':
return en ? 'Automatic repair queued' : '自动修复已提交';
case 'noop':
return en ? 'No-op reconcile' : '无需修复';
default:
@@ -244,6 +246,22 @@ export default function PairPlanningPanel({
>
{pairingActionResult.error ? (
<div style={{ color: '#b91c1c' }}>{pairingActionResult.error}</div>
) : pairingActionResult.queued ? (
<>
<div style={{ color: '#0f172a', fontWeight: 600 }}>
{formatActionMode(pairingActionResult.mode, en)}
</div>
<div>
{en
? `Task queued: ${pairingActionResult.task_id || '-'}`
: `任务已提交：${pairingActionResult.task_id || '-'}`}
</div>
<div>
{en
? 'Track progress in the task center. Refresh this status after the task completes.'
: '请在任务中心查看进度，任务完成后刷新这里的状态。'}
</div>
</>
) : (
<>
<div style={{ color: '#0f172a', fontWeight: 600 }}>
frontend/src/panels/PairsListPanel.jsx
+35 -3
View File
@@ -1,4 +1,4 @@
import { useCallback, useMemo } from 'react';
import { useCallback, useMemo, useState } from 'react';
import { useShallow } from 'zustand/react/shallow';
import { usePairingStore, useAuthStore } from '../store';
import VirtualizedList from '../components/common/VirtualizedList';
@@ -19,6 +19,7 @@ function PairsListPanel({
})));
const { currentUser } = useAuthStore();
const isReadOnlyUser = !!currentUser && currentUser.role !== 'admin';
const [batchSizeInput, setBatchSizeInput] = useState('100');
const handlePairSelectionChange = useCallback((index) => {
setFoundPairs((prevPairs) => {
@@ -45,6 +46,19 @@ function PairsListPanel({
() => foundPairs.filter((pair) => pair.isSelected).length,
[foundPairs]
);
const batchSize = useMemo(() => {
const parsed = Number(batchSizeInput);
return Number.isInteger(parsed) && parsed > 0 ? parsed : 0;
}, [batchSizeInput]);
const plannedBatchCount = useMemo(() => (
selectedPairsCount > 0 && batchSize > 0
? Math.ceil(selectedPairsCount / batchSize)
: 0
), [batchSize, selectedPairsCount]);
const handleCreateDinsarBatch = useCallback(() => {
if (typeof onCreateDinsarBatch !== 'function') return;
onCreateDinsarBatch({ chunkSize: batchSize });
}, [batchSize, onCreateDinsarBatch]);
const mapPreviewPairs = useMemo(() => {
const visible = foundPairs.filter((pair) => pair.isVis);
return visible.slice(0, 24);
@@ -141,9 +155,27 @@ function PairsListPanel({
</div>
</div>
<footer className="panel-footer">
<button
onClick={onCreateDinsarBatch}
<label style={{ display: 'inline-flex', alignItems: 'center', gap: 6, marginRight: 10 }}>
每批
<input
type="number"
min="1"
max={Math.max(1, selectedPairsCount)}
value={batchSizeInput}
onChange={(event) => setBatchSizeInput(event.target.value)}
style={{ width: 88 }}
disabled={selectedPairsCount === 0 || isReadOnlyUser}
/>
条
</label>
{selectedPairsCount > 0 && batchSize > 0 && (
<span style={{ marginRight: 10, color: '#64748b', fontSize: 12 }}>
将创建 {plannedBatchCount} 个批次
</span>
)}
<button
onClick={handleCreateDinsarBatch}
disabled={selectedPairsCount === 0 || batchSize <= 0 || isReadOnlyUser}
className="footer-button"
title="保存选中的配对为任务批次"
>
frontend/src/store/taskStore.js
+2
View File
@@ -5,9 +5,11 @@ const s = (set, key) => (v) =>
export const useTaskStore = create((set) => ({
activeTasks: [],
runtimeSummary: null,
isCheckingTasks: true, // 初始化时假设正在检查任务，避免闪烁
pendingTaskIds: [],
setActiveTasks: s(set, 'activeTasks'),
setRuntimeSummary: s(set, 'runtimeSummary'),
setIsCheckingTasks: s(set, 'isCheckingTasks'),
setPendingTaskIds: s(set, 'pendingTaskIds'),
}));
nginx/nginx.conf
+13
View File
@@ -130,6 +130,19 @@ http {
}
# SSE: disable buffering so events are pushed immediately
location /api/tasks/runtime-summary/stream {
proxy_pass http://127.0.0.1:18000;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_cache off;
proxy_read_timeout 3600s;
proxy_set_header Connection '';
}
location /api/tasks/active/stream {
proxy_pass http://127.0.0.1:18000;
proxy_http_version 1.1;
scripts/start_app.ps1
+5
View File
@@ -709,6 +709,11 @@ if (Test-Path -LiteralPath "$NginxConfPath") {
'(location\s+/api/tasks/active/stream\s*\{[\s\S]*?proxy_pass\s+)http://(127\.0\.0\.1|localhost):\d+(;)',
"`${1}$BackendProxy`${3}"
)
$NewConfContent = [regex]::Replace(
$NewConfContent,
'(location\s+/api/tasks/runtime-summary/stream\s*\{[\s\S]*?proxy_pass\s+)http://(127\.0\.0\.1|localhost):\d+(;)',
"`${1}$BackendProxy`${3}"
)
$NewConfContent = [regex]::Replace(
$NewConfContent,
'(location\s+/api/cluster/\s*\{[\s\S]*?proxy_pass\s+)http://(127\.0\.0\.1|localhost):\d+(;)',
tools/lt1_data_sync/LT1DataSync.spec
+38
View File
@@ -0,0 +1,38 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['lt1_data_sync_gui.py'],
pathex=[],
binaries=[('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\tcl86t.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\tk86t.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\libcrypto-3-x64.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\liblzma.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\libbz2.dll', '.')],
datas=[],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name='LT1DataSync',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
tools/lt1_data_sync/README.md
+189
View File
@@ -0,0 +1,189 @@
# LT1AssetTool
独立 EXE 工具，只做三件事：
1. 扫描服务器资产路径，列出服务器现在有哪些资产，生成一个随身 JSON。
2. 扫描一个或多个 UNC 路径，读取随身 JSON，服务器已有资产不参与复制，剩下的复制到一个或多个指定路径。
3. 将一个或多个磁盘路径下的资产复制或剪切到服务器资产路径，并提示重新执行第 1 步。
## 资产识别
当前只识别平铺文件：
- `LT1*.tar.gz`
- `LT1*.tgz`
- `LT1*.tar`
- `LT1*.zip`
- `LT1*.txt`
判断是否已有资产使用：
```text
资产类型 + 文件名 + 文件大小
```
## 第 1 步：扫描服务器资产路径
输入：
```text
服务器资产路径：
D:\LuTan1_Image_Pool_Zip
D:\LT1_data_lsarorbit
服务器资产 JSON 保存为：
E:\server_assets.json
```
输出 JSON 示例：
```json
{
"schema": "lt1_asset_inventory.v2",
"generated_at": "2026-06-22 16:00:00",
"roots": [
"D:\\LuTan1_Image_Pool_Zip",
"D:\\LT1_data_lsarorbit"
],
"asset_count": 2,
"assets": [
{
"kind": "lt1_archive",
"name": "LT1A_xxx.tar.gz",
"path": "D:\\LuTan1_Image_Pool_Zip\\LT1A_xxx.tar.gz",
"size": 123456789,
"mtime": 1782100000.0
},
{
"kind": "lt1_orbit",
"name": "LT1A_GpsData_GAS_C_20240101.txt",
"path": "D:\\LT1_data_lsarorbit\\LT1A_GpsData_GAS_C_20240101.txt",
"size": 345678,
"mtime": 1782100100.0
}
]
}
```
以后就带着这个 JSON 去内网机器。
## 第 2 步：从 UNC 补拷缺失资产
输入：
```text
读取服务器资产 JSON：
E:\server_assets.json
UNC 源路径：
\\server01\lt1_archives
\\server02\lt1_orbits
复制目标路径：
E:\LT1_TRANSFER
F:\LT1_TRANSFER
```
规则：
- JSON 中已有且大小一致：跳过
- JSON 中有同名资产但大小不同：报告冲突，不复制
- JSON 中没有：复制到指定目标路径
- 多个目标路径时，选择第一个空间足够的路径
- 如果目标路径已有同名文件且大小一致：跳过
- 复制过程先写 `.part`，完成并校验大小后再改名
## 第 3 步：磁盘导入服务器
输入：
```text
磁盘资产路径：
E:\LT1_TRANSFER
F:\LT1_TRANSFER
服务器资产路径：
D:\LuTan1_Image_Pool_Zip
D:\LT1_data_lsarorbit
```
可选：
```text
剪切到服务器
```
规则：
- 服务器已有且大小一致：跳过
- 服务器有同名资产但大小不同：报告冲突，不覆盖
- 服务器没有：复制或剪切到服务器路径
- 多个服务器路径时，选择第一个空间足够的路径
执行第 3 步之后，回到服务器重新执行第 1 步，生成新的随身 JSON。
## 报告和日志
需要指定 `报告/日志目录`，例如：
```text
E:\LT1AssetToolReports
```
工具会生成：
```text
E:\LT1AssetToolReports\
logs\
run_YYYYMMDD_HHMMSS.log
reports\
unc_copy_report_YYYYMMDD_HHMMSS.csv
disk_import_report_YYYYMMDD_HHMMSS.csv
```
CSV 可以用 Excel 打开。
## 命令行
扫描服务器：
```powershell
python lt1_data_sync_cli.py scan-server `
--report-dir "E:\LT1AssetToolReports" `
--server-roots "D:\LuTan1_Image_Pool_Zip;D:\LT1_data_lsarorbit" `
--output-json "E:\server_assets.json"
```
从 UNC 复制缺失资产：
```powershell
python lt1_data_sync_cli.py copy-unc `
--report-dir "E:\LT1AssetToolReports" `
--server-json "E:\server_assets.json" `
--unc-roots "\\server01\lt1_archives;\\server02\lt1_orbits" `
--targets "E:\LT1_TRANSFER;F:\LT1_TRANSFER" `
--execute
```
磁盘导入服务器：
```powershell
python lt1_data_sync_cli.py import-disk `
--report-dir "E:\LT1AssetToolReports" `
--disk-roots "E:\LT1_TRANSFER;F:\LT1_TRANSFER" `
--server-roots "D:\LuTan1_Image_Pool_Zip;D:\LT1_data_lsarorbit" `
--execute
```
## 打包 EXE
```powershell
cd D:\Code\Insar_management_system_v2\tools\lt1_data_sync
.\build_exe.ps1 -Python "C:\ProgramData\anaconda3\envs\InSAR\python.exe"
```
输出：
```text
dist\LT1DataSync.exe
```
tools/lt1_data_sync/build_exe.ps1
+37
View File
@@ -0,0 +1,37 @@
param(
[string]$Python = "python"
)
$ErrorActionPreference = "Stop"
$ToolDir = Split-Path -Parent $MyInvocation.MyCommand.Path
Set-Location $ToolDir
try {
& $Python --version
} catch {
throw "Python was not found. Install Python 3.10+ or pass -Python with a full python.exe path."
}
$PythonExe = (Get-Command $Python).Source
$EnvRoot = Split-Path -Parent $PythonExe
$CondaBin = Join-Path $EnvRoot "Library\bin"
$ExtraArgs = @()
foreach ($DllName in @("tcl86t.dll", "tk86t.dll", "libcrypto-3-x64.dll", "liblzma.dll", "libbz2.dll")) {
$DllPath = Join-Path $CondaBin $DllName
if (Test-Path $DllPath) {
$ExtraArgs += "--add-binary"
$ExtraArgs += "$DllPath;."
}
}
& $Python -m pip install --upgrade pyinstaller
& $Python -m PyInstaller `
--noconfirm `
--onefile `
--windowed `
--name LT1DataSync `
@ExtraArgs `
lt1_data_sync_gui.py
Write-Host ""
Write-Host "EXE built at: $ToolDir\dist\LT1DataSync.exe"
tools/lt1_data_sync/lt1_data_sync_cli.py
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import argparse
from sync_core import (
FileLogger,
copy_unc_missing_assets,
import_disk_assets_to_server,
normalize_path,
parse_path_list,
scan_assets,
stamp_text,
write_inventory,
)
def add_common(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--report-dir", required=True)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="LT1 asset transfer tool")
sub = parser.add_subparsers(dest="command", required=True)
scan = sub.add_parser("scan-server", help="Scan server asset paths and write a JSON inventory")
add_common(scan)
scan.add_argument("--server-roots", required=True, help="Server asset paths separated by semicolon/newline")
scan.add_argument("--output-json", required=True)
copy = sub.add_parser("copy-unc", help="Copy UNC assets that are missing from server inventory")
add_common(copy)
copy.add_argument("--server-json", required=True)
copy.add_argument("--unc-roots", required=True, help="UNC source paths separated by semicolon/newline")
copy.add_argument("--targets", required=True, help="Copy target paths separated by semicolon/newline")
copy.add_argument("--execute", action="store_true")
imp = sub.add_parser("import-disk", help="Copy or move disk assets into server asset paths")
add_common(imp)
imp.add_argument("--disk-roots", required=True, help="Disk asset paths separated by semicolon/newline")
imp.add_argument("--server-roots", required=True, help="Server asset paths separated by semicolon/newline")
imp.add_argument("--execute", action="store_true")
imp.add_argument("--move", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
report_dir = normalize_path(args.report_dir)
logger = FileLogger(report_dir / "logs" / f"cli_{stamp_text()}.log", print)
if args.command == "scan-server":
roots = parse_path_list(args.server_roots)
assets = scan_assets(roots, log=logger)
write_inventory(normalize_path(args.output_json), assets, [str(root) for root in roots])
logger(f"服务器资产 JSON 已生成：{args.output_json}，资产 {len(assets)} 个")
return 0
if args.command == "copy-unc":
report = copy_unc_missing_assets(
parse_path_list(args.unc_roots),
normalize_path(args.server_json),
parse_path_list(args.targets),
report_dir / "reports",
execute=args.execute,
log=logger,
)
failed = sum(1 for item in report if item.action == "failed")
return 1 if failed else 0
if args.command == "import-disk":
report = import_disk_assets_to_server(
parse_path_list(args.disk_roots),
parse_path_list(args.server_roots),
report_dir / "reports",
execute=args.execute,
move=args.move,
log=logger,
)
failed = sum(1 for item in report if item.action == "failed")
logger("导入后请重新执行 scan-server，生成新的随身 JSON。")
return 1 if failed else 0
raise ValueError(f"Unsupported command: {args.command}")
if __name__ == "__main__":
raise SystemExit(main())
tools/lt1_data_sync/lt1_data_sync_gui.py
+327
View File
@@ -0,0 +1,327 @@
from __future__ import annotations
import json
import queue
import threading
from pathlib import Path
from tkinter import BooleanVar, StringVar, Tk, filedialog, messagebox, ttk
from tkinter.scrolledtext import ScrolledText
from sync_core import (
FileLogger,
copy_unc_missing_assets,
import_disk_assets_to_server,
normalize_path,
parse_path_list,
scan_assets,
stamp_text,
write_inventory,
)
APP_DIR = Path(__file__).resolve().parent
CONFIG_PATH = APP_DIR / "config.json"
DEFAULT_CONFIG = {
"server_asset_roots": r"D:\LuTan1_Image_Pool_Zip" + "\n" + r"D:\LT1_data_lsarorbit",
"inventory_output": "",
"server_inventory_json": "",
"unc_roots": "",
"unc_copy_targets": "",
"disk_roots": "",
"import_server_roots": r"D:\LuTan1_Image_Pool_Zip" + "\n" + r"D:\LT1_data_lsarorbit",
"report_dir": "",
"move_on_import": False,
}
class LT1AssetTool:
def __init__(self, root: Tk) -> None:
self.root = root
self.root.title("LT1 资产搬运工具")
self.root.geometry("1120x820")
self.log_queue: queue.Queue[str] = queue.Queue()
self.worker: threading.Thread | None = None
self.text_widgets: dict[str, ScrolledText] = {}
config = self.load_config()
self.vars = {
"server_asset_roots": StringVar(value=config["server_asset_roots"]),
"inventory_output": StringVar(value=config["inventory_output"]),
"server_inventory_json": StringVar(value=config["server_inventory_json"]),
"unc_roots": StringVar(value=config["unc_roots"]),
"unc_copy_targets": StringVar(value=config["unc_copy_targets"]),
"disk_roots": StringVar(value=config["disk_roots"]),
"import_server_roots": StringVar(value=config["import_server_roots"]),
"report_dir": StringVar(value=config["report_dir"]),
}
self.move_on_import = BooleanVar(value=bool(config["move_on_import"]))
self.summary_text = StringVar(value="准备就绪")
self.build_ui()
self.root.after(100, self.drain_logs)
def load_config(self) -> dict:
if not CONFIG_PATH.exists():
return dict(DEFAULT_CONFIG)
try:
payload = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
return {**DEFAULT_CONFIG, **payload}
except Exception:
return dict(DEFAULT_CONFIG)
def sync_text_vars(self) -> None:
for key, widget in self.text_widgets.items():
self.vars[key].set(widget.get("1.0", "end").strip())
def save_config(self) -> None:
self.sync_text_vars()
payload = {key: var.get() for key, var in self.vars.items()}
payload["move_on_import"] = self.move_on_import.get()
CONFIG_PATH.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def build_ui(self) -> None:
outer = ttk.Frame(self.root, padding=12)
outer.pack(fill="both", expand=True)
common = ttk.LabelFrame(outer, text="公共输出", padding=10)
common.pack(fill="x")
self.add_path_row(common, 0, "报告/日志目录", "report_dir")
scan_box = ttk.LabelFrame(outer, text="1. 扫描服务器资产路径，生成随身 JSON", padding=10)
scan_box.pack(fill="x", pady=(10, 0))
self.add_multi_path_row(scan_box, 0, "服务器资产路径（多个用分号或换行）", "server_asset_roots")
self.add_file_save_row(scan_box, 1, "服务器资产 JSON 保存为", "inventory_output")
ttk.Button(scan_box, text="执行 1：扫描服务器并生成 JSON", command=lambda: self.start_worker(self.run_scan_server)).grid(
row=2,
column=1,
sticky="w",
pady=(8, 0),
)
unc_box = ttk.LabelFrame(outer, text="2. 扫描 UNC 多路径，按服务器 JSON 跳过已有，剩余复制到指定路径", padding=10)
unc_box.pack(fill="x", pady=(10, 0))
self.add_file_row(unc_box, 0, "读取服务器资产 JSON", "server_inventory_json")
self.add_multi_path_row(unc_box, 1, "UNC 源路径（多个用分号或换行）", "unc_roots")
self.add_multi_path_row(unc_box, 2, "复制目标路径（多个用分号或换行）", "unc_copy_targets")
ttk.Button(unc_box, text="预览 2：只生成复制报告", command=lambda: self.start_worker(lambda logger: self.run_unc_copy(logger, execute=False))).grid(
row=3,
column=1,
sticky="w",
pady=(8, 0),
)
ttk.Button(unc_box, text="执行 2：复制缺失资产", command=lambda: self.start_worker(lambda logger: self.run_unc_copy(logger, execute=True))).grid(
row=3,
column=1,
sticky="w",
padx=(180, 0),
pady=(8, 0),
)
import_box = ttk.LabelFrame(outer, text="3. 将指定磁盘路径资产复制或剪切到服务器资产路径", padding=10)
import_box.pack(fill="x", pady=(10, 0))
self.add_multi_path_row(import_box, 0, "磁盘资产路径（多个用分号或换行）", "disk_roots")
self.add_multi_path_row(import_box, 1, "服务器资产路径（多个用分号或换行）", "import_server_roots")
ttk.Checkbutton(import_box, text="剪切到服务器（不勾选则复制）", variable=self.move_on_import).grid(row=2, column=1, sticky="w")
ttk.Button(import_box, text="预览 3：只生成导入报告", command=lambda: self.start_worker(lambda logger: self.run_import(logger, execute=False))).grid(
row=3,
column=1,
sticky="w",
pady=(8, 0),
)
ttk.Button(import_box, text="执行 3：导入到服务器", command=lambda: self.start_worker(lambda logger: self.run_import(logger, execute=True))).grid(
row=3,
column=1,
sticky="w",
padx=(180, 0),
pady=(8, 0),
)
ttk.Label(import_box, text="执行 3 后，请回到服务器重新执行 1，生成新的随身 JSON。", foreground="#92400e").grid(
row=4,
column=1,
sticky="w",
pady=(8, 0),
)
control = ttk.Frame(outer)
control.pack(fill="x", pady=10)
ttk.Button(control, text="保存配置", command=self.handle_save_config).pack(side="left")
ttk.Label(control, textvariable=self.summary_text, font=("Microsoft YaHei UI", 10, "bold")).pack(side="left", padx=16)
log_box = ttk.LabelFrame(outer, text="日志", padding=10)
log_box.pack(fill="both", expand=True)
self.log_view = ScrolledText(log_box, height=14, wrap="word")
self.log_view.pack(fill="both", expand=True)
def add_path_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_dir(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_file_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_file(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_file_save_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_save_file(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_multi_path_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="nw", padx=(0, 8), pady=4)
text = ScrolledText(frame, width=105, height=3, wrap="none")
text.insert("1.0", self.vars[key].get())
text.grid(row=row, column=1, sticky="ew", pady=4)
self.text_widgets[key] = text
ttk.Button(frame, text="追加目录", command=lambda: self.append_dir(key)).grid(row=row, column=2, padx=(8, 0), pady=4, sticky="n")
frame.columnconfigure(1, weight=1)
def choose_dir(self, key: str) -> None:
selected = filedialog.askdirectory()
if selected:
self.vars[key].set(selected)
def append_dir(self, key: str) -> None:
selected = filedialog.askdirectory()
if not selected:
return
widget = self.text_widgets.get(key)
if not widget:
self.vars[key].set(selected)
return
current = widget.get("1.0", "end").strip()
widget.delete("1.0", "end")
widget.insert("1.0", f"{current}\n{selected}" if current else selected)
def choose_file(self, key: str) -> None:
selected = filedialog.askopenfilename(filetypes=[("JSON 文件", "*.json"), ("所有文件", "*.*")])
if selected:
self.vars[key].set(selected)
def choose_save_file(self, key: str) -> None:
selected = filedialog.asksaveasfilename(
defaultextension=".json",
filetypes=[("JSON 文件", "*.json"), ("所有文件", "*.*")],
initialfile=f"server_assets_{stamp_text()}.json",
)
if selected:
self.vars[key].set(selected)
def report_dir(self) -> Path:
text = self.vars["report_dir"].get().strip()
if not text:
raise ValueError("请指定报告/日志目录")
return normalize_path(text)
def logger(self) -> FileLogger:
return FileLogger(self.report_dir() / "logs" / f"run_{stamp_text()}.log", self.log)
def handle_save_config(self) -> None:
self.save_config()
messagebox.showinfo("已保存", f"配置已保存到：{CONFIG_PATH}")
def log(self, message: str) -> None:
self.log_queue.put(message)
def drain_logs(self) -> None:
while True:
try:
message = self.log_queue.get_nowait()
except queue.Empty:
break
self.log_view.insert("end", message + "\n")
self.log_view.see("end")
self.root.after(100, self.drain_logs)
def start_worker(self, action) -> None:
if self.worker and self.worker.is_alive():
messagebox.showwarning("正在运行", "已有任务正在运行。")
return
self.save_config()
self.log_view.delete("1.0", "end")
self.worker = threading.Thread(target=self.run_action, args=(action,), daemon=True)
self.worker.start()
def run_action(self, action) -> None:
try:
action(self.logger())
except Exception as exc:
self.log(f"任务失败：{exc}")
self.root.after(0, lambda: messagebox.showerror("任务失败", str(exc)))
def run_scan_server(self, logger: FileLogger) -> None:
roots = parse_path_list(self.vars["server_asset_roots"].get())
if not roots:
raise ValueError("请填写服务器资产路径")
output = self.vars["inventory_output"].get().strip()
if not output:
raise ValueError("请指定服务器资产 JSON 保存路径")
logger("开始扫描服务器资产路径")
assets = scan_assets(roots, log=logger)
write_inventory(normalize_path(output), assets, [str(root) for root in roots])
self.root.after(0, lambda: self.summary_text.set(f"服务器资产 {len(assets)} 个，JSON 已生成"))
logger(f"完成：{output}")
def run_unc_copy(self, logger: FileLogger, *, execute: bool) -> None:
inventory = self.vars["server_inventory_json"].get().strip()
unc_roots = parse_path_list(self.vars["unc_roots"].get())
targets = parse_path_list(self.vars["unc_copy_targets"].get())
if not inventory:
raise ValueError("请指定服务器资产 JSON")
if not unc_roots:
raise ValueError("请填写 UNC 源路径")
if not targets:
raise ValueError("请填写复制目标路径")
logger("开始扫描 UNC 并按服务器 JSON 跳过已有资产")
report = copy_unc_missing_assets(
unc_roots,
normalize_path(inventory),
targets,
self.report_dir() / "reports",
execute=execute,
log=logger,
)
copied = sum(1 for item in report if item.action == "copied")
planned = sum(1 for item in report if item.action == "planned")
skipped = sum(1 for item in report if item.action == "skip")
failed = sum(1 for item in report if item.action == "failed")
text = f"UNC 处理完成：copied={copied}, planned={planned}, skipped={skipped}, failed={failed}"
self.root.after(0, lambda: self.summary_text.set(text))
logger(text)
def run_import(self, logger: FileLogger, *, execute: bool) -> None:
disk_roots = parse_path_list(self.vars["disk_roots"].get())
server_roots = parse_path_list(self.vars["import_server_roots"].get())
if not disk_roots:
raise ValueError("请填写磁盘资产路径")
if not server_roots:
raise ValueError("请填写服务器资产路径")
logger("开始从磁盘导入资产到服务器")
report = import_disk_assets_to_server(
disk_roots,
server_roots,
self.report_dir() / "reports",
execute=execute,
move=self.move_on_import.get(),
log=logger,
)
imported = sum(1 for item in report if item.action in {"copied", "moved"})
planned = sum(1 for item in report if item.action == "planned")
skipped = sum(1 for item in report if item.action == "skip")
failed = sum(1 for item in report if item.action == "failed")
text = f"磁盘导入完成：imported={imported}, planned={planned}, skipped={skipped}, failed={failed}。请重新执行 1。"
self.root.after(0, lambda: self.summary_text.set(text))
logger(text)
def main() -> None:
root = Tk()
LT1AssetTool(root)
root.mainloop()
if __name__ == "__main__":
main()
tools/lt1_data_sync/sync_core.py
+316
View File
@@ -0,0 +1,316 @@
from __future__ import annotations
import csv
import json
import os
import shutil
from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import Callable, Iterable
LT1_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
LT1_ORBIT_SUFFIXES = (".txt",)
ProgressCallback = Callable[[str], None]
def now_text() -> str:
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def stamp_text() -> str:
return datetime.now().strftime("%Y%m%d_%H%M%S")
def normalize_path(value: str) -> Path:
return Path(value.strip().strip('"')).expanduser()
def parse_path_list(value: str) -> list[Path]:
paths: list[Path] = []
for line in str(value or "").replace(";", "\n").splitlines():
text = line.strip().strip('"')
if text:
paths.append(normalize_path(text))
return paths
def safe_mkdir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def is_asset_file(path: Path) -> bool:
lower = path.name.lower()
if lower.startswith("lt1") and lower.endswith(LT1_ARCHIVE_SUFFIXES):
return True
if lower.startswith("lt1") and lower.endswith(LT1_ORBIT_SUFFIXES):
return True
return False
def classify_asset(path: Path) -> str:
lower = path.name.lower()
if lower.endswith(LT1_ARCHIVE_SUFFIXES):
return "lt1_archive"
if lower.endswith(LT1_ORBIT_SUFFIXES):
return "lt1_orbit"
return "unknown"
@dataclass(frozen=True)
class AssetRecord:
kind: str
name: str
path: str
size: int
mtime: float
@dataclass(frozen=True)
class CopyRecord:
name: str
kind: str
source_path: str
target_path: str
size: int
action: str
reason: str
class FileLogger:
def __init__(self, path: Path, ui_log: ProgressCallback | None = None) -> None:
self.path = path
self.ui_log = ui_log
safe_mkdir(path.parent)
def __call__(self, message: str) -> None:
line = f"{now_text()} {message}"
with self.path.open("a", encoding="utf-8") as handle:
handle.write(line + "\n")
if self.ui_log:
self.ui_log(line)
def scan_assets(paths: Iterable[Path], *, log: ProgressCallback | None = None) -> list[AssetRecord]:
records: list[AssetRecord] = []
for root in paths:
if not root.exists():
if log:
log(f"路径不存在，跳过：{root}")
continue
if not root.is_dir():
if log:
log(f"不是目录，跳过：{root}")
continue
count = 0
for entry in root.iterdir():
if not entry.is_file() or not is_asset_file(entry):
continue
stat = entry.stat()
records.append(
AssetRecord(
kind=classify_asset(entry),
name=entry.name,
path=str(entry),
size=stat.st_size,
mtime=stat.st_mtime,
)
)
count += 1
if log:
log(f"扫描完成：{root}，资产 {count} 个")
records.sort(key=lambda item: (item.kind, item.name.lower(), item.path.lower()))
return records
def asset_key(record: AssetRecord) -> tuple[str, str]:
return record.kind, record.name.lower()
def build_asset_index(records: Iterable[AssetRecord]) -> dict[tuple[str, str], AssetRecord]:
index: dict[tuple[str, str], AssetRecord] = {}
for record in records:
key = asset_key(record)
if key not in index:
index[key] = record
return index
def read_inventory(path: Path) -> list[AssetRecord]:
payload = json.loads(path.read_text(encoding="utf-8"))
records: list[AssetRecord] = []
for item in payload.get("assets", payload.get("files", [])):
try:
records.append(
AssetRecord(
kind=str(item.get("kind") or ""),
name=str(item.get("name") or ""),
path=str(item.get("path") or ""),
size=int(item.get("size") or 0),
mtime=float(item.get("mtime") or 0),
)
)
except (TypeError, ValueError):
continue
return [item for item in records if item.kind and item.name]
def write_inventory(path: Path, records: list[AssetRecord], roots: list[str]) -> None:
safe_mkdir(path.parent)
payload = {
"schema": "lt1_asset_inventory.v2",
"generated_at": now_text(),
"roots": roots,
"asset_count": len(records),
"assets": [asdict(item) for item in records],
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def write_csv(path: Path, records: list[CopyRecord]) -> None:
safe_mkdir(path.parent)
with path.open("w", newline="", encoding="utf-8-sig") as handle:
writer = csv.DictWriter(
handle,
fieldnames=["name", "kind", "source_path", "target_path", "size", "action", "reason"],
)
writer.writeheader()
for record in records:
writer.writerow(asdict(record))
def choose_target_root(target_roots: list[Path], filename: str, size: int) -> tuple[Path | None, str]:
for root in target_roots:
dest = root / filename
if dest.exists():
try:
if dest.stat().st_size == size:
return None, f"目标已存在且大小一致：{dest}"
return None, f"目标已存在但大小不同：{dest}"
except OSError:
return None, f"无法读取目标文件状态：{dest}"
for root in target_roots:
try:
safe_mkdir(root)
free_bytes = shutil.disk_usage(root).free
if free_bytes > size:
return root, "选择第一个空间足够的目标路径"
except OSError:
continue
return None, "没有空间足够的目标路径"
def copy_file_atomic(source: Path, target: Path, *, move: bool = False) -> None:
safe_mkdir(target.parent)
part = target.with_name(target.name + ".part")
if part.exists():
part.unlink()
if target.exists():
raise FileExistsError(str(target))
if move:
shutil.copy2(source, part)
if part.stat().st_size != source.stat().st_size:
raise OSError(f"复制后大小不一致：{source} -> {target}")
os.replace(part, target)
source.unlink()
else:
shutil.copy2(source, part)
if part.stat().st_size != source.stat().st_size:
raise OSError(f"复制后大小不一致：{source} -> {target}")
os.replace(part, target)
def copy_unc_missing_assets(
unc_roots: list[Path],
server_inventory_json: Path,
target_roots: list[Path],
report_dir: Path,
*,
execute: bool,
log: ProgressCallback | None = None,
) -> list[CopyRecord]:
server_assets = read_inventory(server_inventory_json)
server_index = build_asset_index(server_assets)
source_assets = scan_assets(unc_roots, log=log)
report: list[CopyRecord] = []
for asset in source_assets:
server_asset = server_index.get(asset_key(asset))
if server_asset and server_asset.size == asset.size:
report.append(
CopyRecord(asset.name, asset.kind, asset.path, server_asset.path, asset.size, "skip", "服务器清单已有且大小一致")
)
continue
if server_asset and server_asset.size != asset.size:
report.append(
CopyRecord(asset.name, asset.kind, asset.path, server_asset.path, asset.size, "conflict", "服务器清单有同名资产但大小不同")
)
continue
target_root, reason = choose_target_root(target_roots, asset.name, asset.size)
if target_root is None:
report.append(CopyRecord(asset.name, asset.kind, asset.path, "", asset.size, "skip", reason))
continue
target_path = target_root / asset.name
if not execute:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "planned", reason))
continue
try:
copy_file_atomic(Path(asset.path), target_path)
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "copied", reason))
if log:
log(f"已复制：{asset.name} -> {target_path}")
except Exception as exc:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "failed", str(exc)))
if log:
log(f"复制失败：{asset.name}，{exc}")
write_csv(report_dir / f"unc_copy_report_{stamp_text()}.csv", report)
return report
def import_disk_assets_to_server(
disk_roots: list[Path],
server_asset_roots: list[Path],
report_dir: Path,
*,
execute: bool,
move: bool,
log: ProgressCallback | None = None,
) -> list[CopyRecord]:
source_assets = scan_assets(disk_roots, log=log)
server_assets = scan_assets(server_asset_roots, log=log)
server_index = build_asset_index(server_assets)
report: list[CopyRecord] = []
for asset in source_assets:
existing = server_index.get(asset_key(asset))
if existing and existing.size == asset.size:
report.append(CopyRecord(asset.name, asset.kind, asset.path, existing.path, asset.size, "skip", "服务器已存在且大小一致"))
continue
if existing and existing.size != asset.size:
report.append(CopyRecord(asset.name, asset.kind, asset.path, existing.path, asset.size, "conflict", "服务器有同名资产但大小不同"))
continue
target_root, reason = choose_target_root(server_asset_roots, asset.name, asset.size)
if target_root is None:
report.append(CopyRecord(asset.name, asset.kind, asset.path, "", asset.size, "skip", reason))
continue
target_path = target_root / asset.name
if not execute:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "planned", reason))
continue
try:
copy_file_atomic(Path(asset.path), target_path, move=move)
action = "moved" if move else "copied"
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, action, reason))
if log:
log(f"已{('剪切' if move else '复制')}：{asset.name} -> {target_path}")
except Exception as exc:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "failed", str(exc)))
if log:
log(f"导入失败：{asset.name}，{exc}")
write_csv(report_dir / f"disk_import_report_{stamp_text()}.csv", report)
return report
tools/lt1_data_sync/test_sync_core.py
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import tempfile
from pathlib import Path
from sync_core import (
copy_unc_missing_assets,
import_disk_assets_to_server,
parse_path_list,
read_inventory,
scan_assets,
write_inventory,
)
def write_file(path: Path, content: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
def test_scan_server_writes_inventory_json() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
server = root / "server"
output = root / "server_assets.json"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
assets = scan_assets([server])
write_inventory(output, assets, [str(server)])
loaded = read_inventory(output)
assert len(loaded) == 1
assert loaded[0].name == "LT1A_EXIST.tar.gz"
def test_unc_copy_uses_server_json_to_skip_existing() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
server = root / "server"
unc = root / "unc"
target = root / "disk_a"
report_dir = root / "reports"
inventory = root / "server_assets.json"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
write_file(unc / "LT1A_EXIST.tar.gz", b"same")
write_file(unc / "LT1A_MISSING.tar.gz", b"new")
write_inventory(inventory, scan_assets([server]), [str(server)])
report = copy_unc_missing_assets([unc], inventory, [target], report_dir, execute=True)
assert (target / "LT1A_MISSING.tar.gz").read_bytes() == b"new"
assert not (target / "LT1A_EXIST.tar.gz").exists()
assert sum(1 for item in report if item.action == "skip") == 1
assert sum(1 for item in report if item.action == "copied") == 1
def test_import_disk_copies_or_moves_to_server_and_skips_existing() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
disk = root / "disk"
server = root / "server"
report_dir = root / "reports"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
write_file(disk / "LT1A_EXIST.tar.gz", b"same")
write_file(disk / "LT1A_NEW.tar.gz", b"new")
report = import_disk_assets_to_server([disk], [server], report_dir, execute=True, move=True)
assert (server / "LT1A_NEW.tar.gz").read_bytes() == b"new"
assert not (disk / "LT1A_NEW.tar.gz").exists()
assert sum(1 for item in report if item.action == "skip") == 1
assert sum(1 for item in report if item.action == "moved") == 1
def test_parse_path_list_accepts_semicolon_and_newline() -> None:
paths = parse_path_list(r"E:\;F:\Data" + "\n" + r"\\server\share")
assert len(paths) == 3
assert str(paths[1]) == r"F:\Data"
assert "server" in str(paths[2])
if __name__ == "__main__":
test_scan_server_writes_inventory_json()
test_unc_copy_uses_server_json_to_skip_existing()
test_import_disk_copies_or_moves_to_server_and_skips_existing()
test_parse_path_list_accepts_semicolon_and_newline()
print("ok")