Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor local InSAR asset and production workflows

Browse Source
This commit is contained in:
Harmon
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routers/ai.py
+12 -6
View File
@@ -15,7 +15,9 @@ from typing import List, Optional
from ..ai_service import (
analyze_map_with_vlm,
get_ollama_models,
get_model_info,
is_likely_vlm_model,
is_model_trained,
predict_quality,
train_quality_model,
@@ -146,12 +148,12 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
counts = await dinsar_read_service.get_ai_status_counts(db)
ollama_online = False
ollama_models: List[str] = []
ollama_vlm_models: List[str] = []
try:
import httpx
async with httpx.AsyncClient(timeout=1.0) as client:
ollama_base = settings.OLLAMA_BASE_URL
response = await client.get(f"{ollama_base.rstrip('/')}/api/tags")
ollama_online = response.status_code == 200
ollama_models = await get_ollama_models(timeout=1.0)
ollama_vlm_models = [model for model in ollama_models if is_likely_vlm_model(model)]
ollama_online = True
except Exception:
ollama_online = False
@@ -161,7 +163,11 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)):
"labeled_count": counts["labeled_count"],
"good_count": counts["good_count"],
"bad_count": counts["bad_count"],
"ollama_online": ollama_online
"ollama_online": ollama_online,
"ollama_models": ollama_models,
"ollama_vlm_models": ollama_vlm_models,
"ollama_base_url": settings.OLLAMA_BASE_URL,
"default_vlm_model": settings.DEFAULT_VLM_MODEL,
}
backend/app/routers/assets.py
+53 -4
View File
@@ -22,21 +22,29 @@ router = APIRouter(prefix="/assets", tags=["assets"])
class AssetScanRequest(BaseModel):
inventory_types: List[str] = Field(default_factory=list)
root_ids: List[int] = Field(default_factory=list)
families: List[str] = Field(default_factory=list)
bind_orbits: bool = True
build_previews: bool = True
class ArchiveIntegrityAuditRequest(BaseModel):
families: List[str] = Field(default_factory=list)
source_formats: List[str] = Field(default_factory=list)
asset_ids: List[int] = Field(default_factory=list)
force: bool = False
limit: Optional[int] = Field(default=None, ge=0)
class S1UnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
class S1BatchUnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
scan_before_unpack: bool = True
@@ -89,7 +97,50 @@ async def run_asset_inventory_scan_now(
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
families=payload.get("families") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
build_previews=bool(payload.get("build_previews", True)),
)
@router.post("/inventory/archive-integrity-audit", status_code=202)
async def run_archive_integrity_audit(
request: Optional[ArchiveIntegrityAuditRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
try:
task_id = await task_service.create_task(
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"Source archive integrity audit",
params=payload,
)
job_id = await job_queue_service.create_job(
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
payload=payload,
task_id=task_id,
)
return {"message": "Source archive integrity audit queued", "task_id": task_id, "job_id": job_id}
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/inventory/archive-integrity-audit-now")
async def run_archive_integrity_audit_now(
request: Optional[ArchiveIntegrityAuditRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = admin_user
payload = (request or ArchiveIntegrityAuditRequest()).model_dump()
return await asset_inventory_service.audit_source_archive_integrity(
db,
families=payload.get("families") or None,
source_formats=payload.get("source_formats") or None,
asset_ids=payload.get("asset_ids") or None,
force=bool(payload.get("force", False)),
limit=payload.get("limit"),
)
@@ -186,8 +237,6 @@ async def unpack_sentinel1_source_asset(
}
if request_data.min_disk_space_gb is not None:
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
if request_data.delete_archive is not None:
payload["delete_archive"] = request_data.delete_archive
try:
task_id = await task_service.create_task(
"UNPACK_SENTINEL1",
backend/app/routers/dinsar_production.py
+89
View File
@@ -2,7 +2,9 @@
from __future__ import annotations
import asyncio
import os
import re
from datetime import datetime
from typing import Any, Dict, Literal, Optional
from fastapi import APIRouter, Depends, HTTPException
@@ -65,6 +67,43 @@ class PreviewInputAssetsRequest(BaseModel):
num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all")
def _is_dinsar_task_dir(path: str) -> bool:
return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave"))
def _is_landsar_task_dir(path: str) -> bool:
return os.path.isdir(os.path.join(path, "Input_Data")) or _is_dinsar_task_dir(path)
def _count_candidate_tasks(root_dir: str) -> Dict[str, Any]:
try:
if _is_dinsar_task_dir(root_dir) or _is_landsar_task_dir(root_dir):
return {"task_count": 1, "invalid_child_count": 0, "mode": "single_task"}
task_count = 0
invalid_child_count = 0
with os.scandir(root_dir) as entries:
for entry in entries:
if not entry.is_dir() or entry.name.startswith("._"):
continue
child_path = entry.path
if _is_dinsar_task_dir(child_path) or _is_landsar_task_dir(child_path):
task_count += 1
elif entry.name.lower().startswith("task_"):
invalid_child_count += 1
return {
"task_count": task_count,
"invalid_child_count": invalid_child_count,
"mode": "task_root",
}
except OSError as exc:
return {
"task_count": 0,
"invalid_child_count": 0,
"mode": "unreadable",
"error": str(exc),
}
def _get_registry():
from ..dinsar_engines import registry
@@ -136,6 +175,56 @@ async def run_wsl_check(
return report.to_dict()
@router.get("/task-roots")
async def list_task_roots(
current_user: AuthUserORM = Depends(_get_current_user),
):
_ = current_user
root_dir = os.path.normpath(os.path.abspath(str(settings.DINSAR_TASK_POOL_ROOT or "").strip()))
root_exists = bool(root_dir and os.path.isdir(root_dir))
items = []
child_dir_count = 0
if root_exists:
try:
with os.scandir(root_dir) as entries:
child_dirs = sorted(
[entry for entry in entries if entry.is_dir() and not entry.name.startswith("._")],
key=lambda entry: entry.name.lower(),
)
except OSError as exc:
raise HTTPException(status_code=500, detail=f"无法读取 D-InSAR Task_Pool: {exc}") from exc
child_dir_count = len(child_dirs)
for entry in child_dirs:
path = os.path.normpath(entry.path)
stat = entry.stat()
summary = _count_candidate_tasks(path)
task_count = int(summary.get("task_count") or 0)
invalid_child_count = int(summary.get("invalid_child_count") or 0)
items.append(
{
"name": entry.name,
"path": path,
"task_count": task_count,
"invalid_child_count": invalid_child_count,
"valid": task_count > 0,
"mode": summary.get("mode"),
"error": summary.get("error"),
"mtime": stat.st_mtime,
"updated_at": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"),
}
)
items.sort(key=lambda item: str(item.get("name") or "").lower())
return {
"root": root_dir,
"root_exists": root_exists,
"items": items,
"count": len(items),
"child_dir_count": child_dir_count,
}
@router.post("/engines/pyint/preview-input-assets")
async def preview_pyint_input_assets(
req: PreviewInputAssetsRequest,
backend/app/routers/monitor.py
+409 -120
View File
@@ -1,9 +1,12 @@
from __future__ import annotations
from typing import List, Optional
import os
import shutil
from typing import Any, Iterable, List, Optional
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from pydantic import BaseModel, Field
from sqlalchemy import delete, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
@@ -17,6 +20,15 @@ from .dependencies import _require_admin
router = APIRouter()
MONITOR_ALLOWED_TARGETS = {"radar", "orbit", "dinsar", "gf3"}
MONITOR_SCAN_TASK_TYPES = {
"SCAN_ASSET_INVENTORY",
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"SCAN_DATA",
"SCAN_DINSAR",
"GF3_SARSCAPE_SYNC",
"GF3_QUICKLOOK_WEBP",
}
MONITOR_TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
MONITOR_LOG_DEFAULT_LIMIT = read_int_env(
"MONITOR_LOG_DEFAULT_LIMIT",
50,
@@ -37,10 +49,272 @@ MONITOR_LOG_MAX_OFFSET = read_int_env(
)
def _path_kind(path: str) -> str:
text = str(path or "").strip()
if text.startswith("\\\\"):
return "unc"
drive, _tail = os.path.splitdrive(os.path.normpath(text))
if drive:
return "windows"
if text.startswith("/mnt/"):
return "wsl_mount"
if text.startswith("/"):
return "posix"
return "relative"
def _nearest_existing_path(path: str) -> str:
candidate = os.path.normpath(str(path or "").strip())
while candidate and not os.path.exists(candidate):
parent = os.path.dirname(candidate)
if not parent or parent == candidate:
break
candidate = parent
return candidate
def _storage_status(label: str, path: str, role: str) -> dict[str, Any]:
text = str(path or "").strip()
item: dict[str, Any] = {
"label": label,
"role": role,
"path": text,
"path_kind": _path_kind(text),
"exists": bool(text and os.path.exists(text)),
"probe_path": "",
"total_gb": None,
"used_gb": None,
"free_gb": None,
"free_ratio": None,
"status": "missing" if text else "empty",
"message": "",
}
if not text:
item["message"] = "Path is not configured."
return item
if item["path_kind"] == "unc":
item["status"] = "blocked"
item["message"] = "UNC paths are not allowed for active local production."
return item
probe = _nearest_existing_path(text)
if not probe or not os.path.exists(probe):
item["message"] = "No existing parent path found."
return item
item["probe_path"] = probe
try:
total, used, free = shutil.disk_usage(probe)
except OSError as exc:
item["status"] = "error"
item["message"] = str(exc)
return item
gb = 1024 ** 3
free_ratio = float(free) / float(total or 1)
item.update(
{
"total_gb": round(total / gb, 2),
"used_gb": round(used / gb, 2),
"free_gb": round(free / gb, 2),
"free_ratio": round(free_ratio, 4),
"status": "ok",
"message": "",
}
)
if not item["exists"]:
item["status"] = "missing"
item["message"] = f"Path does not exist; disk usage probed from {probe}."
elif free_ratio < 0.10:
item["status"] = "critical"
item["message"] = "Free space is below 10%."
elif free_ratio < 0.20:
item["status"] = "warning"
item["message"] = "Free space is below 20%."
return item
def _add_storage_roots(rows: list[dict[str, str]], label: str, paths: Iterable[str], role: str) -> None:
for path in paths:
text = str(path or "").strip()
if text:
rows.append({"label": label, "path": text, "role": role})
def _storage_group_identity(item: dict[str, Any]) -> tuple[str, str, str]:
path = str(item.get("path") or "").strip()
path_kind = str(item.get("path_kind") or "")
if path_kind == "windows":
base = str(item.get("probe_path") or path).strip()
drive, _tail = os.path.splitdrive(os.path.normpath(base))
if drive:
drive = drive.upper()
volume_path = f"{drive}\\"
return f"windows:{drive}", volume_path, f"本机磁盘 {volume_path}"
if path_kind == "unc":
key = os.path.normcase(os.path.normpath(path))
return f"unc:{key}", path, "UNC path"
if path_kind in {"posix", "wsl_mount"}:
base = str(item.get("probe_path") or path or "/").strip()
root = "/mnt/" + base.split("/")[2] if base.startswith("/mnt/") and len(base.split("/")) > 2 else "/"
return f"{path_kind}:{root}", root, root
key = os.path.normcase(os.path.normpath(path))
return f"path:{key}", path, str(item.get("label") or item.get("role") or path or "Storage")
def _capacity_status(free_ratio: Any) -> str:
if free_ratio is None:
return "missing"
try:
ratio = float(free_ratio)
except (TypeError, ValueError):
return "missing"
if ratio < 0.10:
return "critical"
if ratio < 0.20:
return "warning"
return "ok"
def _group_storage_statuses(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
groups: dict[str, dict[str, Any]] = {}
order: list[str] = []
for item in items:
group_key, volume_path, volume_label = _storage_group_identity(item)
group = groups.get(group_key)
if group is None:
group = {
"label": volume_label,
"role": "storage_volume",
"path": volume_path,
"path_kind": item.get("path_kind"),
"exists": False,
"probe_path": item.get("probe_path") or "",
"total_gb": None,
"used_gb": None,
"free_gb": None,
"free_ratio": None,
"status": "missing",
"message": "",
"paths": [],
"configured_path_count": 0,
"existing_path_count": 0,
"missing_path_count": 0,
"blocked_path_count": 0,
"error_path_count": 0,
}
groups[group_key] = group
order.append(group_key)
group["configured_path_count"] += 1
if item.get("exists"):
group["existing_path_count"] += 1
if item.get("status") == "missing":
group["missing_path_count"] += 1
elif item.get("status") == "blocked":
group["blocked_path_count"] += 1
elif item.get("status") == "error":
group["error_path_count"] += 1
group["paths"].append(
{
"label": item.get("label"),
"role": item.get("role"),
"path": item.get("path"),
"exists": item.get("exists"),
"status": item.get("status"),
"message": item.get("message"),
}
)
if item.get("total_gb") is not None:
group["exists"] = True
group["probe_path"] = item.get("probe_path") or group["probe_path"]
group["total_gb"] = item.get("total_gb")
group["used_gb"] = item.get("used_gb")
group["free_gb"] = item.get("free_gb")
group["free_ratio"] = item.get("free_ratio")
result: list[dict[str, Any]] = []
for key in order:
group = groups[key]
status = _capacity_status(group.get("free_ratio"))
if group["blocked_path_count"]:
status = "blocked"
elif group["error_path_count"]:
status = "error"
elif group["missing_path_count"] and status == "ok":
status = "partial"
group["status"] = status
messages = [f"{group['configured_path_count']} 个配置路径"]
if group["missing_path_count"]:
messages.append(f"{group['missing_path_count']} 个路径缺失")
if group["blocked_path_count"]:
messages.append(f"{group['blocked_path_count']} 个 UNC 被禁用")
if group["error_path_count"]:
messages.append(f"{group['error_path_count']} 个探测失败")
if status == "critical":
messages.append("剩余空间低于 10%")
elif status == "warning" and not group["missing_path_count"]:
messages.append("剩余空间低于 20%")
group["message"] = "；".join(messages) + "。"
result.append(group)
return result
def _collect_storage_roots() -> list[dict[str, Any]]:
rows: list[dict[str, str]] = []
_add_storage_roots(rows, "Task_Pool", [settings.TASK_POOL_ROOT], "task_pool")
_add_storage_roots(rows, "D-InSAR Task_Pool", [settings.DINSAR_TASK_POOL_ROOT], "dinsar_task_pool")
_add_storage_roots(rows, "SBAS Task_Pool", [settings.SBAS_TASK_POOL_ROOT], "sbas_task_pool")
_add_storage_roots(rows, "Data distribution root", [settings.DATA_DISTRIBUTION_ROOT], "data_distribution")
source_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
source_path_keys = {os.path.normcase(os.path.normpath(path)) for path in source_paths}
s1_storage_paths = [
path
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
if os.path.normcase(os.path.normpath(path)) not in source_path_keys
]
_add_storage_roots(rows, "LT/S1 local source pools", source_paths, "source_local")
_add_storage_roots(rows, "Sentinel-1 local source pool", s1_storage_paths, "source_local")
_add_storage_roots(rows, "Orbit source pool", split_env_paths(settings.ORBIT_SOURCE_DIRS), "orbit_source")
_add_storage_roots(rows, "LT-1 radar scan pool", split_env_paths(settings.MONITOR_RADAR_DIRS), "lt1_storage")
_add_storage_roots(rows, "D-InSAR product root", [settings.DINSAR_PRODUCT_DIR], "dinsar_product")
_add_storage_roots(rows, "GF3 native _geo", split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), "gf3_native")
_add_storage_roots(rows, "GF3 standard/index", split_env_paths(settings.GF3_STORAGE_DIRS), "gf3_storage")
_add_storage_roots(rows, "GF3 task/runtime pool", [settings.GF3_TASK_POOL_ROOT], "gf3_task_pool")
_add_storage_roots(rows, "GF3 SARscape runtime", [settings.GF3_SARSCAPE_RUNTIME_DIR], "gf3_runtime")
_add_storage_roots(rows, "Result publish root", [settings.RESULT_PUBLISH_ROOT], "result_publish")
_add_storage_roots(rows, "SBAS work root", [settings.GAMMA_SBAS_WORK_ROOT], "sbas_work")
_add_storage_roots(rows, "SBAS product root", [settings.GAMMA_SBAS_PRODUCT_ROOT], "sbas_product")
_add_storage_roots(rows, "SAR analysis ready", [settings.SAR_ANALYSIS_READY_ROOT], "analysis_ready")
_add_storage_roots(rows, "SAR analysis work", [settings.SAR_ANALYSIS_WORK_ROOT], "analysis_work")
seen: set[str] = set()
result: list[dict[str, Any]] = []
for row in rows:
key = os.path.normcase(os.path.normpath(row["path"]))
if key in seen:
continue
seen.add(key)
result.append(_storage_status(row["label"], row["path"], row["role"]))
return _group_storage_statuses(result)
class MonitorConfig(BaseModel):
radar_dirs: List[str] = []
orbit_dir: Optional[str] = None
orbit_source_dirs: List[str] = []
orbit_production_txt_pool: Optional[str] = None
dinsar_dirs: List[str] = []
dinsar_product_dir: Optional[str] = None
sbas_product_root: Optional[str] = None
task_pool_root: Optional[str] = None
dinsar_task_pool_root: Optional[str] = None
sbas_task_pool_root: Optional[str] = None
gf3_task_pool_root: Optional[str] = None
data_distribution_root: Optional[str] = None
gf3_archive_source_dirs: List[str] = []
gf3_source_dirs: List[str] = []
gf3_legacy_gdal_enabled: bool = False
@@ -53,6 +327,7 @@ class MonitorConfig(BaseModel):
gf3_sarscape_polarizations: Optional[str] = None
gf3_sarscape_auto_standardize: bool = True
gf3_sarscape_clean_after_success: bool = True
storage_roots: List[dict[str, Any]] = []
# Manual-only: config is read from .env
@@ -70,6 +345,13 @@ class GF3UnpackRunRequest(BaseModel):
class GF3SarscapeSyncRequest(BaseModel):
force: bool = False
register: bool = True
quicklook_only: bool = False
native_dirs: List[str] = []
class GF3QuicklookWebpRequest(BaseModel):
force: bool = False
max_records: Optional[int] = Field(default=None, ge=0)
class GF3SarscapeProduceRequest(BaseModel):
@@ -91,7 +373,7 @@ class GF3SarscapeCleanRequest(BaseModel):
@router.post("/monitor/config")
async def update_monitor_config(config: MonitorConfig):
"""
更新数据监控配置（雷达、精轨和Dinsar结果）。
鏇存柊鏁版嵁鐩戞帶閰嶇疆锛堥浄杈俱€佺簿杞ㄥ拰Dinsar缁撴灉锛夈€?
"""
raise HTTPException(
status_code=403,
@@ -102,8 +384,8 @@ async def update_monitor_config(config: MonitorConfig):
@router.post("/monitor/run-now")
async def run_monitor_now(target: Optional[str] = None, background_tasks: BackgroundTasks = None, admin_user: AuthUserORM = Depends(_require_admin)):
"""
手动触发一次监控任务（扫描所有配置的目录）。
target: 'radar', 'orbit', 'dinsar' 或 None (全部)
鎵嬪姩瑙﹀彂涓€娆＄洃鎺т换鍔★紙鎵弿鎵€鏈夐厤缃殑鐩綍锛夈€?
target: 'radar', 'orbit', 'dinsar' 鎴?None (鍏ㄩ儴)
"""
normalized_target = (target or "").strip().lower() or None
if normalized_target and normalized_target not in MONITOR_ALLOWED_TARGETS:
@@ -127,7 +409,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
raise HTTPException(status_code=400, detail="Monitor paths are not configured in .env.")
task_type = "SCAN_DATA" if normalized_target in ["radar", "orbit", "gf3", None] else "SCAN_DINSAR"
task_name = f"手动触发扫描 ({normalized_target or '全部'})"
task_name = f"Manual scan ({normalized_target or 'all'})"
try:
task_id = await task_service.create_task(task_type, task_name)
@@ -135,7 +417,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
payload = {"dirs": dinsar_dirs}
job_type = "SCAN_DINSAR"
elif normalized_target == "gf3":
# GF3 扫描：用 gf3_storage_dirs 作为 radar_dirs
# GF3 scan uses gf3_storage_dirs as radar_dirs.
payload = {
"radar_dirs": gf3_storage_dirs,
"target": "radar",
@@ -150,7 +432,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
await job_queue_service.create_job(job_type, payload=payload, task_id=task_id)
return {
"message": f"已触发{normalized_target or '全部'}手动扫描任务（已进入队列）",
"message": f"Manual scan queued for {normalized_target or 'all'}.",
"task_id": task_id
}
except ValueError as e:
@@ -160,14 +442,14 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr
@router.post("/monitor/gf3-process")
async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin)):
"""
批量 GF3 L1A→L2 处理：扫描 GF3_SOURCE_DIRS，过滤已处理的，逐个处理并自动入库。
Batch GF3 legacy L1A to L2 processing.
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 Python/GDAL preprocessing is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or []
@@ -175,7 +457,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.")
task_type = "GF3_BATCH_PROCESS"
task_name = "GF3 批量 L1A→L2 处理"
task_name = "GF3 legacy batch process"
try:
task_id = await task_service.create_task(task_type, task_name)
@@ -185,7 +467,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin
task_id=task_id,
)
return {
"message": "GF3 批量处理任务已提交",
"message": "GF3 batch process task submitted",
"task_id": task_id,
}
except ValueError as e:
@@ -198,147 +480,100 @@ async def run_gf3_sarscape_sync(
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
扫描 GF3 SARscape 原生 _geo 二进制池，转换为标准 GeoTIFF，并登记入库。
Scan GF3 SARscape native _geo outputs.
"""
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
options = request_data or GF3SarscapeSyncRequest()
gf3_native_dirs = options.native_dirs or MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
options = request_data or GF3SarscapeSyncRequest()
task_type = "GF3_SARSCAPE_SYNC"
task_name = "GF3 SARscape 原生结果标准化"
task_name = "GF3 SARscape native result inventory" if options.quicklook_only else "GF3 SARscape native standardize"
payload = {
"native_dirs": gf3_native_dirs,
"storage_root": gf3_storage_dirs[0],
"force": bool(options.force),
"register": bool(options.register),
"quicklook_only": bool(options.quicklook_only),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape 原生结果标准化任务已提交",
"message": (
"GF3 SARscape native result inventory task submitted"
if options.quicklook_only
else "GF3 SARscape native standardize task submitted"
),
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-quicklook-webp", status_code=202)
async def run_gf3_quicklook_webp(
request_data: GF3QuicklookWebpRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Generate local WebP cache files from registered GF3 SARscape native _geo records.
"""
options = request_data or GF3QuicklookWebpRequest()
task_type = "GF3_QUICKLOOK_WEBP"
task_name = "GF3 native _geo WebP cache"
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
payload = {
"force": bool(options.force),
"max_records": int(options.max_records or 0),
"native_dirs": gf3_native_dirs,
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 native _geo WebP cache task submitted",
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.post("/monitor/gf3-sarscape-produce", status_code=202)
async def run_gf3_sarscape_produce(
request_data: GF3SarscapeProduceRequest | None = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates.
GF3 SARscape production is disabled on this management machine.
"""
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or []
gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
if not gf3_native_dirs:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.")
if not gf3_storage_dirs:
raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.")
if not settings.GF3_SARSCAPE_WRAPPER_EXE:
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.")
if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH):
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
options = request_data or GF3SarscapeProduceRequest()
selected_dates = []
for raw_date in options.selected_dates or []:
text = str(raw_date or "").strip()
if not text:
continue
normalized = text.replace("-", "").replace("_", "")
if len(normalized) != 8 or not normalized.isdigit():
raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}")
if normalized not in selected_dates:
selected_dates.append(normalized)
task_type = "GF3_SARSCAPE_PRODUCE"
task_name = "GF3 SARscape production"
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success)
payload = {
"source_dirs": gf3_archive_source_dirs,
"native_dirs": gf3_native_dirs,
"native_root": gf3_native_dirs[0],
"storage_root": gf3_storage_dirs[0],
"wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE,
"idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH,
"dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH,
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
"max_archives_per_run": int(options.max_archives_per_run or 0),
"selected_dates": selected_dates,
"local_staging_root": settings.GF3_TASK_POOL_ROOT,
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
"auto_standardize": bool(auto_standardize),
"clean_after_success": bool(clean_after_success),
"force_standardize": bool(options.force_standardize),
"register": bool(options.register),
"cleanup_require_standardized": True,
"cleanup_dry_run": bool(options.cleanup_dry_run),
}
try:
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": (
f"GF3 SARscape production task submitted for {', '.join(selected_dates)}"
if selected_dates
else "GF3 SARscape production task submitted"
),
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
raise HTTPException(
status_code=409,
detail=(
"GF3 SARscape production is disabled on this management machine. "
"Run GF3 production on the SARscape host, copy completed _geo results to local "
"GF3_SARSCAPE_NATIVE_DIRS, then use GF3 native result registration and GF3 _geo WebP generation."
),
)
@router.get("/monitor/gf3-sarscape-dates")
async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)):
"""
List available GF3 SARscape source dates from configured raw archive roots.
GF3 SARscape production date selection is disabled on this management machine.
"""
from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
discovery = discover_gf3_sarscape_inputs(
gf3_archive_source_dirs,
archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS),
raise HTTPException(
status_code=409,
detail="GF3 SARscape production date selection is disabled. Register local _geo native results instead.",
)
by_date: dict[str, dict[str, object]] = {}
undated = 0
for item in discovery.get("inputs") or []:
scene_name = str(item.get("scene_name") or "")
date_text = str(item.get("scene_date") or "")
if not date_text:
undated += 1
continue
bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []})
bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1
scenes = bucket.get("scenes")
if isinstance(scenes, list) and len(scenes) < 20:
scenes.append(scene_name)
dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True)
return {
"dates": dates,
"input_count": discovery.get("input_count") or 0,
"undated_count": undated,
"missing_roots": discovery.get("missing_roots") or [],
}
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
@@ -394,14 +629,14 @@ async def run_gf3_unpack(
admin_user: AuthUserORM = Depends(_require_admin),
):
"""
将 GF3 压缩包池解包到 GF3_SOURCE_DIRS，作为后续 L1A→L2 预处理输入。
Unpack GF3 archives into GF3_SOURCE_DIRS for the legacy pipeline.
"""
if not settings.GF3_LEGACY_GDAL_ENABLED:
raise HTTPException(
status_code=409,
detail=(
"Legacy GF3 archive unpack is disabled. "
"Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
"Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly."
),
)
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
@@ -416,7 +651,7 @@ async def run_gf3_unpack(
max_files = max(0, int(request_data.max_files_per_run))
task_type = "GF3_UNPACK"
task_name = "GF3 压缩包解包"
task_name = "GF3 archive unpack"
payload = {
"source_dirs": gf3_archive_source_dirs,
"target_dirs": gf3_source_dirs,
@@ -429,7 +664,7 @@ async def run_gf3_unpack(
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 解包任务已提交",
"message": "GF3 unpack task submitted",
"task_id": task_id,
}
except ValueError as e:
@@ -439,9 +674,24 @@ async def run_gf3_unpack(
@router.get("/monitor/status")
async def get_monitor_status():
"""
获取当前监控状态。
鑾峰彇褰撳墠鐩戞帶鐘舵€併€?
"""
return MONITOR_CONFIG
config = dict(MONITOR_CONFIG)
config.update(
{
"task_pool_root": settings.TASK_POOL_ROOT,
"dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT,
"sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT,
"gf3_task_pool_root": settings.GF3_TASK_POOL_ROOT,
"data_distribution_root": settings.DATA_DISTRIBUTION_ROOT,
"dinsar_product_dir": settings.DINSAR_PRODUCT_DIR,
"sbas_product_root": settings.GAMMA_SBAS_PRODUCT_ROOT,
"orbit_source_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS) or split_env_paths(settings.MONITOR_ORBIT_DIR),
"orbit_production_txt_pool": settings.ORBIT_POOL_ENVI,
"storage_roots": _collect_storage_roots(),
}
)
return config
@router.get("/monitor/logs")
@@ -451,7 +701,7 @@ async def get_monitor_logs(
db: AsyncSession = Depends(get_db),
):
"""
获取最新的监控日志 (来自任务日志表)。
鑾峰彇鏈€鏂扮殑鐩戞帶鏃ュ織 (鏉ヨ嚜浠诲姟鏃ュ織琛?銆?
"""
safe_limit = min(MONITOR_LOG_MAX_LIMIT, max(1, int(limit or MONITOR_LOG_DEFAULT_LIMIT)))
safe_offset = min(MONITOR_LOG_MAX_OFFSET, max(0, int(offset or 0)))
@@ -478,3 +728,42 @@ async def get_monitor_logs(
"count": len(logs),
"logs": logs,
}
@router.delete("/monitor/scan-task-history")
async def clear_monitor_scan_task_history(
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
"""
Clear finished scan task records and their logs from the monitor panel.
Running and pending tasks are intentionally preserved.
"""
task_rows = await db.execute(
select(SystemTaskORM.task_id)
.where(SystemTaskORM.task_type.in_(MONITOR_SCAN_TASK_TYPES))
.where(SystemTaskORM.status.in_(MONITOR_TERMINAL_TASK_STATUSES))
)
task_ids = [str(task_id) for task_id in task_rows.scalars().all() if task_id]
if not task_ids:
return {
"deleted_task_count": 0,
"deleted_log_count": 0,
"preserved_active": True,
}
log_count_result = await db.execute(
select(func.count(TaskLogORM.id)).where(TaskLogORM.task_id.in_(task_ids))
)
deleted_log_count = int(log_count_result.scalar_one() or 0)
await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids)))
task_delete_result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids)))
await db.commit()
return {
"deleted_task_count": int(task_delete_result.rowcount or len(task_ids)),
"deleted_log_count": deleted_log_count,
"preserved_active": True,
}
backend/app/routers/orbit.py
+18 -10
View File
@@ -35,6 +35,8 @@ class OrbitPoolActionRequest(BaseModel):
async def _build_orbit_database_stats(
db: AsyncSession,
pool_inventory: Dict[str, Any],
*,
isce2_enabled: bool = False,
) -> Dict[str, Any]:
total_radar_count = (
await db.execute(select(func.count(RadarDataORM.id)))
@@ -94,9 +96,9 @@ async def _build_orbit_database_stats(
}
envi_stems = set(pool_inventory["envi"]["files"].keys())
isce2_stems = set(pool_inventory["isce2"]["files"].keys())
isce2_stems = set(pool_inventory["isce2"]["files"].keys()) if isce2_enabled else set()
missing_in_envi = sorted(db_expected_stems - envi_stems)
missing_in_isce2 = sorted(db_expected_stems - isce2_stems)
missing_in_isce2 = sorted(db_expected_stems - isce2_stems) if isce2_enabled else []
return {
"total_radar_count": int(total_radar_count or 0),
@@ -110,6 +112,7 @@ async def _build_orbit_database_stats(
"db_expected_stem_count": len(db_expected_stems),
"stems_missing_in_envi_count": len(missing_in_envi),
"stems_missing_in_isce2_count": len(missing_in_isce2),
"isce2_enabled": bool(isce2_enabled),
"sample_missing_in_envi": missing_in_envi[:20],
"sample_missing_in_isce2": missing_in_isce2[:20],
"path_errors": db_path_errors,
@@ -123,26 +126,27 @@ async def get_orbit_status(
):
"""Return source, pool, consistency, and database orbit status."""
source_dir = settings.MONITOR_ORBIT_DIR
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
source_stats = await asyncio.to_thread(scan_orbit_dir, source_dir)
pool_inventory = await asyncio.to_thread(
get_orbit_pool_inventory,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
True,
)
consistency = await asyncio.to_thread(
check_orbit_consistency,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
)
source_gap_summary = await asyncio.to_thread(
summarize_source_orbit_gaps,
source_dir,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_QUARANTINE_DIR,
)
database_stats = await _build_orbit_database_stats(db, pool_inventory)
database_stats = await _build_orbit_database_stats(db, pool_inventory, isce2_enabled=bool(settings.ISCE2_ENABLED))
return {
"orbit_root": source_stats.orbit_root,
@@ -184,7 +188,8 @@ async def get_orbit_status(
"errors": pool_inventory["envi"]["errors"],
},
"isce2": {
"path": settings.ORBIT_POOL_ISCE2,
"path": isce2_pool,
"enabled": bool(settings.ISCE2_ENABLED),
"total": pool_inventory["isce2"]["total"],
"duplicate_count": pool_inventory["isce2"]["duplicate_count"],
"errors": pool_inventory["isce2"]["errors"],
@@ -207,25 +212,28 @@ async def sync_orbit_pool_action(
Check pool consistency, or repair missing entries when repair=true.
"""
if payload and payload.quarantine_bad:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
quarantine_bad_orbits,
settings.MONITOR_ORBIT_DIR,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_QUARANTINE_DIR,
)
if payload and payload.repair:
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
repair_orbit_pools,
settings.MONITOR_ORBIT_DIR,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
settings.ORBIT_POOL_LANDSAR,
)
isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else ""
return await asyncio.to_thread(
check_orbit_consistency,
settings.ORBIT_POOL_ENVI,
settings.ORBIT_POOL_ISCE2,
isce2_pool,
)
backend/app/routers/pairing.py
+5 -5
View File
@@ -40,10 +40,10 @@ router = APIRouter()
def get_pairing_request_from_form(
time_baseline_min: int = Form(1),
time_baseline_max: int = Form(90),
time_baseline_max: int = Form(30),
overlap_threshold: float = Form(0.5),
spatial_baseline_max_meters: int = Form(3000),
limit_footprint_center_distance: bool = Form(False),
spatial_baseline_max_meters: int = Form(5000),
limit_footprint_center_distance: bool = Form(True),
max_temporal_baseline_days: Optional[int] = Form(None),
pair_footprint_overlap_min_ratio: Optional[float] = Form(None),
footprint_center_distance_max_meters: Optional[int] = Form(None),
@@ -57,7 +57,7 @@ def get_pairing_request_from_form(
master_date_to: Optional[str] = Form(None),
slave_date_from: Optional[str] = Form(None),
slave_date_to: Optional[str] = Form(None),
strategy: str = Form("all"),
strategy: str = Form("dinsar_production"),
num_connections: int = Form(1),
reference_image_id: Optional[int] = Form(None),
allowed_satellites: Optional[str] = Form(None), # JSON string
@@ -91,7 +91,7 @@ def get_pairing_request_from_form(
master_date_to=master_date_to,
slave_date_from=slave_date_from,
slave_date_to=slave_date_to,
strategy=strategy,
strategy="dinsar_production",
num_connections=num_connections,
reference_image_id=reference_image_id,
allowed_satellites=satellites_list,
backend/app/routers/radar.py
+65
View File
@@ -41,6 +41,7 @@ from .dependencies import (
router = APIRouter()
logger = logging.getLogger(__name__)
GF3_NATIVE_PREVIEW_SOURCE_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW"
LIST_QUERY_MAX_LIMIT = read_int_env(
"LIST_QUERY_MAX_LIMIT",
2000,
@@ -188,6 +189,38 @@ def _radar_preview_paths(record: RadarDataORM) -> Tuple[str, str]:
return raw_cache_path, geo_cache_path
def _is_gf3_native_preview_record(record: RadarDataORM) -> bool:
return str(record.source_format or "") == GF3_NATIVE_PREVIEW_SOURCE_FORMAT
def _build_gf3_native_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo:
preview_path = str(record.preview_cache_path or "")
has_native_cache = (
(record.preview_cache_status or "NONE") == "READY"
and preview_path.lower().endswith(".webp")
and os.path.exists(preview_path)
)
metadata = record.metadata_json or {}
source_path = str(metadata.get("default_native_path") or "")
source_found = bool(source_path and os.path.exists(source_path))
return RadarPreviewStatusInfo(
radar_id=record.id,
status="READY" if has_native_cache else (record.preview_cache_status or "NONE"),
cache_version=record.preview_cache_version,
cache_updated_at=record.preview_cache_updated_at,
has_geo_cache=has_native_cache,
has_raw_cache=has_native_cache,
source_found=source_found,
fallback_in_use=False,
message=(
"GF3 native _geo WebP cache is available."
if has_native_cache
else "GF3 native _geo WebP cache has not been generated."
),
error=None if has_native_cache else record.preview_cache_error,
)
def _build_radar_preview_status(
record: RadarDataORM,
source_found: bool,
@@ -228,6 +261,9 @@ async def _build_radar_preview_cache(
db: AsyncSession,
force: bool = False,
) -> RadarPreviewStatusInfo:
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
@@ -355,7 +391,33 @@ async def _get_cached_radar_preview(data_id: int, db: AsyncSession):
if not record:
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
if _is_gf3_native_preview_record(record):
preview_path = str(record.preview_cache_path or "")
if (
(record.preview_cache_status or "NONE") == "READY"
and preview_path.lower().endswith(".webp")
and os.path.exists(preview_path)
):
return FileResponse(
preview_path,
media_type="image/webp",
headers={"Cache-Control": "public, max-age=31536000"},
)
raise HTTPException(status_code=404, detail="GF3 native _geo WebP cache has not been generated.")
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
if (
(record.preview_cache_status or "NONE") == "READY"
and record.preview_cache_path
and str(record.preview_cache_path).lower().endswith(".webp")
and os.path.exists(record.preview_cache_path)
):
return FileResponse(
record.preview_cache_path,
media_type="image/webp",
headers={"Cache-Control": "public, max-age=31536000"},
)
if os.path.exists(geo_cache_path):
return FileResponse(
geo_cache_path,
@@ -664,6 +726,9 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep
if not record:
raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。")
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
backend/app/routers/tools.py
+77 -19
View File
@@ -1,12 +1,14 @@
from __future__ import annotations
import os
import re
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field, field_validator
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import read_int_env
from ..config import read_int_env, settings
from ..database import get_db
from ..services.job_queue_service import job_queue_service
from ..services.task_service import TASK_LOG_DEFAULT_LIMIT, TASK_LOG_MAX_LIMIT, TASK_QUERY_MAX_OFFSET, task_service
@@ -33,12 +35,14 @@ COPY_BATCH_MAX_COPY_ITEMS = read_int_env(
minimum=1,
maximum=200000,
)
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "task_zip", "source_bundle"}
COPY_DINSAR_PACKAGE_MODES = {"task_folder", "source_bundle"}
_COPY_TARGET_NAME_RE = re.compile(r"[^0-9A-Za-z\u4e00-\u9fff._-]+")
class CopyBatchRequest(BaseModel):
batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
dest_dir: str = Field(default="", max_length=COPY_BATCH_TEXT_MAX_LENGTH)
target_name: Optional[str] = Field(default=None, max_length=255)
copy_statuses: Optional[List[str]] = None
include_orbit_files: bool = False
export_zip: bool = False
@@ -46,7 +50,7 @@ class CopyBatchRequest(BaseModel):
skip_existing: bool = True
max_items: Optional[int] = None
@field_validator("batch_id", "dest_dir", mode="before")
@field_validator("batch_id", mode="before")
@classmethod
def _normalize_required_text(cls, value):
normalized = str(value or "").strip()
@@ -54,6 +58,17 @@ class CopyBatchRequest(BaseModel):
raise ValueError("Field must not be empty.")
return normalized
@field_validator("dest_dir", mode="before")
@classmethod
def _normalize_optional_dest_dir(cls, value):
return str(value or "").strip()
@field_validator("target_name", mode="before")
@classmethod
def _normalize_optional_target_name(cls, value):
normalized = str(value or "").strip()
return normalized or None
@field_validator("copy_statuses", mode="before")
@classmethod
def _validate_copy_statuses_length(cls, value):
@@ -88,6 +103,10 @@ class CopyBatchRequest(BaseModel):
@classmethod
def _normalize_package_mode(cls, value):
normalized = str(value or "task_folder").strip().lower()
if normalized == "task_zip":
normalized = "task_folder"
if normalized in {"bundle", "dedupe_source"}:
normalized = "source_bundle"
if normalized not in COPY_DINSAR_PACKAGE_MODES:
raise ValueError(
f"package_mode must be one of: {sorted(COPY_DINSAR_PACKAGE_MODES)}."
@@ -115,6 +134,28 @@ def _normalize_copy_batch_statuses(copy_statuses: Optional[List[str]]) -> List[s
return normalized or ["COMPLETED"]
def _safe_copy_target_name(value: Optional[str]) -> str:
raw = str(value or "").strip()
if not raw:
raise HTTPException(status_code=400, detail="target_name 不能为空")
if os.path.isabs(raw) or os.path.splitdrive(raw)[0] or "\\" in raw or "/" in raw:
raise HTTPException(status_code=400, detail="target_name 只能是任务名，不能包含路径")
normalized = _COPY_TARGET_NAME_RE.sub("_", raw).strip("._ ")
if not normalized:
raise HTTPException(status_code=400, detail="target_name 不合法")
if normalized.upper() in {"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "LPT1", "LPT2", "LPT3"}:
raise HTTPException(status_code=400, detail="target_name 是 Windows 保留名称")
return normalized[:120]
def _server_copy_destination(package_mode: str, target_name: Optional[str]) -> str:
safe_name = _safe_copy_target_name(target_name)
root = settings.DINSAR_TASK_POOL_ROOT if package_mode == "task_folder" else settings.DATA_DISTRIBUTION_ROOT
if not root:
raise HTTPException(status_code=500, detail="服务器目标根目录未配置")
return os.path.normpath(os.path.join(root, safe_name))
@router.post("/tools/copy-ps-stack")
async def copy_ps_stack_endpoint(
request: CopyBatchRequest,
@@ -124,20 +165,26 @@ async def copy_ps_stack_endpoint(
"""
Start PS-InSAR copy task from a batch.
"""
_validate_export_path(request.dest_dir, "dest_dir")
try:
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
if request.target_name:
dest_dir = _server_copy_destination("source_bundle", request.target_name)
else:
dest_dir = request.dest_dir
_validate_export_path(dest_dir, "dest_dir")
params = {
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"file_type": "PS_STACK",
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
}
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.dest_dir}", params=params)
task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.target_name or dest_dir}", params=params)
payload = {
"file_type": "PS_STACK",
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
}
@@ -150,7 +197,8 @@ async def copy_ps_stack_endpoint(
detail={
"task_id": task_id,
"batch_id": request.batch_id,
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"copy_statuses": copy_statuses,
},
)
@@ -169,32 +217,41 @@ async def copy_dinsar_pairs_endpoint(
"""
Start D-InSAR copy task from a batch.
"""
_validate_export_path(request.dest_dir, "dest_dir")
try:
copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses)
package_mode = request.package_mode
if bool(request.export_zip) and package_mode == "task_folder":
package_mode = "task_zip"
if request.target_name:
dest_dir = _server_copy_destination(package_mode, request.target_name)
else:
dest_dir = request.dest_dir
_validate_export_path(dest_dir, "dest_dir")
params = {
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"file_type": "DINSAR_PAIRS",
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,
}
task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params)
task_name = (
f"D-InSAR 生产数据准备: {request.target_name or dest_dir}"
if package_mode == "task_folder"
else f"D-InSAR 数据分发: {request.target_name or dest_dir}"
)
task_id = await task_service.create_task("COPY_DATA", task_name, params=params)
payload = {
"file_type": "DINSAR_PAIRS",
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,
@@ -208,10 +265,11 @@ async def copy_dinsar_pairs_endpoint(
detail={
"task_id": task_id,
"batch_id": request.batch_id,
"dest_dir": request.dest_dir,
"dest_dir": dest_dir,
"target_name": request.target_name,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": package_mode == "task_zip",
"export_zip": False,
"package_mode": package_mode,
"skip_existing": bool(request.skip_existing),
"max_items": request.max_items,