chore: initialize insar management system v2

This commit is contained in:
2026-04-14 13:16:01 +08:00
commit ecc72ec9cd
361 changed files with 2142522 additions and 0 deletions
+25
View File
@@ -0,0 +1,25 @@
# Services module
# 空间计算服务
from .spatial_service import spatial_service
from .spatial_service import SpatialService
# 数据导入服务
from .data_service import data_service
from .data_service import DataService
# 图像处理服务
from .image_service import image_service
from .image_service import ImageService
# 任务管理服务
from .task_service import task_service
__all__ = [
"spatial_service",
"SpatialService",
"data_service",
"DataService",
"image_service",
"ImageService",
"task_service",
]
+154
View File
@@ -0,0 +1,154 @@
"""
AI 诊断 Prompt 模板配置。
支持三种预设模板:quick(快速)、standard(标准)、detailed(详细)。
"""
from typing import Dict
# Prompt 模板字典
PROMPT_TEMPLATES: Dict[str, str] = {
"quick": """请快速分析这张 D-InSAR 形变图,重点关注:
1. 是否存在明显的地表形变区域(红色/蓝色异常区)
2. 形变的大致范围和强度
3. 初步的风险等级评估(低/中/高)
请用简洁的语言(3-5 句话)给出结论。""",
"standard": """请分析这张 D-InSAR 地表形变监测结果图,提供以下信息:
**1. 形变特征识别**
- 识别图中的形变区域(红色表示远离卫星/沉降,蓝色表示靠近卫星/抬升,绿色表示稳定)
- 描述形变的空间分布模式、强度范围和影响面积
- 判断形变是否连续、是否存在突变边界、是否有明显的形变中心
- 分析形变的几何形态特征(线性、圆形、不规则等)
**2. 风险评估**
- 根据形变幅度(参考色标 ±0.1m)、分布特征和梯度,评估风险等级
* 低风险:形变量 < 2cm,分布零散
* 中风险:形变量 2-5cm,局部集中
* 高风险:形变量 5-10cm,连续分布
* 极高风险:形变量 > 10cm,急剧变化
- 识别可能的地质灾害类型(滑坡、地面沉降、地裂缝、采空区塌陷等)
- 结合已知隐患点位置,分析形变与灾害点的空间关联性
- 指出需要重点关注的区域和紧急程度
**3. 数据质量评价**
- 评估图像的整体相干性(观察是否有大面积失相干区域)
- 判断噪声水平(是否有明显的随机噪声或条纹状伪影)
- 识别可能的误差来源(大气延迟、轨道误差、解缠错误等)
- 给出结果可靠性评分(质量评分: X/10)
- 说明评分依据和结果的适用性
**4. 监测建议**
- 针对识别的高风险区域,建议后续监测频率
- 推荐需要补充的地面调查或其他监测手段
- 提出预警阈值建议
**输出格式要求:**
- 使用 Markdown 格式,包含清晰的章节标题(使用 ### 三级标题)
- 在风险评估部分明确标注:**风险等级: [低/中/高/极高]**
- 在质量评价部分明确标注:**质量评分: X/10**
- 如果有高置信度的判断,标注:**置信度: XX%**
- 使用列表和加粗强调关键信息
- 保持专业、客观、准确的语言风格""",
"detailed": """请对这张 D-InSAR 地表形变监测结果进行全面深入的专业分析:
**一、形变场特征分析**
1. **形变区域识别**
- 详细描述所有可识别的形变区域(位置、范围、形态)
- 区分主要形变区和次要形变区
- 分析形变的空间连续性和边界特征
2. **形变量级评估**
- 估算最大形变量(视线向位移)
- 分析形变梯度和变化率
- 识别形变集中区和过渡区
3. **形变模式判读**
- 判断形变类型(沉降、抬升、水平位移等)
- 分析形变的对称性和方向性
- 识别可能的形变驱动机制
**二、地质灾害风险评估**
1. **灾害类型识别**
- 根据形变特征推断可能的地质灾害类型
- 分析灾害的发育阶段和活动性
- 评估灾害的潜在影响范围
2. **风险等级评定**
- 综合形变幅度、速率、分布给出风险等级(低/中/高/极高)
- 指出高风险区域和需要紧急关注的位置
- 提供风险分级的依据和置信度
3. **监测建议**
- 建议后续监测的重点区域
- 推荐监测频率和技术手段
- 提出需要补充的地面调查内容
**三、数据质量与可靠性评价**
1. **相干性分析**
- 评估整体相干性水平
- 识别失相干区域及其原因
- 判断结果的空间覆盖完整性
2. **噪声与误差**
- 识别大气延迟、轨道误差等系统性误差
- 评估随机噪声水平
- 判断是否存在解缠错误
3. **可靠性评分**
- 给出数据质量综合评分(1-10 分)
- 说明评分依据
- 指出结果的适用范围和局限性
**四、综合结论与建议**
- 总结关键发现
- 给出明确的风险判断
- 提供决策支持建议
请以专业、严谨的语言输出完整的 Markdown 格式报告。"""
}
def get_prompt_template(template_name: str) -> str:
"""
获取 Prompt 模板。
Args:
template_name: 模板名称(quick/standard/detailed)
Returns:
Prompt 文本
Raises:
ValueError: 模板名称不存在
"""
if template_name not in PROMPT_TEMPLATES:
raise ValueError(
f"Unknown prompt template: {template_name}. "
f"Available templates: {', '.join(PROMPT_TEMPLATES.keys())}"
)
return PROMPT_TEMPLATES[template_name]
def list_prompt_templates() -> Dict[str, Dict[str, str]]:
"""
列出所有可用的 Prompt 模板及其描述。
Returns:
模板字典,格式:{name: {description, preview}}
"""
return {
"quick": {
"description": "快速分析(3-5 句话,适合批量筛查)",
"preview": PROMPT_TEMPLATES["quick"][:100] + "..."
},
"standard": {
"description": "标准分析(包含形变识别、风险评估、质量评价)",
"preview": PROMPT_TEMPLATES["standard"][:100] + "..."
},
"detailed": {
"description": "详细分析(全面深入的专业报告,适合重点区域)",
"preview": PROMPT_TEMPLATES["detailed"][:100] + "..."
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,313 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import DinsarResultORM, ResultProductORM
_BBOX_TOLERANCE = 1e-6
@dataclass(frozen=True)
class _ProjectedProduct:
product: ResultProductORM
name: str
file_path: str
def _safe_text(value: Any, *, default: str = "") -> str:
text = str(value or "").strip()
return text or default
def _safe_float(value: Any) -> Optional[float]:
try:
if value in (None, ""):
return None
return float(value)
except (TypeError, ValueError):
return None
def _same_float(left: Any, right: Any, *, tol: float = _BBOX_TOLERANCE) -> bool:
left_num = _safe_float(left)
right_num = _safe_float(right)
if left_num is None and right_num is None:
return True
if left_num is None or right_num is None:
return False
return abs(left_num - right_num) <= tol
def _same_bbox(row: DinsarResultORM, product: ResultProductORM) -> bool:
return (
_same_float(row.min_lon, product.min_lon)
and _same_float(row.min_lat, product.min_lat)
and _same_float(row.max_lon, product.max_lon)
and _same_float(row.max_lat, product.max_lat)
)
class DinsarCompatService:
def _project_product(self, product: ResultProductORM) -> Optional[_ProjectedProduct]:
file_path = _safe_text(
product.primary_asset_path
or product.source_primary_path
or product.manifest_path
)
if not file_path:
return None
name = _safe_text(
product.display_name
or product.task_alias
or product.task_name
or product.product_id,
default=product.product_id,
)
return _ProjectedProduct(
product=product,
name=name,
file_path=file_path,
)
def _apply_product_fields(self, row: DinsarResultORM, projected: _ProjectedProduct) -> bool:
product = projected.product
changed = False
for field_name, field_value in (
("compat_product_id", product.product_id),
("name", projected.name),
("file_path", projected.file_path),
("min_lon", product.min_lon),
("min_lat", product.min_lat),
("max_lon", product.max_lon),
("max_lat", product.max_lat),
("coverage_polygon", product.coverage_polygon),
("geom", product.geom),
):
if getattr(row, field_name) != field_value:
setattr(row, field_name, field_value)
changed = True
return changed
def _mirror_legacy_annotations(self, row: DinsarResultORM, product: ResultProductORM) -> bool:
changed = False
for field_name in ("ai_score", "user_label"):
row_value = getattr(row, field_name)
if getattr(product, field_name) != row_value:
setattr(product, field_name, row_value)
changed = True
return changed
def _mirror_product_annotations_to_row(self, product: ResultProductORM, row: DinsarResultORM) -> bool:
changed = False
for field_name in ("ai_score", "user_label"):
product_value = getattr(product, field_name)
if getattr(row, field_name) != product_value:
setattr(row, field_name, product_value)
changed = True
return changed
async def sync_from_catalog(
self,
db: AsyncSession,
*,
prune_missing: bool = True,
) -> Dict[str, Any]:
product_result = await db.execute(
select(ResultProductORM)
.where(ResultProductORM.catalog_name == "dinsar")
.order_by(ResultProductORM.id.asc())
)
products = product_result.scalars().all()
row_result = await db.execute(
select(DinsarResultORM).order_by(DinsarResultORM.id.asc())
)
rows = row_result.scalars().all()
by_product_id = {
str(row.compat_product_id): row
for row in rows
if _safe_text(row.compat_product_id)
}
by_file_path = {
_safe_text(row.file_path): row
for row in rows
if _safe_text(row.file_path)
}
by_name: Dict[str, List[DinsarResultORM]] = {}
for row in rows:
key = _safe_text(row.name)
if not key:
continue
by_name.setdefault(key, []).append(row)
touched_row_ids: set[int] = set()
created = 0
updated = 0
skipped = 0
mirrored_product_fields = 0
for product in products:
projected = self._project_product(product)
if projected is None:
skipped += 1
continue
row = by_product_id.get(product.product_id)
if row is None:
row = by_file_path.get(projected.file_path)
if row is None:
for candidate in by_name.get(projected.name, []):
if candidate.id in touched_row_ids:
continue
if _same_bbox(candidate, product):
row = candidate
break
if row is None:
row = DinsarResultORM(
compat_product_id=product.product_id,
name=projected.name,
file_path=projected.file_path,
min_lon=product.min_lon,
min_lat=product.min_lat,
max_lon=product.max_lon,
max_lat=product.max_lat,
coverage_polygon=product.coverage_polygon,
geom=product.geom,
is_cached=False,
ai_score=product.ai_score,
user_label=product.user_label,
)
db.add(row)
await db.flush()
created += 1
else:
if self._apply_product_fields(row, projected):
updated += 1
if self._mirror_legacy_annotations(row, product):
mirrored_product_fields += 1
touched_row_ids.add(int(row.id))
deleted = 0
if prune_missing:
for row in rows:
if row.id in touched_row_ids:
continue
await db.delete(row)
deleted += 1
await db.commit()
return {
"catalog_count": len(products),
"compat_count": len(touched_row_ids),
"created": created,
"updated": updated,
"deleted": deleted,
"skipped": skipped,
"mirrored_product_fields": mirrored_product_fields,
}
async def sync_product_annotations_from_result(
self,
db: AsyncSession,
*,
result_id: int,
) -> Optional[Dict[str, Any]]:
result = await db.execute(
select(DinsarResultORM).where(DinsarResultORM.id == int(result_id))
)
row = result.scalar_one_or_none()
if row is None:
return None
product_id = _safe_text(row.compat_product_id)
if not product_id:
return None
product_result = await db.execute(
select(ResultProductORM).where(ResultProductORM.product_id == product_id)
)
product = product_result.scalar_one_or_none()
if product is None:
return None
changed = self._mirror_legacy_annotations(row, product)
if changed:
await db.commit()
return {
"result_id": row.id,
"product_id": product.product_id,
"changed": changed,
}
async def sync_result_annotations_from_products(
self,
db: AsyncSession,
*,
product_ids: Optional[List[str]] = None,
) -> Dict[str, Any]:
stmt = select(ResultProductORM).where(ResultProductORM.catalog_name == "dinsar")
normalized_product_ids = [
str(product_id or "").strip()
for product_id in (product_ids or [])
if str(product_id or "").strip()
]
if normalized_product_ids:
stmt = stmt.where(ResultProductORM.product_id.in_(normalized_product_ids))
product_result = await db.execute(stmt.order_by(ResultProductORM.id.asc()))
products = product_result.scalars().all()
if not products:
return {
"requested": len(normalized_product_ids),
"synced": 0,
"updated": 0,
"missing_rows": 0,
}
product_keys = [product.product_id for product in products]
row_result = await db.execute(
select(DinsarResultORM)
.where(DinsarResultORM.compat_product_id.in_(product_keys))
.order_by(DinsarResultORM.id.asc())
)
rows = row_result.scalars().all()
rows_by_product_id: Dict[str, DinsarResultORM] = {}
for row in rows:
product_id = _safe_text(row.compat_product_id)
if product_id and product_id not in rows_by_product_id:
rows_by_product_id[product_id] = row
synced = 0
updated = 0
missing_rows = 0
for product in products:
row = rows_by_product_id.get(product.product_id)
if row is None:
missing_rows += 1
continue
synced += 1
if self._mirror_product_annotations_to_row(product, row):
updated += 1
if updated:
await db.commit()
return {
"requested": len(normalized_product_ids) or len(products),
"synced": synced,
"updated": updated,
"missing_rows": missing_rows,
}
dinsar_compat_service = DinsarCompatService()
+102
View File
@@ -0,0 +1,102 @@
"""D-InSAR 结果导出服务:将结果文件复制到用户指定目录。"""
from __future__ import annotations
import logging
import os
import shutil
from typing import Any, Dict, List
logger = logging.getLogger(__name__)
# 允许的文件扩展名(主文件 + 附属文件)
_ASSOCIATED_EXTENSIONS = ["", ".hdr", ".sml", ".xml", ".aux.xml", ".prj"]
def export_dinsar_results(
file_paths: List[str],
target_dir: str,
) -> Dict[str, Any]:
"""
将指定的 D-InSAR 结果文件复制到目标目录。
支持本地路径和 UNC 路径(如 \\\\server\\share\\path)。
对每个主文件,同时复制同名的附属文件(.hdr, .sml 等)。
返回复制统计信息。
"""
# 规范化目标路径
target_dir = target_dir.strip()
if not target_dir:
raise ValueError("目标路径不能为空")
# 创建目标目录
try:
os.makedirs(target_dir, exist_ok=True)
except OSError as e:
raise ValueError(f"无法创建目标目录 '{target_dir}': {e}")
copied = 0
skipped = 0
failed = 0
details: List[Dict[str, Any]] = []
for file_path in file_paths:
file_path = file_path.strip()
if not file_path:
continue
if not os.path.isfile(file_path):
failed += 1
details.append({
"source": file_path,
"status": "error",
"message": "源文件不存在",
})
continue
base_name = os.path.basename(file_path)
src_dir = os.path.dirname(file_path)
item_copied = 0
item_skipped = 0
item_failed = 0
for ext in _ASSOCIATED_EXTENSIONS:
src = os.path.join(src_dir, base_name + ext) if ext else file_path
if not os.path.isfile(src):
continue
dst = os.path.join(target_dir, base_name + ext)
try:
if os.path.isfile(dst):
# 大小相同则跳过
if os.path.getsize(src) == os.path.getsize(dst):
item_skipped += 1
skipped += 1
continue
shutil.copy2(src, dst)
item_copied += 1
copied += 1
except OSError as e:
item_failed += 1
failed += 1
logger.warning("复制文件失败 %s -> %s: %s", src, dst, e)
status = "ok" if item_failed == 0 else "partial"
details.append({
"source": file_path,
"name": base_name,
"status": status,
"copied": item_copied,
"skipped": item_skipped,
"failed": item_failed,
})
return {
"total_files": len(file_paths),
"copied": copied,
"skipped": skipped,
"failed": failed,
"target_dir": target_dir,
"details": details,
}
+154
View File
@@ -0,0 +1,154 @@
from __future__ import annotations
import hashlib
import json
import os
import re
from datetime import datetime
from typing import Any, Dict, Optional
PAIR_META_FILENAME = ".dinsar_pair.json"
RUN_META_FILENAME = ".dinsar_run.json"
_DATE_RE = re.compile(r"^\d{8}$")
_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+")
def normalize_path(value: Any) -> str:
text = str(value or "").strip().strip('"').strip("'")
if not text:
return ""
return os.path.normcase(os.path.normpath(os.path.abspath(text)))
def slugify(value: Any, *, default: str = "item", max_len: int = 48) -> str:
text = _SAFE_TEXT_RE.sub("_", str(value or "").strip()).strip("._")
if not text:
text = default
return text[:max_len]
def _normalize_date(value: Any) -> str:
text = str(value or "").strip()
if _DATE_RE.match(text):
return text
digits = re.sub(r"\D", "", text)
if len(digits) >= 8:
return digits[:8]
return "unknown"
def build_task_alias(master_date: Any, slave_date: Any) -> str:
return f"Task_{_normalize_date(master_date)}_{_normalize_date(slave_date)}"
def build_pair_key(
master_path: Any,
slave_path: Any,
master_date: Any = None,
slave_date: Any = None,
) -> str:
master_date_text = _normalize_date(master_date)
slave_date_text = _normalize_date(slave_date)
payload = "||".join(
[
normalize_path(master_path),
normalize_path(slave_path),
master_date_text,
slave_date_text,
]
)
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{master_date_text}_{slave_date_text}_{digest}"
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str:
alias = str(task_alias or "").strip() or "Task_unknown_unknown"
parts = alias.split("_")
master_date = parts[1] if len(parts) > 2 else "unknown"
slave_date = parts[2] if len(parts) > 2 else "unknown"
payload = "||".join([alias, normalize_path(source_hint)])
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
def build_run_key(
engine_code: str,
profile_code: str,
*,
started_at: Optional[datetime] = None,
) -> str:
timestamp = started_at or datetime.utcnow()
ts_text = timestamp.strftime("%Y%m%dT%H%M%SZ")
engine_slug = slugify(engine_code, default="engine", max_len=16)
profile_slug = slugify(profile_code, default="profile", max_len=24)
return f"run_{ts_text}_{engine_slug}_{profile_slug}"
def ensure_unique_task_aliases(pairs: list[Any]) -> list[Any]:
counts: Dict[str, int] = {}
final_pairs: list[Any] = []
for pair in pairs:
base_alias = str(
getattr(pair, "task_alias", None)
or getattr(pair, "task_name", None)
or build_task_alias(
getattr(getattr(pair, "master", None), "imaging_date", None),
getattr(getattr(pair, "slave", None), "imaging_date", None),
)
).strip()
count = counts.get(base_alias, 0) + 1
counts[base_alias] = count
unique_alias = base_alias if count == 1 else f"{base_alias}_{count - 1}"
setattr(pair, "task_alias", unique_alias)
setattr(pair, "task_name", unique_alias)
final_pairs.append(pair)
return final_pairs
def load_json_sidecar(directory: Any, filename: str) -> Optional[Dict[str, Any]]:
base_dir = normalize_path(directory)
if not base_dir or not os.path.isdir(base_dir):
return None
path = os.path.join(base_dir, filename)
if not os.path.isfile(path):
return None
try:
with open(path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
return payload if isinstance(payload, dict) else None
except Exception:
return None
def find_json_sidecar(directory: Any, filename: str, *, max_levels: int = 4) -> Optional[Dict[str, Any]]:
current = normalize_path(directory)
levels = 0
while current and levels <= max_levels:
payload = load_json_sidecar(current, filename)
if payload is not None:
return payload
parent = os.path.dirname(current)
if not parent or parent == current:
break
current = parent
levels += 1
return None
def write_json_sidecar(directory: Any, filename: str, payload: Dict[str, Any]) -> str:
target_dir = normalize_path(directory)
os.makedirs(target_dir, exist_ok=True)
target_path = os.path.join(target_dir, filename)
with open(target_path, "w", encoding="utf-8") as fp:
json.dump(payload, fp, ensure_ascii=False, indent=2)
return target_path
def write_pair_metadata(directory: Any, payload: Dict[str, Any]) -> str:
return write_json_sidecar(directory, PAIR_META_FILENAME, payload)
def write_run_metadata(directory: Any, payload: Dict[str, Any]) -> str:
return write_json_sidecar(directory, RUN_META_FILENAME, payload)
+291
View File
@@ -0,0 +1,291 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Dict, List, Optional
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import defer
from ..models import DinsarResultORM, ResultProductORM
from .data_service import data_service
DINSAR_CATALOG_NAME = "dinsar"
_PIL_COMPATIBLE_EXTENSIONS = {
".bmp",
".gif",
".jpeg",
".jpg",
".png",
".tif",
".tiff",
".webp",
}
@dataclass(frozen=True)
class DinsarCatalogReadRecord:
product: ResultProductORM
compat_row: Optional[DinsarResultORM]
display_name: str
image_path: Optional[str]
def _safe_path(value: object) -> Optional[str]:
text = str(value or "").strip()
return text or None
def _is_supported_image_path(path: Optional[str]) -> bool:
if not path or not os.path.isfile(path):
return False
suffix = os.path.splitext(path)[1].lower()
return suffix in _PIL_COMPATIBLE_EXTENSIONS
class DinsarReadService:
def _build_record(
self,
product: ResultProductORM,
compat_row: Optional[DinsarResultORM],
) -> DinsarCatalogReadRecord:
return DinsarCatalogReadRecord(
product=product,
compat_row=compat_row,
display_name=self.get_display_name(product, compat_row),
image_path=self.resolve_image_path(product, compat_row),
)
def get_display_name(
self,
product: ResultProductORM,
compat_row: Optional[DinsarResultORM] = None,
) -> str:
for value in (
product.display_name,
product.task_alias,
product.task_name,
getattr(compat_row, "name", None),
product.product_id,
):
text = str(value or "").strip()
if text:
return text
return "unknown"
def resolve_preview_path(
self,
product: ResultProductORM,
compat_row: Optional[DinsarResultORM] = None,
) -> Optional[str]:
preview_path = _safe_path(product.preview_path)
if _is_supported_image_path(preview_path):
return preview_path
if compat_row is not None:
cache_path = data_service.get_dinsar_cache_path(compat_row.id, compat_row.name)
if _is_supported_image_path(cache_path):
return cache_path
return None
def resolve_image_path(
self,
product: ResultProductORM,
compat_row: Optional[DinsarResultORM] = None,
) -> Optional[str]:
preview_path = self.resolve_preview_path(product, compat_row)
if preview_path:
return preview_path
for candidate in (
_safe_path(product.primary_asset_path),
_safe_path(product.source_primary_path),
):
if _is_supported_image_path(candidate):
return candidate
return None
async def list_catalog_records(
self,
db: AsyncSession,
*,
labeled_only: bool = False,
include_geom: bool = False,
) -> List[DinsarCatalogReadRecord]:
stmt = select(ResultProductORM).where(
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME
)
if labeled_only:
stmt = stmt.where(ResultProductORM.user_label.is_not(None))
if not include_geom:
stmt = stmt.options(defer(ResultProductORM.geom))
stmt = stmt.order_by(ResultProductORM.id.asc())
product_result = await db.execute(stmt)
products = product_result.scalars().all()
product_ids = [product.product_id for product in products if str(product.product_id or "").strip()]
compat_by_product_id: Dict[str, DinsarResultORM] = {}
if product_ids:
compat_result = await db.execute(
select(DinsarResultORM)
.where(DinsarResultORM.compat_product_id.in_(product_ids))
.order_by(DinsarResultORM.id.asc())
)
for row in compat_result.scalars().all():
product_id = str(row.compat_product_id or "").strip()
if product_id and product_id not in compat_by_product_id:
compat_by_product_id[product_id] = row
return [
self._build_record(product, compat_by_product_id.get(product.product_id))
for product in products
]
async def count_compat_records(self, db: AsyncSession) -> int:
result = await db.execute(
select(func.count(DinsarResultORM.id))
.select_from(DinsarResultORM)
.join(
ResultProductORM,
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
)
.where(ResultProductORM.catalog_name == DINSAR_CATALOG_NAME)
)
return int(result.scalar_one() or 0)
async def list_compat_records(
self,
db: AsyncSession,
*,
limit: int,
offset: int,
include_geom: bool = False,
) -> List[DinsarCatalogReadRecord]:
stmt = (
select(DinsarResultORM, ResultProductORM)
.join(
ResultProductORM,
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
)
.where(ResultProductORM.catalog_name == DINSAR_CATALOG_NAME)
.order_by(
ResultProductORM.published_at.desc().nullslast(),
ResultProductORM.id.desc(),
)
.offset(max(0, int(offset or 0)))
.limit(max(1, int(limit or 1)))
)
if not include_geom:
stmt = stmt.options(
defer(DinsarResultORM.geom),
defer(ResultProductORM.geom),
)
result = await db.execute(stmt)
return [
self._build_record(product, compat_row)
for compat_row, product in result.all()
]
async def get_compat_record(
self,
db: AsyncSession,
*,
compat_result_id: int,
include_geom: bool = False,
) -> Optional[DinsarCatalogReadRecord]:
stmt = (
select(DinsarResultORM, ResultProductORM)
.join(
ResultProductORM,
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
)
.where(
DinsarResultORM.id == int(compat_result_id),
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME,
)
)
if not include_geom:
stmt = stmt.options(
defer(DinsarResultORM.geom),
defer(ResultProductORM.geom),
)
result = await db.execute(stmt)
row = result.first()
if row is None:
return None
compat_row, product = row
return self._build_record(product, compat_row)
async def list_compat_records_by_ids(
self,
db: AsyncSession,
*,
compat_result_ids: List[int],
include_geom: bool = False,
) -> List[DinsarCatalogReadRecord]:
normalized_ids = sorted({
int(value)
for value in compat_result_ids
if value is not None
})
if not normalized_ids:
return []
stmt = (
select(DinsarResultORM, ResultProductORM)
.join(
ResultProductORM,
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
)
.where(
DinsarResultORM.id.in_(normalized_ids),
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME,
)
.order_by(DinsarResultORM.id.asc())
)
if not include_geom:
stmt = stmt.options(
defer(DinsarResultORM.geom),
defer(ResultProductORM.geom),
)
result = await db.execute(stmt)
return [
self._build_record(product, compat_row)
for compat_row, product in result.all()
]
async def get_ai_status_counts(self, db: AsyncSession) -> Dict[str, int]:
total_labeled_res = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME,
ResultProductORM.user_label.is_not(None),
)
)
good_res = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME,
ResultProductORM.user_label == 1,
)
)
bad_res = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == DINSAR_CATALOG_NAME,
ResultProductORM.user_label == 0,
)
)
return {
"labeled_count": int(total_labeled_res.scalar_one() or 0),
"good_count": int(good_res.scalar_one() or 0),
"bad_count": int(bad_res.scalar_one() or 0),
}
dinsar_read_service = DinsarReadService()
@@ -0,0 +1,98 @@
from __future__ import annotations
import os
from typing import Any, Dict, List, Optional
from sqlalchemy.ext.asyncio import AsyncSession
from .manifest_inventory_service import manifest_inventory_service
from .result_catalog_service import result_catalog_service
from .task_service import task_service
def _dedupe_existing_dirs(paths: Optional[List[str]]) -> tuple[List[str], List[str]]:
normalized: List[str] = []
ignored: List[str] = []
for raw_path in paths or []:
path = os.path.normpath(os.path.abspath(str(raw_path or "").strip()))
if not path:
continue
if not os.path.isdir(path):
ignored.append(path)
continue
if path in normalized:
continue
normalized.append(path)
return normalized, ignored
class DinsarScanService:
async def _update_task(self, task_id: Optional[str], *, message: str, progress: int) -> None:
if not task_id:
return
await task_service.update_task(
task_id,
message=message,
progress=max(0, min(100, int(progress))),
)
async def run_scan(
self,
db: AsyncSession,
*,
source_directories: Optional[List[str]] = None,
publish_root: Optional[str] = None,
task_id: Optional[str] = None,
) -> Dict[str, Any]:
source_dirs, ignored_dirs = _dedupe_existing_dirs(source_directories)
publish_result = None
if source_dirs:
await self._update_task(
task_id,
message="正在将 D-InSAR 原始结果发布为 package...",
progress=20,
)
publish_result = await result_catalog_service.publish_from_sources(
db,
source_dirs,
publish_root=publish_root,
)
await self._update_task(
task_id,
message="正在同步 manifest inventory...",
progress=55,
)
inventory_result = await manifest_inventory_service.sync_manifest_roots(db)
await self._update_task(
task_id,
message="正在重建 D-InSAR catalog 与兼容视图...",
progress=75,
)
rebuild_result = await result_catalog_service.rebuild_catalog(
db,
publish_root=publish_root,
full_rebuild=True,
)
compat_result = rebuild_result.get("compat_sync")
compat_error = rebuild_result.get("compat_error")
return {
"source_directories": source_dirs,
"ignored_source_directories": ignored_dirs,
"publish": publish_result,
"manifest_inventory": inventory_result,
"rebuild": rebuild_result,
"compat_sync": compat_result,
"compat_error": compat_error,
"message": (
"D-InSAR unified scan completed"
if not compat_error
else f"D-InSAR unified scan completed with compat error: {compat_error}"
),
}
dinsar_scan_service = DinsarScanService()
+926
View File
@@ -0,0 +1,926 @@
"""D-InSAR workflow functions extracted from envi_service.py."""
from __future__ import annotations
import math
import os
import time
import defusedxml.ElementTree as ET
from glob import glob
from typing import Any, Dict, List, Optional
from .envi_service import (
DEFAULT_TIMEOUT,
DEM_BASE_FILE,
RUNTIME_DIR,
CUSTOM_TARGET_RESOLUTION_M,
CUSTOM_FILTER_METHOD,
CUSTOM_UNWRAP_COH_THRESHOLD,
CUSTOM_GCP_COH_THRESHOLD,
CUSTOM_GCP_NUMBER,
CUSTOM_GEOCODING_COH_THRESHOLD,
CUSTOM_GEOCODING_PIXEL_SIZE_M,
_read_env,
_normalize_path,
_to_local_path,
_collect_task_folders,
_find_meta_files,
_has_sml,
_first_sml_base,
_build_sarscapedata,
_unwrap_sarscapedata,
execute_envi_task,
_write_progress,
)
# Stability check configuration
_STABILITY_INTERVAL = int(_read_env("ENVI_STABILITY_CHECK_INTERVAL", "15") or 15)
_STABILITY_ROUNDS = int(_read_env("ENVI_STABILITY_ROUNDS", "3") or 3)
_STABILITY_MAX_WAIT = int(_read_env("ENVI_STABILITY_MAX_WAIT", "3600") or 3600)
def run_dinsar_workflow(
root_dir: str,
num_to_process: int = 0,
timeout: int = DEFAULT_TIMEOUT,
) -> Dict[str, Any]:
"""Run D-InSAR metatask on Task_* folders.
Smart chaining: for each Task_* folder, if master/slave lack .sml files,
automatically run Import first, then proceed with D-InSAR.
DEM path is read from .env (IDL_DINSAR_DEM_BASE_FILE).
"""
root_dir = _to_local_path(root_dir)
dem_base_file = DEM_BASE_FILE
if not root_dir or not os.path.isdir(root_dir):
raise ValueError(f"D-InSAR root directory does not exist: {root_dir}")
if not dem_base_file:
raise ValueError(
"DEM path not configured. Set IDL_DINSAR_DEM_BASE_FILE in .env"
)
task_folders = _collect_task_folders(root_dir)
log_lines: List[str] = [
f"[envi] dinsar metatask",
f"[envi] root_dir={root_dir}",
f"[envi] dem={dem_base_file}",
f"[envi] Task_* folders={len(task_folders)}",
]
if not task_folders:
return {
"summary": {
"task_folders": 0,
"processed": 0,
"failed": 0,
"skipped": 0,
"auto_imported": 0,
},
"log_lines": log_lines,
}
processed = 0
failed = 0
skipped = 0
auto_imported = 0
for folder in task_folders:
if num_to_process > 0 and processed >= num_to_process:
log_lines.append(f"[envi] reached limit={num_to_process}")
break
task_name = os.path.basename(folder)
master_dir = os.path.join(folder, "master")
slave_dir = os.path.join(folder, "slave")
if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir):
skipped += 1
log_lines.append(f"[skip] {task_name}: master/slave dir missing")
continue
# --- Smart chaining: auto-import if .sml missing ---
for side, side_dir in [("master", master_dir), ("slave", slave_dir)]:
if not _has_sml(side_dir):
meta_files = _find_meta_files(side_dir)
if not meta_files:
log_lines.append(
f"[warn] {task_name}/{side}: no .sml and no .meta.xml"
)
continue
log_lines.append(
f"[auto-import] {task_name}/{side}: "
f"importing {len(meta_files)} file(s)"
)
for mf in meta_files:
start = time.time()
try:
execute_envi_task(
"SARsImportLuTan1",
{
"INPUT_FILE_LIST": [mf],
"ROOT_URI_FOR_OUTPUT": side_dir,
},
)
elapsed = round(time.time() - start, 1)
auto_imported += 1
log_lines.append(
f"[auto-import ok] {task_name}/{side}: "
f"{os.path.basename(mf)} ({elapsed}s)"
)
except Exception as exc:
elapsed = round(time.time() - start, 1)
log_lines.append(
f"[auto-import err] {task_name}/{side}: "
f"{os.path.basename(mf)} ({elapsed}s): {exc}"
)
# After auto-import, check .sml again
master_base = _first_sml_base(master_dir)
slave_base = _first_sml_base(slave_dir)
if not master_base or not slave_base:
skipped += 1
log_lines.append(
f"[skip] {task_name}: still missing .sml after import "
f"(master={'yes' if master_base else 'no'} "
f"slave={'yes' if slave_base else 'no'})"
)
continue
output_dir = os.path.join(folder, "dinsar_results")
os.makedirs(output_dir, exist_ok=True)
start = time.time()
try:
execute_envi_task(
"SARsMetataskInSARDisplacementGeneration",
{
"REFERENCE_SARSCAPEDATA": _build_sarscapedata(master_base),
"SECONDARY_SARSCAPEDATA": _build_sarscapedata(slave_base),
"DEM_SARSCAPEDATA": _build_sarscapedata(dem_base_file),
"OUTPUT_FOLDER": _normalize_path(output_dir),
},
)
elapsed = round(time.time() - start, 1)
processed += 1
log_lines.append(f"[ok] dinsar {task_name} ({elapsed}s)")
except Exception as exc:
elapsed = round(time.time() - start, 1)
failed += 1
log_lines.append(
f"[err] dinsar {task_name} failed ({elapsed}s): {exc}"
)
if failed > 0 and processed == 0 and len(task_folders) > 0:
detail = "\n".join(log_lines[-20:])
raise RuntimeError(
f"All D-InSAR tasks failed. failed={failed}, "
f"skipped={skipped}.\n{detail}"
)
return {
"summary": {
"task_folders": len(task_folders),
"processed": processed,
"failed": failed,
"skipped": skipped,
"auto_imported": auto_imported,
},
"log_lines": log_lines,
}
def _read_sml_parameter(sml_file: str, param_name: str) -> Optional[str]:
"""Read a parameter value from a SARscape .sml XML file."""
sml_path = _to_local_path(sml_file)
if not os.path.isfile(sml_path):
return None
try:
tree = ET.parse(sml_path)
root = tree.getroot()
tag_upper = param_name.upper()
for elem in root.iter():
local_tag = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag
if local_tag.upper() == tag_upper and elem.text:
return elem.text.strip()
except Exception as exc:
print(f"[WARN] _read_sml: {exc}")
return None
def _calculate_looks(
master_sml: str,
slave_sml: str,
target_resolution: float,
) -> tuple:
"""Calculate range and azimuth looks from SML pixel spacing."""
m_rg = _read_sml_parameter(master_sml, "PixelSpacingRg")
m_az = _read_sml_parameter(master_sml, "PixelSpacingAz")
m_inc = _read_sml_parameter(master_sml, "IncidenceAngle")
s_rg = _read_sml_parameter(slave_sml, "PixelSpacingRg")
s_az = _read_sml_parameter(slave_sml, "PixelSpacingAz")
s_inc = _read_sml_parameter(slave_sml, "IncidenceAngle")
if not all([m_rg, m_az, m_inc, s_rg, s_az, s_inc]):
raise ValueError(
"Cannot read pixel spacing / incidence angle from SML files. "
f"master={master_sml} slave={slave_sml}"
)
m_rg_f, m_az_f, m_inc_f = float(m_rg), float(m_az), float(m_inc)
s_rg_f, s_az_f, s_inc_f = float(s_rg), float(s_az), float(s_inc)
avg_az = (m_az_f + s_az_f) / 2.0
azimuth_looks = max(1, int(target_resolution / avg_az))
m_ground_rg = m_rg_f / math.sin(math.radians(m_inc_f))
s_ground_rg = s_rg_f / math.sin(math.radians(s_inc_f))
avg_ground_rg = (m_ground_rg + s_ground_rg) / 2.0
range_looks = max(1, int(target_resolution / avg_ground_rg))
return range_looks, azimuth_looks
def _find_latest_sarscapedata(
directory: str,
pattern_fragment: str,
log_lines: Optional[List[str]] = None,
) -> Optional[Dict[str, Any]]:
"""Scan directory for the latest SARscape file matching a pattern."""
pattern = os.path.join(directory, f"*{pattern_fragment}*.sml")
candidates = sorted(glob(pattern), key=os.path.getmtime, reverse=True)
if log_lines is not None:
log_lines.append(f"[scan] pattern={pattern} found={len(candidates)}")
if not candidates:
return None
sml_path = candidates[0]
base = sml_path[:-4] # strip .sml
if log_lines is not None:
log_lines.append(f"[scan] using: {os.path.basename(base)}")
return _build_sarscapedata(base)
def _wait_files_stable(
directory: str,
log_lines: Optional[List[str]] = None,
) -> None:
"""Wait until all files in directory have stable sizes."""
if not directory or not os.path.isdir(directory):
return
def _snapshot() -> Dict[str, int]:
sizes: Dict[str, int] = {}
try:
for root, _dirs, files in os.walk(directory):
for f in files:
fp = os.path.join(root, f)
try:
sizes[fp] = os.path.getsize(fp)
except OSError as exc:
print(f"[WARN] _snapshot getsize: {exc}")
except Exception as exc:
print(f"[WARN] _snapshot walk: {exc}")
return sizes
stable_count = 0
prev = _snapshot()
wait_start = time.time()
while stable_count < _STABILITY_ROUNDS:
elapsed = time.time() - wait_start
if elapsed > _STABILITY_MAX_WAIT:
if log_lines is not None:
log_lines.append(
f"[stability] max wait {_STABILITY_MAX_WAIT}s reached, proceeding"
)
break
time.sleep(_STABILITY_INTERVAL)
cur = _snapshot()
if cur == prev:
stable_count += 1
else:
stable_count = 0
prev = cur
total_wait = round(time.time() - wait_start, 1)
if log_lines is not None:
log_lines.append(
f"[stability] files stable after {total_wait}s "
f"({stable_count}/{_STABILITY_ROUNDS} rounds, "
f"{len(prev)} files)"
)
def _wait_for_disp_stable(
directory: str,
log_lines: Optional[List[str]] = None,
) -> bool:
"""Wait for *_rsp_disp file to appear and stabilize."""
if not directory or not os.path.isdir(directory):
return False
wait_start = time.time()
disp_path = None
while (time.time() - wait_start) < _STABILITY_MAX_WAIT:
for f in os.listdir(directory):
if f.endswith("_rsp_disp") and not f.endswith(".hdr") and not f.endswith(".sml"):
disp_path = os.path.join(directory, f)
break
if disp_path:
break
if log_lines is not None and int(time.time() - wait_start) % 60 == 0:
log_lines.append(
f"[wait_disp] waiting for _rsp_disp file... "
f"({int(time.time() - wait_start)}s)"
)
time.sleep(_STABILITY_INTERVAL)
if not disp_path:
if log_lines is not None:
log_lines.append(
f"[wait_disp] _rsp_disp file not found after "
f"{int(time.time() - wait_start)}s"
)
return False
if log_lines is not None:
log_lines.append(
f"[wait_disp] found {os.path.basename(disp_path)} "
f"after {int(time.time() - wait_start)}s"
)
stable_count = 0
prev_size = None
while stable_count < _STABILITY_ROUNDS:
if (time.time() - wait_start) > _STABILITY_MAX_WAIT:
if log_lines is not None:
log_lines.append("[wait_disp] max wait reached, proceeding")
break
time.sleep(_STABILITY_INTERVAL)
try:
cur_size = os.path.getsize(disp_path)
except OSError:
cur_size = -1
if cur_size >= 0 and cur_size == prev_size:
stable_count += 1
else:
stable_count = 0
prev_size = cur_size
total_wait = round(time.time() - wait_start, 1)
if log_lines is not None:
log_lines.append(
f"[wait_disp] stable after {total_wait}s "
f"(size={prev_size} bytes)"
)
return True
def _generate_gcps(
coherence_file: str,
output_shp: str,
coh_threshold: float = 0.7,
num_points: int = 100,
log_lines: Optional[List[str]] = None,
) -> bool:
"""Generate GCP shapefile from coherence raster."""
try:
import numpy as np
import rasterio
import geopandas as gpd
from shapely.geometry import Point
except ImportError as exc:
raise RuntimeError(
"rasterio, geopandas, and shapely are required for GCP generation. "
f"Missing: {exc}"
) from exc
coh_path = _to_local_path(coherence_file)
if not os.path.isfile(coh_path):
for ext in [".hdr", ""]:
candidate = coh_path + ext
if os.path.isfile(candidate):
coh_path = candidate
break
if not os.path.isfile(coh_path):
if log_lines is not None:
log_lines.append(f"[gcp] coherence file not found: {coh_path}")
return False
with rasterio.open(coh_path) as src:
data = src.read(1)
ns, nl = src.width, src.height
grid_dim = math.ceil(math.sqrt(num_points))
x_step = nl // grid_dim
y_step = ns // grid_dim
points = []
for j in range(grid_dim):
for i in range(grid_dim):
x_start = i * y_step
y_start = j * x_step
x_end = min((i + 1) * y_step, ns)
y_end = min((j + 1) * x_step, nl)
if x_start >= ns or y_start >= nl:
continue
cell = data[y_start:y_end, x_start:x_end]
max_val = float(np.nanmax(cell)) if cell.size > 0 else 0.0
if max_val >= coh_threshold:
idx = int(np.nanargmax(cell))
cell_h, cell_w = cell.shape
max_row = idx // cell_w
max_col = idx % cell_w
px_col = x_start + max_col
px_row = y_start + max_row
points.append((px_col, px_row))
if not points:
if log_lines is not None:
log_lines.append(
f"[gcp] no points found above threshold {coh_threshold}"
)
return False
records = []
for idx, (col, row) in enumerate(points):
records.append({
"SHP_ID": idx,
"GCP_LABEL": f"GCP_{idx + 1}",
"GCP_TYPE": "undefined",
"GCP_COLUMN": float(col),
"GCP_ROW": float(row),
"GCP_OTHER_": "",
"geometry": Point(float(col), float(row)),
})
gdf = gpd.GeoDataFrame(records)
out_path = _to_local_path(output_shp)
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
gdf.to_file(out_path, driver="ESRI Shapefile")
if log_lines is not None:
log_lines.append(f"[gcp] created {len(points)} GCPs -> {out_path}")
return True
def _run_dinsar_custom_single(
master_base: str,
slave_base: str,
dem_base: str,
output_root: str,
log_lines: List[str],
job_id: Optional[str] = None,
pair_index: int = 0,
total_pairs: int = 0,
pair_name: str = "",
) -> bool:
"""Execute the 6-step custom D-InSAR workflow for one pair."""
master_sml = master_base + ".sml"
slave_sml = slave_base + ".sml"
master_sd = _build_sarscapedata(master_base)
slave_sd = _build_sarscapedata(slave_base)
dem_sd = _build_sarscapedata(dem_base)
out_dir = _normalize_path(os.path.dirname(output_root))
try:
range_looks, azimuth_looks = _calculate_looks(
master_sml, slave_sml, CUSTOM_TARGET_RESOLUTION_M
)
log_lines.append(
f"[custom] looks: range={range_looks} azimuth={azimuth_looks} "
f"(target_res={CUSTOM_TARGET_RESOLUTION_M}m)"
)
except Exception as exc:
log_lines.append(f"[custom] looks calculation failed: {exc}")
return False
# === STEP 1: Interferogram Generation ===
log_lines.append("[custom] step 1/6: Interferogram Generation")
_write_progress(job_id, 1, 6, "Interferogram Generation", out_dir, pair_index, total_pairs, pair_name)
start = time.time()
try:
r1 = execute_envi_task(
"SARsInSARInterferogramGeneration",
{
"REFERENCE_SARSCAPEDATA": master_sd,
"SECONDARY_SARSCAPEDATA": slave_sd,
"DEM_SARSCAPEDATA": dem_sd,
"ROOT_URI_FOR_OUTPUT": out_dir,
"RG_LOOKS_NBR": float(range_looks),
"AZ_LOOKS_NBR": float(azimuth_looks),
"COREGISTRATION_WITH_DEM": True,
},
)
log_lines.append(f"[custom] step 1 ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
log_lines.append(
f"[custom] step 1 failed ({round(time.time() - start, 1)}s): {exc}"
)
return False
s1_dint = _unwrap_sarscapedata(r1.get("DINT_SARSCAPEDATA"))
s1_ref_pwr = _unwrap_sarscapedata(r1.get("REFERENCE_POWER_SARSCAPEDATA"))
s1_sec_pwr = _unwrap_sarscapedata(r1.get("SECONDARY_POWER_SARSCAPEDATA"))
s1_sint = _unwrap_sarscapedata(r1.get("SINT_SARSCAPEDATA"))
s1_srdem = _unwrap_sarscapedata(r1.get("SRDEM_SARSCAPEDATA"))
# === STEP 2: Filtering and Coherence ===
log_lines.append("[custom] step 2/6: Filtering and Coherence")
_write_progress(job_id, 2, 6, "Filtering and Coherence", out_dir, pair_index, total_pairs, pair_name)
start = time.time()
try:
r2 = execute_envi_task(
"SARsInSARFilterAndCoherence",
{
"DINT_SARSCAPEDATA": s1_dint,
"REFERENCE_SARSCAPEDATA": s1_ref_pwr,
"SECONDARY_SARSCAPEDATA": s1_sec_pwr,
"ROOT_URI_FOR_OUTPUT": out_dir,
"FILTERING_METHOD": CUSTOM_FILTER_METHOD,
"COHERENCE": True,
"INTERF_FILT": True,
},
)
log_lines.append(f"[custom] step 2 ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
log_lines.append(
f"[custom] step 2 failed ({round(time.time() - start, 1)}s): {exc}"
)
return False
s2_fint = _unwrap_sarscapedata(r2.get("FINT_SARSCAPEDATA"))
s2_cc = _unwrap_sarscapedata(r2.get("COHERENCE_SARSCAPEDATA"))
# === STEP 3: Remove Residual Phase Frequency ===
log_lines.append("[custom] step 3/6: Orbital Trend Removal")
_write_progress(job_id, 3, 6, "Orbital Trend Removal", out_dir, pair_index, total_pairs, pair_name)
start = time.time()
s3_rrpf = None
try:
r3 = execute_envi_task(
"SARsInSARRemoveResidualPhaseFrequency",
{
"INTERFEROGRAM_SARSCAPEDATA": s2_fint,
"COHERENCE_FILE_NAME": s2_cc,
"ROOT_URI_FOR_OUTPUT": out_dir,
},
)
s3_rrpf = _unwrap_sarscapedata(r3.get("RRPF_DINT_SARSCAPEDATA"))
log_lines.append(f"[custom] step 3 ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
elapsed = round(time.time() - start, 1)
log_lines.append(f"[custom] step 3 engine error ({elapsed}s): {exc}")
log_lines.append("[custom] step 3 scanning for generated RRPF file...")
s3_rrpf = _find_latest_sarscapedata(
_to_local_path(out_dir), "ISARRRPF", log_lines
)
if not s3_rrpf:
log_lines.append("[custom] step 3 failed: no RRPF output found")
return False
# === STEP 4: Phase Unwrapping ===
log_lines.append("[custom] step 4/6: Phase Unwrapping")
_write_progress(job_id, 4, 6, "Phase Unwrapping", out_dir, pair_index, total_pairs, pair_name)
start = time.time()
s4_upha = None
try:
r4 = execute_envi_task(
"SARsInSARPhaseUnwrapping",
{
"INFILE_NAME": s3_rrpf,
"COHERENCEFILE_NAME": s2_cc,
"ROOT_URI_FOR_OUTPUT": out_dir,
"UPHA_COH_THRESHOLD": CUSTOM_UNWRAP_COH_THRESHOLD,
},
)
s4_upha = _unwrap_sarscapedata(r4.get("OUTFILE_NAME"))
log_lines.append(f"[custom] step 4 ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
elapsed = round(time.time() - start, 1)
log_lines.append(f"[custom] step 4 engine error ({elapsed}s): {exc}")
log_lines.append("[custom] step 4 scanning for generated UPHA file...")
s4_upha = _find_latest_sarscapedata(
_to_local_path(out_dir), "ISARPU", log_lines
)
if not s4_upha:
log_lines.append("[custom] step 4 failed: no UPHA output found")
return False
# === STEP 5: Refinement and Reflattening ===
log_lines.append("[custom] step 5a/6: GCP Generation")
_write_progress(job_id, 5, 6, "GCP Generation + Refinement", out_dir, pair_index, total_pairs, pair_name)
cc_url = s2_cc.get("url", "") if isinstance(s2_cc, dict) else str(s2_cc)
cc_local = _to_local_path(cc_url)
auto_gcp_shp = os.path.join(os.path.dirname(cc_local) or out_dir, "auto_gcp.shp")
gcp_ok = _generate_gcps(
coherence_file=cc_local,
output_shp=auto_gcp_shp,
coh_threshold=CUSTOM_GCP_COH_THRESHOLD,
num_points=CUSTOM_GCP_NUMBER,
log_lines=log_lines,
)
if not gcp_ok:
log_lines.append("[custom] step 5a failed: GCP generation returned no points")
return False
log_lines.append("[custom] step 5b/6: Refinement and Reflattening")
start = time.time()
s5_upha = None
try:
r5 = execute_envi_task(
"SARsInSARRefinementAndReflattening",
{
"INPUT_UPHA_FILE_NAME": s4_upha,
"REFERENCE_SARSCAPEDATA": s1_ref_pwr,
"SECONDARY_SARSCAPEDATA": s1_sec_pwr,
"SLANT_RANGE_DEM_FILE_NAME": s1_srdem,
"SYNTHETIC_FILE_NAME": s1_sint,
"COHERENCE_FILE_NAME": s2_cc,
"DEM_SARSCAPEDATA": dem_sd,
"ROOT_URI_FOR_OUTPUT": out_dir,
"REFINEMENT_GCP_FILE_NAME": _normalize_path(auto_gcp_shp),
},
)
s5_upha = _unwrap_sarscapedata(r5.get("UPHA_REFLAT_SARSCAPEDATA"))
log_lines.append(f"[custom] step 5b ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
elapsed = round(time.time() - start, 1)
log_lines.append(f"[custom] step 5b engine error ({elapsed}s): {exc}")
log_lines.append("[custom] step 5b scanning for generated REFLAT UPHA file...")
s5_upha = _find_latest_sarscapedata(
_to_local_path(out_dir), "ISARRF", log_lines
)
if not s5_upha:
s5_upha = _find_latest_sarscapedata(
_to_local_path(out_dir), "_reflat_upha", log_lines
)
if not s5_upha:
log_lines.append("[custom] step 5b failed: no REFLAT UPHA output found")
return False
# === STEP 6: Phase to Displacement and Geocoding ===
log_lines.append("[custom] step 6/6: Phase to Displacement + Geocoding")
_write_progress(job_id, 6, 6, "Phase to Displacement + Geocoding", out_dir, pair_index, total_pairs, pair_name)
start = time.time()
try:
execute_envi_task(
"SARsInSARPhaseToDisplacement",
{
"INPUT_SARSCAPEDATA": s5_upha,
"COHERNCE_SARSCAPEDATA": s2_cc,
"DEM_SARSCAPEDATA": dem_sd,
"ROOT_URI_FOR_OUTPUT": out_dir,
"COHERENCE_THRESHOLD": CUSTOM_GEOCODING_COH_THRESHOLD,
"GEOCODE_RG_GRID_SIZE": CUSTOM_GEOCODING_PIXEL_SIZE_M,
"GEOCODE_AZ_GRID_SIZE": CUSTOM_GEOCODING_PIXEL_SIZE_M,
},
)
log_lines.append(f"[custom] step 6 ok ({round(time.time() - start, 1)}s)")
except Exception as exc:
elapsed = round(time.time() - start, 1)
log_lines.append(f"[custom] step 6 engine error ({elapsed}s): {exc}")
log_lines.append("[custom] step 6: waiting for _rsp_disp file...")
disp_ok = _wait_for_disp_stable(_to_local_path(out_dir), log_lines)
if not disp_ok:
log_lines.append("[custom] step 6 failed: _rsp_disp never appeared")
return False
log_lines.append("[custom] final stability check on output directory...")
_wait_files_stable(_to_local_path(out_dir), log_lines)
_write_progress(job_id, 6, 6, "Completed", out_dir, pair_index, total_pairs, pair_name)
return True
def run_dinsar_custom_workflow(
root_dir: str,
num_to_process: int = 0,
timeout: int = DEFAULT_TIMEOUT,
job_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Run custom 6-step D-InSAR on Task_* folders."""
root_dir = _to_local_path(root_dir)
dem_base_file = DEM_BASE_FILE
if not root_dir or not os.path.isdir(root_dir):
raise ValueError(f"D-InSAR root directory does not exist: {root_dir}")
if not dem_base_file:
raise ValueError(
"DEM path not configured. Set IDL_DINSAR_DEM_BASE_FILE in .env"
)
task_folders = _collect_task_folders(root_dir)
log_lines: List[str] = [
f"[envi] dinsar custom (6-step)",
f"[envi] root_dir={root_dir}",
f"[envi] dem={dem_base_file}",
f"[envi] target_resolution={CUSTOM_TARGET_RESOLUTION_M}m",
f"[envi] filter={CUSTOM_FILTER_METHOD}",
f"[envi] unwrap_coh={CUSTOM_UNWRAP_COH_THRESHOLD}",
f"[envi] gcp_coh={CUSTOM_GCP_COH_THRESHOLD} gcp_n={CUSTOM_GCP_NUMBER}",
f"[envi] geocode_coh={CUSTOM_GEOCODING_COH_THRESHOLD} "
f"geocode_px={CUSTOM_GEOCODING_PIXEL_SIZE_M}m",
f"[envi] Task_* folders={len(task_folders)}",
]
if not task_folders:
return {
"summary": {
"task_folders": 0,
"processed": 0,
"failed": 0,
"skipped": 0,
"auto_imported": 0,
},
"log_lines": log_lines,
}
processed = 0
failed = 0
skipped = 0
auto_imported = 0
effective_total = len(task_folders) if num_to_process <= 0 else min(num_to_process, len(task_folders))
pair_counter = 0
for folder in task_folders:
if num_to_process > 0 and processed >= num_to_process:
log_lines.append(f"[envi] reached limit={num_to_process}")
break
task_name = os.path.basename(folder)
master_dir = os.path.join(folder, "master")
slave_dir = os.path.join(folder, "slave")
if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir):
skipped += 1
log_lines.append(f"[skip] {task_name}: master/slave dir missing")
continue
for side, side_dir in [("master", master_dir), ("slave", slave_dir)]:
if not _has_sml(side_dir):
meta_files = _find_meta_files(side_dir)
if not meta_files:
log_lines.append(
f"[warn] {task_name}/{side}: no .sml and no .meta.xml"
)
continue
log_lines.append(
f"[auto-import] {task_name}/{side}: "
f"importing {len(meta_files)} file(s)"
)
for mf in meta_files:
imp_start = time.time()
try:
execute_envi_task(
"SARsImportLuTan1",
{
"INPUT_FILE_LIST": [mf],
"ROOT_URI_FOR_OUTPUT": side_dir,
},
)
elapsed = round(time.time() - imp_start, 1)
auto_imported += 1
log_lines.append(
f"[auto-import ok] {task_name}/{side}: "
f"{os.path.basename(mf)} ({elapsed}s)"
)
except Exception as exc:
elapsed = round(time.time() - imp_start, 1)
log_lines.append(
f"[auto-import err] {task_name}/{side}: "
f"{os.path.basename(mf)} ({elapsed}s): {exc}"
)
master_base = _first_sml_base(master_dir)
slave_base = _first_sml_base(slave_dir)
if not master_base or not slave_base:
skipped += 1
log_lines.append(
f"[skip] {task_name}: still missing .sml after import "
f"(master={'yes' if master_base else 'no'} "
f"slave={'yes' if slave_base else 'no'})"
)
continue
output_dir = os.path.join(folder, "dinsar_results")
os.makedirs(output_dir, exist_ok=True)
output_root = os.path.join(output_dir, "workflow")
pair_start = time.time()
pair_counter += 1
log_lines.append(f"[custom] === {task_name} start ({pair_counter}/{effective_total}) ===")
try:
success = _run_dinsar_custom_single(
master_base, slave_base, dem_base_file, output_root, log_lines,
job_id=job_id,
pair_index=pair_counter,
total_pairs=effective_total,
pair_name=task_name,
)
except Exception as exc:
success = False
log_lines.append(f"[custom] {task_name} crashed: {exc}")
elapsed = round(time.time() - pair_start, 1)
if success:
processed += 1
log_lines.append(f"[ok] custom dinsar {task_name} ({elapsed}s)")
else:
failed += 1
log_lines.append(f"[err] custom dinsar {task_name} failed ({elapsed}s)")
try:
os.makedirs(RUNTIME_DIR, exist_ok=True)
_interim_log = os.path.join(RUNTIME_DIR, "dinsar_custom_progress.log")
with open(_interim_log, "w", encoding="utf-8") as _fp:
_fp.write("\n".join(log_lines))
except Exception as exc:
print(f"[WARN] dinsar log write: {exc}")
if failed > 0 and processed == 0 and len(task_folders) > 0:
detail = "\n".join(log_lines[-20:])
raise RuntimeError(
f"All custom D-InSAR tasks failed. failed={failed}, "
f"skipped={skipped}.\n{detail}"
)
return {
"summary": {
"task_folders": len(task_folders),
"processed": processed,
"failed": failed,
"skipped": skipped,
"auto_imported": auto_imported,
},
"log_lines": log_lines,
}
def inspect_dinsar(root_dir: str) -> Dict[str, Any]:
"""Pre-check D-InSAR readiness. Includes Import status detection."""
root_dir = _to_local_path(root_dir)
dem_base_file = DEM_BASE_FILE
dem_ok = bool(
dem_base_file
and (os.path.isfile(dem_base_file) or os.path.isdir(dem_base_file))
)
result: Dict[str, Any] = {
"workflow": "dinsar",
"root_dir": root_dir,
"exists": bool(root_dir and os.path.isdir(root_dir)),
"ready": False,
"summary": {},
"warnings": [],
}
if not result["exists"]:
result["warnings"].append("root_dir does not exist.")
return result
task_folders = _collect_task_folders(root_dir)
ready_count = 0
need_import_count = 0
missing_structure = 0
for folder in task_folders:
master_dir = os.path.join(folder, "master")
slave_dir = os.path.join(folder, "slave")
if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir):
missing_structure += 1
continue
master_has_sml = _has_sml(master_dir)
slave_has_sml = _has_sml(slave_dir)
if master_has_sml and slave_has_sml:
ready_count += 1
else:
master_has_meta = bool(_find_meta_files(master_dir))
slave_has_meta = bool(_find_meta_files(slave_dir))
if (master_has_sml or master_has_meta) and (
slave_has_sml or slave_has_meta
):
need_import_count += 1
else:
missing_structure += 1
result["summary"] = {
"task_folder_count": len(task_folders),
"ready_for_dinsar": ready_count,
"need_import_first": need_import_count,
"missing_structure": missing_structure,
"dem_base_file": dem_base_file or "(not configured)",
"dem_exists": dem_ok,
}
result["ready"] = (ready_count + need_import_count) > 0 and dem_ok
if not dem_ok:
result["warnings"].append(
"DEM path not configured or does not exist. "
"Set IDL_DINSAR_DEM_BASE_FILE in .env"
)
if len(task_folders) == 0:
result["warnings"].append("No Task_* folders found.")
if need_import_count > 0:
result["warnings"].append(
f"{need_import_count} folder(s) need Import first "
f"(will be auto-imported during D-InSAR)."
)
return result
+468
View File
@@ -0,0 +1,468 @@
"""D-InSAR result extraction and task overview helpers."""
from __future__ import annotations
import os
import re
import shutil
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from ..config import settings
from .envi_service import _read_env, _to_local_path
_ENVI_DISP_PATTERN = re.compile(r"^out_ISARPTD_(\d{14})_rsp_disp$")
_ISCE2_DISP_PATTERN = re.compile(r"^(?P<prefix>.+)_disp\.(?:tif|tiff)$", re.IGNORECASE)
_ISCE2_SKIP_PATTERN = re.compile(r".+_disp_full\.(?:tif|tiff)$", re.IGNORECASE)
def _get_dinsar_target_dir() -> str:
raw = _read_env("MONITOR_DINSAR_DIRS", "")
dirs = [p.strip() for p in raw.replace(";", ",").split(",") if p.strip()]
if not dirs:
raise ValueError("MONITOR_DINSAR_DIRS 未在 .env 中配置")
return dirs[0]
def _normalize_directory(path: str, label: str) -> str:
local_path = os.path.normpath(os.path.abspath(_to_local_path(path)))
if not os.path.isdir(local_path):
raise ValueError(f"{label}不存在: {path}")
return local_path
def _normalize_output_directory(path: str) -> str:
return os.path.normpath(os.path.abspath(_to_local_path(path)))
def _same_path(left: Optional[str], right: Optional[str]) -> bool:
if not left or not right:
return False
try:
return os.path.normcase(os.path.realpath(left)) == os.path.normcase(os.path.realpath(right))
except OSError:
return os.path.normcase(os.path.abspath(left)) == os.path.normcase(os.path.abspath(right))
def _looks_like_task_dir(path: str) -> bool:
return (
os.path.isdir(os.path.join(path, "master"))
and os.path.isdir(os.path.join(path, "slave"))
)
def _iter_task_dirs(root_dir: str) -> List[Tuple[str, str]]:
if _looks_like_task_dir(root_dir):
return [(os.path.basename(os.path.normpath(root_dir)), root_dir)]
task_dirs: List[Tuple[str, str]] = []
try:
with os.scandir(root_dir) as entries:
for entry in entries:
if not entry.is_dir():
continue
if not entry.name.startswith("Task_"):
continue
task_dirs.append((entry.name, entry.path))
except OSError:
return []
task_dirs.sort(key=lambda item: item[0].lower())
return task_dirs
def _root_has_isce2_products(root_dir: str) -> bool:
try:
with os.scandir(root_dir) as entries:
for entry in entries:
if not entry.is_file():
continue
if _ISCE2_SKIP_PATTERN.match(entry.name):
continue
if _ISCE2_DISP_PATTERN.match(entry.name):
return True
except OSError:
return False
return False
def _find_envi_task_result(task_dir: str) -> Optional[Dict[str, Any]]:
dinsar_results_dir = os.path.join(task_dir, "dinsar_results")
if not os.path.isdir(dinsar_results_dir):
return None
candidates: List[Tuple[str, str]] = []
try:
for entry in os.scandir(dinsar_results_dir):
if not entry.is_file():
continue
match = _ENVI_DISP_PATTERN.match(entry.name)
if match:
candidates.append((match.group(1), entry.name))
except OSError:
return None
if not candidates:
return None
candidates.sort(key=lambda item: item[0], reverse=True)
_, best_name = candidates[0]
base_path = os.path.join(dinsar_results_dir, best_name)
source_files = [base_path]
for ext in (".hdr", ".sml"):
candidate = base_path + ext
if os.path.isfile(candidate):
source_files.append(candidate)
return {
"engine": "envi",
"task_name": os.path.basename(os.path.normpath(task_dir)),
"source_dir": dinsar_results_dir,
"source_files": source_files,
}
def _find_isce2_coherence_file(directory: str, prefix: str) -> Optional[str]:
for name in (f"{prefix}_coh.tif", f"{prefix}_coh.tiff"):
path = os.path.join(directory, name)
if os.path.isfile(path):
return path
return None
def _build_isce2_product_index(root_dir: Optional[str]) -> Dict[str, Dict[str, Any]]:
if not root_dir or not os.path.isdir(root_dir):
return {}
index: Dict[str, Dict[str, Any]] = {}
for current_root, _, files in os.walk(root_dir):
for file_name in files:
if _ISCE2_SKIP_PATTERN.match(file_name):
continue
match = _ISCE2_DISP_PATTERN.match(file_name)
if not match:
continue
prefix = match.group("prefix")
disp_path = os.path.join(current_root, file_name)
try:
stat = os.stat(disp_path)
scan_ts = max(stat.st_mtime, stat.st_ctime)
except OSError:
scan_ts = 0.0
existing = index.get(prefix)
if existing and existing.get("scan_ts", 0.0) >= scan_ts:
continue
coh_path = _find_isce2_coherence_file(current_root, prefix)
source_files = [disp_path]
if coh_path:
source_files.append(coh_path)
index[prefix] = {
"engine": "isce2",
"task_name": prefix,
"source_dir": current_root,
"source_files": source_files,
"scan_ts": scan_ts,
}
return index
def _merge_isce2_indexes(*indexes: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
merged: Dict[str, Dict[str, Any]] = {}
for index in indexes:
for prefix, item in index.items():
existing = merged.get(prefix)
if not existing or item.get("scan_ts", 0.0) >= existing.get("scan_ts", 0.0):
merged[prefix] = item
return merged
def _extract_envi_result(
*,
task_name: str,
result: Dict[str, Any],
target_dir: str,
) -> Dict[str, Any]:
target_base = os.path.join(target_dir, f"{task_name}_geo_disp")
details = {
"task": task_name,
"engine": "envi",
"target_name": f"{task_name}_geo_disp",
"source_dir": result.get("source_dir", ""),
"source_files": list(result.get("source_files", [])),
"target_files": [],
"status": "ok",
"copied": 0,
"skipped": 0,
"overwritten": 0,
"failed": 0,
}
for source_path in result.get("source_files", []):
suffix = source_path[len(result["source_files"][0]):]
target_path = target_base + suffix
details["target_files"].append(target_path)
try:
if os.path.isfile(target_path):
if os.path.getsize(source_path) == os.path.getsize(target_path):
details["skipped"] += 1
continue
shutil.copy2(source_path, target_path)
details["overwritten"] += 1
else:
shutil.copy2(source_path, target_path)
details["copied"] += 1
except OSError as exc:
details["failed"] += 1
details["status"] = f"error: {exc}"
return details
def _extract_isce2_result(
*,
task_name: str,
result: Dict[str, Any],
target_dir: str,
) -> Dict[str, Any]:
details = {
"task": task_name,
"engine": "isce2",
"target_name": task_name,
"source_dir": result.get("source_dir", ""),
"source_files": list(result.get("source_files", [])),
"target_files": [],
"status": "ok",
"copied": 0,
"skipped": 0,
"overwritten": 0,
"failed": 0,
}
for source_path in result.get("source_files", []):
target_path = os.path.join(target_dir, os.path.basename(source_path))
details["target_files"].append(target_path)
try:
if os.path.isfile(target_path):
if os.path.getsize(source_path) == os.path.getsize(target_path):
details["skipped"] += 1
continue
shutil.copy2(source_path, target_path)
details["overwritten"] += 1
else:
shutil.copy2(source_path, target_path)
details["copied"] += 1
except OSError as exc:
details["failed"] += 1
details["status"] = f"error: {exc}"
return details
def _collect_extracted_paths(target_dir: Optional[str], task_name: str) -> List[str]:
if not target_dir or not os.path.isdir(target_dir):
return []
candidates = [
os.path.join(target_dir, f"{task_name}_geo_disp"),
os.path.join(target_dir, f"{task_name}_disp.tif"),
os.path.join(target_dir, f"{task_name}_disp.tiff"),
os.path.join(target_dir, f"{task_name}_coh.tif"),
os.path.join(target_dir, f"{task_name}_coh.tiff"),
]
return [path for path in candidates if os.path.isfile(path)]
def _build_task_status(
*,
task_name: str,
task_path: str,
target_dir: Optional[str],
isce2_index: Dict[str, Dict[str, Any]],
) -> Dict[str, Any]:
has_master = os.path.isdir(os.path.join(task_path, "master"))
has_slave = os.path.isdir(os.path.join(task_path, "slave"))
has_structure = has_master and has_slave
imported = False
for subdir in ("master", "slave"):
local_dir = os.path.join(task_path, subdir)
if not os.path.isdir(local_dir):
continue
try:
if any(file_name.endswith(".sml") for file_name in os.listdir(local_dir)):
imported = True
break
except OSError:
continue
envi_result = _find_envi_task_result(task_path)
isce2_result = isce2_index.get(task_name)
result_engines: List[str] = []
result_paths: List[str] = []
if envi_result:
result_engines.append("envi")
result_paths.extend(envi_result.get("source_files", []))
if isce2_result:
result_engines.append("isce2")
result_paths.extend(isce2_result.get("source_files", []))
extracted_paths = _collect_extracted_paths(target_dir, task_name)
try:
mtime = os.path.getmtime(task_path)
last_modified = datetime.utcfromtimestamp(mtime).strftime("%Y-%m-%d %H:%M")
except OSError:
last_modified = "-"
return {
"task_name": task_name,
"has_structure": has_structure,
"imported": imported,
"dinsar_done": bool(result_engines),
"extracted": bool(extracted_paths),
"result_engines": result_engines,
"result_paths": result_paths,
"extracted_paths": extracted_paths,
"last_modified": last_modified,
}
def extract_disp_results(root_dir: str, dest_dir: Optional[str] = None) -> Dict[str, Any]:
"""Extract ENVI and ISCE2 displacement results into the monitor directory."""
source_root = _normalize_directory(root_dir, "结果根目录")
if dest_dir:
target_dir = _normalize_output_directory(dest_dir)
else:
target_dir = _normalize_output_directory(_get_dinsar_target_dir())
os.makedirs(target_dir, exist_ok=True)
task_dirs = _iter_task_dirs(source_root)
configured_isce2_root = ""
if settings.ISCE2_OUTPUT_ROOT:
candidate_root = os.path.normpath(os.path.abspath(settings.ISCE2_OUTPUT_ROOT))
if os.path.isdir(candidate_root):
configured_isce2_root = candidate_root
scan_source_root_for_isce2 = (not task_dirs) or _root_has_isce2_products(source_root)
direct_isce2_index = _build_isce2_product_index(source_root) if scan_source_root_for_isce2 else {}
configured_isce2_index = {}
if configured_isce2_root and not _same_path(configured_isce2_root, source_root):
configured_isce2_index = _build_isce2_product_index(configured_isce2_root)
isce2_index = _merge_isce2_indexes(direct_isce2_index, configured_isce2_index)
details: List[Dict[str, Any]] = []
copied = 0
skipped = 0
overwritten = 0
failed = 0
if task_dirs:
for task_name, task_path in task_dirs:
envi_result = _find_envi_task_result(task_path)
if envi_result:
detail = _extract_envi_result(task_name=task_name, result=envi_result, target_dir=target_dir)
details.append(detail)
isce2_result = isce2_index.get(task_name)
if isce2_result:
detail = _extract_isce2_result(task_name=task_name, result=isce2_result, target_dir=target_dir)
details.append(detail)
else:
for task_name in sorted(isce2_index):
detail = _extract_isce2_result(task_name=task_name, result=isce2_index[task_name], target_dir=target_dir)
details.append(detail)
for item in details:
copied += int(item.get("copied", 0))
skipped += int(item.get("skipped", 0))
overwritten += int(item.get("overwritten", 0))
failed += int(item.get("failed", 0))
return {
"processed": len(details),
"copied": copied,
"skipped": skipped,
"overwritten": overwritten,
"failed": failed,
"target_dir": target_dir,
"source_root": source_root,
"details": details,
}
def get_task_overview(root_dir: str) -> Dict[str, Any]:
"""Return task overview for both ENVI and ISCE2 D-InSAR products."""
source_root = _normalize_directory(root_dir, "任务根目录")
task_dirs = _iter_task_dirs(source_root)
try:
target_dir = _normalize_output_directory(_get_dinsar_target_dir())
except ValueError:
target_dir = None
configured_isce2_root = ""
if settings.ISCE2_OUTPUT_ROOT:
candidate_root = os.path.normpath(os.path.abspath(settings.ISCE2_OUTPUT_ROOT))
if os.path.isdir(candidate_root):
configured_isce2_root = candidate_root
direct_isce2_index = _build_isce2_product_index(source_root) if _root_has_isce2_products(source_root) else {}
configured_isce2_index = {}
if configured_isce2_root and not _same_path(configured_isce2_root, source_root):
configured_isce2_index = _build_isce2_product_index(configured_isce2_root)
isce2_index = _merge_isce2_indexes(direct_isce2_index, configured_isce2_index)
tasks: List[Dict[str, Any]] = []
summary = {
"total": 0,
"imported": 0,
"dinsar_done": 0,
"extracted": 0,
"envi_done": 0,
"isce2_done": 0,
"envi_extracted": 0,
"isce2_extracted": 0,
}
for task_name, task_path in task_dirs:
task_status = _build_task_status(
task_name=task_name,
task_path=task_path,
target_dir=target_dir,
isce2_index=isce2_index,
)
tasks.append(task_status)
summary["total"] += 1
if task_status["imported"]:
summary["imported"] += 1
if task_status["dinsar_done"]:
summary["dinsar_done"] += 1
if task_status["extracted"]:
summary["extracted"] += 1
if "envi" in task_status["result_engines"]:
summary["envi_done"] += 1
if "isce2" in task_status["result_engines"]:
summary["isce2_done"] += 1
extracted_paths = task_status.get("extracted_paths", [])
if any(path.endswith("_geo_disp") for path in extracted_paths):
summary["envi_extracted"] += 1
if any(_ISCE2_DISP_PATTERN.match(os.path.basename(path)) for path in extracted_paths):
summary["isce2_extracted"] += 1
return {
"tasks": tasks,
"summary": summary,
"target_dir": target_dir or "",
"source_root": source_root,
"isce2_output_root": configured_isce2_root,
}
+202
View File
@@ -0,0 +1,202 @@
"""Import workflow functions extracted from envi_service.py."""
from __future__ import annotations
import os
from glob import glob
from typing import Any, Dict, List
from .envi_service import (
DEFAULT_TIMEOUT,
_collect_task_folders,
_find_meta_files,
_has_sml,
_to_local_path,
execute_envi_task,
)
def _import_single_dir(
display_name: str,
data_dir: str,
log_lines: List[str],
) -> tuple:
"""Import all .meta.xml in one directory. Returns (processed, failed, skipped)."""
import time
if _has_sml(data_dir):
log_lines.append(f"[skip] {display_name}: .sml already exists")
return 0, 0, 1
meta_files = _find_meta_files(data_dir)
if not meta_files:
log_lines.append(f"[skip] {display_name}: no *.meta.xml")
return 0, 0, 0
processed = 0
failed = 0
for meta_file in meta_files:
start = time.time()
try:
execute_envi_task(
"SARsImportLuTan1",
{
"INPUT_FILE_LIST": [meta_file],
"ROOT_URI_FOR_OUTPUT": data_dir,
},
)
elapsed = round(time.time() - start, 1)
processed += 1
log_lines.append(
f"[ok] import {display_name}: "
f"{os.path.basename(meta_file)} ({elapsed}s)"
)
except Exception as exc:
elapsed = round(time.time() - start, 1)
failed += 1
log_lines.append(
f"[err] import {display_name}: "
f"{os.path.basename(meta_file)} failed ({elapsed}s): {exc}"
)
return processed, failed, 0
def run_import_workflow(
root_dir: str,
num_to_process: int = 0,
timeout: int = DEFAULT_TIMEOUT,
) -> Dict[str, Any]:
"""Scan root_dir for raw SAR data and batch-import via SARsImportLuTan1."""
root_dir = _to_local_path(root_dir)
if not root_dir or not os.path.isdir(root_dir):
raise ValueError(f"Import root directory does not exist: {root_dir}")
task_folders = _collect_task_folders(root_dir)
scan_mode = "task_folder"
if not task_folders:
task_folders = sorted(
p for p in glob(os.path.join(root_dir, "*")) if os.path.isdir(p)
)
scan_mode = "flat_folder"
log_lines: List[str] = [
f"[envi] import task=SARsImportLuTan1",
f"[envi] root_dir={root_dir}",
f"[envi] scan_mode={scan_mode}",
f"[envi] found folders={len(task_folders)}",
]
if not task_folders:
return {
"summary": {
"scan_mode": scan_mode,
"task_folders": 0,
"processed": 0,
"failed": 0,
"skipped": 0,
},
"log_lines": log_lines,
}
total_processed = 0
total_failed = 0
total_skipped = 0
for folder in task_folders:
if num_to_process > 0 and total_processed >= num_to_process:
log_lines.append(f"[envi] reached limit={num_to_process}")
break
name = os.path.basename(folder)
if scan_mode == "flat_folder":
candidates = [(name, folder)]
else:
candidates = []
for side in ("master", "slave"):
d = os.path.join(folder, side)
if os.path.isdir(d):
candidates.append((f"{name}/{side}", d))
else:
log_lines.append(f"[warn] {name}/{side} missing, skip")
for display, data_dir in candidates:
if num_to_process > 0 and total_processed >= num_to_process:
break
p, f, s = _import_single_dir(display, data_dir, log_lines)
total_processed += p
total_failed += f
total_skipped += s
if total_failed > 0 and total_processed == 0:
detail = "\n".join(log_lines[-20:])
raise RuntimeError(
f"All import tasks failed. failed={total_failed}.\n{detail}"
)
return {
"summary": {
"scan_mode": scan_mode,
"task_folders": len(task_folders),
"processed": total_processed,
"failed": total_failed,
"skipped": total_skipped,
},
"log_lines": log_lines,
}
def inspect_import(root_dir: str) -> Dict[str, Any]:
"""Pre-check Import readiness for a root directory."""
root_dir = _to_local_path(root_dir)
result: Dict[str, Any] = {
"workflow": "import",
"root_dir": root_dir,
"exists": bool(root_dir and os.path.isdir(root_dir)),
"ready": False,
"summary": {},
"warnings": [],
}
if not result["exists"]:
result["warnings"].append("root_dir does not exist.")
return result
task_folders = _collect_task_folders(root_dir)
scan_mode = "task_folder"
scan_targets = task_folders
if not task_folders:
scan_mode = "flat_folder"
scan_targets = sorted(
p for p in glob(os.path.join(root_dir, "*")) if os.path.isdir(p)
)
meta_count = 0
sml_count = 0
candidate_count = 0
for folder in scan_targets:
if scan_mode == "flat_folder":
candidates = [folder]
else:
candidates = [
os.path.join(folder, "master"),
os.path.join(folder, "slave"),
]
for candidate in candidates:
if not os.path.isdir(candidate):
continue
metas = glob(os.path.join(candidate, "*.meta.xml"))
smls = glob(os.path.join(candidate, "*.sml"))
meta_count += len(metas)
sml_count += len(smls)
if metas and not smls:
candidate_count += len(metas)
result["summary"] = {
"scan_mode": scan_mode,
"folder_count": len(scan_targets),
"meta_file_count": meta_count,
"existing_sml_count": sml_count,
"import_candidate_count": candidate_count,
}
result["ready"] = candidate_count > 0
if candidate_count == 0:
result["warnings"].append(
"No import candidates (*.meta.xml without *.sml)."
)
return result
+64
View File
@@ -0,0 +1,64 @@
"""CLI runner for ENVI workflows in a subprocess.
Usage (called by job_handlers.py):
python -m backend.app.services.envi_runner_cli \
--workflow import \
--root-dir Z:/Test_data/Test_IDL_1 \
[--num-to-process 0] \
[--timeout-seconds 14400]
Outputs a single JSON line to stdout on success.
Errors go to stderr with non-zero exit code.
"""
from __future__ import annotations
import argparse
import json
import sys
from ..config import ensure_project_env_loaded
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Run ENVI/SARscape workflow in isolated process."
)
parser.add_argument(
"--workflow",
required=True,
choices=["import", "dinsar", "dinsar_custom"],
)
parser.add_argument("--root-dir", required=True)
parser.add_argument("--num-to-process", type=int, default=0)
parser.add_argument("--timeout-seconds", type=int, default=None)
parser.add_argument("--job-id", type=str, default=None)
return parser.parse_args()
def main() -> int:
ensure_project_env_loaded()
args = _parse_args()
try:
from .envi_service import run_workflow
record = run_workflow(
workflow=args.workflow,
root_dir=args.root_dir,
num_to_process=args.num_to_process,
timeout=args.timeout_seconds or 14400,
job_id=args.job_id,
)
print(json.dumps(record, ensure_ascii=False))
return 0
except Exception as exc:
err = {
"ok": False,
"error": str(exc),
"workflow": args.workflow,
}
print(json.dumps(err, ensure_ascii=False), file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
File diff suppressed because it is too large Load Diff
+582
View File
@@ -0,0 +1,582 @@
"""
GF3 (GaoFen-3) L1A -> L2 processing service.
Pipeline: Extract archive -> Parse XML metadata -> Radiometric calibration -> RPC geometric correction.
Pure Python implementation using GDAL/numpy, no ENVI/SARscape dependency.
"""
from __future__ import annotations
import json
import logging
import math
import os
import shutil
import tarfile
import zipfile
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Archive extraction
# ---------------------------------------------------------------------------
def _extract_archive(path: str, dest: str) -> str:
"""Extract .tar.gz or .zip archive to *dest*, return extracted directory path."""
os.makedirs(dest, exist_ok=True)
if tarfile.is_tarfile(path):
with tarfile.open(path, "r:*") as tf:
# Security: prevent path traversal
for member in tf.getmembers():
if member.name.startswith("/") or ".." in member.name:
raise ValueError(f"Unsafe path in archive: {member.name}")
tf.extractall(dest)
elif zipfile.is_zipfile(path):
with zipfile.ZipFile(path, "r") as zf:
for info in zf.infolist():
if info.filename.startswith("/") or ".." in info.filename:
raise ValueError(f"Unsafe path in archive: {info.filename}")
zf.extractall(dest)
else:
raise ValueError(f"Unsupported archive format: {path}")
return dest
# ---------------------------------------------------------------------------
# GF3 XML metadata parser
# ---------------------------------------------------------------------------
def _parse_gf3_meta(xml_path: str) -> Dict[str, Any]:
"""Parse GF3 product XML metadata.
Returns dict with keys per polarization:
polarizations: list of str (e.g. ["HH", "VV"])
calibration: {pol: {"QualifyValue": float, "CalibrationConst": float}}
"""
import defusedxml.ElementTree as ET
tree = ET.parse(xml_path)
root = tree.getroot()
result: Dict[str, Any] = {"polarizations": [], "calibration": {}}
# Find all imageinfo or channel elements
# GF3 XML structure varies; search by tag name for robustness
def _find_all_recursive(element, tag):
found = []
for child in element.iter():
if tag.lower() in child.tag.lower():
found.append(child)
return found
# Try to find QualifyValue and CalibrationConst
# Typical GF3 XML has <imageinfo> -> <QualifyValue> and <CalibrationConst>
# per polarization channel
qualify_values = {}
cal_consts = {}
# Search for elements containing polarization-specific calibration info
for elem in root.iter():
tag = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag # strip namespace
if tag == "QualifyValue" and elem.text:
# Parent should indicate which polarization this belongs to
parent = _find_parent(root, elem)
pol = _extract_polarization(parent, elem)
if pol:
try:
qualify_values[pol] = float(elem.text.strip())
except ValueError:
pass
if tag == "CalibrationConst" and elem.text:
parent = _find_parent(root, elem)
pol = _extract_polarization(parent, elem)
if pol:
try:
cal_consts[pol] = float(elem.text.strip())
except ValueError:
pass
# If per-polarization search didn't work, try flat extraction
if not qualify_values:
# Fallback: find all QualifyValue elements in order
qv_elems = [e for e in root.iter() if e.tag.split("}")[-1] == "QualifyValue" and e.text]
cc_elems = [e for e in root.iter() if e.tag.split("}")[-1] == "CalibrationConst" and e.text]
# Find polarization list
pol_elems = [e for e in root.iter() if e.tag.split("}")[-1] == "Polarisation" and e.text]
if not pol_elems:
pol_elems = [e for e in root.iter() if e.tag.split("}")[-1] == "polarization" and e.text]
pols = [e.text.strip().upper() for e in pol_elems]
for i, pol in enumerate(pols):
if i < len(qv_elems):
try:
qualify_values[pol] = float(qv_elems[i].text.strip())
except ValueError:
pass
if i < len(cc_elems):
try:
cal_consts[pol] = float(cc_elems[i].text.strip())
except ValueError:
pass
polarizations = sorted(set(list(qualify_values.keys()) + list(cal_consts.keys())))
if not polarizations:
# Last resort: guess from TIFF filenames in same directory
xml_dir = os.path.dirname(xml_path)
for f in os.listdir(xml_dir):
fl = f.upper()
for pol in ("HH", "HV", "VH", "VV"):
if pol in fl and f.lower().endswith((".tif", ".tiff")):
if pol not in polarizations:
polarizations.append(pol)
polarizations.sort()
calibration = {}
for pol in polarizations:
calibration[pol] = {
"QualifyValue": qualify_values.get(pol, 1.0),
"CalibrationConst": cal_consts.get(pol, 0.0),
}
result["polarizations"] = polarizations
result["calibration"] = calibration
logger.info("[GF3] Parsed metadata: polarizations=%s, calibration=%s", polarizations, calibration)
return result
def _find_parent(root, target):
"""Find the parent element of *target* in the tree."""
for parent in root.iter():
for child in parent:
if child is target:
return parent
return None
def _extract_polarization(parent, elem) -> Optional[str]:
"""Try to extract polarization from context around an XML element."""
if parent is None:
return None
# Check parent tag or sibling elements
tag = parent.tag.split("}")[-1] if "}" in parent.tag else parent.tag
for pol in ("HH", "HV", "VH", "VV"):
if pol in tag.upper():
return pol
# Check sibling text
for child in parent:
child_tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
if "polaris" in child_tag.lower() and child.text:
return child.text.strip().upper()
return None
# ---------------------------------------------------------------------------
# RPC file parser
# ---------------------------------------------------------------------------
def _read_rpb(rpb_path: str) -> Dict[str, Any]:
"""Parse a .rpb (Rational Polynomial Coefficients) file into GDAL metadata dict."""
rpc = {}
current_key = None
values = []
with open(rpb_path, "r") as f:
for line in f:
line = line.strip().rstrip(";")
if "=" in line:
if current_key and values:
rpc[current_key] = values
values = []
key, _, val = line.partition("=")
current_key = key.strip()
val = val.strip().strip("(").strip(")")
if val:
for v in val.replace(",", " ").split():
try:
values.append(float(v))
except ValueError:
values.append(v)
else:
# continuation of values
val = line.strip("()").strip()
if val:
for v in val.replace(",", " ").split():
try:
values.append(float(v))
except ValueError:
values.append(v)
if current_key and values:
rpc[current_key] = values
# Map to GDAL RPC metadata keys
gdal_rpc = {}
key_map = {
"lineOffset": "LINE_OFF",
"sampOffset": "SAMP_OFF",
"latOffset": "LAT_OFF",
"longOffset": "LONG_OFF",
"heightOffset": "HEIGHT_OFF",
"lineScale": "LINE_SCALE",
"sampScale": "SAMP_SCALE",
"latScale": "LAT_SCALE",
"longScale": "LONG_SCALE",
"heightScale": "HEIGHT_SCALE",
"lineNumCoef": "LINE_NUM_COEFF",
"lineDenCoef": "LINE_DEN_COEFF",
"sampNumCoef": "SAMP_NUM_COEFF",
"sampDenCoef": "SAMP_DEN_COEFF",
}
for rpb_key, gdal_key in key_map.items():
if rpb_key in rpc:
val = rpc[rpb_key]
if isinstance(val, list):
if len(val) == 1:
gdal_rpc[gdal_key] = str(val[0])
else:
gdal_rpc[gdal_key] = " ".join(str(v) for v in val)
else:
gdal_rpc[gdal_key] = str(val)
return gdal_rpc
# ---------------------------------------------------------------------------
# Radiometric calibration
# ---------------------------------------------------------------------------
def _radiometric_calibration(tiff_path: str, qv: float, cal: float, output_path: str) -> str:
"""L1A -> L1B single-polarization radiometric calibration.
Formula: A = sqrt(I^2 + Q^2), dB = 20*log10(A * QV / 65535) - Cal
For amplitude-only TIFF: dB = 20*log10(A * QV / 65535) - Cal
"""
from osgeo import gdal
ds = gdal.Open(tiff_path, gdal.GA_ReadOnly)
if ds is None:
raise FileNotFoundError(f"Cannot open TIFF: {tiff_path}")
n_bands = ds.RasterCount
width = ds.RasterXSize
height = ds.RasterYSize
if n_bands >= 2:
# Complex I/Q data
band_i = ds.GetRasterBand(1).ReadAsArray().astype(np.float64)
band_q = ds.GetRasterBand(2).ReadAsArray().astype(np.float64)
amplitude = np.sqrt(band_i ** 2 + band_q ** 2)
else:
# Amplitude only
amplitude = ds.GetRasterBand(1).ReadAsArray().astype(np.float64)
# Avoid log of zero
amplitude = np.where(amplitude > 0, amplitude, np.nan)
db_values = 20.0 * np.log10(amplitude * qv / 65535.0) - cal
db_values = np.where(np.isfinite(db_values), db_values, 0).astype(np.float32)
# Write calibrated result preserving georeference
driver = gdal.GetDriverByName("GTiff")
out_ds = driver.Create(output_path, width, height, 1, gdal.GDT_Float32,
options=["COMPRESS=DEFLATE"])
out_ds.SetGeoTransform(ds.GetGeoTransform())
out_ds.SetProjection(ds.GetProjection())
# Copy RPC metadata if present
rpc_md = ds.GetMetadata("RPC")
if rpc_md:
out_ds.SetMetadata(rpc_md, "RPC")
out_ds.GetRasterBand(1).WriteArray(db_values)
out_ds.GetRasterBand(1).SetNoDataValue(0)
out_ds.FlushCache()
out_ds = None
ds = None
logger.info("[GF3] Calibration done: %s -> %s (QV=%.2f, Cal=%.2f)", tiff_path, output_path, qv, cal)
return output_path
# ---------------------------------------------------------------------------
# Geometric correction
# ---------------------------------------------------------------------------
def _geometric_correction(
l1b_path: str,
rpb_path: Optional[str],
output_path: str,
resolution: float,
dem_path: str,
) -> str:
"""L1B -> L2 RPC geometric correction using GDAL Warp."""
from osgeo import gdal
ds = gdal.Open(l1b_path, gdal.GA_ReadOnly)
if ds is None:
raise FileNotFoundError(f"Cannot open L1B: {l1b_path}")
# If RPC not already in dataset, load from .rpb
rpc_md = ds.GetMetadata("RPC")
if not rpc_md and rpb_path and os.path.isfile(rpb_path):
rpc_md = _read_rpb(rpb_path)
ds.SetMetadata(rpc_md, "RPC")
logger.info("[GF3] Loaded RPC from %s", rpb_path)
warp_options = gdal.WarpOptions(
dstSRS="EPSG:4326",
format="GTiff",
xRes=resolution,
yRes=resolution,
rpc=True,
creationOptions=["COMPRESS=DEFLATE"],
)
# Use DEM if available
if dem_path and os.path.isfile(dem_path):
warp_options = gdal.WarpOptions(
dstSRS="EPSG:4326",
format="GTiff",
xRes=resolution,
yRes=resolution,
rpc=True,
transformerOptions=[f"RPC_DEM={dem_path}"],
creationOptions=["COMPRESS=DEFLATE"],
)
result = gdal.Warp(output_path, ds, options=warp_options)
ds = None
if result is None:
raise RuntimeError(f"GDAL Warp failed for {l1b_path}")
result = None
logger.info("[GF3] Geometric correction done: %s -> %s", l1b_path, output_path)
return output_path
# ---------------------------------------------------------------------------
# Main pipeline
# ---------------------------------------------------------------------------
def run_gf3_l1a_to_l2(
input_dir: str,
output_dir: str,
resolution: float = 0.0002,
job_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Run GF3 L1A -> L2 pipeline: extract -> calibrate -> geometric correction.
Args:
input_dir: Path to GF3 L1A product directory (or archive file)
output_dir: Output directory for L2 products
resolution: Output resolution in degrees (default 0.0002 ~ 20m)
job_id: Optional job ID for progress tracking
Returns:
dict with keys: ok, l2_paths, polarizations, output_dir
"""
from ..config import settings
dem_path = settings.GF3_GEO_DEM_PATH
os.makedirs(output_dir, exist_ok=True)
# Step 1: Extract if archive
work_dir = input_dir
if os.path.isfile(input_dir):
logger.info("[GF3] Extracting archive: %s", input_dir)
extract_dir = os.path.join(output_dir, "_extracted")
_extract_archive(input_dir, extract_dir)
# Find actual data directory (may be nested)
subdirs = [d for d in os.listdir(extract_dir)
if os.path.isdir(os.path.join(extract_dir, d))]
work_dir = os.path.join(extract_dir, subdirs[0]) if subdirs else extract_dir
# Step 2: Scan for XML + TIFF + RPB files
xml_path = None
tiff_files: Dict[str, str] = {} # pol -> tiff path
rpb_files: Dict[str, str] = {} # pol -> rpb path
for f in os.listdir(work_dir):
fl = f.lower()
fp = os.path.join(work_dir, f)
if fl.endswith(".meta.xml") or (fl.endswith(".xml") and "meta" in fl):
xml_path = fp
elif fl.endswith(".xml") and xml_path is None:
xml_path = fp
elif fl.endswith((".tif", ".tiff")):
for pol in ("HH", "HV", "VH", "VV"):
if pol in f.upper():
tiff_files[pol] = fp
break
elif fl.endswith(".rpb"):
for pol in ("HH", "HV", "VH", "VV"):
if pol in f.upper():
rpb_files[pol] = fp
break
if not xml_path:
# Try subdirectories
for sub in os.listdir(work_dir):
sub_path = os.path.join(work_dir, sub)
if os.path.isdir(sub_path):
for f in os.listdir(sub_path):
fl = f.lower()
fp = os.path.join(sub_path, f)
if fl.endswith(".xml") and not xml_path:
xml_path = fp
elif fl.endswith((".tif", ".tiff")):
for pol in ("HH", "HV", "VH", "VV"):
if pol in f.upper():
tiff_files[pol] = fp
break
elif fl.endswith(".rpb"):
for pol in ("HH", "HV", "VH", "VV"):
if pol in f.upper():
rpb_files[pol] = fp
break
if not tiff_files:
return {"ok": False, "error": f"No TIFF files found in {work_dir}"}
# Step 3: Parse XML metadata
meta = {"polarizations": list(tiff_files.keys()), "calibration": {}}
if xml_path:
try:
meta = _parse_gf3_meta(xml_path)
except Exception as e:
logger.warning("[GF3] Failed to parse XML %s: %s, using defaults", xml_path, e)
polarizations = meta.get("polarizations", list(tiff_files.keys()))
calibration = meta.get("calibration", {})
# Step 4-5: Process each polarization
l2_paths = []
for pol in polarizations:
if pol not in tiff_files:
logger.warning("[GF3] No TIFF found for polarization %s, skipping", pol)
continue
tiff_path = tiff_files[pol]
cal_info = calibration.get(pol, {"QualifyValue": 1.0, "CalibrationConst": 0.0})
qv = cal_info["QualifyValue"]
cal = cal_info["CalibrationConst"]
# L1A -> L1B (calibration)
l1b_path = os.path.join(output_dir, f"{pol}_L1B.tif")
logger.info("[GF3] Calibrating %s (QV=%.2f, Cal=%.2f)", pol, qv, cal)
_radiometric_calibration(tiff_path, qv, cal, l1b_path)
# L1B -> L2 (geometric correction)
l2_path = os.path.join(output_dir, f"{pol}_L2.tif")
rpb_path = rpb_files.get(pol)
logger.info("[GF3] Geometric correction %s (resolution=%.6f)", pol, resolution)
_geometric_correction(l1b_path, rpb_path, l2_path, resolution, dem_path)
l2_paths.append(l2_path)
# Clean up intermediate L1B
try:
os.remove(l1b_path)
except OSError:
pass
if not l2_paths:
return {"ok": False, "error": "No polarization channels processed successfully"}
logger.info("[GF3] Pipeline complete: %d L2 products", len(l2_paths))
return {
"ok": True,
"l2_paths": l2_paths,
"polarizations": polarizations,
"output_dir": output_dir,
"input_dir_name": os.path.basename(input_dir),
}
# ---------------------------------------------------------------------------
# Auto-register L2 result into radar_data table
# ---------------------------------------------------------------------------
async def register_l2_to_radar_data(l2_dir: str, input_dir_name: str, polarizations: List[str], db) -> Optional[int]:
"""Register a GF3 L2 output directory as a radar_data record.
Args:
l2_dir: Path to the L2 output directory
input_dir_name: Original L1A input directory name (for metadata extraction)
polarizations: List of polarization channels processed
db: AsyncSession
Returns:
radar_data.id if successfully registered, else None
"""
from ..utils import parse_gf3_l2_dirname
from ..models import RadarDataORM
from .data_service import extract_geotiff_bounds
from geoalchemy2.shape import from_shape
from shapely.geometry import Polygon as ShapelyPolygon
from sqlalchemy.future import select
# Check if already registered (by file_path)
result = await db.execute(
select(RadarDataORM).where(RadarDataORM.file_path == l2_dir).limit(1)
)
if result.scalar_one_or_none():
logger.info("[GF3] L2 dir already registered: %s", l2_dir)
return None
# Parse metadata from the input directory name
meta = parse_gf3_l2_dirname(input_dir_name)
if not meta:
# Fallback: minimal metadata
meta = {
"satellite": "GF3",
"imaging_date": None,
"polarization": ",".join(polarizations) if polarizations else None,
}
# Try to extract polygon from the first L2 GeoTIFF
polygon = None
try:
for f in os.listdir(l2_dir):
if f.lower().endswith((".tif", ".tiff")) and "L2" in f:
tiff_path = os.path.join(l2_dir, f)
polygon = extract_geotiff_bounds(tiff_path)
if polygon:
break
except OSError:
pass
coverage_geom = None
if polygon and len(polygon) >= 4:
try:
shp = ShapelyPolygon(polygon)
if shp.is_valid:
coverage_geom = from_shape(shp, srid=4326)
except Exception:
pass
radar = RadarDataORM(
file_path=l2_dir,
satellite=meta.get("satellite", "GF3"),
imaging_date=meta.get("imaging_date"),
imaging_mode=meta.get("imaging_mode"),
polarization=meta.get("polarization") or (",".join(polarizations) if polarizations else None),
scene_center_lon=meta.get("scene_center_lon"),
scene_center_lat=meta.get("scene_center_lat"),
coverage_polygon=coverage_geom,
)
db.add(radar)
await db.flush()
radar_id = radar.id
await db.commit()
logger.info("[GF3] Registered L2 in radar_data: id=%s, path=%s", radar_id, l2_dir)
return radar_id
+857
View File
@@ -0,0 +1,857 @@
import os
import asyncio
from datetime import datetime, timedelta
from typing import Any, Dict, Optional
import httpx
from sqlalchemy import and_, func, literal, or_, select, text
from .. import database
from ..config import read_int_env, settings, split_env_paths
from ..db_maintenance import inspect_database_structure
from ..models import (
AiDiagnosisORM,
DinsarResultORM,
ResultCatalogStateORM,
ResultProductORM,
SystemWorkerHeartbeatORM,
)
from ..idl_service import get_idl_status
from .pairing_state_service import pairing_state_service
DEFAULT_WORKER_TIMEOUT_SECONDS = 60
DEFAULT_SCHEMA_CACHE_SECONDS = 120
_SCHEMA_CACHE_LOCK = asyncio.Lock()
_SCHEMA_CACHE: Dict[str, Any] = {
"payload": None,
"expires_at": None,
}
def _get_session_factory():
if database.AsyncSessionLocal is None:
database.init_db()
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal
def _default_schema_status() -> Dict[str, Any]:
return {
"schema_ok": False,
"missing_tables": [],
"required_tables": [],
"extra_tables": [],
"required_table_count": 0,
"current_table_count": 0,
"missing_columns": {},
"extra_columns": {},
"type_mismatches": [],
"nullable_mismatches": [],
"schema_reasons": [],
"schema_issue_count": 0,
}
def _apply_schema_details(status: Dict[str, Any], schema_details: Dict[str, Any]) -> None:
status["required_tables"] = schema_details.get("required_tables", [])
status["required_table_count"] = int(schema_details.get("required_table_count") or 0)
status["current_table_count"] = int(schema_details.get("current_table_count") or 0)
status["missing_tables"] = schema_details.get("missing_tables", [])
status["extra_tables"] = schema_details.get("extra_tables", [])
status["missing_columns"] = schema_details.get("missing_columns", {})
status["extra_columns"] = schema_details.get("extra_columns", {})
status["type_mismatches"] = schema_details.get("type_mismatches", [])
status["nullable_mismatches"] = schema_details.get("nullable_mismatches", [])
status["schema_reasons"] = schema_details.get("reasons", [])
status["schema_issue_count"] = int(schema_details.get("reason_count") or 0)
status["schema_ok"] = not bool(schema_details.get("mismatch"))
async def _get_cached_schema_details(force_refresh: bool = False) -> Dict[str, Any]:
ttl_seconds = read_int_env(
"HEALTH_SCHEMA_CACHE_SECONDS",
DEFAULT_SCHEMA_CACHE_SECONDS,
minimum=5,
maximum=3600,
)
now = datetime.utcnow()
cached_payload = _SCHEMA_CACHE.get("payload")
expires_at = _SCHEMA_CACHE.get("expires_at")
if (
not force_refresh
and cached_payload is not None
and expires_at is not None
and expires_at > now
):
return cached_payload
async with _SCHEMA_CACHE_LOCK:
cached_payload = _SCHEMA_CACHE.get("payload")
expires_at = _SCHEMA_CACHE.get("expires_at")
now = datetime.utcnow()
if (
not force_refresh
and cached_payload is not None
and expires_at is not None
and expires_at > now
):
return cached_payload
try:
session_factory = _get_session_factory()
async with session_factory() as db:
schema_details = await db.run_sync(
lambda sync_session: inspect_database_structure(sync_session.connection())
)
except Exception:
if cached_payload is not None:
return cached_payload
raise
_SCHEMA_CACHE["payload"] = schema_details
_SCHEMA_CACHE["expires_at"] = now + timedelta(seconds=ttl_seconds)
return schema_details
async def _check_database(force_schema_refresh: bool = False) -> Dict[str, Any]:
status = {
"ok": False,
"postgis_ok": False,
**_default_schema_status(),
"error": None,
}
try:
session_factory = _get_session_factory()
async with session_factory() as db:
await db.execute(text("SELECT 1"))
status["ok"] = True
ext = await db.execute(
text("SELECT 1 FROM pg_extension WHERE extname = 'postgis'")
)
status["postgis_ok"] = ext.first() is not None
schema_details = await _get_cached_schema_details(force_refresh=force_schema_refresh)
_apply_schema_details(status, schema_details)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_worker(timeout_seconds: int = DEFAULT_WORKER_TIMEOUT_SECONDS) -> Dict[str, Any]:
status = {
"ok": False,
"worker_count": 0,
"workers": [],
"timeout_seconds": timeout_seconds,
}
try:
session_factory = _get_session_factory()
async with session_factory() as db:
threshold = datetime.utcnow() - timedelta(seconds=timeout_seconds)
result = await db.execute(
select(SystemWorkerHeartbeatORM)
.where(SystemWorkerHeartbeatORM.last_seen >= threshold)
.order_by(SystemWorkerHeartbeatORM.last_seen.desc())
)
rows = result.scalars().all()
status["worker_count"] = len(rows)
status["workers"] = [
{
"worker_id": r.worker_id,
"hostname": r.hostname,
"pid": r.pid,
"last_seen": r.last_seen,
}
for r in rows
]
status["ok"] = len(rows) > 0
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_ollama() -> Dict[str, Any]:
status = {"ok": False, "error": None, "status_code": None, "models": []}
ollama_base_url = settings.OLLAMA_BASE_URL
try:
async with httpx.AsyncClient(timeout=1.5) as client:
resp = await client.get(f"{ollama_base_url}/api/tags")
status["status_code"] = resp.status_code
if resp.status_code == 200:
data = resp.json()
status["models"] = [m.get("name") for m in data.get("models", [])]
status["ok"] = True
except Exception as exc:
status["error"] = str(exc)
return status
def _build_catalog_status(
*,
catalog_name: str,
storage_root: str,
enabled: bool = True,
) -> Dict[str, Any]:
return {
"ok": False,
"catalog_name": catalog_name,
"enabled": enabled,
"storage_root": storage_root,
"storage_root_exists": os.path.isdir(storage_root),
"state_present": False,
"catalog_status": None,
"needs_rebuild": None,
"manifest_count": 0,
"db_count": 0,
"issue_count": 0,
"last_boot_check_at": None,
"last_full_rebuild_at": None,
"last_incremental_scan_at": None,
"last_message": None,
"error": None,
}
async def _check_catalog(
*,
catalog_name: str,
storage_root: str,
enabled: bool = True,
) -> Dict[str, Any]:
status = _build_catalog_status(
catalog_name=catalog_name,
storage_root=storage_root,
enabled=enabled,
)
if not enabled:
status["ok"] = True
status["catalog_status"] = "DISABLED"
status["needs_rebuild"] = False
status["last_message"] = "catalog disabled by config"
return status
try:
session_factory = _get_session_factory()
async with session_factory() as db:
result = await db.execute(
select(ResultCatalogStateORM).where(
ResultCatalogStateORM.catalog_name == catalog_name
)
)
state = result.scalar_one_or_none()
if state is not None:
status["state_present"] = True
status["catalog_status"] = state.status
status["needs_rebuild"] = bool(state.needs_rebuild)
status["manifest_count"] = int(state.manifest_count or 0)
status["issue_count"] = int(state.issue_count or 0)
status["last_boot_check_at"] = state.last_boot_check_at
status["last_full_rebuild_at"] = state.last_full_rebuild_at
status["last_incremental_scan_at"] = state.last_incremental_scan_at
status["last_message"] = state.last_message
if state.storage_root:
status["storage_root"] = state.storage_root
status["storage_root_exists"] = os.path.isdir(state.storage_root)
count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == catalog_name
)
)
status["db_count"] = int(count_result.scalar_one() or 0)
manifest_count = int(status["manifest_count"] or 0)
needs_rebuild = bool(status["needs_rebuild"])
status["ok"] = bool(status["storage_root_exists"]) and not (
manifest_count > 0 and needs_rebuild
)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_result_catalog() -> Dict[str, Any]:
return await _check_catalog(
catalog_name="dinsar",
storage_root=settings.DINSAR_PRODUCT_DIR,
enabled=True,
)
async def _check_psinsar_result_catalog() -> Dict[str, Any]:
return await _check_catalog(
catalog_name="psinsar",
storage_root=settings.PSINSAR_PRODUCT_DIR,
enabled=bool(settings.TIMESERIES_ENABLED),
)
async def _check_nginx() -> Dict[str, Any]:
status = {"ok": False, "error": None, "status_code": None}
nginx_health_url = settings.NGINX_HEALTH_URL
try:
async with httpx.AsyncClient(timeout=1.0) as client:
resp = await client.get(nginx_health_url)
status["status_code"] = resp.status_code
status["ok"] = True
except Exception as exc:
status["error"] = str(exc)
return status
def _as_optional_bool(value: Any) -> Optional[bool]:
if value is None:
return None
return bool(value)
def _sanitize_catalog_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"catalog_name": payload.get("catalog_name"),
"enabled": _as_optional_bool(payload.get("enabled")),
"ok": bool(payload.get("ok")),
"needs_rebuild": _as_optional_bool(payload.get("needs_rebuild")),
"catalog_status": payload.get("catalog_status"),
"manifest_count": int(payload.get("manifest_count") or 0),
"db_count": int(payload.get("db_count") or 0),
"issue_count": int(payload.get("issue_count") or 0),
}
def _sanitize_bridge_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
"catalog_count": int(payload.get("catalog_count") or 0),
"compat_count": int(payload.get("compat_count") or 0),
"matched_count": int(payload.get("matched_count") or 0),
"missing_compat_count": int(payload.get("missing_compat_count") or 0),
"orphan_compat_count": int(payload.get("orphan_compat_count") or 0),
"duplicate_compat_product_count": int(payload.get("duplicate_compat_product_count") or 0),
"annotation_drift_count": int(payload.get("annotation_drift_count") or 0),
"diagnosis_total_count": int(payload.get("diagnosis_total_count") or 0),
"diagnosis_missing_product_row_count": int(payload.get("diagnosis_missing_product_row_count") or 0),
"diagnosis_product_identity_mismatch_count": int(payload.get("diagnosis_product_identity_mismatch_count") or 0),
"diagnosis_result_product_mismatch_count": int(payload.get("diagnosis_result_product_mismatch_count") or 0),
"result_trace_missing_count": int(payload.get("result_trace_missing_count") or 0),
"result_trace_orphan_count": int(payload.get("result_trace_orphan_count") or 0),
"result_trace_pair_mismatch_count": int(payload.get("result_trace_pair_mismatch_count") or 0),
}
def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
"configured_count": int(payload.get("configured_count") or 0),
"accessible_count": int(payload.get("accessible_count") or 0),
"inaccessible_count": int(payload.get("inaccessible_count") or 0),
}
def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
return {
"ok": bool(payload.get("ok")),
"state_present": bool(payload.get("state_present")),
"status": payload.get("status"),
"metric_version": payload.get("metric_version"),
"scene_count": int(payload.get("scene_count") or 0),
"pair_count": int(payload.get("pair_count") or 0),
"dirty_scene_count": int(payload.get("dirty_scene_count") or 0),
"needs_rebuild": bool(payload.get("needs_rebuild")),
"cache_ready": bool(payload.get("cache_ready")),
"network_run_count": int(payload.get("network_run_count") or 0),
"network_edge_count": int(payload.get("network_edge_count") or 0),
"duplicate_reverse_pair_count": int(payload.get("duplicate_reverse_pair_count") or 0),
"orphan_edge_count": int(payload.get("orphan_edge_count") or 0),
}
def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
database = payload.get("database", {}) or {}
worker = payload.get("worker", {}) or {}
result_catalog = payload.get("result_catalog", {}) or {}
dinsar_result_catalog = payload.get("dinsar_result_catalog", {}) or result_catalog
psinsar_result_catalog = payload.get("psinsar_result_catalog", {}) or {}
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
source_roots = payload.get("source_roots", {}) or {}
pairing_system = payload.get("pairing_system", {}) or {}
idl = payload.get("idl", {}) or {}
idl_status = idl.get("status", {}) or {}
ollama = payload.get("ollama", {}) or {}
nginx = payload.get("nginx", {}) or {}
sanitized_dinsar_catalog = _sanitize_catalog_status(dinsar_result_catalog)
sanitized_psinsar_catalog = _sanitize_catalog_status(psinsar_result_catalog)
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
return {
"ok": bool(payload.get("ok")),
"timestamp": payload.get("timestamp"),
"database": {
"ok": bool(database.get("ok")),
"postgis_ok": bool(database.get("postgis_ok")),
"schema_ok": bool(database.get("schema_ok")),
},
"worker": {
"ok": bool(worker.get("ok")),
"worker_count": int(worker.get("worker_count") or 0),
"timeout_seconds": int(worker.get("timeout_seconds") or DEFAULT_WORKER_TIMEOUT_SECONDS),
},
"result_catalog": sanitized_dinsar_catalog,
"dinsar_result_catalog": sanitized_dinsar_catalog,
"psinsar_result_catalog": sanitized_psinsar_catalog,
"catalogs": {
"dinsar": sanitized_dinsar_catalog,
"psinsar": sanitized_psinsar_catalog,
},
"dinsar_bridge": sanitized_dinsar_bridge,
"source_roots": sanitized_source_roots,
"pairing_system": sanitized_pairing_system,
"idl": {
"ok": bool(idl.get("ok")),
"status": {
"is_running": bool(idl_status.get("is_running")),
},
},
"ollama": {
"ok": _as_optional_bool(ollama.get("ok")),
},
"nginx": {
"ok": _as_optional_bool(nginx.get("ok")),
},
}
async def _check_dinsar_engines() -> Dict[str, Any]:
"""检查所有 D-InSAR 引擎可用性(快速,不做 WSL 深度校验)。"""
try:
from ..dinsar_engines import registry
engines = registry.list_engines()
results = []
any_available = False
for engine in engines:
avail = await asyncio.to_thread(engine.check_available)
results.append({
"engine_code": avail.engine_code,
"status": avail.status,
"available": avail.available,
"message": avail.message,
})
if avail.available:
any_available = True
# 整体状态:至少一个引擎可用即为 ok/degraded
non_placeholder = [r for r in results if r["status"] != "not_implemented"]
all_available = all(r["available"] for r in non_placeholder)
if all_available and any_available:
overall = "ok"
elif any_available:
overall = "degraded"
else:
overall = "unavailable"
return {
"ok": any_available,
"overall": overall,
"engines": results,
}
except Exception as exc:
return {"ok": False, "overall": "error", "engines": [], "error": str(exc)}
async def _check_dinsar_bridge() -> Dict[str, Any]:
status = {
"ok": False,
"catalog_count": 0,
"compat_count": 0,
"matched_count": 0,
"missing_compat_count": 0,
"orphan_compat_count": 0,
"duplicate_compat_product_count": 0,
"annotation_drift_count": 0,
"diagnosis_total_count": 0,
"diagnosis_with_product_ref_count": 0,
"diagnosis_missing_product_ref_count": 0,
"diagnosis_with_product_id_count": 0,
"diagnosis_missing_product_id_count": 0,
"diagnosis_missing_product_row_count": 0,
"diagnosis_product_identity_mismatch_count": 0,
"diagnosis_result_product_mismatch_count": 0,
"result_trace_missing_count": 0,
"result_trace_orphan_count": 0,
"result_trace_pair_mismatch_count": 0,
"error": None,
}
try:
session_factory = _get_session_factory()
async with session_factory() as db:
catalog_count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == "dinsar"
)
)
status["catalog_count"] = int(catalog_count_result.scalar_one() or 0)
compat_count_result = await db.execute(
select(func.count(DinsarResultORM.id)).where(
DinsarResultORM.compat_product_id.is_not(None)
)
)
status["compat_count"] = int(compat_count_result.scalar_one() or 0)
matched_count_result = await db.execute(
select(func.count(DinsarResultORM.id))
.select_from(DinsarResultORM)
.join(
ResultProductORM,
and_(
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
ResultProductORM.catalog_name == "dinsar",
),
)
)
status["matched_count"] = int(matched_count_result.scalar_one() or 0)
missing_compat_count_result = await db.execute(
select(func.count(ResultProductORM.id))
.select_from(ResultProductORM)
.outerjoin(
DinsarResultORM,
DinsarResultORM.compat_product_id == ResultProductORM.product_id,
)
.where(
ResultProductORM.catalog_name == "dinsar",
DinsarResultORM.id.is_(None),
)
)
status["missing_compat_count"] = int(missing_compat_count_result.scalar_one() or 0)
orphan_compat_count_result = await db.execute(
select(func.count(DinsarResultORM.id))
.select_from(DinsarResultORM)
.outerjoin(
ResultProductORM,
and_(
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
ResultProductORM.catalog_name == "dinsar",
),
)
.where(
DinsarResultORM.compat_product_id.is_not(None),
ResultProductORM.id.is_(None),
)
)
status["orphan_compat_count"] = int(orphan_compat_count_result.scalar_one() or 0)
duplicate_subquery = (
select(DinsarResultORM.compat_product_id)
.where(DinsarResultORM.compat_product_id.is_not(None))
.group_by(DinsarResultORM.compat_product_id)
.having(func.count(DinsarResultORM.id) > 1)
.subquery()
)
duplicate_count_result = await db.execute(
select(func.count()).select_from(duplicate_subquery)
)
status["duplicate_compat_product_count"] = int(duplicate_count_result.scalar_one() or 0)
annotation_drift_result = await db.execute(
select(func.count(DinsarResultORM.id))
.select_from(DinsarResultORM)
.join(
ResultProductORM,
and_(
ResultProductORM.product_id == DinsarResultORM.compat_product_id,
ResultProductORM.catalog_name == "dinsar",
),
)
.where(
or_(
func.coalesce(ResultProductORM.ai_score, literal(-1.0))
!= func.coalesce(DinsarResultORM.ai_score, literal(-1.0)),
func.coalesce(ResultProductORM.user_label, literal(-999999))
!= func.coalesce(DinsarResultORM.user_label, literal(-999999)),
)
)
)
status["annotation_drift_count"] = int(annotation_drift_result.scalar_one() or 0)
diagnosis_total_result = await db.execute(
select(func.count(AiDiagnosisORM.id))
)
status["diagnosis_total_count"] = int(diagnosis_total_result.scalar_one() or 0)
diagnosis_with_product_ref_result = await db.execute(
select(func.count(AiDiagnosisORM.id)).where(AiDiagnosisORM.product_ref_id.is_not(None))
)
status["diagnosis_with_product_ref_count"] = int(diagnosis_with_product_ref_result.scalar_one() or 0)
status["diagnosis_missing_product_ref_count"] = (
status["diagnosis_total_count"] - status["diagnosis_with_product_ref_count"]
)
diagnosis_with_product_id_result = await db.execute(
select(func.count(AiDiagnosisORM.id)).where(AiDiagnosisORM.product_id.is_not(None))
)
status["diagnosis_with_product_id_count"] = int(diagnosis_with_product_id_result.scalar_one() or 0)
status["diagnosis_missing_product_id_count"] = (
status["diagnosis_total_count"] - status["diagnosis_with_product_id_count"]
)
diagnosis_missing_product_row_result = await db.execute(
select(func.count(AiDiagnosisORM.id))
.select_from(AiDiagnosisORM)
.outerjoin(ResultProductORM, ResultProductORM.id == AiDiagnosisORM.product_ref_id)
.where(
AiDiagnosisORM.product_ref_id.is_not(None),
ResultProductORM.id.is_(None),
)
)
status["diagnosis_missing_product_row_count"] = int(
diagnosis_missing_product_row_result.scalar_one() or 0
)
diagnosis_product_identity_mismatch_result = await db.execute(
select(func.count(AiDiagnosisORM.id))
.select_from(AiDiagnosisORM)
.join(ResultProductORM, ResultProductORM.id == AiDiagnosisORM.product_ref_id)
.where(
AiDiagnosisORM.product_id.is_not(None),
AiDiagnosisORM.product_id != ResultProductORM.product_id,
)
)
status["diagnosis_product_identity_mismatch_count"] = int(
diagnosis_product_identity_mismatch_result.scalar_one() or 0
)
diagnosis_result_product_mismatch_result = await db.execute(
select(func.count(AiDiagnosisORM.id))
.select_from(AiDiagnosisORM)
.join(DinsarResultORM, DinsarResultORM.id == AiDiagnosisORM.result_id)
.where(
or_(
DinsarResultORM.compat_product_id.is_(None),
and_(
AiDiagnosisORM.product_id.is_not(None),
DinsarResultORM.compat_product_id != AiDiagnosisORM.product_id,
),
)
)
)
status["diagnosis_result_product_mismatch_count"] = int(
diagnosis_result_product_mismatch_result.scalar_one() or 0
)
result_trace_missing_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == "dinsar",
or_(
ResultProductORM.pair_uid.is_(None),
ResultProductORM.network_run_id.is_(None),
ResultProductORM.network_edge_id.is_(None),
ResultProductORM.policy_version.is_(None),
),
)
)
status["result_trace_missing_count"] = int(
result_trace_missing_result.scalar_one() or 0
)
result_trace_orphan_result = await db.execute(
text(
"""
SELECT COUNT(*)
FROM result_products rp
LEFT JOIN pairing_network_runs pnr
ON pnr.network_run_id = rp.network_run_id
LEFT JOIN pairing_network_edges pne
ON pne.id = rp.network_edge_id
AND pne.network_run_ref_id = pnr.id
WHERE rp.catalog_name = 'dinsar'
AND COALESCE(rp.pair_uid, '') <> ''
AND COALESCE(rp.network_run_id, '') <> ''
AND rp.network_edge_id IS NOT NULL
AND (pnr.id IS NULL OR pne.id IS NULL)
"""
)
)
status["result_trace_orphan_count"] = int(
result_trace_orphan_result.scalar_one() or 0
)
result_trace_pair_mismatch_result = await db.execute(
text(
"""
SELECT COUNT(*)
FROM result_products rp
JOIN pairing_network_runs pnr
ON pnr.network_run_id = rp.network_run_id
JOIN pairing_network_edges pne
ON pne.id = rp.network_edge_id
AND pne.network_run_ref_id = pnr.id
JOIN pairing_metric_cache pmc
ON pmc.id = pne.metric_cache_ref_id
WHERE rp.catalog_name = 'dinsar'
AND COALESCE(rp.pair_uid, '') <> ''
AND COALESCE(pmc.pair_uid, '') <> ''
AND rp.pair_uid <> pmc.pair_uid
"""
)
)
status["result_trace_pair_mismatch_count"] = int(
result_trace_pair_mismatch_result.scalar_one() or 0
)
status["ok"] = all(
[
status["missing_compat_count"] == 0,
status["orphan_compat_count"] == 0,
status["duplicate_compat_product_count"] == 0,
status["annotation_drift_count"] == 0,
status["diagnosis_missing_product_row_count"] == 0,
status["diagnosis_product_identity_mismatch_count"] == 0,
status["diagnosis_result_product_mismatch_count"] == 0,
status["result_trace_orphan_count"] == 0,
status["result_trace_pair_mismatch_count"] == 0,
]
)
except Exception as exc:
status["error"] = str(exc)
return status
def _probe_directory_status(path: str) -> Dict[str, Any]:
normalized = str(path or "").strip()
payload = {
"path": normalized,
"exists": False,
"accessible": False,
"error": None,
}
if not normalized:
payload["error"] = "empty path"
return payload
try:
if os.path.isdir(normalized):
payload["exists"] = True
try:
with os.scandir(normalized) as iterator:
next(iterator, None)
payload["accessible"] = True
except Exception as exc:
payload["error"] = str(exc)
return payload
os.stat(normalized)
payload["error"] = "path exists but is not a directory"
except Exception as exc:
payload["error"] = str(exc)
return payload
async def _check_source_roots() -> Dict[str, Any]:
items = []
for path in split_env_paths(settings.MONITOR_RADAR_DIRS):
status = _probe_directory_status(path)
status["role"] = "radar_source"
items.append(status)
orbit_dir = str(settings.MONITOR_ORBIT_DIR or "").strip()
if orbit_dir:
status = _probe_directory_status(orbit_dir)
status["role"] = "orbit_source"
items.append(status)
for path in split_env_paths(settings.MONITOR_DINSAR_DIRS):
status = _probe_directory_status(path)
status["role"] = "dinsar_source"
items.append(status)
configured_count = len(items)
accessible_count = sum(1 for item in items if item.get("accessible"))
inaccessible_count = configured_count - accessible_count
return {
"ok": inaccessible_count == 0,
"configured_count": configured_count,
"accessible_count": accessible_count,
"inaccessible_count": inaccessible_count,
"items": items,
}
async def get_health_status(
include_external: bool = True,
include_details: bool = False,
full: bool = False,
refresh: bool = False,
) -> Dict[str, Any]:
db_status = await _check_database(force_schema_refresh=refresh)
worker_timeout = read_int_env("JOB_WORKER_HEALTH_TIMEOUT", DEFAULT_WORKER_TIMEOUT_SECONDS, minimum=5, maximum=86400)
worker_status = await _check_worker(worker_timeout)
idl_status = get_idl_status()
idl_ok = bool(idl_status.get("is_installed"))
nginx_status = {"ok": None}
ollama_status = {"ok": None}
if include_external:
nginx_status = await _check_nginx()
ollama_status = await _check_ollama()
result_catalog_status = await _check_result_catalog()
psinsar_result_catalog_status = await _check_psinsar_result_catalog()
dinsar_bridge_status = await _check_dinsar_bridge()
source_roots_status = await _check_source_roots()
pairing_system_status = await pairing_state_service.get_pairing_system_status()
engines_status = {"ok": None, "overall": None, "engines": []}
if full or include_details:
engines_status = await _check_dinsar_engines()
overall_ok = all(
[
db_status.get("ok"),
db_status.get("schema_ok"),
db_status.get("postgis_ok"),
worker_status.get("ok"),
result_catalog_status.get("ok"),
dinsar_bridge_status.get("ok"),
source_roots_status.get("ok"),
pairing_system_status.get("ok"),
(not settings.TIMESERIES_ENABLED) or psinsar_result_catalog_status.get("ok"),
]
)
full_payload = {
"ok": overall_ok,
"timestamp": datetime.utcnow().isoformat() + "Z",
"database": db_status,
"worker": worker_status,
"result_catalog": result_catalog_status,
"dinsar_result_catalog": result_catalog_status,
"psinsar_result_catalog": psinsar_result_catalog_status,
"catalogs": {
"dinsar": result_catalog_status,
"psinsar": psinsar_result_catalog_status,
},
"dinsar_bridge": dinsar_bridge_status,
"source_roots": source_roots_status,
"pairing_system": pairing_system_status,
"idl": {
"ok": idl_ok,
"status": idl_status,
},
"ollama": ollama_status,
"nginx": nginx_status,
"dinsar_engines": engines_status,
}
if include_details:
return full_payload
return _sanitize_health_status(full_payload)
+685
View File
@@ -0,0 +1,685 @@
"""
图像处理服务
提供 D-InSAR 结果图像处理功能:
- 生成可视化图像
- 提取影像 footprint
- 加载颜色表
- 生成缩略图
优化策略:
- 使用 GDAL 降采样提取 footprint,减少内存占用
- 颜色表使用单例模式缓存
- 支持自动透明边缘裁剪
"""
import os
import time
import json
from typing import Tuple, Optional, Dict, Any, List
from PIL import Image
import rasterio
from rasterio.enums import Resampling
from rasterio.warp import transform as transform_coords, transform_bounds
from rasterio.features import shapes
from rasterio.transform import Affine
import numpy as np
import matplotlib.cm as matplotlib_cm
import matplotlib.colors as mcolors
from shapely.geometry import shape, Polygon, mapping
from shapely.ops import unary_union, transform as shapely_transform
from pyproj import Transformer
from ..config import settings
class ImageService:
"""
图像处理服务
提供 D-InSAR 结果图像的读取、处理和可视化功能。
使用 GDAL 降采样策略优化大数据量图像处理。
"""
# 颜色表缓存(单例模式)
_colormap_cache: Optional[Tuple[mcolors.LinearSegmentedColormap, float, float]] = None
_colormap_filename: Optional[str] = None
# 默认缩略图尺寸
DEFAULT_THUMBNAIL_SIZE = (
settings.DINSAR_THUMBNAIL_MAX_SIZE,
settings.DINSAR_THUMBNAIL_MAX_SIZE
)
@staticmethod
def create_dinsar_image(
file_path: str,
auto_stretch: bool = False,
max_size: Optional[Tuple[int, int]] = None
) -> Image.Image:
"""
读取 D-InSAR 结果文件,应用颜色表,返回 PIL.Image 对象。
Args:
file_path: 结果文件路径
auto_stretch: 是否使用分位数自动拉伸
Returns:
处理后的 PIL.Image 对象
"""
if not os.path.exists(file_path):
raise FileNotFoundError(f"结果文件不存在: {file_path}")
# 加载颜色表
try:
custom_cmap, vmin, vmax = ImageService.load_colormap()
if auto_stretch:
vmin, vmax = None, None
except (FileNotFoundError, ValueError) as e:
print(f"错误: 无法加载自定义颜色表, 将回退到默认值. 错误: {e}")
try:
custom_cmap = mcolors.colormaps.get('viridis')
except AttributeError:
custom_cmap = matplotlib_cm.get_cmap('viridis')
vmin, vmax = None, None
with rasterio.open(file_path) as dataset:
if max_size:
max_w, max_h = max_size
scale = max(dataset.width / max_w, dataset.height / max_h, 1)
out_w = max(1, int(dataset.width / scale))
out_h = max(1, int(dataset.height / scale))
data = dataset.read(
1,
out_shape=(out_h, out_w),
resampling=Resampling.bilinear
).astype(float)
mask = dataset.dataset_mask(out_shape=(out_h, out_w))
else:
data = dataset.read(1).astype(float)
mask = dataset.dataset_mask()
if np.all(mask == 0):
img_array = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8)
else:
valid_data = data[mask == 255]
if vmin is not None and vmax is not None and vmax > vmin:
data_normalized = (data - vmin) / (vmax - vmin)
else:
p2, p98 = np.nanpercentile(valid_data, (2, 98))
if p98 > p2:
data_normalized = (data - p2) / (p98 - p2)
else:
data_normalized = np.zeros_like(data, dtype=float)
data_normalized = np.clip(data_normalized, 0, 1)
colored_data = custom_cmap(data_normalized)
# 强制 NoData 区域透明
colored_data[mask == 0] = (0, 0, 0, 0)
img_array = (colored_data * 255).astype(np.uint8)
return Image.fromarray(img_array)
@staticmethod
def generate_thumbnail(
image: Image.Image,
max_size: Optional[Tuple[int, int]] = None
) -> Image.Image:
"""
生成缩略图。
Args:
image: 原始 PIL.Image
max_size: 最大尺寸 (宽, 高),默认使用 DEFAULT_THUMBNAIL_SIZE
Returns:
缩略图
"""
if max_size is None:
max_size = ImageService.DEFAULT_THUMBNAIL_SIZE
# 自动裁剪透明边缘
bbox = image.getbbox()
if bbox:
image = image.crop(bbox)
# 生成缩略图
thumb = image.copy()
thumb.thumbnail(max_size, Image.Resampling.LANCZOS)
return thumb
@staticmethod
def load_colormap(
file_name: str = "qgis_color.txt"
) -> Tuple[mcolors.LinearSegmentedColormap, float, float]:
"""
加载 QGIS 导出的颜色表(带缓存)。
Args:
file_name: 颜色表文件名
Returns:
(colormap, vmin, vmax) 元组
"""
# 检查缓存
if (
ImageService._colormap_cache is not None and
ImageService._colormap_filename == file_name
):
return ImageService._colormap_cache
# 构建文件路径
cmap_path = os.path.join(settings.COLORMAPS_DIR, file_name)
if not os.path.exists(cmap_path):
raise FileNotFoundError(f"色彩映射文件未找到: {cmap_path}")
# 解析颜色表
colors_data = []
with open(cmap_path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#') or line.startswith('INTERPOLATION'):
continue
parts = line.split(',')
try:
value = float(parts[0])
r, g, b, a = [int(p) for p in parts[1:5]]
colors_data.append((value, (r / 255.0, g / 255.0, b / 255.0, a / 255.0)))
except (ValueError, IndexError):
print(f"警告: 无法解析颜色行: {line}")
continue
if not colors_data:
raise ValueError("色彩映射文件中没有有效的颜色数据。")
# 按值排序
colors_data.sort(key=lambda x: x[0])
vmin = colors_data[0][0]
vmax = colors_data[-1][0]
# 归一化颜色点位置
if vmax == vmin:
norm_points = [0.0] * len(colors_data)
else:
norm_points = [(item[0] - vmin) / (vmax - vmin) for item in colors_data]
colors = [item[1] for item in colors_data]
# 构建颜色列表
cmap_list = []
last_pos = -1.0
for i, pos in enumerate(norm_points):
pos = max(pos, last_pos + 1e-6)
cmap_list.append((min(pos, 1.0), colors[i]))
last_pos = pos
custom_cmap = mcolors.LinearSegmentedColormap.from_list('qgis_custom', cmap_list)
# 更新缓存
ImageService._colormap_cache = (custom_cmap, vmin, vmax)
ImageService._colormap_filename = file_name
return custom_cmap, vmin, vmax
@staticmethod
def clear_colormap_cache():
"""清除颜色表缓存(用于测试或强制刷新)"""
ImageService._colormap_cache = None
ImageService._colormap_filename = None
@staticmethod
def extract_footprint(file_path: str) -> Dict[str, Any]:
"""
提取 D-InSAR 结果的 footprint(使用 GDAL 降采样策略)。
极速优化版 V2:借鉴 QGIS/GDAL 降采样策略,仅读取缩略图掩码计算 Footprint。
Args:
file_path: 影像文件路径
Returns:
{
"min_lon": float,
"min_lat": float,
"max_lon": float,
"max_lat": float,
"coverage_polygon": dict # GeoJSON 格式
}
"""
start_time = time.time()
with rasterio.open(file_path) as dataset:
# 1. 智能降采样读取掩码
MAX_SIZE = settings.DINSAR_FOOTPRINT_MAX_SIZE
if dataset.width > MAX_SIZE or dataset.height > MAX_SIZE:
scale = max(dataset.width, dataset.height) / MAX_SIZE
new_width = max(1, int(dataset.width / scale))
new_height = max(1, int(dataset.height / scale))
mask = dataset.dataset_mask(out_shape=(new_height, new_width))
rescale_transform = dataset.transform * Affine.scale(
dataset.width / new_width,
dataset.height / new_height
)
else:
mask = dataset.dataset_mask()
rescale_transform = dataset.transform
# 2. 提取有效区域多边形
mask_shapes = list(shapes(mask, mask=(mask == 255), transform=rescale_transform))
if not mask_shapes:
# 回退到全图范围
footprint_poly = Polygon([
dataset.transform * (0, 0),
dataset.transform * (dataset.width, 0),
dataset.transform * (dataset.width, dataset.height),
dataset.transform * (0, dataset.height),
dataset.transform * (0, 0)
])
else:
# 合并形状并取凸包
polys = [shape(s) for s, v in mask_shapes]
footprint_poly = unary_union(polys).convex_hull
if footprint_poly.geom_type == 'Polygon':
tolerance = abs(rescale_transform[0]) * 1.5
footprint_poly = footprint_poly.simplify(
tolerance,
preserve_topology=True
)
elapsed = (time.time() - start_time) * 1000
print(f" [性能] Footprint 提取耗时: {elapsed:.1f}ms (文件: {os.path.basename(file_path)})")
# 3. 坐标系转换到 WGS84
if dataset.crs and dataset.crs.to_epsg() != 4326:
transformer = Transformer.from_crs(
dataset.crs,
"EPSG:4326",
always_xy=True
)
footprint_poly = shapely_transform(
transformer.transform,
footprint_poly
)
# 4. 返回紧凑边界框
left, bottom, right, top = footprint_poly.bounds
return {
"min_lon": left,
"min_lat": bottom,
"max_lon": right,
"max_lat": top,
"coverage_polygon": mapping(footprint_poly)
}
@staticmethod
def save_image_as_webp(
image: Image.Image,
output_path: str,
quality: int = 80
) -> None:
"""
保存图像为 WebP 格式。
Args:
image: PIL.Image 对象
output_path: 输出文件路径
quality: WebP 质量 (1-100)
"""
os.makedirs(os.path.dirname(output_path), exist_ok=True)
image.save(output_path, format='WEBP', quality=quality)
@staticmethod
def create_cached_image(
file_path: str,
cache_path: str,
thumbnail_size: Optional[Tuple[int, int]] = None
) -> bool:
"""
创建并保存缓存图像。
Args:
file_path: 原始文件路径
cache_path: 缓存文件路径
thumbnail_size: 缩略图尺寸
Returns:
是否成功
"""
try:
# 生成图像
full_res_img = ImageService.create_dinsar_image(
file_path,
max_size=thumbnail_size or ImageService.DEFAULT_THUMBNAIL_SIZE
)
# 自动裁剪透明边缘
bbox = full_res_img.getbbox()
if bbox:
full_res_img = full_res_img.crop(bbox)
# 生成缩略图
thumbnail = ImageService.generate_thumbnail(
full_res_img,
thumbnail_size or ImageService.DEFAULT_THUMBNAIL_SIZE
)
# 保存为 WebP
ImageService.save_image_as_webp(thumbnail, cache_path, quality=80)
return True
except Exception as e:
print(f"创建缓存图像失败: {file_path}, 错误: {e}")
return False
@staticmethod
def _extract_quadrilateral_points(coverage_polygon: Any) -> Optional[np.ndarray]:
points: List[Tuple[float, float]] = []
if isinstance(coverage_polygon, dict):
coordinates = coverage_polygon.get("coordinates") if coverage_polygon else None
if isinstance(coordinates, list) and coordinates:
first_ring = coordinates[0]
if isinstance(first_ring, list):
for point in first_ring:
if isinstance(point, (list, tuple)) and len(point) >= 2:
points.append((float(point[0]), float(point[1])))
elif isinstance(coverage_polygon, list):
for point in coverage_polygon:
if isinstance(point, (list, tuple)) and len(point) >= 2:
points.append((float(point[0]), float(point[1])))
if len(points) < 4:
return None
dedup: List[Tuple[float, float]] = []
for lon, lat in points:
if not dedup:
dedup.append((lon, lat))
continue
prev_lon, prev_lat = dedup[-1]
if abs(prev_lon - lon) < 1e-10 and abs(prev_lat - lat) < 1e-10:
continue
dedup.append((lon, lat))
if len(dedup) >= 5:
first_lon, first_lat = dedup[0]
last_lon, last_lat = dedup[-1]
if abs(first_lon - last_lon) < 1e-10 and abs(first_lat - last_lat) < 1e-10:
dedup = dedup[:-1]
if len(dedup) != 4:
return None
return np.asarray(dedup, dtype=np.float64)
@staticmethod
def _extract_source_corner_mapping_points(source_corner_mapping: Any) -> Optional[np.ndarray]:
if not isinstance(source_corner_mapping, dict):
return None
ordered_keys = ["bottom_left", "bottom_right", "top_right", "top_left"]
points: List[Tuple[float, float]] = []
for key in ordered_keys:
value = source_corner_mapping.get(key)
if not isinstance(value, (list, tuple)) or len(value) < 2:
return None
try:
points.append((float(value[0]), float(value[1])))
except (TypeError, ValueError):
return None
return np.asarray(points, dtype=np.float64)
@staticmethod
def _compute_homography(src_points: np.ndarray, dst_points: np.ndarray) -> Optional[np.ndarray]:
if src_points.shape != (4, 2) or dst_points.shape != (4, 2):
return None
matrix_a: List[List[float]] = []
matrix_b: List[float] = []
for (sx, sy), (dx, dy) in zip(src_points, dst_points):
matrix_a.append([sx, sy, 1.0, 0.0, 0.0, 0.0, -dx * sx, -dx * sy])
matrix_b.append(float(dx))
matrix_a.append([0.0, 0.0, 0.0, sx, sy, 1.0, -dy * sx, -dy * sy])
matrix_b.append(float(dy))
try:
solved, _, rank, _ = np.linalg.lstsq(
np.asarray(matrix_a, dtype=np.float64),
np.asarray(matrix_b, dtype=np.float64),
rcond=None,
)
except np.linalg.LinAlgError:
return None
if rank < 8:
return None
return np.array(
[
[solved[0], solved[1], solved[2]],
[solved[3], solved[4], solved[5]],
[solved[6], solved[7], 1.0],
],
dtype=np.float64,
)
@staticmethod
def _estimate_geo_canvas_size(
bbox: Tuple[float, float, float, float],
max_size: Tuple[int, int],
) -> Optional[Tuple[int, int]]:
min_lon, min_lat, max_lon, max_lat = bbox
lon_span = max_lon - min_lon
lat_span = max_lat - min_lat
if lon_span <= 0 or lat_span <= 0:
return None
mean_lat_rad = np.deg2rad((min_lat + max_lat) * 0.5)
x_span_scaled = max(lon_span * max(np.cos(mean_lat_rad), 1e-3), 1e-9)
y_span_scaled = max(lat_span, 1e-9)
ratio = x_span_scaled / y_span_scaled
limit = max(64, int(max(max_size[0], max_size[1])))
if ratio >= 1:
width = limit
height = max(64, int(round(width / ratio)))
else:
height = limit
width = max(64, int(round(height * ratio)))
return width, height
@staticmethod
def _warp_preview_to_geo_bbox(
source_rgb: np.ndarray,
inverse_h: np.ndarray,
bbox: Tuple[float, float, float, float],
out_size: Tuple[int, int],
) -> np.ndarray:
src_h, src_w = source_rgb.shape[:2]
out_w, out_h = out_size
min_lon, min_lat, max_lon, max_lat = bbox
lon_span = max_lon - min_lon
lat_span = max_lat - min_lat
if src_h < 1 or src_w < 1 or out_h < 1 or out_w < 1:
return np.zeros((max(out_h, 1), max(out_w, 1), 4), dtype=np.uint8)
grid_x, grid_y = np.meshgrid(
np.arange(out_w, dtype=np.float64),
np.arange(out_h, dtype=np.float64),
)
lon = min_lon + ((grid_x + 0.5) / out_w) * lon_span
lat = max_lat - ((grid_y + 0.5) / out_h) * lat_span
lon_flat = lon.reshape(-1)
lat_flat = lat.reshape(-1)
denom = inverse_h[2, 0] * lon_flat + inverse_h[2, 1] * lat_flat + inverse_h[2, 2]
valid = np.abs(denom) > 1e-8
u = np.zeros_like(lon_flat)
v = np.zeros_like(lat_flat)
u[valid] = (
inverse_h[0, 0] * lon_flat[valid]
+ inverse_h[0, 1] * lat_flat[valid]
+ inverse_h[0, 2]
) / denom[valid]
v[valid] = (
inverse_h[1, 0] * lon_flat[valid]
+ inverse_h[1, 1] * lat_flat[valid]
+ inverse_h[1, 2]
) / denom[valid]
if src_w >= 2 and src_h >= 2:
valid &= (u >= 0) & (u < (src_w - 1)) & (v >= 0) & (v < (src_h - 1))
else:
valid &= (u >= 0) & (u <= (src_w - 1)) & (v >= 0) & (v <= (src_h - 1))
output = np.zeros((out_h, out_w, 4), dtype=np.uint8)
if not np.any(valid):
return output
valid_idx = np.where(valid)[0]
u_valid = u[valid_idx]
v_valid = v[valid_idx]
if src_w >= 2 and src_h >= 2:
x0 = np.floor(u_valid).astype(np.int32)
y0 = np.floor(v_valid).astype(np.int32)
x1 = np.clip(x0 + 1, 0, src_w - 1)
y1 = np.clip(y0 + 1, 0, src_h - 1)
du = (u_valid - x0).astype(np.float32)
dv = (v_valid - y0).astype(np.float32)
src_float = source_rgb.astype(np.float32, copy=False)
s00 = src_float[y0, x0]
s10 = src_float[y0, x1]
s01 = src_float[y1, x0]
s11 = src_float[y1, x1]
samples = (
s00 * (1 - du)[:, None] * (1 - dv)[:, None]
+ s10 * du[:, None] * (1 - dv)[:, None]
+ s01 * (1 - du)[:, None] * dv[:, None]
+ s11 * du[:, None] * dv[:, None]
)
rgb = np.clip(samples, 0, 255).astype(np.uint8)
else:
nearest_x = np.clip(np.round(u_valid).astype(np.int32), 0, src_w - 1)
nearest_y = np.clip(np.round(v_valid).astype(np.int32), 0, src_h - 1)
rgb = source_rgb[nearest_y, nearest_x]
flat = output.reshape(-1, 4)
flat[valid_idx, :3] = rgb
flat[valid_idx, 3] = 255
return output
@staticmethod
def create_geocorrected_radar_cached_image(
source_image_path: str,
cache_path: str,
coverage_polygon: Any,
bbox: Tuple[float, float, float, float],
source_corner_mapping: Optional[Dict[str, Any]] = None,
thumbnail_size: Optional[Tuple[int, int]] = None,
quality: Optional[int] = None,
) -> Tuple[bool, Optional[str]]:
try:
if not os.path.exists(source_image_path):
return False, "preview_source_not_found"
polygon_points = ImageService._extract_source_corner_mapping_points(source_corner_mapping)
if polygon_points is None:
polygon_points = ImageService._extract_quadrilateral_points(coverage_polygon)
if polygon_points is None:
return False, "invalid_coverage_polygon"
max_size = thumbnail_size or (
settings.RADAR_THUMBNAIL_MAX_SIZE,
settings.RADAR_THUMBNAIL_MAX_SIZE,
)
out_size = ImageService._estimate_geo_canvas_size(bbox, max_size)
if out_size is None:
return False, "invalid_bbox"
with Image.open(source_image_path) as image:
source_rgb = np.asarray(image.convert("RGB"), dtype=np.uint8)
src_h, src_w = source_rgb.shape[:2]
if src_h < 1 or src_w < 1:
return False, "invalid_source_image_size"
source_points = np.asarray(
[
[0.0, float(src_h - 1)],
[float(src_w - 1), float(src_h - 1)],
[float(src_w - 1), 0.0],
[0.0, 0.0],
],
dtype=np.float64,
)
homography = ImageService._compute_homography(source_points, polygon_points)
if homography is None:
return False, "homography_solve_failed"
try:
inverse_h = np.linalg.inv(homography)
except np.linalg.LinAlgError:
return False, "homography_invert_failed"
warped_rgba = ImageService._warp_preview_to_geo_bbox(
source_rgb=source_rgb,
inverse_h=inverse_h,
bbox=bbox,
out_size=out_size,
)
target_quality = quality if quality is not None else settings.RADAR_GEO_CACHE_QUALITY
output_image = Image.fromarray(warped_rgba, mode="RGBA")
ImageService.save_image_as_webp(output_image, cache_path, quality=target_quality)
return True, None
except Exception as e:
return False, str(e)
@staticmethod
def create_radar_cached_image(
source_image_path: str,
cache_path: str,
thumbnail_size: Optional[Tuple[int, int]] = None
) -> bool:
"""
为源雷达数据包中的预览图(jpg/png 等)创建 WebP 缓存。
Args:
source_image_path: 原始预览图路径
cache_path: 缓存输出路径
thumbnail_size: 缩略图尺寸
Returns:
是否成功
"""
try:
if not os.path.exists(source_image_path):
return False
max_size = thumbnail_size or (
settings.RADAR_THUMBNAIL_MAX_SIZE,
settings.RADAR_THUMBNAIL_MAX_SIZE,
)
with Image.open(source_image_path) as image:
image = image.convert("RGB")
image.thumbnail(max_size, Image.Resampling.LANCZOS)
ImageService.save_image_as_webp(image, cache_path, quality=82)
return True
except Exception as e:
print(f"创建源影像缓存失败: {source_image_path}, 错误: {e}")
return False
# 全局服务实例
image_service = ImageService()
File diff suppressed because it is too large Load Diff
+388
View File
@@ -0,0 +1,388 @@
import asyncio
import json
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Optional
from sqlalchemy import and_, or_, select, update, text
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..config import read_int_env
from ..models import SystemJobORM
JOB_STATUS_READY = "READY"
JOB_STATUS_RUNNING = "RUNNING"
JOB_STATUS_COMPLETED = "COMPLETED"
JOB_STATUS_FAILED = "FAILED"
JOB_STATUS_RETRY = "RETRY"
JOB_QUEUE_MAX_PAYLOAD_BYTES = read_int_env(
"JOB_QUEUE_MAX_PAYLOAD_BYTES",
512 * 1024,
minimum=1024,
maximum=10 * 1024 * 1024,
)
JOB_QUEUE_MAX_ATTEMPTS = read_int_env(
"JOB_QUEUE_MAX_ATTEMPTS",
10,
minimum=1,
maximum=100,
)
JOB_QUEUE_MAX_PRIORITY_ABS = read_int_env(
"JOB_QUEUE_MAX_PRIORITY_ABS",
1000,
minimum=1,
maximum=100000,
)
JOB_QUEUE_MAX_ID_LENGTH = read_int_env(
"JOB_QUEUE_MAX_ID_LENGTH",
128,
minimum=16,
maximum=512,
)
JOB_QUEUE_MAX_TYPE_LENGTH = read_int_env(
"JOB_QUEUE_MAX_TYPE_LENGTH",
64,
minimum=8,
maximum=256,
)
JOB_QUEUE_STALE_RUNNING_SECONDS = read_int_env(
"JOB_QUEUE_STALE_RUNNING_SECONDS",
300,
minimum=30,
maximum=7 * 24 * 60 * 60,
)
def _new_session() -> AsyncSession:
if database.AsyncSessionLocal is None:
database.init_db()
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
def _normalize_job_type(job_type: str) -> str:
normalized = str(job_type or "").strip().upper()
if not normalized:
raise ValueError("job_type must not be empty.")
if len(normalized) > JOB_QUEUE_MAX_TYPE_LENGTH:
raise ValueError(
f"job_type is too long (max {JOB_QUEUE_MAX_TYPE_LENGTH} chars)."
)
return normalized
def _normalize_optional_text(value: Optional[str], field_name: str) -> Optional[str]:
if value is None:
return None
normalized = str(value).strip()
if not normalized:
return None
if len(normalized) > JOB_QUEUE_MAX_ID_LENGTH:
raise ValueError(f"{field_name} is too long (max {JOB_QUEUE_MAX_ID_LENGTH} chars).")
return normalized
def _normalize_priority(value: int) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
parsed = 0
return min(JOB_QUEUE_MAX_PRIORITY_ABS, max(-JOB_QUEUE_MAX_PRIORITY_ABS, parsed))
def _normalize_max_attempts(value: int) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
parsed = 3
return min(JOB_QUEUE_MAX_ATTEMPTS, max(1, parsed))
def _normalize_payload(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if payload is None:
return {}
if not isinstance(payload, dict):
raise ValueError("Job payload must be a JSON object (dict).")
try:
serialized = json.dumps(
payload,
ensure_ascii=False,
separators=(",", ":"),
allow_nan=False,
)
except (TypeError, ValueError) as exc:
raise ValueError("Job payload must be JSON-serializable.") from exc
payload_bytes = len(serialized.encode("utf-8"))
if payload_bytes > JOB_QUEUE_MAX_PAYLOAD_BYTES:
raise ValueError(
f"Job payload too large ({payload_bytes} bytes), "
f"max allowed is {JOB_QUEUE_MAX_PAYLOAD_BYTES} bytes."
)
return payload
class JobQueueService:
"""
DB-backed job queue service (PostgreSQL).
"""
async def create_job(
self,
job_type: str,
payload: Optional[Dict[str, Any]] = None,
priority: int = 0,
max_attempts: int = 3,
workflow_run_id: Optional[str] = None,
workflow_step_id: Optional[str] = None,
task_id: Optional[str] = None,
db: Optional[AsyncSession] = None,
) -> str:
gen_db = db is None
if gen_db:
db = _new_session()
try:
normalized_job_type = _normalize_job_type(job_type)
normalized_payload = _normalize_payload(payload)
normalized_priority = _normalize_priority(priority)
normalized_max_attempts = _normalize_max_attempts(max_attempts)
normalized_workflow_run_id = _normalize_optional_text(workflow_run_id, "workflow_run_id")
normalized_workflow_step_id = _normalize_optional_text(workflow_step_id, "workflow_step_id")
normalized_task_id = _normalize_optional_text(task_id, "task_id")
job_id = str(uuid.uuid4())
job = SystemJobORM(
job_id=job_id,
job_type=normalized_job_type,
status=JOB_STATUS_READY,
priority=normalized_priority,
payload=normalized_payload,
max_attempts=normalized_max_attempts,
workflow_run_id=normalized_workflow_run_id,
workflow_step_id=normalized_workflow_step_id,
task_id=normalized_task_id,
)
db.add(job)
if gen_db:
await db.commit()
else:
await db.flush()
return job_id
finally:
if gen_db:
await db.close()
async def claim_next_job(
self,
worker_id: str,
lock_timeout_seconds: int = 1800,
db: Optional[AsyncSession] = None,
) -> Optional[SystemJobORM]:
gen_db = db is None
if gen_db:
db = _new_session()
try:
lock_timeout_seconds = int(lock_timeout_seconds)
async with db.begin():
result = await db.execute(
text(
"""
SELECT id
FROM system_jobs
WHERE status IN ('READY', 'RETRY')
AND (next_run_at IS NULL OR next_run_at <= NOW())
AND (
locked_by IS NULL OR
locked_at IS NULL OR
locked_at < NOW() - make_interval(secs => :lock_timeout_seconds)
)
ORDER BY priority DESC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1
"""
),
{"lock_timeout_seconds": lock_timeout_seconds},
)
row = result.first()
if not row:
return None
job_pk = row[0]
await db.execute(
update(SystemJobORM)
.where(SystemJobORM.id == job_pk)
.values(
status=JOB_STATUS_RUNNING,
locked_by=worker_id,
locked_at=datetime.now(timezone.utc).replace(tzinfo=None),
heartbeat_at=datetime.now(timezone.utc).replace(tzinfo=None),
started_at=datetime.now(timezone.utc).replace(tzinfo=None),
)
)
job_result = await db.execute(
select(SystemJobORM).where(SystemJobORM.id == job_pk)
)
return job_result.scalar_one_or_none()
finally:
if gen_db:
await db.close()
async def heartbeat(self, job_id: str, db: Optional[AsyncSession] = None) -> None:
gen_db = db is None
if gen_db:
db = _new_session()
try:
await db.execute(
update(SystemJobORM)
.where(SystemJobORM.job_id == job_id)
.values(heartbeat_at=datetime.now(timezone.utc).replace(tzinfo=None))
)
await db.commit()
finally:
if gen_db:
await db.close()
async def recover_stale_running_jobs(
self,
stale_seconds: int = JOB_QUEUE_STALE_RUNNING_SECONDS,
db: Optional[AsyncSession] = None,
) -> Dict[str, int]:
gen_db = db is None
if gen_db:
db = _new_session()
try:
safe_stale_seconds = max(30, int(stale_seconds))
now = datetime.now(timezone.utc).replace(tzinfo=None)
cutoff = now - timedelta(seconds=safe_stale_seconds)
result = await db.execute(
select(SystemJobORM).where(
SystemJobORM.status == JOB_STATUS_RUNNING,
or_(
SystemJobORM.heartbeat_at < cutoff,
and_(
SystemJobORM.heartbeat_at.is_(None),
SystemJobORM.locked_at < cutoff,
),
and_(
SystemJobORM.heartbeat_at.is_(None),
SystemJobORM.locked_at.is_(None),
),
),
)
)
stale_jobs = result.scalars().all()
if not stale_jobs:
return {"recovered": 0, "failed": 0}
recovered = 0
failed = 0
for job in stale_jobs:
attempts = int(job.attempts or 0) + 1
if attempts < int(job.max_attempts or 1):
status = JOB_STATUS_RETRY
next_run_at = now + timedelta(seconds=5)
recovered += 1
finished_at = None
else:
status = JOB_STATUS_FAILED
next_run_at = None
failed += 1
finished_at = now
await db.execute(
update(SystemJobORM)
.where(SystemJobORM.id == job.id)
.values(
status=status,
attempts=attempts,
next_run_at=next_run_at,
last_error=f"Recovered stale RUNNING job after {safe_stale_seconds}s timeout.",
finished_at=finished_at,
locked_by=None,
locked_at=None,
heartbeat_at=None,
)
)
await db.commit()
return {"recovered": recovered, "failed": failed}
finally:
if gen_db:
await db.close()
async def mark_completed(
self,
job_id: str,
db: Optional[AsyncSession] = None,
) -> None:
gen_db = db is None
if gen_db:
db = _new_session()
try:
await db.execute(
update(SystemJobORM)
.where(SystemJobORM.job_id == job_id)
.values(
status=JOB_STATUS_COMPLETED,
finished_at=datetime.now(timezone.utc).replace(tzinfo=None),
last_error=None,
locked_by=None,
locked_at=None,
heartbeat_at=None,
)
)
await db.commit()
finally:
if gen_db:
await db.close()
async def mark_failed(
self,
job: SystemJobORM,
error_message: str,
retry_delay_seconds: int = 30,
db: Optional[AsyncSession] = None,
) -> str:
gen_db = db is None
if gen_db:
db = _new_session()
try:
attempts = (job.attempts or 0) + 1
status = JOB_STATUS_FAILED
next_run_at = None
if attempts < (job.max_attempts or 0):
status = JOB_STATUS_RETRY
backoff = retry_delay_seconds * (2 ** max(attempts - 1, 0))
next_run_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=backoff)
await db.execute(
update(SystemJobORM)
.where(SystemJobORM.job_id == job.job_id)
.values(
status=status,
attempts=attempts,
next_run_at=next_run_at,
last_error=error_message,
finished_at=datetime.now(timezone.utc).replace(tzinfo=None) if status == JOB_STATUS_FAILED else None,
locked_by=None,
locked_at=None,
heartbeat_at=None,
)
)
await db.commit()
return status
finally:
if gen_db:
await db.close()
job_queue_service = JobQueueService()
+186
View File
@@ -0,0 +1,186 @@
import asyncio
import os
import socket
import uuid
import time
from typing import Set
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy import func
from ..models import SystemJobORM
from .job_queue_service import job_queue_service, JOB_STATUS_RETRY
from .job_handlers import get_job_handler
from .task_service import task_service
from .workflow_service import workflow_service
from .. import database
from ..config import settings
from ..models import SystemWorkerHeartbeatORM
IDL_JOB_TYPES = {"IDL_RUN_IMPORT", "IDL_RUN_DINSAR", "WATER_GEOCODE", "WATER_FLOOD"}
def _default_worker_id() -> str:
host = socket.gethostname()
pid = os.getpid()
rand = uuid.uuid4().hex[:6]
return f"{host}:{pid}:{rand}"
def _new_session():
if database.AsyncSessionLocal is None:
database.init_db()
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
async def _touch_worker(worker_id: str) -> None:
host = socket.gethostname()
pid = os.getpid()
async with _new_session() as db:
stmt = pg_insert(SystemWorkerHeartbeatORM).values(
worker_id=worker_id,
hostname=host,
pid=pid,
)
stmt = stmt.on_conflict_do_update(
index_elements=["worker_id"],
set_={
"hostname": host,
"pid": pid,
"last_seen": func.now(),
},
)
await db.execute(stmt)
await db.commit()
async def _run_job(job: SystemJobORM) -> None:
handler = get_job_handler(job.job_type)
if handler is None:
msg = f"No handler registered for job type: {job.job_type}"
status = await job_queue_service.mark_failed(job, msg)
if job.task_id:
if status == JOB_STATUS_RETRY:
await task_service.update_task(job.task_id, message=f"任务失败,稍后重试: {msg}")
else:
await task_service.update_task(job.task_id, status="FAILED", message=msg)
if job.workflow_run_id and job.workflow_step_id and status != JOB_STATUS_RETRY:
await workflow_service.mark_step_failed(
job.workflow_run_id,
job.workflow_step_id,
msg,
)
return
try:
await handler(job)
await job_queue_service.mark_completed(job.job_id)
if job.workflow_run_id and job.workflow_step_id:
await workflow_service.mark_step_completed(
job.workflow_run_id,
job.workflow_step_id,
)
except Exception as exc:
err = f"{exc}"
retry_delay_seconds = 30
if (job.job_type or "").upper() in IDL_JOB_TYPES:
retry_delay_seconds = int(settings.IDL_JOB_RETRY_DELAY_SECONDS)
status = await job_queue_service.mark_failed(
job,
err,
retry_delay_seconds=retry_delay_seconds,
)
if job.task_id:
if status == JOB_STATUS_RETRY:
await task_service.update_task(job.task_id, message=f"任务失败,稍后重试: {err}")
else:
await task_service.update_task(job.task_id, status="FAILED", message=err)
if job.workflow_run_id and job.workflow_step_id and status != JOB_STATUS_RETRY:
await workflow_service.mark_step_failed(
job.workflow_run_id,
job.workflow_step_id,
err,
)
async def run_worker_loop(
poll_interval: float = 1.0,
concurrency: int = 1,
worker_id: str = "",
) -> None:
"""
Main loop for DB-backed job worker.
"""
if not worker_id:
worker_id = _default_worker_id()
concurrency = max(1, int(concurrency))
sem = asyncio.Semaphore(concurrency)
active: Set[asyncio.Task] = set()
job_heartbeat_interval = float(settings.JOB_WORKER_JOB_HEARTBEAT_INTERVAL)
stale_recover_interval = float(settings.JOB_WORKER_STALE_RECOVER_INTERVAL)
stale_running_seconds = int(settings.JOB_WORKER_STALE_RUNNING_SECONDS)
async def _job_heartbeat_loop(job_id: str) -> None:
safe_interval = max(1.0, job_heartbeat_interval)
while True:
await asyncio.sleep(safe_interval)
try:
await job_queue_service.heartbeat(job_id)
except Exception as exc:
print(f"[WARN] heartbeat: {exc}")
async def _wrap(job: SystemJobORM):
heartbeat_task = asyncio.create_task(_job_heartbeat_loop(job.job_id))
try:
async with sem:
await _run_job(job)
finally:
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
last_heartbeat = 0.0
last_recover = 0.0
heartbeat_interval = float(settings.JOB_WORKER_HEARTBEAT_INTERVAL)
while True:
now = time.monotonic()
if now - last_heartbeat >= heartbeat_interval:
try:
await _touch_worker(worker_id)
except Exception as exc:
print(f"[WARN] worker cleanup: {exc}")
last_heartbeat = now
if now - last_recover >= max(5.0, stale_recover_interval):
try:
recovered = await job_queue_service.recover_stale_running_jobs(stale_running_seconds)
if (recovered.get("recovered", 0) or recovered.get("failed", 0)):
print(
f"[*] Recovered stale jobs: retry={recovered.get('recovered', 0)} "
f"failed={recovered.get('failed', 0)}"
)
except Exception as exc:
print(f"[WARN] recover_stale: {exc}")
last_recover = now
done = {t for t in active if t.done()}
for t in done:
active.remove(t)
try:
t.result()
except Exception as exc:
print(f"[WARN] worker poll: {exc}")
if len(active) < concurrency:
job = await job_queue_service.claim_next_job(worker_id)
if job:
task = asyncio.create_task(_wrap(job))
active.add(task)
continue
await asyncio.sleep(poll_interval)
@@ -0,0 +1,411 @@
from __future__ import annotations
import asyncio
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
from .manifest_snapshot_service import ManifestEntrySnapshot, build_manifest_snapshot
def _utcnow() -> datetime:
return datetime.utcnow()
def _new_session() -> AsyncSession:
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
def _normcase_path(path: str) -> str:
return os.path.normcase(os.path.normpath(os.path.abspath(str(path or "").strip())))
def _is_parent_path(parent: str, child: str) -> bool:
if not parent or not child or parent == child:
return False
try:
return os.path.commonpath([parent, child]) == parent
except ValueError:
return False
def _manifest_status(metadata: Dict[str, Any]) -> str:
return "INVALID" if metadata.get("parse_error") else "DISCOVERED"
def _read_manifest_metadata(path: str, *, root: ManagedRootORM) -> Dict[str, Any]:
metadata: Dict[str, Any] = {
"root_role": root.root_role,
"owner_engine": root.owner_engine,
}
try:
with open(path, "r", encoding="utf-8") as stream:
payload = json.load(stream)
if not isinstance(payload, dict):
metadata["parse_error"] = "manifest root must be a JSON object"
return metadata
except Exception as exc:
metadata["parse_error"] = f"{exc.__class__.__name__}: {exc}"
return metadata
for key in (
"schema_version",
"catalog_name",
"product_type",
"product_id",
"display_name",
"run_key",
"run_id",
"group_key",
"reference_date",
"published_at",
"produced_at",
):
value = payload.get(key)
if value not in (None, ""):
metadata[key] = value
return metadata
class ManifestInventoryService:
async def _get_manifest_roots(
self,
db: AsyncSession,
*,
root_id: Optional[int] = None,
) -> Tuple[List[ManagedRootORM], List[Dict[str, Any]]]:
stmt = select(ManagedRootORM).where(ManagedRootORM.scan_mode == "manifest_tree")
if root_id is None:
stmt = stmt.where(ManagedRootORM.enabled == True) # noqa: E712
else:
stmt = stmt.where(ManagedRootORM.id == int(root_id))
stmt = stmt.order_by(ManagedRootORM.path.asc(), ManagedRootORM.id.asc())
result = await db.execute(stmt)
roots = result.scalars().all()
if root_id is not None:
return roots, []
normalized_paths = {
root.id: _normcase_path(root.path)
for root in roots
if str(root.path or "").strip()
}
effective_roots: List[ManagedRootORM] = []
skipped_roots: List[Dict[str, Any]] = []
for root in roots:
root_path = normalized_paths.get(root.id, "")
nested_child = next(
(
other
for other in roots
if other.id != root.id
and _is_parent_path(root_path, normalized_paths.get(other.id, ""))
),
None,
)
if nested_child is not None:
skipped_roots.append(
{
"root_id": root.id,
"root_code": root.root_code,
"path": root.path,
"reason": "covered_by_nested_root",
"nested_root_id": nested_child.id,
"nested_root_code": nested_child.root_code,
"nested_root_path": nested_child.path,
}
)
continue
effective_roots.append(root)
return effective_roots, skipped_roots
async def _ensure_default_cursor(self, db: AsyncSession, root: ManagedRootORM) -> ScanCursorORM:
result = await db.execute(
select(ScanCursorORM).where(
ScanCursorORM.root_ref_id == root.id,
ScanCursorORM.cursor_key == "default",
)
)
cursor = result.scalar_one_or_none()
if cursor is None:
cursor = ScanCursorORM(
root_ref_id=root.id,
cursor_key="default",
cursor_type="manifest_tree",
scan_scope="root",
status="IDLE",
)
db.add(cursor)
await db.flush()
return cursor
async def _mark_root_removed(self, db: AsyncSession, *, root_id: int, seen_at: datetime) -> int:
result = await db.execute(
select(PathInventoryORM).where(
PathInventoryORM.root_ref_id == root_id,
PathInventoryORM.status != "REMOVED",
)
)
rows = result.scalars().all()
removed = 0
for row in rows:
row.status = "REMOVED"
row.last_seen_at = seen_at
removed += 1
return removed
async def _sync_single_root(self, db: AsyncSession, root: ManagedRootORM) -> Dict[str, Any]:
started_at = _utcnow()
cursor = await self._ensure_default_cursor(db, root)
cursor.status = "RUNNING"
cursor.last_scan_started_at = started_at
cursor.last_error = None
await db.flush()
root_exists = os.path.isdir(root.path)
root.exists_flag = root_exists
if not root_exists:
removed = await self._mark_root_removed(db, root_id=root.id, seen_at=started_at)
cursor.last_seen_entry_count = 0
cursor.last_seen_fingerprint = ""
cursor.last_seen_mtime = None
cursor.status = "IDLE"
cursor.last_scan_finished_at = _utcnow()
cursor.last_error = "root_missing"
await db.commit()
return {
"root_id": root.id,
"root_code": root.root_code,
"path": root.path,
"manifest_count": 0,
"created": 0,
"updated": 0,
"removed": removed,
"invalid": 0,
"fingerprint": "",
"status": "root_missing",
}
snapshot = await asyncio.to_thread(build_manifest_snapshot, root.path)
seen_at = _utcnow()
existing_result = await db.execute(
select(PathInventoryORM).where(PathInventoryORM.root_ref_id == root.id)
)
existing_rows = existing_result.scalars().all()
existing_by_rel = {row.relative_path: row for row in existing_rows}
seen_paths: set[str] = set()
created = 0
updated = 0
removed = 0
invalid = 0
for entry in snapshot.entries:
seen_paths.add(entry.relative_path)
metadata = _read_manifest_metadata(entry.absolute_path, root=root)
status = _manifest_status(metadata)
if status == "INVALID":
invalid += 1
row = existing_by_rel.get(entry.relative_path)
values = {
"basename": os.path.basename(entry.relative_path),
"extension": os.path.splitext(entry.relative_path)[1].lower() or None,
"size_bytes": entry.size_bytes,
"mtime": entry.mtime,
"ctime": entry.ctime,
"fingerprint": entry.fingerprint,
"status": status,
"metadata_json": metadata,
"last_seen_at": seen_at,
"last_parsed_at": seen_at,
}
if row is None:
row = PathInventoryORM(
root_ref_id=root.id,
relative_path=entry.relative_path,
path_type="file",
first_seen_at=seen_at,
**values,
)
db.add(row)
created += 1
continue
changed = False
for field_name, field_value in values.items():
if getattr(row, field_name) != field_value:
setattr(row, field_name, field_value)
changed = True
if changed:
updated += 1
for row in existing_rows:
if row.relative_path in seen_paths:
continue
if row.status != "REMOVED":
row.status = "REMOVED"
row.last_seen_at = seen_at
removed += 1
latest_mtime = max(
(float(entry.mtime) for entry in snapshot.entries if entry.mtime is not None),
default=None,
)
cursor.last_seen_entry_count = snapshot.manifest_count
cursor.last_seen_fingerprint = snapshot.tree_fingerprint
cursor.last_seen_mtime = latest_mtime
cursor.status = "IDLE"
cursor.last_scan_finished_at = _utcnow()
cursor.last_error = None
await db.commit()
return {
"root_id": root.id,
"root_code": root.root_code,
"path": root.path,
"manifest_count": snapshot.manifest_count,
"created": created,
"updated": updated,
"removed": removed,
"invalid": invalid,
"fingerprint": snapshot.tree_fingerprint,
"status": "ok",
}
async def sync_manifest_roots(
self,
db: Optional[AsyncSession] = None,
*,
root_id: Optional[int] = None,
) -> Dict[str, Any]:
generated_session = db is None
if generated_session:
db = _new_session()
assert db is not None
try:
roots, skipped_roots = await self._get_manifest_roots(db, root_id=root_id)
results: List[Dict[str, Any]] = []
errors: List[Dict[str, Any]] = []
total_created = 0
total_updated = 0
total_removed = 0
total_invalid = 0
total_manifests = 0
for root in roots:
try:
item = await self._sync_single_root(db, root)
results.append(item)
total_created += int(item["created"])
total_updated += int(item["updated"])
total_removed += int(item["removed"])
total_invalid += int(item["invalid"])
total_manifests += int(item["manifest_count"])
except Exception as exc:
await db.rollback()
errors.append(
{
"root_id": root.id,
"root_code": root.root_code,
"path": root.path,
"error": str(exc),
}
)
return {
"root_id": root_id,
"scanned_roots": len(results),
"skipped_roots": len(skipped_roots),
"manifest_count": total_manifests,
"created": total_created,
"updated": total_updated,
"removed": total_removed,
"invalid": total_invalid,
"errors": errors,
"results": results,
"skipped": skipped_roots,
}
finally:
if generated_session:
await db.close()
async def list_inventory(
self,
db: AsyncSession,
*,
root_id: int,
limit: int = 200,
offset: int = 0,
include_removed: bool = False,
) -> Dict[str, Any]:
safe_limit = max(1, min(int(limit or 200), 1000))
safe_offset = max(0, int(offset or 0))
root_result = await db.execute(
select(ManagedRootORM).where(ManagedRootORM.id == int(root_id))
)
root = root_result.scalar_one_or_none()
if root is None:
raise ValueError(f"Managed root not found: {root_id}")
stmt = select(PathInventoryORM).where(PathInventoryORM.root_ref_id == root.id)
count_stmt = select(func.count(PathInventoryORM.id)).where(PathInventoryORM.root_ref_id == root.id)
if not include_removed:
stmt = stmt.where(PathInventoryORM.status != "REMOVED")
count_stmt = count_stmt.where(PathInventoryORM.status != "REMOVED")
total_result = await db.execute(count_stmt)
total = int(total_result.scalar_one() or 0)
rows_result = await db.execute(
stmt.order_by(PathInventoryORM.relative_path.asc()).offset(safe_offset).limit(safe_limit)
)
rows = rows_result.scalars().all()
return {
"root": {
"id": root.id,
"root_code": root.root_code,
"root_role": root.root_role,
"display_name": root.display_name,
"path": root.path,
"scan_mode": root.scan_mode,
"owner_engine": root.owner_engine,
"enabled": bool(root.enabled),
"exists_flag": bool(root.exists_flag),
},
"items": [
{
"id": row.id,
"relative_path": row.relative_path,
"path_type": row.path_type,
"basename": row.basename,
"extension": row.extension,
"size_bytes": row.size_bytes,
"mtime": row.mtime,
"ctime": row.ctime,
"fingerprint": row.fingerprint,
"status": row.status,
"metadata_json": row.metadata_json,
"first_seen_at": row.first_seen_at,
"last_seen_at": row.last_seen_at,
"last_parsed_at": row.last_parsed_at,
}
for row in rows
],
"total": total,
"limit": safe_limit,
"offset": safe_offset,
"has_more": safe_offset + len(rows) < total,
}
manifest_inventory_service = ManifestInventoryService()
@@ -0,0 +1,142 @@
from __future__ import annotations
import hashlib
import os
from dataclasses import dataclass
from typing import List, Optional, Tuple
MANIFEST_FILENAME = "manifest.json"
def _normalize_path(path: str) -> str:
return os.path.normpath(os.path.abspath(path))
def _hash_file(path: str, *, chunk_size: int = 1024 * 1024) -> str:
digest = hashlib.sha1()
with open(path, "rb") as stream:
while True:
chunk = stream.read(chunk_size)
if not chunk:
break
digest.update(chunk)
return digest.hexdigest()
@dataclass(frozen=True)
class ManifestEntrySnapshot:
absolute_path: str
relative_path: str
fingerprint: str
size_bytes: Optional[int]
mtime: Optional[float]
ctime: Optional[float]
@dataclass(frozen=True)
class ManifestSnapshot:
root: str
manifest_count: int
tree_fingerprint: str
entries: Tuple[ManifestEntrySnapshot, ...]
manifest_paths: Tuple[str, ...]
def iter_manifest_paths(root: str) -> List[str]:
normalized_root = _normalize_path(root)
if not os.path.isdir(normalized_root):
return []
manifest_paths: List[str] = []
stack = [normalized_root]
while stack:
current = stack.pop()
try:
with os.scandir(current) as entries:
for entry in entries:
try:
if entry.is_dir(follow_symlinks=False):
stack.append(entry.path)
continue
if entry.is_file(follow_symlinks=False) and entry.name == MANIFEST_FILENAME:
manifest_paths.append(_normalize_path(entry.path))
except OSError:
continue
except OSError:
continue
manifest_paths.sort()
return manifest_paths
def build_manifest_snapshot(root: str) -> ManifestSnapshot:
normalized_root = _normalize_path(root)
manifest_paths = tuple(iter_manifest_paths(normalized_root))
digest = hashlib.sha1()
entries: List[ManifestEntrySnapshot] = []
for manifest_path in manifest_paths:
try:
relative_path = os.path.relpath(manifest_path, normalized_root).replace("\\", "/")
except ValueError:
relative_path = os.path.basename(manifest_path)
try:
file_fingerprint = _hash_file(manifest_path)
except OSError as exc:
file_fingerprint = f"!read_error:{exc.__class__.__name__}"
try:
stat = os.stat(manifest_path)
size_bytes = int(stat.st_size)
mtime = float(stat.st_mtime)
ctime = float(stat.st_ctime)
except OSError:
size_bytes = None
mtime = None
ctime = None
digest.update(relative_path.encode("utf-8", errors="ignore"))
digest.update(b"\0")
digest.update(file_fingerprint.encode("utf-8", errors="ignore"))
digest.update(b"\n")
entries.append(
ManifestEntrySnapshot(
absolute_path=manifest_path,
relative_path=relative_path,
fingerprint=file_fingerprint,
size_bytes=size_bytes,
mtime=mtime,
ctime=ctime,
)
)
return ManifestSnapshot(
root=normalized_root,
manifest_count=len(manifest_paths),
tree_fingerprint=digest.hexdigest() if manifest_paths else "",
entries=tuple(entries),
manifest_paths=manifest_paths,
)
def evaluate_manifest_reconcile(
*,
manifest_count: int,
db_count: int,
current_fingerprint: str,
indexed_fingerprint: Optional[str],
) -> dict:
reasons: List[str] = []
normalized_indexed_fingerprint = str(indexed_fingerprint or "").strip()
normalized_current_fingerprint = str(current_fingerprint or "").strip()
if int(db_count or 0) != int(manifest_count or 0):
reasons.append("count_mismatch")
if manifest_count > 0 and normalized_indexed_fingerprint != normalized_current_fingerprint:
reasons.append("fingerprint_mismatch")
return {
"needs_rebuild": bool(reasons),
"reasons": reasons,
"current_fingerprint": normalized_current_fingerprint,
"indexed_fingerprint": normalized_indexed_fingerprint,
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,482 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, List, Optional, Sequence
from sqlalchemy import delete, func, or_, select, text, update
from sqlalchemy.ext.asyncio import AsyncSession
from ..models import PairingCacheStateORM, PairingDirtySceneORM, PairingMetricCacheORM, RadarDataORM
from .pairing_state_service import (
PAIRING_CACHE_SCOPE_GLOBAL,
pairing_state_service,
)
FULL_REBUILD_DIRTY_SCENE_THRESHOLD = 64
FULL_REBUILD_DIRTY_SCENE_RATIO = 0.25
def _utcnow() -> datetime:
return datetime.utcnow()
def _scene_uid_expr(alias: str) -> str:
return (
f"COALESCE(NULLIF({alias}.unique_id, ''), NULLIF({alias}.file_path, ''), "
f"'scene:' || {alias}.id::text)"
)
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
left_uid = _scene_uid_expr(left_alias)
right_uid = _scene_uid_expr(right_alias)
return (
f"({left_alias}.imaging_date < {right_alias}.imaging_date "
f"OR ({left_alias}.imaging_date = {right_alias}.imaging_date AND "
f"({left_uid} < {right_uid} OR ({left_uid} = {right_uid} AND {left_alias}.id < {right_alias}.id))))"
)
def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
return (
f"{left_alias}.id <> {right_alias}.id "
f"AND {left_alias}.geom IS NOT NULL "
f"AND {right_alias}.geom IS NOT NULL "
f"AND {left_alias}.imaging_date ~ '^[0-9]{{8}}$' "
f"AND {right_alias}.imaging_date ~ '^[0-9]{{8}}$' "
f"AND {left_alias}.orbit_direction IS NOT NULL "
f"AND {right_alias}.orbit_direction IS NOT NULL "
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
)
def _full_rebuild_insert_sql() -> str:
master_uid = _scene_uid_expr("m")
slave_uid = _scene_uid_expr("s")
return f"""
INSERT INTO pairing_metric_cache (
master_scene_ref_id,
slave_scene_ref_id,
master_scene_uid,
slave_scene_uid,
pair_uid,
metric_version,
orientation_rule_version,
time_baseline_days,
spatial_baseline_meters,
scene_overlap_ratio,
orbit_direction,
same_satellite,
same_imaging_mode,
same_polarization,
master_imaging_date,
slave_imaging_date,
master_satellite,
slave_satellite,
master_imaging_mode,
slave_imaging_mode,
master_polarization,
slave_polarization,
master_file_path,
slave_file_path,
status,
computed_at
)
SELECT
m.id AS master_scene_ref_id,
s.id AS slave_scene_ref_id,
{master_uid} AS master_scene_uid,
{slave_uid} AS slave_scene_uid,
md5({master_uid} || '|' || {slave_uid}) AS pair_uid,
:metric_version AS metric_version,
:orientation_rule_version AS orientation_rule_version,
ABS(to_date(s.imaging_date, 'YYYYMMDD') - to_date(m.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision AS spatial_baseline_meters,
(
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
m.orbit_direction,
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
(
NULLIF(m.imaging_mode, '') IS NOT NULL
AND NULLIF(s.imaging_mode, '') IS NOT NULL
AND m.imaging_mode = s.imaging_mode
) AS same_imaging_mode,
(
NULLIF(m.polarization, '') IS NOT NULL
AND NULLIF(s.polarization, '') IS NOT NULL
AND m.polarization = s.polarization
) AS same_polarization,
m.imaging_date AS master_imaging_date,
s.imaging_date AS slave_imaging_date,
m.satellite AS master_satellite,
s.satellite AS slave_satellite,
m.imaging_mode AS master_imaging_mode,
s.imaging_mode AS slave_imaging_mode,
m.polarization AS master_polarization,
s.polarization AS slave_polarization,
m.file_path AS master_file_path,
s.file_path AS slave_file_path,
'READY' AS status,
NOW() AS computed_at
FROM radar_data m
JOIN radar_data s
ON { _hard_constraints_expr('m', 's') }
AND { _orientation_is_left_master_expr('m', 's') }
"""
def _incremental_insert_sql() -> str:
dirty_uid = _scene_uid_expr("d")
other_uid = _scene_uid_expr("o")
dirty_is_master = _orientation_is_left_master_expr("d", "o")
return f"""
INSERT INTO pairing_metric_cache (
master_scene_ref_id,
slave_scene_ref_id,
master_scene_uid,
slave_scene_uid,
pair_uid,
metric_version,
orientation_rule_version,
time_baseline_days,
spatial_baseline_meters,
scene_overlap_ratio,
orbit_direction,
same_satellite,
same_imaging_mode,
same_polarization,
master_imaging_date,
slave_imaging_date,
master_satellite,
slave_satellite,
master_imaging_mode,
slave_imaging_mode,
master_polarization,
slave_polarization,
master_file_path,
slave_file_path,
status,
computed_at
)
SELECT
CASE WHEN {dirty_is_master} THEN d.id ELSE o.id END AS master_scene_ref_id,
CASE WHEN {dirty_is_master} THEN o.id ELSE d.id END AS slave_scene_ref_id,
CASE WHEN {dirty_is_master} THEN {dirty_uid} ELSE {other_uid} END AS master_scene_uid,
CASE WHEN {dirty_is_master} THEN {other_uid} ELSE {dirty_uid} END AS slave_scene_uid,
md5(
CASE WHEN {dirty_is_master}
THEN {dirty_uid} || '|' || {other_uid}
ELSE {other_uid} || '|' || {dirty_uid}
END
) AS pair_uid,
:metric_version AS metric_version,
:orientation_rule_version AS orientation_rule_version,
ABS(to_date(o.imaging_date, 'YYYYMMDD') - to_date(d.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision AS spatial_baseline_meters,
(
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
d.orbit_direction,
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
(
NULLIF(d.imaging_mode, '') IS NOT NULL
AND NULLIF(o.imaging_mode, '') IS NOT NULL
AND d.imaging_mode = o.imaging_mode
) AS same_imaging_mode,
(
NULLIF(d.polarization, '') IS NOT NULL
AND NULLIF(o.polarization, '') IS NOT NULL
AND d.polarization = o.polarization
) AS same_polarization,
CASE WHEN {dirty_is_master} THEN d.imaging_date ELSE o.imaging_date END AS master_imaging_date,
CASE WHEN {dirty_is_master} THEN o.imaging_date ELSE d.imaging_date END AS slave_imaging_date,
CASE WHEN {dirty_is_master} THEN d.satellite ELSE o.satellite END AS master_satellite,
CASE WHEN {dirty_is_master} THEN o.satellite ELSE d.satellite END AS slave_satellite,
CASE WHEN {dirty_is_master} THEN d.imaging_mode ELSE o.imaging_mode END AS master_imaging_mode,
CASE WHEN {dirty_is_master} THEN o.imaging_mode ELSE d.imaging_mode END AS slave_imaging_mode,
CASE WHEN {dirty_is_master} THEN d.polarization ELSE o.polarization END AS master_polarization,
CASE WHEN {dirty_is_master} THEN o.polarization ELSE d.polarization END AS slave_polarization,
CASE WHEN {dirty_is_master} THEN d.file_path ELSE o.file_path END AS master_file_path,
CASE WHEN {dirty_is_master} THEN o.file_path ELSE d.file_path END AS slave_file_path,
'READY' AS status,
NOW() AS computed_at
FROM radar_data d
JOIN radar_data o
ON d.id = :dirty_scene_id
AND { _hard_constraints_expr('d', 'o') }
ON CONFLICT (master_scene_ref_id, slave_scene_ref_id, metric_version)
DO NOTHING
"""
class PairingCacheService:
async def _get_state_row(self, db: AsyncSession) -> PairingCacheStateORM:
payload = await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
result = await db.execute(
select(PairingCacheStateORM).where(
PairingCacheStateORM.cache_scope == PAIRING_CACHE_SCOPE_GLOBAL
)
)
state = result.scalar_one_or_none()
if state is None:
raise RuntimeError(f"Pairing cache state row missing after bootstrap: {payload}")
return state
async def _count_pair_rows(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(PairingMetricCacheORM.id)))
return int(result.scalar_one() or 0)
async def _count_scene_rows(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(RadarDataORM.id)))
return int(result.scalar_one() or 0)
async def _count_pending_dirty_rows(self, db: AsyncSession) -> int:
result = await db.execute(
select(func.count(PairingDirtySceneORM.id)).where(
PairingDirtySceneORM.status == "PENDING"
)
)
return int(result.scalar_one() or 0)
async def _resolve_dirty_rows(
self,
db: AsyncSession,
*,
scene_ids: Optional[Sequence[int]] = None,
) -> int:
stmt = (
update(PairingDirtySceneORM)
.where(PairingDirtySceneORM.status == "PENDING")
.values(status="RESOLVED", resolved_at=_utcnow())
)
if scene_ids is not None:
normalized_ids = [int(value) for value in scene_ids if value is not None]
if not normalized_ids:
return 0
stmt = stmt.where(PairingDirtySceneORM.scene_ref_id.in_(normalized_ids))
result = await db.execute(stmt)
return int(result.rowcount or 0)
async def _set_state_rebuilding(self, db: AsyncSession, *, last_error: Optional[str] = None) -> None:
state = await self._get_state_row(db)
state.status = "REBUILDING"
state.last_error = last_error
await db.flush()
async def _finalize_state_success(
self,
db: AsyncSession,
*,
full_rebuild: bool,
) -> Dict[str, Any]:
state = await self._get_state_row(db)
pair_count = await self._count_pair_rows(db)
scene_count = await self._count_scene_rows(db)
dirty_scene_count = await self._count_pending_dirty_rows(db)
state.metric_version = pairing_state_service.metric_version
state.scene_count = scene_count
state.pair_count = pair_count
state.dirty_scene_count = dirty_scene_count
state.status = (
"READY"
if dirty_scene_count == 0 and not (scene_count > 1 and pair_count == 0)
else "DIRTY"
)
state.last_error = None
if full_rebuild:
state.last_full_rebuild_at = _utcnow()
else:
state.last_incremental_reconcile_at = _utcnow()
await db.flush()
return {
"status": state.status,
"scene_count": scene_count,
"pair_count": pair_count,
"dirty_scene_count": dirty_scene_count,
"metric_version": state.metric_version,
"cache_ready": state.status == "READY",
"needs_rebuild": dirty_scene_count > 0 or (scene_count > 1 and pair_count == 0),
}
async def _finalize_state_failure(self, db: AsyncSession, *, error: Exception) -> None:
state = await self._get_state_row(db)
state.status = "FAILED"
state.last_error = str(error)
await db.flush()
def _should_full_rebuild(
self,
*,
dirty_scene_count: int,
scene_count: int,
pair_count: int,
force_full: bool,
) -> bool:
if force_full:
return True
if pair_count == 0 and scene_count > 1:
return True
if dirty_scene_count >= FULL_REBUILD_DIRTY_SCENE_THRESHOLD:
return True
if scene_count > 0 and dirty_scene_count / max(scene_count, 1) >= FULL_REBUILD_DIRTY_SCENE_RATIO:
return True
return False
async def rebuild_metric_cache(
self,
db: AsyncSession,
*,
commit: bool = True,
) -> Dict[str, Any]:
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
await self._set_state_rebuilding(db)
try:
delete_result = await db.execute(delete(PairingMetricCacheORM))
await db.execute(
text(_full_rebuild_insert_sql()),
{
"metric_version": pairing_state_service.metric_version,
"orientation_rule_version": pairing_state_service.orientation_rule_version,
},
)
resolved_dirty = await self._resolve_dirty_rows(db)
summary = await self._finalize_state_success(db, full_rebuild=True)
if commit:
await db.commit()
else:
await db.flush()
return {
"ok": True,
"mode": "full_rebuild",
"deleted_pair_rows": int(delete_result.rowcount or 0),
"resolved_dirty_rows": resolved_dirty,
**summary,
}
except Exception as exc:
await db.rollback()
await self._finalize_state_failure(db, error=exc)
if commit:
await db.commit()
raise
async def reconcile_dirty_scenes(
self,
db: AsyncSession,
*,
force_full: bool = False,
commit: bool = True,
) -> Dict[str, Any]:
await pairing_state_service.ensure_pairing_cache_state(db, commit=False)
dirty_result = await db.execute(
select(PairingDirtySceneORM.scene_ref_id)
.where(PairingDirtySceneORM.status == "PENDING")
.order_by(PairingDirtySceneORM.marked_at.asc(), PairingDirtySceneORM.id.asc())
)
dirty_scene_ids = [int(value) for value in dirty_result.scalars().all() if value is not None]
dirty_scene_count = len(dirty_scene_ids)
pair_count = await self._count_pair_rows(db)
scene_count = await self._count_scene_rows(db)
if dirty_scene_count == 0:
summary = await self._finalize_state_success(db, full_rebuild=False)
if commit:
await db.commit()
return {
"ok": True,
"mode": "noop",
"dirty_scene_count": 0,
"deleted_pair_rows": 0,
"resolved_dirty_rows": 0,
"insert_attempts": 0,
**summary,
}
if self._should_full_rebuild(
dirty_scene_count=dirty_scene_count,
scene_count=scene_count,
pair_count=pair_count,
force_full=force_full,
):
result = await self.rebuild_metric_cache(db, commit=commit)
result["trigger_dirty_scene_count"] = dirty_scene_count
result["forced"] = force_full
return result
await self._set_state_rebuilding(db)
try:
delete_result = await db.execute(
delete(PairingMetricCacheORM).where(
or_(
PairingMetricCacheORM.master_scene_ref_id.in_(dirty_scene_ids),
PairingMetricCacheORM.slave_scene_ref_id.in_(dirty_scene_ids),
)
)
)
insert_attempts = 0
insert_sql = text(_incremental_insert_sql())
for dirty_scene_id in dirty_scene_ids:
insert_result = await db.execute(
insert_sql,
{
"dirty_scene_id": int(dirty_scene_id),
"metric_version": pairing_state_service.metric_version,
"orientation_rule_version": pairing_state_service.orientation_rule_version,
},
)
insert_attempts += int(insert_result.rowcount or 0)
resolved_dirty = await self._resolve_dirty_rows(db, scene_ids=dirty_scene_ids)
summary = await self._finalize_state_success(db, full_rebuild=False)
if commit:
await db.commit()
else:
await db.flush()
return {
"ok": True,
"mode": "incremental_reconcile",
"dirty_scene_count": dirty_scene_count,
"deleted_pair_rows": int(delete_result.rowcount or 0),
"resolved_dirty_rows": resolved_dirty,
"insert_attempts": insert_attempts,
**summary,
}
except Exception as exc:
await db.rollback()
await self._finalize_state_failure(db, error=exc)
if commit:
await db.commit()
raise
async def get_admin_summary(self, db: AsyncSession) -> Dict[str, Any]:
payload = await pairing_state_service.get_pairing_system_status(db)
pending_result = await db.execute(
select(
PairingDirtySceneORM.scene_ref_id,
PairingDirtySceneORM.scene_uid,
PairingDirtySceneORM.reason,
PairingDirtySceneORM.marked_at,
)
.where(PairingDirtySceneORM.status == "PENDING")
.order_by(PairingDirtySceneORM.marked_at.asc(), PairingDirtySceneORM.id.asc())
.limit(20)
)
pending_rows = pending_result.all()
payload["pending_dirty_scenes"] = [
{
"scene_ref_id": int(scene_ref_id),
"scene_uid": scene_uid,
"reason": reason,
"marked_at": marked_at,
}
for scene_ref_id, scene_uid, reason, marked_at in pending_rows
]
return payload
pairing_cache_service = PairingCacheService()
@@ -0,0 +1,370 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, Optional, Sequence
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..models import (
PairingCacheStateORM,
PairingDirtySceneORM,
PairingMetricCacheORM,
PairingNetworkEdgeORM,
PairingNetworkRunORM,
RadarDataORM,
)
PAIRING_CACHE_SCOPE_GLOBAL = "global"
DEFAULT_PAIRING_METRIC_VERSION = "2026.04.v1"
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
def _new_session() -> AsyncSession:
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
def _utcnow() -> datetime:
return datetime.utcnow()
def _normalize_scene_uid(
*,
scene_ref_id: Optional[int],
unique_id: Optional[str],
file_path: Optional[str],
) -> str:
unique_text = str(unique_id or "").strip()
if unique_text:
return unique_text
file_text = str(file_path or "").strip()
if file_text:
return file_text
if scene_ref_id is not None:
return f"scene:{int(scene_ref_id)}"
return "scene:unknown"
class PairingStateService:
metric_version = DEFAULT_PAIRING_METRIC_VERSION
orientation_rule_version = PAIRING_ORIENTATION_RULE_VERSION
async def _count_radar_scenes(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(RadarDataORM.id)))
return int(result.scalar_one() or 0)
async def _count_pending_dirty_scenes(self, db: AsyncSession) -> int:
result = await db.execute(
select(func.count(PairingDirtySceneORM.id)).where(
PairingDirtySceneORM.status == "PENDING"
)
)
return int(result.scalar_one() or 0)
async def _count_metric_cache_rows(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(PairingMetricCacheORM.id)))
return int(result.scalar_one() or 0)
async def _get_global_state(self, db: AsyncSession) -> Optional[PairingCacheStateORM]:
result = await db.execute(
select(PairingCacheStateORM).where(
PairingCacheStateORM.cache_scope == PAIRING_CACHE_SCOPE_GLOBAL
)
)
return result.scalar_one_or_none()
def _build_state_payload(
self,
*,
state: Optional[PairingCacheStateORM],
scene_count: int,
dirty_scene_count: int,
metric_cache_count: int,
) -> Dict[str, Any]:
status = state.status if state is not None else "UNINITIALIZED"
last_error = state.last_error if state is not None else None
needs_rebuild = bool(
dirty_scene_count > 0
or status in {"DIRTY", "DEGRADED", "FAILED", "UNINITIALIZED"}
or (scene_count > 1 and metric_cache_count == 0)
)
cache_ready = not needs_rebuild and status == "READY"
state_dict = getattr(state, "__dict__", {}) if state is not None else {}
return {
"state_present": state is not None,
"cache_scope": state.cache_scope if state is not None else PAIRING_CACHE_SCOPE_GLOBAL,
"metric_version": (state.metric_version if state is not None else None) or self.metric_version,
"status": status,
"scene_count": int(scene_count),
"pair_count": int(metric_cache_count),
"dirty_scene_count": int(dirty_scene_count),
"needs_rebuild": needs_rebuild,
"cache_ready": cache_ready,
"last_full_rebuild_at": state_dict.get("last_full_rebuild_at"),
"last_incremental_reconcile_at": state_dict.get("last_incremental_reconcile_at"),
"last_error": last_error,
"updated_at": state_dict.get("updated_at"),
}
async def ensure_pairing_cache_state(
self,
db: AsyncSession,
*,
commit: bool = False,
) -> Dict[str, Any]:
scene_count = await self._count_radar_scenes(db)
dirty_scene_count = await self._count_pending_dirty_scenes(db)
metric_cache_count = await self._count_metric_cache_rows(db)
state = await self._get_global_state(db)
created = False
if state is None:
state = PairingCacheStateORM(
cache_scope=PAIRING_CACHE_SCOPE_GLOBAL,
metric_version=self.metric_version,
status="DIRTY" if scene_count > 1 else "READY",
scene_count=scene_count,
pair_count=metric_cache_count,
dirty_scene_count=dirty_scene_count,
)
db.add(state)
await db.flush()
created = True
state.metric_version = state.metric_version or self.metric_version
state.scene_count = scene_count
state.pair_count = metric_cache_count
state.dirty_scene_count = dirty_scene_count
if state.status == "READY" and (
dirty_scene_count > 0 or (scene_count > 1 and metric_cache_count == 0)
):
state.status = "DIRTY"
elif state.status in {None, ""}:
state.status = "DIRTY" if scene_count > 1 else "READY"
if commit:
await db.commit()
await db.refresh(state)
else:
await db.flush()
payload = self._build_state_payload(
state=state,
scene_count=scene_count,
dirty_scene_count=dirty_scene_count,
metric_cache_count=metric_cache_count,
)
payload["created"] = created
return payload
async def bootstrap_pairing_cache_state(self) -> Dict[str, Any]:
async with _new_session() as db:
return await self.ensure_pairing_cache_state(db, commit=True)
async def mark_global_dirty(
self,
db: AsyncSession,
*,
reason: str = "manual",
commit: bool = False,
) -> Dict[str, Any]:
payload = await self.ensure_pairing_cache_state(db, commit=False)
state = await self._get_global_state(db)
if state is None:
raise RuntimeError("Pairing cache state row is missing after bootstrap.")
state.status = "DIRTY"
if state.last_error and reason != "error_recovery":
state.last_error = None
if commit:
await db.commit()
else:
await db.flush()
payload = await self.ensure_pairing_cache_state(db, commit=False)
payload["dirty_mark_mode"] = "global"
payload["reason"] = reason
return payload
async def mark_scenes_dirty(
self,
db: AsyncSession,
*,
scene_ids: Sequence[int],
reason: str = "scan",
commit: bool = False,
) -> Dict[str, Any]:
normalized_ids = sorted(
{
int(scene_id)
for scene_id in (scene_ids or [])
if scene_id is not None and int(scene_id) > 0
}
)
if not normalized_ids:
return await self.mark_global_dirty(db, reason=reason, commit=commit)
result = await db.execute(
select(RadarDataORM).where(RadarDataORM.id.in_(normalized_ids))
)
radar_rows = result.scalars().all()
if not radar_rows:
return await self.mark_global_dirty(db, reason=reason, commit=commit)
existing_result = await db.execute(
select(PairingDirtySceneORM.scene_ref_id).where(
PairingDirtySceneORM.scene_ref_id.in_([row.id for row in radar_rows]),
PairingDirtySceneORM.status == "PENDING",
)
)
existing_pending_ids = {int(value) for value in existing_result.scalars().all()}
created = 0
now = _utcnow()
for row in radar_rows:
if int(row.id) in existing_pending_ids:
continue
db.add(
PairingDirtySceneORM(
scene_ref_id=row.id,
scene_uid=_normalize_scene_uid(
scene_ref_id=row.id,
unique_id=row.unique_id,
file_path=row.file_path,
),
reason=reason,
status="PENDING",
marked_at=now,
)
)
created += 1
await db.flush()
state = await self._get_global_state(db)
if state is None:
await self.ensure_pairing_cache_state(db, commit=False)
state = await self._get_global_state(db)
if state is None:
raise RuntimeError("Pairing cache state row is missing after bootstrap.")
state.status = "DIRTY"
state.last_error = None
if commit:
await db.commit()
else:
await db.flush()
payload = await self.ensure_pairing_cache_state(db, commit=False)
payload["dirty_mark_mode"] = "scene"
payload["reason"] = reason
payload["requested_scene_count"] = len(normalized_ids)
payload["resolved_scene_count"] = len(radar_rows)
payload["created_dirty_rows"] = created
return payload
async def get_pairing_system_status(
self,
db: Optional[AsyncSession] = None,
) -> Dict[str, Any]:
own_session = db is None
session = db or _new_session()
try:
scene_count = await self._count_radar_scenes(session)
dirty_scene_count = await self._count_pending_dirty_scenes(session)
metric_cache_count = await self._count_metric_cache_rows(session)
state = await self._get_global_state(session)
run_count_result = await session.execute(select(func.count(PairingNetworkRunORM.id)))
network_run_count = int(run_count_result.scalar_one() or 0)
edge_count_result = await session.execute(select(func.count(PairingNetworkEdgeORM.id)))
network_edge_count = int(edge_count_result.scalar_one() or 0)
reverse_duplicate_result = await session.execute(
text(
"""
SELECT COUNT(*)
FROM pairing_metric_cache a
JOIN pairing_metric_cache b
ON a.master_scene_ref_id = b.slave_scene_ref_id
AND a.slave_scene_ref_id = b.master_scene_ref_id
AND a.metric_version = b.metric_version
AND a.id < b.id
"""
)
)
duplicate_reverse_pair_count = int(reverse_duplicate_result.scalar_one() or 0)
orphan_edge_result = await session.execute(
text(
"""
SELECT COUNT(*)
FROM pairing_network_edges e
LEFT JOIN pairing_network_runs r
ON r.id = e.network_run_ref_id
LEFT JOIN pairing_metric_cache m
ON m.id = e.metric_cache_ref_id
WHERE r.id IS NULL OR m.id IS NULL
"""
)
)
orphan_edge_count = int(orphan_edge_result.scalar_one() or 0)
payload = self._build_state_payload(
state=state,
scene_count=scene_count,
dirty_scene_count=dirty_scene_count,
metric_cache_count=metric_cache_count,
)
payload.update(
{
"ok": bool(
payload["status"] not in {"FAILED", "UNINITIALIZED", "ERROR"}
and orphan_edge_count == 0
and duplicate_reverse_pair_count == 0
),
"network_run_count": network_run_count,
"network_edge_count": network_edge_count,
"duplicate_reverse_pair_count": duplicate_reverse_pair_count,
"orphan_edge_count": orphan_edge_count,
"orientation_rule_version": self.orientation_rule_version,
"error": None,
}
)
return payload
except Exception as exc:
return {
"ok": False,
"state_present": False,
"cache_scope": PAIRING_CACHE_SCOPE_GLOBAL,
"metric_version": self.metric_version,
"status": "ERROR",
"scene_count": 0,
"pair_count": 0,
"dirty_scene_count": 0,
"needs_rebuild": True,
"cache_ready": False,
"last_full_rebuild_at": None,
"last_incremental_reconcile_at": None,
"last_error": None,
"updated_at": None,
"network_run_count": 0,
"network_edge_count": 0,
"duplicate_reverse_pair_count": 0,
"orphan_edge_count": 0,
"orientation_rule_version": self.orientation_rule_version,
"error": str(exc),
}
finally:
if own_session:
await session.close()
pairing_state_service = PairingStateService()
@@ -0,0 +1,806 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import os
from datetime import datetime
from typing import Any, Dict, List, Optional
from geoalchemy2.shape import from_shape
from shapely.geometry import Polygon
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
from ..models import ResultAssetORM, ResultCatalogStateORM, ResultIssueORM, ResultProductORM
from .manifest_snapshot_service import (
build_manifest_snapshot,
evaluate_manifest_reconcile,
iter_manifest_paths,
)
PSINSAR_CATALOG_NAME = "psinsar"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
TASK_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
_ARTIFACT_ROLE_MAP = {
"timeseries_cube": "timeseries_cube",
"velocity_map": "velocity_map",
"velocity_geotiff": "velocity_geotiff",
"temporal_coherence": "temporal_coherence",
"temporal_coherence_geotiff": "temporal_coherence_geotiff",
"quality_mask": "quality_mask",
"quality_mask_geotiff": "quality_mask_geotiff",
"preview_png": "preview_png",
"diagnostic_png": "diagnostic_png",
}
_PRIMARY_PRODUCT_TYPES = {"timeseries_cube", "velocity_geotiff"}
_PREFERRED_PRIMARY_PRODUCT_TYPES = ("velocity_geotiff", "timeseries_cube", "velocity_map")
_PREVIEW_PRODUCT_TYPES = {"preview_png"}
def _utcnow() -> datetime:
return datetime.utcnow()
def _normalize_path(path: str) -> str:
return os.path.normpath(os.path.abspath(path))
def _stable_digest(*parts: Any, length: int = 20) -> str:
payload = "||".join(str(part or "") for part in parts)
return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length]
def _parse_datetime(value: Optional[str]) -> Optional[datetime]:
text = str(value or "").strip()
if not text:
return None
if text.endswith("Z"):
text = text[:-1] + "+00:00"
try:
return datetime.fromisoformat(text).replace(tzinfo=None)
except ValueError:
return None
def _safe_float(value: Any) -> Optional[float]:
try:
return float(value)
except (TypeError, ValueError):
return None
def _safe_int(value: Any) -> Optional[int]:
try:
return int(float(value))
except (TypeError, ValueError):
return None
def _resolve_relative_path(base_dir: str, relative_path: str) -> str:
base_dir = _normalize_path(base_dir)
target = _normalize_path(os.path.join(base_dir, relative_path))
if not target.startswith(base_dir + os.sep) and target != base_dir:
raise ValueError(f"Invalid relative path outside package: {relative_path}")
return target
def _build_bbox_polygon(
min_lon: Optional[float],
min_lat: Optional[float],
max_lon: Optional[float],
max_lat: Optional[float],
):
if None in (min_lon, min_lat, max_lon, max_lat):
return None
return Polygon(
[
(min_lon, min_lat),
(max_lon, min_lat),
(max_lon, max_lat),
(min_lon, max_lat),
(min_lon, min_lat),
]
)
def _artifact_format(path: str) -> Optional[str]:
ext = os.path.splitext(str(path or "").lower())[1]
return {
".h5": "hdf5",
".tif": "geotiff",
".tiff": "geotiff",
".png": "png",
".json": "json",
".cfg": "cfg",
}.get(ext)
def _artifact_media_type(path: str) -> Optional[str]:
ext = os.path.splitext(str(path or "").lower())[1]
return {
".h5": "application/x-hdf5",
".tif": "image/tiff",
".tiff": "image/tiff",
".png": "image/png",
".json": "application/json",
".cfg": "text/plain",
}.get(ext)
def _derive_bbox_from_summary(manifest: Dict[str, Any]) -> Dict[str, Optional[float]]:
summaries = manifest.get("summaries") or {}
for key in ("geo_velocity", "geo_timeseries", "velocity"):
summary = summaries.get(key) or {}
attrs = summary.get("attrs") or {}
x_first = _safe_float(attrs.get("X_FIRST"))
y_first = _safe_float(attrs.get("Y_FIRST"))
x_step = _safe_float(attrs.get("X_STEP"))
y_step = _safe_float(attrs.get("Y_STEP"))
width = _safe_int(attrs.get("WIDTH"))
length = _safe_int(attrs.get("LENGTH"))
if None in (x_first, y_first, x_step, y_step, width, length):
continue
x_last = x_first + x_step * max(width - 1, 0)
y_last = y_first + y_step * max(length - 1, 0)
return {
"min_lon": min(x_first, x_last),
"max_lon": max(x_first, x_last),
"min_lat": min(y_first, y_last),
"max_lat": max(y_first, y_last),
}
return {
"min_lon": None,
"max_lon": None,
"min_lat": None,
"max_lat": None,
}
class PsinsarCatalogService:
def get_publish_root(self, publish_root: Optional[str] = None) -> str:
root = publish_root or settings.PSINSAR_PRODUCT_DIR
normalized = _normalize_path(root)
os.makedirs(normalized, exist_ok=True)
return normalized
async def _get_or_create_catalog_state(
self,
db: AsyncSession,
*,
storage_root: Optional[str] = None,
) -> ResultCatalogStateORM:
root = self.get_publish_root(storage_root)
result = await db.execute(
select(ResultCatalogStateORM).where(
ResultCatalogStateORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
state = result.scalar_one_or_none()
if state is None:
state = ResultCatalogStateORM(
catalog_name=PSINSAR_CATALOG_NAME,
storage_root=root,
status="READY",
needs_rebuild=False,
)
db.add(state)
await db.flush()
elif state.storage_root != root:
state.storage_root = root
return state
def _iter_manifest_paths(self, publish_root: str) -> List[str]:
return iter_manifest_paths(self.get_publish_root(publish_root))
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
with open(manifest_path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
schema_version = str(payload.get("schema_version") or "").strip().lower()
catalog_name = str(payload.get("catalog_name") or "").strip().lower()
if schema_version != "psinsar.publish.v1":
raise ValueError("manifest schema_version is not psinsar.publish.v1")
if catalog_name != PSINSAR_CATALOG_NAME:
raise ValueError("manifest catalog_name is not psinsar")
return payload
def _build_rows_from_manifest(
self,
manifest_path: str,
manifest: Dict[str, Any],
) -> ResultProductORM:
package_dir = _normalize_path(os.path.dirname(manifest_path))
basename = os.path.basename(package_dir)
group_key = str(manifest.get("group_key") or "").strip() or None
reference_date = str(manifest.get("reference_date") or "").strip() or None
stack_dates = [str(item).strip() for item in (manifest.get("stack_dates") or []) if str(item).strip()]
run_key = str(manifest.get("run_id") or "").strip() or basename
display_name = group_key or basename
product_id = "psinsar_" + _stable_digest(package_dir, run_key, reference_date, length=20)
bbox = _derive_bbox_from_summary(manifest)
poly = _build_bbox_polygon(
bbox.get("min_lon"),
bbox.get("min_lat"),
bbox.get("max_lon"),
bbox.get("max_lat"),
)
summary_json = {
"group_key": group_key,
"reference_date": reference_date,
"reference_point": manifest.get("reference_point"),
"stack_dates": stack_dates,
"stack_size": len(stack_dates),
"mode": manifest.get("mode"),
"processor_code": manifest.get("processor_code"),
"quality": manifest.get("quality"),
"summaries": manifest.get("summaries"),
}
published_at = _parse_datetime(manifest.get("published_at"))
if published_at is None:
try:
published_at = datetime.utcfromtimestamp(os.path.getmtime(manifest_path))
except OSError:
published_at = _utcnow()
product = ResultProductORM(
product_id=product_id,
catalog_name=PSINSAR_CATALOG_NAME,
product_type="psinsar_bundle",
display_name=display_name,
task_name=display_name,
task_alias=group_key or basename,
pair_key=None,
run_key=run_key,
profile_code=str(manifest.get("processor_code") or "").strip() or None,
engine_code=str(manifest.get("engine_code") or "unknown"),
engine_version=None,
status="READY",
health_status="OK",
publish_dir=package_dir,
manifest_path=_normalize_path(manifest_path),
source_primary_path=None,
preview_path=None,
primary_asset_path=None,
summary_json=summary_json,
tags_json=None,
min_lon=bbox.get("min_lon"),
min_lat=bbox.get("min_lat"),
max_lon=bbox.get("max_lon"),
max_lat=bbox.get("max_lat"),
geom=from_shape(poly, srid=4326) if poly is not None else None,
coverage_polygon={
"type": "Polygon",
"coordinates": [[
[bbox["min_lon"], bbox["min_lat"]],
[bbox["max_lon"], bbox["min_lat"]],
[bbox["max_lon"], bbox["max_lat"]],
[bbox["min_lon"], bbox["max_lat"]],
[bbox["min_lon"], bbox["min_lat"]],
]],
} if None not in (bbox["min_lon"], bbox["min_lat"], bbox["max_lon"], bbox["max_lat"]) else None,
produced_at=_parse_datetime(manifest.get("produced_at")) or published_at,
published_at=published_at,
registered_at=_utcnow(),
)
has_warn = False
has_error = False
artifacts = manifest.get("artifacts") or []
chosen_primary_path = None
for candidate_type in _PREFERRED_PRIMARY_PRODUCT_TYPES:
for artifact in artifacts:
if str(artifact.get("product_type") or "").strip() == candidate_type:
chosen_primary_path = str(artifact.get("path") or "").strip()
break
if chosen_primary_path:
break
for artifact in artifacts:
relative_path = str(artifact.get("path") or "").strip()
if not relative_path:
continue
absolute_path = _resolve_relative_path(package_dir, relative_path)
exists_flag = os.path.exists(absolute_path)
try:
file_size = os.path.getsize(absolute_path) if exists_flag else None
except OSError:
file_size = None
product_type = str(artifact.get("product_type") or "asset").strip()
asset_role = _ARTIFACT_ROLE_MAP.get(product_type, product_type or "asset")
is_required = product_type in _PRIMARY_PRODUCT_TYPES or product_type in _PREVIEW_PRODUCT_TYPES
is_primary = relative_path == chosen_primary_path
asset = ResultAssetORM(
asset_role=asset_role,
asset_name=os.path.basename(relative_path) or asset_role,
relative_path=relative_path,
absolute_path=absolute_path,
format=_artifact_format(relative_path),
media_type=_artifact_media_type(relative_path),
is_required=is_required,
is_primary=is_primary,
exists_flag=exists_flag,
file_size=file_size,
)
product.assets.append(asset)
if product_type == "timeseries_cube" and exists_flag:
product.source_primary_path = absolute_path
if is_primary and exists_flag:
product.primary_asset_path = absolute_path
if product_type in _PREVIEW_PRODUCT_TYPES and exists_flag:
product.preview_path = absolute_path
if is_required and not exists_flag:
has_error = True
product.issues.append(
ResultIssueORM(
issue_code="MISSING_REQUIRED_ASSET",
severity="ERROR",
status="OPEN",
scope="file",
message=f"required asset missing: {relative_path}",
repair_action="rebuild_catalog",
)
)
if not product.preview_path:
has_warn = True
product.issues.append(
ResultIssueORM(
issue_code="MISSING_PREVIEW",
severity="WARN",
status="OPEN",
scope="file",
message="preview image is missing",
repair_action="rebuild_preview",
)
)
if has_error:
product.status = "QUARANTINED"
product.health_status = "ERROR"
elif has_warn:
product.status = "PARTIAL"
product.health_status = "WARN"
return product
async def _catalog_counts(self, db: AsyncSession) -> tuple[int, int]:
db_count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
issue_count_result = await db.execute(
select(func.count(ResultIssueORM.id))
.select_from(ResultIssueORM)
.join(ResultProductORM, ResultIssueORM.product_ref_id == ResultProductORM.id)
.where(ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME)
)
return (
int(db_count_result.scalar_one() or 0),
int(issue_count_result.scalar_one() or 0),
)
async def register_manifest(
self,
db: AsyncSession,
*,
manifest_path: str,
publish_root: Optional[str] = None,
) -> Dict[str, Any]:
root = self.get_publish_root(publish_root)
normalized_manifest_path = _normalize_path(manifest_path)
if not os.path.isfile(normalized_manifest_path):
raise FileNotFoundError(f"PS-InSAR manifest not found: {normalized_manifest_path}")
if not (
normalized_manifest_path == root
or normalized_manifest_path.startswith(root + os.sep)
):
raise ValueError("Manifest path is outside the configured PS-InSAR publish root.")
state = await self._get_or_create_catalog_state(db, storage_root=root)
state.status = "UPDATING"
state.last_message = f"registering manifest: {normalized_manifest_path}"
state.needs_rebuild = False
await db.flush()
manifest = await asyncio.to_thread(self._load_manifest, normalized_manifest_path)
product = self._build_rows_from_manifest(normalized_manifest_path, manifest)
existing_result = await db.execute(
select(ResultProductORM).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME,
or_(
ResultProductORM.manifest_path == product.manifest_path,
ResultProductORM.publish_dir == product.publish_dir,
ResultProductORM.run_key == product.run_key,
),
)
)
existing_products = existing_result.scalars().all()
deleted_existing = 0
for existing in existing_products:
await db.delete(existing)
deleted_existing += 1
if existing_products:
await db.flush()
db.add(product)
await db.flush()
snapshot = await asyncio.to_thread(build_manifest_snapshot, root)
db_count, issue_count = await self._catalog_counts(db)
state.manifest_count = snapshot.manifest_count
state.manifest_fingerprint = snapshot.tree_fingerprint
state.db_count = db_count
state.issue_count = issue_count
state.needs_rebuild = False
state.status = "READY" if product.health_status == "OK" else "WARN"
state.last_incremental_scan_at = _utcnow()
state.last_message = (
f"registered manifest: run_key={product.run_key}, "
f"product_id={product.product_id}, replaced={deleted_existing}"
)
await db.commit()
return {
"manifest_path": normalized_manifest_path,
"publish_dir": product.publish_dir,
"product_db_id": product.id,
"product_id": product.product_id,
"run_key": product.run_key,
"manifest_fingerprint": snapshot.tree_fingerprint,
"status": product.status,
"health_status": product.health_status,
"deleted_existing": deleted_existing,
}
async def rebuild_catalog(
self,
db: AsyncSession,
*,
publish_root: Optional[str] = None,
full_rebuild: bool = True,
) -> Dict[str, Any]:
root = self.get_publish_root(publish_root)
state = await self._get_or_create_catalog_state(db, storage_root=root)
state.status = "REBUILDING"
state.last_message = "catalog rebuild in progress"
state.needs_rebuild = False
await db.commit()
snapshot = await asyncio.to_thread(build_manifest_snapshot, root)
manifest_paths = list(snapshot.manifest_paths)
if full_rebuild:
await db.execute(
delete(ResultProductORM).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
await db.commit()
created = 0
failed = 0
issue_count = 0
details: List[Dict[str, Any]] = []
for manifest_path in manifest_paths:
try:
manifest = await asyncio.to_thread(self._load_manifest, manifest_path)
product = self._build_rows_from_manifest(manifest_path, manifest)
db.add(product)
await db.flush()
issue_count += len(product.issues)
created += 1
details.append(
{
"manifest_path": manifest_path,
"product_id": product.product_id,
"status": product.status,
}
)
except Exception as exc:
failed += 1
details.append(
{
"manifest_path": manifest_path,
"status": "error",
"message": str(exc),
}
)
await db.commit()
db_count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
db_count = int(db_count_result.scalar_one() or 0)
state.manifest_count = snapshot.manifest_count
state.manifest_fingerprint = snapshot.tree_fingerprint
state.db_count = db_count
state.issue_count = issue_count + failed
state.needs_rebuild = False
state.status = "READY" if failed == 0 else "WARN"
now = _utcnow()
state.last_full_rebuild_at = now
state.last_incremental_scan_at = now
state.last_message = (
f"catalog rebuild finished: manifests={snapshot.manifest_count}, "
f"registered={created}, failed={failed}, issues={state.issue_count}"
)
await db.commit()
return {
"publish_root": root,
"manifest_count": snapshot.manifest_count,
"manifest_fingerprint": snapshot.tree_fingerprint,
"registered": created,
"failed": failed,
"issue_count": state.issue_count,
"details": details,
}
async def list_products(
self,
db: AsyncSession,
*,
limit: int = 100,
offset: int = 0,
status: Optional[str] = None,
query: Optional[str] = None,
) -> Dict[str, Any]:
safe_limit = max(1, min(int(limit or 100), 500))
safe_offset = max(0, int(offset or 0))
stmt = select(ResultProductORM).where(ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME)
count_stmt = select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
if status:
stmt = stmt.where(ResultProductORM.status == status)
count_stmt = count_stmt.where(ResultProductORM.status == status)
if query:
like_value = f"%{query.strip()}%"
stmt = stmt.where(
ResultProductORM.display_name.ilike(like_value)
| ResultProductORM.product_id.ilike(like_value)
| ResultProductORM.run_key.ilike(like_value)
)
count_stmt = count_stmt.where(
ResultProductORM.display_name.ilike(like_value)
| ResultProductORM.product_id.ilike(like_value)
| ResultProductORM.run_key.ilike(like_value)
)
total_result = await db.execute(count_stmt)
total = int(total_result.scalar_one() or 0)
result = await db.execute(
stmt.order_by(
ResultProductORM.published_at.desc().nullslast(),
ResultProductORM.id.desc(),
)
.offset(safe_offset)
.limit(safe_limit)
)
items = result.scalars().all()
payload_items: List[Dict[str, Any]] = []
for item in items:
summary = item.summary_json or {}
payload_items.append(
{
"id": item.id,
"product_id": item.product_id,
"display_name": item.display_name,
"run_key": item.run_key,
"profile_code": item.profile_code,
"engine_code": item.engine_code,
"status": item.status,
"health_status": item.health_status,
"preview_path": item.preview_path,
"primary_asset_path": item.primary_asset_path,
"reference_date": summary.get("reference_date"),
"stack_dates": summary.get("stack_dates") or [],
"stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []),
"published_at": item.published_at,
}
)
return {
"items": payload_items,
"total": total,
"limit": safe_limit,
"offset": safe_offset,
"has_more": safe_offset + len(payload_items) < total,
}
async def get_product_detail(
self,
db: AsyncSession,
*,
product_db_id: int,
) -> Optional[Dict[str, Any]]:
result = await db.execute(
select(ResultProductORM).where(ResultProductORM.id == product_db_id)
)
product = result.scalar_one_or_none()
if product is None or product.catalog_name != PSINSAR_CATALOG_NAME:
return None
assets_result = await db.execute(
select(ResultAssetORM)
.where(ResultAssetORM.product_ref_id == product.id)
.order_by(ResultAssetORM.asset_role.asc(), ResultAssetORM.id.asc())
)
issues_result = await db.execute(
select(ResultIssueORM)
.where(ResultIssueORM.product_ref_id == product.id)
.order_by(ResultIssueORM.detected_at.desc(), ResultIssueORM.id.desc())
)
summary = product.summary_json or {}
return {
"id": product.id,
"product_id": product.product_id,
"catalog_name": product.catalog_name,
"product_type": product.product_type,
"display_name": product.display_name,
"run_key": product.run_key,
"profile_code": product.profile_code,
"engine_code": product.engine_code,
"status": product.status,
"health_status": product.health_status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"source_primary_path": product.source_primary_path,
"preview_path": product.preview_path,
"primary_asset_path": product.primary_asset_path,
"reference_date": summary.get("reference_date"),
"reference_point": summary.get("reference_point"),
"stack_dates": summary.get("stack_dates") or [],
"stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []),
"quality": summary.get("quality"),
"summaries": summary.get("summaries"),
"coverage_polygon": product.coverage_polygon,
"min_lon": product.min_lon,
"min_lat": product.min_lat,
"max_lon": product.max_lon,
"max_lat": product.max_lat,
"produced_at": product.produced_at,
"published_at": product.published_at,
"registered_at": product.registered_at,
"updated_at": product.updated_at,
"assets": [
{
"id": asset.id,
"asset_role": asset.asset_role,
"asset_name": asset.asset_name,
"relative_path": asset.relative_path,
"absolute_path": asset.absolute_path,
"format": asset.format,
"media_type": asset.media_type,
"is_required": asset.is_required,
"is_primary": asset.is_primary,
"exists_flag": asset.exists_flag,
"file_size": asset.file_size,
}
for asset in assets_result.scalars().all()
],
"issues": [
{
"id": issue.id,
"issue_code": issue.issue_code,
"severity": issue.severity,
"status": issue.status,
"scope": issue.scope,
"message": issue.message,
"detected_at": issue.detected_at,
}
for issue in issues_result.scalars().all()
],
}
async def get_catalog_status(
self,
db: AsyncSession,
*,
publish_root: Optional[str] = None,
) -> Dict[str, Any]:
root = self.get_publish_root(publish_root)
state = await self._get_or_create_catalog_state(db, storage_root=root)
db_count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
db_count = int(db_count_result.scalar_one() or 0)
payload = {
"catalog_name": state.catalog_name,
"storage_root": state.storage_root,
"status": state.status,
"needs_rebuild": state.needs_rebuild,
"manifest_count": state.manifest_count,
"manifest_fingerprint": state.manifest_fingerprint,
"db_count": db_count,
"issue_count": state.issue_count,
"last_message": state.last_message,
"last_boot_check_at": state.last_boot_check_at,
"last_full_rebuild_at": state.last_full_rebuild_at,
"last_incremental_scan_at": state.last_incremental_scan_at,
}
await db.commit()
return payload
async def bootstrap_catalog_on_startup_clean(self) -> Dict[str, Any]:
from ..database import AsyncSessionLocal
from .job_queue_service import job_queue_service
from .task_service import task_service
if AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
async with AsyncSessionLocal() as db:
root = self.get_publish_root()
state = await self._get_or_create_catalog_state(db, storage_root=root)
snapshot = await asyncio.to_thread(build_manifest_snapshot, root)
db_count_result = await db.execute(
select(func.count(ResultProductORM.id)).where(
ResultProductORM.catalog_name == PSINSAR_CATALOG_NAME
)
)
db_count = int(db_count_result.scalar_one() or 0)
reconcile = evaluate_manifest_reconcile(
manifest_count=snapshot.manifest_count,
db_count=db_count,
current_fingerprint=snapshot.tree_fingerprint,
indexed_fingerprint=state.manifest_fingerprint,
)
needs_rebuild = bool(reconcile["needs_rebuild"])
state.manifest_count = snapshot.manifest_count
state.db_count = db_count
state.needs_rebuild = needs_rebuild
state.last_boot_check_at = _utcnow()
state.status = "READY" if not needs_rebuild else "WARN"
if not needs_rebuild:
state.manifest_fingerprint = snapshot.tree_fingerprint
state.last_message = (
"boot check complete"
if not needs_rebuild
else (
f"catalog rebuild required: manifests={snapshot.manifest_count}, "
f"db={db_count}, reasons={','.join(reconcile['reasons'])}"
)
)
await db.commit()
queued = False
if needs_rebuild and settings.RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP:
task_id = await task_service.create_task(
TASK_TYPE_REBUILD_PSINSAR_CATALOG,
"PS-InSAR 结果目录重建",
params={"publish_root": root, "full_rebuild": True},
db=db,
)
await job_queue_service.create_job(
JOB_TYPE_REBUILD_PSINSAR_CATALOG,
payload={"publish_root": root, "full_rebuild": True},
task_id=task_id,
db=db,
)
queued = True
return {
"storage_root": root,
"manifest_count": snapshot.manifest_count,
"current_manifest_fingerprint": snapshot.tree_fingerprint,
"indexed_manifest_fingerprint": state.manifest_fingerprint,
"db_count": db_count,
"needs_rebuild": needs_rebuild,
"reasons": reconcile["reasons"],
"queued": queued,
}
psinsar_catalog_service = PsinsarCatalogService()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,599 @@
from __future__ import annotations
import os
import re
from dataclasses import dataclass
from typing import Any, Dict, Iterable, List, Optional
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..config import settings, split_env_paths
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
_SLUG_RE = re.compile(r"[^a-z0-9]+")
_WINDOWS_DRIVE_RE = re.compile(r"^[a-zA-Z]:[\\/]")
@dataclass(frozen=True)
class RootSpec:
root_code: str
root_role: str
display_name: str
path: str
path_kind: str
source_kind: str
source_ref: Optional[str]
scan_mode: str
owner_engine: Optional[str]
metadata_json: Dict[str, Any]
def _new_session() -> AsyncSession:
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
def _slugify(value: str, *, max_len: int = 96) -> str:
text = _SLUG_RE.sub("_", str(value or "").strip().lower()).strip("_")
return (text or "item")[:max_len]
def _normalize_root_path(path: str) -> str:
text = str(path or "").strip()
if not text:
return ""
if text.startswith("\\\\"):
return os.path.normpath(text)
if _WINDOWS_DRIVE_RE.match(text):
return os.path.normpath(text)
if text.startswith("/"):
return text.replace("\\", "/")
return os.path.normpath(os.path.abspath(text))
def _detect_path_kind(path: str) -> str:
text = str(path or "").strip()
if text.startswith("\\\\"):
return "unc"
if _WINDOWS_DRIVE_RE.match(text):
return "windows"
if text.startswith("/mnt/"):
return "wsl_mount"
if text.startswith("/"):
return "posix"
return "relative"
def _build_root_code(root_role: str, source_ref: Optional[str]) -> str:
role_part = _slugify(root_role, max_len=48)
source_part = _slugify(source_ref or "manual", max_len=40)
return f"{role_part}__{source_part}"[:96]
def _cursor_type_for_scan_mode(scan_mode: str) -> str:
mapping = {
"archive_walk": "archive_walk",
"scene_directory": "directory_walk",
"manifest_tree": "manifest_tree",
"file_pool": "file_pool",
"workspace": "workspace",
"quarantine": "directory_walk",
"directory_walk": "directory_walk",
}
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
def _iter_multi_root_specs(
*,
env_var: str,
paths: Iterable[str],
root_role: str,
display_prefix: str,
scan_mode: str,
owner_engine: Optional[str] = None,
) -> List[RootSpec]:
specs: List[RootSpec] = []
deduped: List[str] = []
for raw_path in paths:
normalized_path = _normalize_root_path(raw_path)
if normalized_path and normalized_path not in deduped:
deduped.append(normalized_path)
for idx, path in enumerate(deduped, start=1):
source_ref = f"{env_var}[{idx}]"
specs.append(
RootSpec(
root_code=_build_root_code(root_role, source_ref),
root_role=root_role,
display_name=f"{display_prefix} {idx}",
path=path,
path_kind=_detect_path_kind(path),
source_kind="env",
source_ref=source_ref,
scan_mode=scan_mode,
owner_engine=owner_engine,
metadata_json={
"env_var": env_var,
"env_index": idx,
"imported_from": "settings",
},
)
)
return specs
def _iter_single_root_specs(
*,
env_var: str,
path: Optional[str],
root_role: str,
display_name: str,
scan_mode: str,
owner_engine: Optional[str] = None,
) -> List[RootSpec]:
normalized_path = _normalize_root_path(path or "")
if not normalized_path:
return []
source_ref = env_var
return [
RootSpec(
root_code=_build_root_code(root_role, source_ref),
root_role=root_role,
display_name=display_name,
path=normalized_path,
path_kind=_detect_path_kind(normalized_path),
source_kind="env",
source_ref=source_ref,
scan_mode=scan_mode,
owner_engine=owner_engine,
metadata_json={
"env_var": env_var,
"imported_from": "settings",
},
)
]
def _build_root_specs_from_settings() -> List[RootSpec]:
specs: List[RootSpec] = []
specs.extend(
_iter_multi_root_specs(
env_var="UNPACK_SOURCE_DIRS",
paths=split_env_paths(settings.UNPACK_SOURCE_DIRS),
root_role="archive_inbox",
display_prefix="Archive Inbox",
scan_mode="archive_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="INSAR_STORAGE_DIRS",
paths=split_env_paths(settings.INSAR_STORAGE_DIRS),
root_role="source_pool_radar",
display_prefix="Radar Source Pool",
scan_mode="scene_directory",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="MONITOR_RADAR_DIRS",
paths=split_env_paths(settings.MONITOR_RADAR_DIRS),
root_role="legacy_scan_root_radar",
display_prefix="Legacy Radar Scan Root",
scan_mode="directory_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="MONITOR_DINSAR_DIRS",
paths=split_env_paths(settings.MONITOR_DINSAR_DIRS),
root_role="legacy_scan_root_dinsar",
display_prefix="Legacy D-InSAR Scan Root",
scan_mode="directory_walk",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_SOURCE_DIRS",
paths=split_env_paths(settings.GF3_SOURCE_DIRS),
root_role="source_pool_gf3_input",
display_prefix="GF3 Input Pool",
scan_mode="scene_directory",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="GF3_STORAGE_DIRS",
paths=split_env_paths(settings.GF3_STORAGE_DIRS),
root_role="source_pool_gf3_output",
display_prefix="GF3 Output Pool",
scan_mode="scene_directory",
)
)
specs.extend(
_iter_single_root_specs(
env_var="MONITOR_ORBIT_DIR",
path=settings.MONITOR_ORBIT_DIR,
root_role="orbit_source",
display_name="Orbit Source",
scan_mode="file_pool",
)
)
specs.extend(
_iter_single_root_specs(
env_var="ORBIT_POOL_ENVI",
path=settings.ORBIT_POOL_ENVI,
root_role="orbit_pool_envi",
display_name="ENVI Orbit Pool",
scan_mode="file_pool",
owner_engine="envi",
)
)
specs.extend(
_iter_single_root_specs(
env_var="ORBIT_POOL_ISCE2",
path=settings.ORBIT_POOL_ISCE2,
root_role="orbit_pool_isce2",
display_name="ISCE2 Orbit Pool",
scan_mode="file_pool",
owner_engine="isce2",
)
)
specs.extend(
_iter_single_root_specs(
env_var="ORBIT_POOL_LANDSAR",
path=settings.ORBIT_POOL_LANDSAR,
root_role="orbit_pool_landsar",
display_name="LandSAR Orbit Pool",
scan_mode="file_pool",
owner_engine="landsar",
)
)
specs.extend(
_iter_single_root_specs(
env_var="RESULT_PUBLISH_ROOT",
path=settings.RESULT_PUBLISH_ROOT,
root_role="result_publish_root",
display_name="Result Publish Root",
scan_mode="manifest_tree",
)
)
specs.extend(
_iter_single_root_specs(
env_var="DINSAR_PRODUCT_DIR",
path=settings.DINSAR_PRODUCT_DIR,
root_role="publish_root_dinsar",
display_name="D-InSAR Publish Root",
scan_mode="manifest_tree",
owner_engine="dinsar",
)
)
specs.extend(
_iter_single_root_specs(
env_var="PSINSAR_PRODUCT_DIR",
path=settings.PSINSAR_PRODUCT_DIR,
root_role="publish_root_psinsar",
display_name="PS-InSAR Publish Root",
scan_mode="manifest_tree",
owner_engine="psinsar",
)
)
specs.extend(
_iter_single_root_specs(
env_var="RESULT_QUARANTINE_ROOT",
path=settings.RESULT_QUARANTINE_ROOT,
root_role="quarantine_root",
display_name="Result Quarantine Root",
scan_mode="quarantine",
)
)
specs.extend(
_iter_single_root_specs(
env_var="ISCE2_WORK_ROOT",
path=settings.ISCE2_WORK_ROOT,
root_role="work_root_isce2",
display_name="ISCE2 Work Root",
scan_mode="workspace",
owner_engine="isce2",
)
)
specs.extend(
_iter_single_root_specs(
env_var="TIMESERIES_WORK_ROOT",
path=settings.TIMESERIES_WORK_ROOT,
root_role="work_root_timeseries",
display_name="Timeseries Work Root",
scan_mode="workspace",
owner_engine="timeseries",
)
)
specs.extend(
_iter_single_root_specs(
env_var="IDL_WORKER_RUNTIME_DIR",
path=settings.IDL_WORKER_RUNTIME_DIR,
root_role="work_root_idl",
display_name="IDL Runtime Root",
scan_mode="workspace",
owner_engine="idl",
)
)
return specs
class RootRegistryService:
async def _ensure_default_cursor(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
cursor_result = await db.execute(
select(ScanCursorORM).where(
ScanCursorORM.root_ref_id == root.id,
ScanCursorORM.cursor_key == "default",
)
)
cursor = cursor_result.scalar_one_or_none()
desired_cursor_type = _cursor_type_for_scan_mode(root.scan_mode)
if cursor is None:
cursor = ScanCursorORM(
root_ref_id=root.id,
cursor_key="default",
cursor_type=desired_cursor_type,
scan_scope="root",
status="IDLE",
)
db.add(cursor)
return "created"
changed = False
if cursor.cursor_type != desired_cursor_type:
cursor.cursor_type = desired_cursor_type
changed = True
if cursor.scan_scope != "root":
cursor.scan_scope = "root"
changed = True
return "updated" if changed else None
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
generated_session = db is None
if generated_session:
db = _new_session()
assert db is not None
try:
specs = _build_root_specs_from_settings()
existing_result = await db.execute(
select(ManagedRootORM).where(ManagedRootORM.source_kind == "env")
)
existing_rows = existing_result.scalars().all()
existing_by_code = {row.root_code: row for row in existing_rows}
created = 0
updated = 0
disabled = 0
cursor_created = 0
cursor_updated = 0
synced_codes: set[str] = set()
for spec in specs:
synced_codes.add(spec.root_code)
exists_flag = os.path.exists(spec.path)
row = existing_by_code.get(spec.root_code)
if row is None:
row = ManagedRootORM(
root_code=spec.root_code,
root_role=spec.root_role,
display_name=spec.display_name,
path=spec.path,
path_kind=spec.path_kind,
source_kind=spec.source_kind,
source_ref=spec.source_ref,
scan_mode=spec.scan_mode,
owner_engine=spec.owner_engine,
enabled=True,
exists_flag=exists_flag,
metadata_json=spec.metadata_json,
)
db.add(row)
await db.flush()
created += 1
else:
changed = False
for field_name, value in (
("root_role", spec.root_role),
("display_name", spec.display_name),
("path", spec.path),
("path_kind", spec.path_kind),
("source_kind", spec.source_kind),
("source_ref", spec.source_ref),
("scan_mode", spec.scan_mode),
("owner_engine", spec.owner_engine),
("exists_flag", exists_flag),
("metadata_json", spec.metadata_json),
):
if getattr(row, field_name) != value:
setattr(row, field_name, value)
changed = True
if not row.enabled:
row.enabled = True
changed = True
if changed:
updated += 1
cursor_change = await self._ensure_default_cursor(db, row)
if cursor_change == "created":
cursor_created += 1
elif cursor_change == "updated":
cursor_updated += 1
for row in existing_rows:
if row.root_code in synced_codes:
continue
if row.enabled:
row.enabled = False
disabled += 1
await db.commit()
summary = await self.get_summary(db)
return {
"synced": len(specs),
"created": created,
"updated": updated,
"disabled": disabled,
"cursor_created_or_updated": cursor_created + cursor_updated,
"summary": summary,
}
except Exception:
await db.rollback()
raise
finally:
if generated_session:
await db.close()
async def list_roots(self, db: AsyncSession, *, include_disabled: bool = False) -> List[Dict[str, Any]]:
stmt = select(ManagedRootORM).order_by(
ManagedRootORM.root_role.asc(),
ManagedRootORM.display_name.asc(),
ManagedRootORM.id.asc(),
)
if not include_disabled:
stmt = stmt.where(ManagedRootORM.enabled == True) # noqa: E712
root_result = await db.execute(stmt)
roots = root_result.scalars().all()
root_ids = [root.id for root in roots]
cursor_by_root_id: Dict[int, ScanCursorORM] = {}
inventory_counts: Dict[int, int] = {}
if root_ids:
cursor_result = await db.execute(
select(ScanCursorORM)
.where(ScanCursorORM.root_ref_id.in_(root_ids))
.where(ScanCursorORM.cursor_key == "default")
)
cursor_by_root_id = {
item.root_ref_id: item
for item in cursor_result.scalars().all()
}
inventory_result = await db.execute(
select(PathInventoryORM.root_ref_id, func.count(PathInventoryORM.id))
.where(PathInventoryORM.root_ref_id.in_(root_ids))
.where(PathInventoryORM.status != "REMOVED")
.group_by(PathInventoryORM.root_ref_id)
)
inventory_counts = {
int(root_ref_id): int(count or 0)
for root_ref_id, count in inventory_result.all()
}
payload: List[Dict[str, Any]] = []
for root in roots:
cursor = cursor_by_root_id.get(root.id)
payload.append(
{
"id": root.id,
"root_code": root.root_code,
"root_role": root.root_role,
"display_name": root.display_name,
"path": root.path,
"path_kind": root.path_kind,
"source_kind": root.source_kind,
"source_ref": root.source_ref,
"scan_mode": root.scan_mode,
"owner_engine": root.owner_engine,
"enabled": bool(root.enabled),
"exists_flag": bool(root.exists_flag),
"metadata_json": root.metadata_json,
"created_at": root.created_at,
"updated_at": root.updated_at,
"inventory_count": inventory_counts.get(root.id, 0),
"cursor": (
{
"id": cursor.id,
"cursor_key": cursor.cursor_key,
"cursor_type": cursor.cursor_type,
"scan_scope": cursor.scan_scope,
"status": cursor.status,
"last_scan_started_at": cursor.last_scan_started_at,
"last_scan_finished_at": cursor.last_scan_finished_at,
"last_seen_mtime": cursor.last_seen_mtime,
"last_seen_entry_count": cursor.last_seen_entry_count,
"last_seen_fingerprint": cursor.last_seen_fingerprint,
"last_error": cursor.last_error,
"updated_at": cursor.updated_at,
}
if cursor is not None
else None
),
}
)
return payload
async def get_summary(self, db: AsyncSession) -> Dict[str, Any]:
total_roots = int(
(await db.execute(select(func.count(ManagedRootORM.id)))).scalar_one() or 0
)
enabled_roots = int(
(
await db.execute(
select(func.count(ManagedRootORM.id)).where(ManagedRootORM.enabled == True) # noqa: E712
)
).scalar_one()
or 0
)
existing_roots = int(
(
await db.execute(
select(func.count(ManagedRootORM.id)).where(ManagedRootORM.exists_flag == True) # noqa: E712
)
).scalar_one()
or 0
)
total_cursors = int(
(await db.execute(select(func.count(ScanCursorORM.id)))).scalar_one() or 0
)
total_inventory = int(
(
await db.execute(
select(func.count(PathInventoryORM.id)).where(PathInventoryORM.status != "REMOVED")
)
).scalar_one()
or 0
)
role_rows = await db.execute(
select(
ManagedRootORM.root_role,
func.count(ManagedRootORM.id),
)
.group_by(ManagedRootORM.root_role)
.order_by(ManagedRootORM.root_role.asc())
)
role_counts = {
str(role): int(count or 0)
for role, count in role_rows.all()
}
return {
"total_roots": total_roots,
"enabled_roots": enabled_roots,
"existing_roots": existing_roots,
"missing_roots": max(0, enabled_roots - existing_roots),
"total_cursors": total_cursors,
"total_inventory_items": total_inventory,
"role_counts": role_counts,
}
async def get_status(self, db: AsyncSession, *, include_disabled: bool = False) -> Dict[str, Any]:
roots = await self.list_roots(db, include_disabled=include_disabled)
summary = await self.get_summary(db)
return {
"summary": summary,
"roots": roots,
}
root_registry_service = RootRegistryService()
File diff suppressed because it is too large Load Diff
+370
View File
@@ -0,0 +1,370 @@
import uuid
import asyncio
import hashlib
from datetime import datetime, timedelta
from typing import List, Optional, Dict, Any
import json
import sys
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import func, and_, text
from ..config import read_int_env
from ..models import SystemTaskORM, TaskLogORM
from .. import database
# --- Configuration ---
# 任务超时时间 (分钟)。如果任务超过此时间未更新,视为僵尸任务。
# ENVI 任务单步可能超过 60 分钟,默认设为 120 分钟,可通过环境变量覆盖。
TASK_TIMEOUT_MINUTES = read_int_env("TASK_TIMEOUT_MINUTES", 120, minimum=10, maximum=1440)
TASK_ACTIVE_DEFAULT_LIMIT = read_int_env(
"TASK_ACTIVE_DEFAULT_LIMIT",
100,
minimum=1,
maximum=2000,
)
TASK_ACTIVE_MAX_LIMIT = read_int_env(
"TASK_ACTIVE_MAX_LIMIT",
500,
minimum=1,
maximum=10000,
)
TASK_LOG_DEFAULT_LIMIT = read_int_env(
"TASK_LOG_DEFAULT_LIMIT",
100,
minimum=1,
maximum=5000,
)
TASK_LOG_MAX_LIMIT = read_int_env(
"TASK_LOG_MAX_LIMIT",
1000,
minimum=1,
maximum=20000,
)
TASK_QUERY_MAX_OFFSET = read_int_env(
"TASK_QUERY_MAX_OFFSET",
500000,
minimum=0,
maximum=20000000,
)
def _clamp_pagination(limit: int, offset: int, *, default_limit: int, max_limit: int) -> tuple[int, int]:
try:
parsed_limit = int(limit)
except (TypeError, ValueError):
parsed_limit = default_limit
try:
parsed_offset = int(offset)
except (TypeError, ValueError):
parsed_offset = 0
safe_limit = min(max_limit, max(1, parsed_limit))
safe_offset = min(TASK_QUERY_MAX_OFFSET, max(0, parsed_offset))
return safe_limit, safe_offset
def _task_type_lock_key(task_type: str) -> int:
normalized = (task_type or "").strip().lower().encode("utf-8")
digest = hashlib.sha256(normalized).digest()
return int.from_bytes(digest[:8], byteorder="big", signed=True)
def _is_postgresql_session(db: AsyncSession) -> bool:
try:
bind = db.get_bind()
dialect_name = getattr(getattr(bind, "dialect", None), "name", "")
return (dialect_name or "").lower() == "postgresql"
except Exception as exc:
print(f"[WARN] _is_postgresql: {exc}")
return False
def get_db_session():
"""
获取数据库会话的辅助函数。
如果数据库尚未初始化,则尝试初始化。
"""
if database.AsyncSessionLocal is None:
print(">>> [TaskService] 检测到 AsyncSessionLocal 未初始化,正在尝试重新初始化数据库...")
try:
database.init_db()
print(">>> [TaskService] 数据库初始化成功。")
except Exception as e:
print(f">>> [TaskService] 数据库初始化失败: {e}")
raise
if database.AsyncSessionLocal is None:
raise RuntimeError("无法获取数据库会话,请检查数据库连接配置。")
return database.AsyncSessionLocal()
class TaskService:
"""
持久化任务服务。
负责管理任务的生命周期、日志记录和状态持久化。
"""
def __init__(self):
pass
async def create_task(
self,
task_type: str,
task_name: str,
params: Optional[Dict[str, Any]] = None,
db: Optional[AsyncSession] = None
) -> str:
"""
创建新任务。
实现逻辑:检查是否存在同类型且运行中的任务。如果存在,则抛出异常。
"""
gen_db = db is None
if gen_db:
db = get_db_session()
try:
# 1. 并发冲突检查:是否已有同类型的任务在运行?
if _is_postgresql_session(db):
lock_key = _task_type_lock_key(task_type)
await db.execute(
text("SELECT pg_advisory_xact_lock(:lock_key)"),
{"lock_key": lock_key},
)
status_check = await db.execute(
select(SystemTaskORM)
.where(
and_(
SystemTaskORM.task_type == task_type,
SystemTaskORM.status.in_(["PENDING", "RUNNING"])
)
)
)
existing_task = status_check.scalar_one_or_none()
if existing_task:
raise ValueError(f"任务冲突: 已存在一个正在运行的 {task_type} 任务 (ID: {existing_task.task_id})。请等待该任务完成。")
# 2. 创建新任务记录
task_id = str(uuid.uuid4())
new_task = SystemTaskORM(
task_id=task_id,
task_type=task_type,
task_name=task_name,
status="PENDING",
params=params
)
db.add(new_task)
# 3. 记录创建日志
log_entry = TaskLogORM(
task_id=task_id,
log_level="INFO",
message=f"任务已创建: {task_name} ({task_type})"
)
db.add(log_entry)
await db.commit()
return task_id
except Exception as e:
await db.rollback()
raise e
finally:
if gen_db:
await db.close()
async def start_task(self, task_id: str, message: str = "任务开始执行", db: Optional[AsyncSession] = None):
"""将任务状态更新为 RUNNING"""
gen_db = db is None
if gen_db:
db = get_db_session()
try:
result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id == task_id)
)
task = result.scalar_one_or_none()
if task:
task.status = "RUNNING"
task.message = message
task.started_at = datetime.now()
await self._add_log(task_id, "INFO", "任务已开始", db=db)
await db.commit()
else:
raise ValueError(f"任务未找到: {task_id}")
finally:
if gen_db:
await db.close()
async def update_task(
self,
task_id: str,
status: Optional[str] = None,
progress: Optional[int] = None,
message: Optional[str] = None,
db: Optional[AsyncSession] = None
):
"""
更新任务状态。
如果状态变为 COMPLETED 或 FAILED,标记结束时间。
"""
gen_db = db is None
if gen_db:
db = get_db_session()
try:
result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id == task_id)
)
task = result.scalar_one_or_none()
if task:
if status:
task.status = status
if progress is not None:
task.progress = progress
if message:
task.message = message
# 如果任务结束,更新结束时间
if status in ["COMPLETED", "FAILED", "CANCELLED"]:
task.ended_at = datetime.now()
await self._add_log(task_id, "INFO", f"任务已结束: {status}", db=db)
else:
# 显式更新心跳时间,防止 SQLAlchemy 因属性未变而跳过 UPDATE
task.updated_at = datetime.now()
await db.commit()
else:
print(f"警告: 尝试更新一个不存在的任务 {task_id}")
finally:
if gen_db:
await db.close()
async def get_active_tasks(
self,
limit: int = TASK_ACTIVE_DEFAULT_LIMIT,
offset: int = 0,
db: Optional[AsyncSession] = None,
) -> List[SystemTaskORM]:
"""
获取所有非结束状态的任务。
注意:这里应该加上超时检测逻辑,如果 updated_at 太旧,应该标记为 FAILED。
"""
gen_db = db is None
if gen_db:
db = get_db_session()
try:
safe_limit, safe_offset = _clamp_pagination(
limit,
offset,
default_limit=TASK_ACTIVE_DEFAULT_LIMIT,
max_limit=TASK_ACTIVE_MAX_LIMIT,
)
# 1. 查找僵尸任务并标记为失败
timeout_threshold = datetime.now() - timedelta(minutes=TASK_TIMEOUT_MINUTES)
result = await db.execute(
select(SystemTaskORM).where(
and_(
SystemTaskORM.status == "RUNNING",
SystemTaskORM.updated_at < timeout_threshold
)
)
)
zombie_tasks = result.scalars().all()
for task in zombie_tasks:
task.status = "FAILED"
task.message = "系统检测超时: 任务被认为已失效 (心跳超时)"
log = TaskLogORM(task_id=task.task_id, log_level="WARNING", message="任务因超时被自动标记为失败")
db.add(log)
if zombie_tasks:
await db.commit()
# 2. 获取活跃任务
active_result = await db.execute(
select(SystemTaskORM)
.where(SystemTaskORM.status.in_(["PENDING", "RUNNING"]))
.order_by(SystemTaskORM.created_at.desc())
.offset(safe_offset)
.limit(safe_limit)
)
return active_result.scalars().all()
finally:
if gen_db:
await db.close()
async def get_task(self, task_id: str, db: Optional[AsyncSession] = None) -> Optional[SystemTaskORM]:
gen_db = db is None
if gen_db:
db = get_db_session()
try:
result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id == task_id)
)
return result.scalar_one_or_none()
finally:
if gen_db:
await db.close()
async def get_logs(
self,
task_id: str,
limit: int = TASK_LOG_DEFAULT_LIMIT,
offset: int = 0,
db: Optional[AsyncSession] = None,
) -> List[TaskLogORM]:
"""获取任务日志"""
gen_db = db is None
if gen_db:
db = get_db_session()
try:
safe_limit, safe_offset = _clamp_pagination(
limit,
offset,
default_limit=TASK_LOG_DEFAULT_LIMIT,
max_limit=TASK_LOG_MAX_LIMIT,
)
result = await db.execute(
select(TaskLogORM)
.where(TaskLogORM.task_id == task_id)
.order_by(TaskLogORM.timestamp.desc())
.offset(safe_offset)
.limit(safe_limit)
)
# 返回正序列表以便阅读
logs = result.scalars().all()
return list(reversed(logs))
finally:
if gen_db:
await db.close()
async def _add_log(self, task_id: str, level: str, message: str, db: AsyncSession):
"""内部方法:添加日志"""
log_entry = TaskLogORM(task_id=task_id, log_level=level, message=message)
db.add(log_entry)
# 不在这里 commit,由上层控制事务
async def add_log(self, task_id: str, level: str, message: str):
"""公开方法:添加日志 (带独立事务)"""
db = get_db_session()
try:
await self._add_log(task_id, level, message, db=db)
await db.commit()
finally:
await db.close()
# 创建全局单例实例
task_service = TaskService()
File diff suppressed because it is too large Load Diff
+89
View File
@@ -0,0 +1,89 @@
import asyncio
import importlib.util
import os
from typing import Any, Dict, Optional
from .task_service import task_service
from ..config import settings
_UNPACK_MODULE = None
def _load_unpack_module():
global _UNPACK_MODULE
if _UNPACK_MODULE is not None:
return _UNPACK_MODULE
script_path = os.path.join(settings.PROJECT_ROOT, "scripts", "unpack_archives.py")
spec = importlib.util.spec_from_file_location("unpack_archives", script_path)
if spec is None or spec.loader is None:
raise RuntimeError("Failed to load unpack_archives.py module.")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
_UNPACK_MODULE = module
return module
def get_unpack_config() -> Dict[str, Any]:
module = _load_unpack_module()
env = module.load_env(module.ENV_PATH)
source_dirs = module.parse_dirs(env.get("UNPACK_SOURCE_DIRS"))
insar_storage_dirs = module.parse_dirs(
env.get("INSAR_STORAGE_DIRS")
or env.get("UNPACK_TARGET_DIRS")
or env.get("UNPACK_STORAGE_DIRS")
)
archive_exts = module.parse_dirs(env.get("UNPACK_ARCHIVE_EXTS", ".tar.gz"))
return {
"source_dirs": source_dirs,
"insar_storage_dirs": insar_storage_dirs,
"min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")),
"delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")),
"tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"),
"archive_exts": archive_exts,
}
async def run_unpack_task(task_id: str):
module = _load_unpack_module()
loop = asyncio.get_running_loop()
def _submit(coro):
try:
future = asyncio.run_coroutine_threadsafe(coro, loop)
except RuntimeError:
return
def _swallow_errors(fut):
try:
fut.result()
except Exception as exc:
print(f"[WARN] unpack callback: {exc}")
future.add_done_callback(_swallow_errors)
try:
await task_service.start_task(task_id, message="Archive unpack started")
def log_cb(level: str, message: str):
_submit(task_service.add_log(task_id, level, message))
def progress_cb(progress: int, message: str):
_submit(task_service.update_task(task_id, progress=progress, message=message))
result: Optional[Dict[str, Any]] = await asyncio.to_thread(
module.run_unpack_job,
log_callback=log_cb,
progress_callback=progress_cb,
)
if not result:
result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0}
summary = (
"Unpack complete: processed {processed}, failed {failed}, skipped {skipped}"
).format(**result)
await task_service.update_task(task_id, status="COMPLETED", progress=100, message=summary)
except Exception as exc:
await task_service.update_task(task_id, status="FAILED", message=f"Unpack failed: {exc}")
@@ -0,0 +1,268 @@
"""
Water body detection service — Otsu adaptive threshold + DEM/slope constraints
+ morphological filtering + connected component analysis.
Python reimplementation of MATLAB WaterDetectProcess.m.
"""
from __future__ import annotations
import logging
import math
import os
import struct
from typing import Any, Dict, Optional, Tuple
import numpy as np
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# SRTM HGT helpers
# ---------------------------------------------------------------------------
def _read_hgt(filepath: str) -> np.ndarray:
"""Read a single SRTM .hgt file. Auto-detect SRTM1 (3601) vs SRTM3 (1201)."""
file_size = os.path.getsize(filepath)
if file_size == 3601 * 3601 * 2:
size = 3601 # SRTM1
elif file_size == 1201 * 1201 * 2:
size = 1201 # SRTM3
else:
raise ValueError(f"Unexpected HGT file size: {file_size} bytes ({filepath})")
with open(filepath, "rb") as f:
raw = f.read()
data = np.frombuffer(raw, dtype=">i2").reshape((size, size)).astype(np.float32)
# SRTM void value
data[data == -32768] = np.nan
return data
def _load_srtm3_dem(
bounds: Tuple[float, float, float, float],
dem_dir: str,
) -> Tuple[Optional[np.ndarray], Optional[Tuple[float, float, float, float]]]:
"""Load and mosaic SRTM HGT tiles covering *bounds* (min_lon, min_lat, max_lon, max_lat).
Returns (dem_array, (dem_min_lon, dem_min_lat, dem_max_lon, dem_max_lat)) or (None, None).
"""
min_lon, min_lat, max_lon, max_lat = bounds
lat_start = int(math.floor(min_lat))
lat_end = int(math.floor(max_lat))
lon_start = int(math.floor(min_lon))
lon_end = int(math.floor(max_lon))
tiles: Dict[Tuple[int, int], np.ndarray] = {}
tile_size = None
for lat in range(lat_start, lat_end + 1):
for lon in range(lon_start, lon_end + 1):
ns = "N" if lat >= 0 else "S"
ew = "E" if lon >= 0 else "W"
fname = f"{ns}{abs(lat):02d}{ew}{abs(lon):03d}.hgt"
fpath = os.path.join(dem_dir, fname)
if not os.path.isfile(fpath):
logger.warning("SRTM tile not found: %s", fpath)
continue
tile = _read_hgt(fpath)
tiles[(lat, lon)] = tile
tile_size = tile.shape[0]
if not tiles or tile_size is None:
logger.warning("No SRTM tiles found for bounds %s in %s", bounds, dem_dir)
return None, None
n_lats = lat_end - lat_start + 1
n_lons = lon_end - lon_start + 1
# Each tile is (tile_size x tile_size), tiles overlap by 1 pixel on edges
effective = tile_size - 1
mosaic_h = n_lats * effective + 1
mosaic_w = n_lons * effective + 1
mosaic = np.full((mosaic_h, mosaic_w), np.nan, dtype=np.float32)
for (lat, lon), tile in tiles.items():
row_offset = (lat_end - lat) * effective # top = highest lat
col_offset = (lon - lon_start) * effective
mosaic[row_offset: row_offset + tile_size, col_offset: col_offset + tile_size] = tile
dem_bounds = (
float(lon_start),
float(lat_start),
float(lon_end + 1),
float(lat_end + 1),
)
return mosaic, dem_bounds
# ---------------------------------------------------------------------------
# Core detection
# ---------------------------------------------------------------------------
def _compute_water_area_km2(mask: np.ndarray, pixel_size_x: float, pixel_size_y: float) -> float:
"""Compute water area in km^2 from boolean mask and pixel sizes in degrees."""
water_count = int(np.count_nonzero(mask))
# Approximate at mid-latitude
lat_km = abs(pixel_size_y) * 111.32
lon_km = abs(pixel_size_x) * 111.32 # rough approximation
return water_count * lat_km * lon_km
def run_water_detection(
geo_tiff_path: str,
output_dir: str,
job_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Run water body detection on a GeoTIFF.
Returns dict with keys: ok, output_path, water_area_km2, water_pixel_count, otsu_threshold_db
"""
import rasterio
from scipy.ndimage import median_filter, gaussian_filter, label, zoom
from skimage.filters import threshold_otsu
from skimage.morphology import disk, binary_dilation, binary_erosion
from skimage.measure import regionprops
from ..config import settings
dem_dir = settings.SRTM_DEM_DIR
os.makedirs(output_dir, exist_ok=True)
logger.info("[WaterDetect] Reading input: %s", geo_tiff_path)
# Step 1: Read SAR GeoTIFF
with rasterio.open(geo_tiff_path) as src:
img = src.read(1).astype(np.float32)
transform = src.transform
crs = src.crs
height, width = img.shape
pixel_size_x = transform.a # degrees per pixel (x)
pixel_size_y = transform.e # degrees per pixel (y, negative)
# Step 2: Compute lon/lat bounds
min_lon = transform.c
max_lon = transform.c + width * pixel_size_x
max_lat = transform.f
min_lat = transform.f + height * pixel_size_y # pixel_size_y is negative
if min_lat > max_lat:
min_lat, max_lat = max_lat, min_lat
if min_lon > max_lon:
min_lon, max_lon = max_lon, min_lon
bounds = (min_lon, min_lat, max_lon, max_lat)
logger.info("[WaterDetect] Image bounds: %s, size: %dx%d", bounds, width, height)
# Step 5: Valid mask
valid = np.isfinite(img) & (img != 0)
if np.count_nonzero(valid) < 100:
return {"ok": False, "error": "Too few valid pixels in input image"}
# Step 6: Otsu threshold on valid pixels
valid_pixels = img[valid]
thresh = threshold_otsu(valid_pixels)
logger.info("[WaterDetect] Otsu threshold: %.4f", thresh)
# Step 7-8: Median + Gaussian filtering
filtered = median_filter(img, size=3)
filtered = gaussian_filter(filtered, sigma=1.0)
# Step 9: Initial water mask
water = (filtered < thresh) & valid
# Step 3-4: Load and resample DEM (if available)
dem_applied = False
if dem_dir and os.path.isdir(dem_dir):
dem, dem_bounds = _load_srtm3_dem(bounds, dem_dir)
if dem is not None and dem_bounds is not None:
# Resample DEM to image resolution
zoom_y = height / dem.shape[0]
zoom_x = width / dem.shape[1]
dem_resampled = zoom(dem, (zoom_y, zoom_x), order=1)
# Clip to match image shape exactly
dem_resampled = dem_resampled[:height, :width]
# Step 10: DEM height constraint (0m <= DEM <= 1000m)
dem_valid = np.isfinite(dem_resampled)
height_mask = dem_valid & (dem_resampled >= 0) & (dem_resampled <= 1000)
water = water & height_mask
# Step 11: Slope constraint — exclude slope > tan(60 deg)
slope_threshold = math.tan(math.radians(60))
dy, dx = np.gradient(dem_resampled)
# Convert gradient from pixels to approximate meters
m_per_pixel_y = abs(pixel_size_y) * 111320
m_per_pixel_x = abs(pixel_size_x) * 111320 * math.cos(math.radians((min_lat + max_lat) / 2))
slope_y = dy / max(m_per_pixel_y, 1)
slope_x = dx / max(m_per_pixel_x, 1)
slope = np.sqrt(slope_y ** 2 + slope_x ** 2)
gentle_slope = slope < slope_threshold
water = water & gentle_slope
dem_applied = True
logger.info("[WaterDetect] DEM constraints applied")
else:
logger.warning("[WaterDetect] DEM not available, skipping DEM constraints")
else:
logger.warning("[WaterDetect] SRTM_DEM_DIR not configured, skipping DEM constraints")
# Step 12: Morphological processing — disk(5) dilate→erode→dilate→erode
selem = disk(5)
water = binary_dilation(water, selem)
water = binary_erosion(water, selem)
water = binary_dilation(water, selem)
water = binary_erosion(water, selem)
# Step 13: Connected component filtering
labeled, num_features = label(water)
if num_features > 0:
# Compute min_area: max(3000m² / pixel_area_m², median_area)
pixel_area_m2 = abs(pixel_size_x) * 111320 * abs(pixel_size_y) * 111320
min_pixels_by_area = max(1, int(3000 / max(pixel_area_m2, 1)))
props = regionprops(labeled)
areas = [p.area for p in props]
if areas:
median_area = float(np.median(areas))
min_area = max(min_pixels_by_area, int(median_area))
else:
min_area = min_pixels_by_area
# Remove small components
for prop in props:
if prop.area < min_area:
water[labeled == prop.label] = False
logger.info("[WaterDetect] Connected component filter: min_area=%d pixels, kept %d/%d components",
min_area, np.count_nonzero(np.unique(labeled[water])), num_features)
# Step 14: Output binary mask GeoTIFF (0/255)
output_path = os.path.join(output_dir, "water_mask.tif")
mask_uint8 = np.where(water, 255, 0).astype(np.uint8)
with rasterio.open(
output_path,
"w",
driver="GTiff",
height=height,
width=width,
count=1,
dtype="uint8",
crs=crs,
transform=transform,
compress="deflate",
) as dst:
dst.write(mask_uint8, 1)
water_pixel_count = int(np.count_nonzero(water))
water_area = _compute_water_area_km2(water, pixel_size_x, pixel_size_y)
logger.info("[WaterDetect] Done: water_pixels=%d, area=%.3f km², output=%s",
water_pixel_count, water_area, output_path)
return {
"ok": True,
"output_path": output_path,
"water_area_km2": round(water_area, 4),
"water_pixel_count": water_pixel_count,
"otsu_threshold_db": round(float(thresh), 4),
}
+385
View File
@@ -0,0 +1,385 @@
"""Water body monitoring service v2 — SARscape-based pipeline.
Single-scene preprocessing:
SARsBasicMultilooking → multi-look intensity image
SARsBasicGeocoding → geocoded + calibrated dB image
Flood detection (two-scene pair):
SARsBasicFeFloodingClassification → flood classification map
SARsBasicFeFloodingClassificationRefinement → MRF refinement (optional)
"""
from __future__ import annotations
import os
import time
from glob import glob
from typing import Any, Dict, Optional
from .envi_service import (
DEM_BASE_FILE,
CUSTOM_GEOCODING_PIXEL_SIZE_M,
CUSTOM_TARGET_RESOLUTION_M,
RUNTIME_DIR,
_build_sarscapedata,
_normalize_path,
_to_local_path,
_unwrap_sarscapedata,
_write_progress,
execute_envi_task,
)
# ---------------------------------------------------------------------------
# Output directory
# ---------------------------------------------------------------------------
from ..config import settings
WATER_RESULTS_DIR: str = settings.WATER_RESULTS_DIR
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _find_slc_base(data_dir: str) -> Optional[str]:
"""Find the SLC ENVI file base path (without extension) in data_dir.
Looks for files matching *_slc (no extension, ENVI format with .hdr/.sml).
Returns the base path (without extension) or None.
"""
data_dir = _to_local_path(data_dir)
# SARscape SLC files have no extension but have a .hdr and .sml companion
for fname in os.listdir(data_dir):
if fname.endswith("_slc") and os.path.isfile(os.path.join(data_dir, fname + ".hdr")):
return os.path.join(data_dir, fname)
# Fallback: look for .sml files whose base ends with _slc
smls = glob(os.path.join(data_dir, "*_slc.sml"))
if smls:
return smls[0][:-4] # strip .sml
return None
def _find_geo_db_output(output_dir: str) -> Optional[str]:
"""Find the geocoded dB output file base path produced by SARsBasicGeocoding.
SARscape names the output with a _geo_db or _geo suffix.
Returns base path (without extension) or None.
"""
output_dir = _to_local_path(output_dir)
if not os.path.isdir(output_dir):
return None
candidates = []
for fname in os.listdir(output_dir):
fpath = os.path.join(output_dir, fname)
if os.path.isfile(fpath) and fname.endswith(".hdr"):
base = fpath[:-4]
if "_geo_db" in fname or "_geo" in fname:
candidates.append(base)
if candidates:
# prefer _geo_db over _geo
db_candidates = [c for c in candidates if "_geo_db" in c]
return (db_candidates or candidates)[0]
return None
# ---------------------------------------------------------------------------
# Single-scene geocoding workflow
# ---------------------------------------------------------------------------
def _find_tiff_file(data_dir: str) -> Optional[str]:
"""Find the LuTan-1 .meta.xml file for SARsImportLuTan1 input."""
data_dir = _to_local_path(data_dir)
metas = glob(os.path.join(data_dir, "*.meta.xml"))
if metas:
return metas[0]
# Fallback: raw tiff (older layout)
for fname in os.listdir(data_dir):
if fname.lower().endswith(".tiff") or fname.lower().endswith(".tif"):
return os.path.join(data_dir, fname)
return None
def run_geocoding_workflow(
file_path: str,
output_dir: str,
job_id: Optional[str] = None,
) -> Dict[str, Any]:
"""Run multilooking + geocoding on a single SAR SLC scene.
Args:
file_path: Path to the radar data directory (contains *_slc file).
output_dir: Directory to write outputs into.
job_id: Optional job ID for progress reporting.
Returns:
{"ok": True, "geo_path": "...", "pixel_size_m": 10.0}
or {"ok": False, "error": "..."}
"""
log: list[str] = []
file_path = _to_local_path(file_path)
output_dir = _to_local_path(output_dir)
os.makedirs(output_dir, exist_ok=True)
# --- Find SLC file (already imported) or TIFF (needs import first) ---
slc_base = _find_slc_base(file_path)
if not slc_base:
# Need to import from TIFF first
tiff_file = _find_tiff_file(file_path)
if not tiff_file:
return {"ok": False, "error": f"No SLC or TIFF file found in {file_path}"}
log.append(f"[water] TIFF found, running import: {tiff_file}")
_write_progress(job_id, 1, 3, "Importing LuTan-1 data", output_dir)
t0 = time.time()
try:
r0 = execute_envi_task(
"SARsImportLuTan1",
{
"INPUT_FILE_LIST": [tiff_file],
"RENAME_THE_FILE_USING_PARAMETERS": True,
"APPLY_CALIBRATION_CONSTANT": True,
"GENERATE_QL": False,
"ROOT_URI_FOR_OUTPUT": _normalize_path(file_path),
},
)
log.append(f"[water] import ok ({round(time.time() - t0, 1)}s)")
except Exception as exc:
return {"ok": False, "error": f"Import failed: {exc}", "log": log}
# After import, find the generated _slc file
slc_base = _find_slc_base(file_path)
if not slc_base:
# Also check output from task result
imported = _unwrap_sarscapedata(r0.get("OUTPUT_SARSCAPEDATA"))
if isinstance(imported, list) and imported:
imported = imported[0]
if isinstance(imported, dict):
slc_base = _to_local_path(imported.get("url", "")) or None
if not slc_base:
return {"ok": False, "error": "Import produced no SLC file", "log": log}
total_steps = 3
step_offset = 1
else:
total_steps = 2
step_offset = 0
log.append(f"[water] SLC base: {slc_base}")
slc_sd = _build_sarscapedata(slc_base)
# --- Multilooking ---
_write_progress(job_id, 1 + step_offset, total_steps, "Multilooking", output_dir)
log.append(f"[water] step {1 + step_offset}/{total_steps}: SARsBasicMultilooking")
t0 = time.time()
try:
r1 = execute_envi_task(
"SARsBasicMultilooking",
{
"INPUT_SARSCAPEDATA": [slc_sd],
"GRID_SIZE_FOR_SUGGESTED_LOOKS": float(CUSTOM_TARGET_RESOLUTION_M),
"ROOT_URI_FOR_OUTPUT": _normalize_path(output_dir),
},
)
log.append(f"[water] multilooking ok ({round(time.time() - t0, 1)}s)")
except Exception as exc:
return {"ok": False, "error": f"Multilooking failed: {exc}", "log": log}
mli_sd = _unwrap_sarscapedata(r1.get("OUTPUT_SARSCAPEDATA"))
if not mli_sd:
return {"ok": False, "error": "Multilooking produced no output", "log": log}
log.append(f"[water] multilooking output: {mli_sd.get('url', '?')}")
# --- Geocoding + Radiometric Calibration ---
_write_progress(job_id, 2 + step_offset, total_steps, "Geocoding & Calibration", output_dir)
log.append(f"[water] step {2 + step_offset}/{total_steps}: SARsBasicGeocoding")
t0 = time.time()
geo_params: Dict[str, Any] = {
"INPUT_SARSCAPEDATA": [mli_sd],
"GEOCODE_GRID_SIZE_X": float(CUSTOM_GEOCODING_PIXEL_SIZE_M),
"GEOCODE_GRID_SIZE_Y": float(CUSTOM_GEOCODING_PIXEL_SIZE_M),
"CALIBRATION": True,
"OUTPUT_TYPE": "output_type_db",
"ROOT_URI_FOR_OUTPUT": _normalize_path(output_dir),
}
if DEM_BASE_FILE and os.path.isfile(DEM_BASE_FILE + ".hdr"):
geo_params["DEM_SARSCAPEDATA"] = _build_sarscapedata(DEM_BASE_FILE)
try:
r2 = execute_envi_task("SARsBasicGeocoding", geo_params)
log.append(f"[water] step 2 ok ({round(time.time() - t0, 1)}s)")
except Exception as exc:
return {"ok": False, "error": f"Geocoding failed: {exc}", "log": log}
# Try to get output path from task result
geo_sd = _unwrap_sarscapedata(
r2.get("OUTPUT_DB_SARSCAPEDATA") or r2.get("OUTPUT_SARSCAPEDATA")
)
if isinstance(geo_sd, list) and geo_sd:
geo_sd = geo_sd[0]
geo_path: Optional[str] = None
if isinstance(geo_sd, dict):
geo_path = _to_local_path(geo_sd.get("url", "")) or None
if not geo_path:
# Fallback: scan output dir for _geo_db file
geo_path = _find_geo_db_output(output_dir)
if not geo_path:
return {"ok": False, "error": "Geocoding produced no output file", "log": log}
log.append(f"[water] geo output: {geo_path}")
return {
"ok": True,
"geo_path": geo_path,
"pixel_size_m": CUSTOM_GEOCODING_PIXEL_SIZE_M,
"log": log,
}
# ---------------------------------------------------------------------------
# Flood detection workflow
# ---------------------------------------------------------------------------
def run_flood_detection(
pre_geo_path: str,
post_geo_path: str,
output_dir: str,
job_id: Optional[str] = None,
refine: bool = False,
) -> Dict[str, Any]:
"""Run flood classification on a pre/post event geocoded dB image pair.
Args:
pre_geo_path: Base path of pre-event geocoded dB image (no extension).
post_geo_path: Base path of post-event geocoded dB image (no extension).
output_dir: Directory to write outputs into.
job_id: Optional job ID for progress reporting.
refine: Whether to run MRF refinement after classification.
Returns:
{"ok": True, "classified_path": "...", "flood_area_km2": ..., "stable_water_area_km2": ...}
or {"ok": False, "error": "..."}
"""
log: list[str] = []
pre_geo_path = _to_local_path(pre_geo_path)
post_geo_path = _to_local_path(post_geo_path)
output_dir = _to_local_path(output_dir)
os.makedirs(output_dir, exist_ok=True)
total_steps = 3 if refine else 2
pre_sd = _build_sarscapedata(pre_geo_path)
post_sd = _build_sarscapedata(post_geo_path)
# --- Step 1: Flood Classification ---
_write_progress(job_id, 1, total_steps, "Flood Classification", output_dir)
log.append("[water] step 1: SARsBasicFeFloodingClassification")
t0 = time.time()
flood_params: Dict[str, Any] = {
"INPUT_SARSCAPEDATA": [pre_sd],
"POST_EVENT_FILE": post_sd,
"ROOT_URI_FOR_OUTPUT": _normalize_path(output_dir),
}
if DEM_BASE_FILE and os.path.isfile(DEM_BASE_FILE + ".hdr"):
flood_params["DEM_FILE"] = _build_sarscapedata(DEM_BASE_FILE)
try:
r1 = execute_envi_task("SARsBasicFeFloodingClassification", flood_params)
log.append(f"[water] step 1 ok ({round(time.time() - t0, 1)}s)")
except Exception as exc:
return {"ok": False, "error": f"Flood classification failed: {exc}", "log": log}
classified_sd = _unwrap_sarscapedata(r1.get("OUTPUT_SARSCAPEDATA"))
ratio_sd = _unwrap_sarscapedata(r1.get("RATIO_SARSCAPEDATA"))
pre_out_sd = _unwrap_sarscapedata(r1.get("PRE_EVENT_SARSCAPEDATA"))
post_out_sd = _unwrap_sarscapedata(r1.get("POST_EVENT_SARSCAPEDATA"))
if not classified_sd:
return {"ok": False, "error": "Flood classification produced no output", "log": log}
classified_path = _to_local_path(
classified_sd.get("url", "") if isinstance(classified_sd, dict) else ""
) or None
# --- Step 2 (optional): MRF Refinement ---
if refine and classified_sd and ratio_sd and pre_out_sd and post_out_sd:
_write_progress(job_id, 2, total_steps, "MRF Refinement", output_dir)
log.append("[water] step 2: SARsBasicFeFloodingClassificationRefinement")
t0 = time.time()
try:
r2 = execute_envi_task(
"SARsBasicFeFloodingClassificationRefinement",
{
"PRE_EVENT_FILE": pre_out_sd,
"POST_EVENT_FILE": post_out_sd,
"CLASSIFIED_FILE": classified_sd,
"RATIO_FILE": ratio_sd,
"ROOT_URI_FOR_OUTPUT": _normalize_path(output_dir),
},
)
refined_sd = _unwrap_sarscapedata(r2.get("OUTPUT_SARSCAPEDATA"))
if refined_sd and isinstance(refined_sd, dict):
classified_path = _to_local_path(refined_sd.get("url", "")) or classified_path
log.append(f"[water] step 2 ok ({round(time.time() - t0, 1)}s)")
except Exception as exc:
log.append(f"[water] step 2 refinement failed (non-fatal): {exc}")
# --- Step 3: Parse classification statistics ---
_write_progress(job_id, total_steps, total_steps, "Parsing results", output_dir)
flood_area_km2, stable_water_area_km2 = _parse_flood_stats(classified_path)
log.append(
f"[water] flood={flood_area_km2} km², stable_water={stable_water_area_km2} km²"
)
return {
"ok": True,
"classified_path": classified_path,
"flood_area_km2": flood_area_km2,
"stable_water_area_km2": stable_water_area_km2,
"log": log,
}
def _parse_flood_stats(classified_path: Optional[str]) -> tuple[Optional[float], Optional[float]]:
"""Read the classified flood map and compute area statistics.
SARscape flood classification output values:
0 = no data / background
1 = stable water (permanent water body)
2 = flood (new water)
3 = high scattering point (urban / double bounce)
4 = non-water
Returns (flood_area_km2, stable_water_area_km2).
"""
if not classified_path:
return None, None
classified_path = _to_local_path(classified_path)
if not os.path.isfile(classified_path):
return None, None
try:
import rasterio
with rasterio.open(classified_path) as ds:
data = ds.read(1)
transform = ds.transform
# Pixel area in m²
px_w = abs(transform.a)
px_h = abs(transform.e)
# If CRS is geographic (degrees), convert to meters approximately
if ds.crs and ds.crs.is_geographic:
import math
lat_center = (ds.bounds.top + ds.bounds.bottom) / 2.0
px_w_m = px_w * math.cos(math.radians(lat_center)) * 111320
px_h_m = px_h * 111320
else:
px_w_m, px_h_m = px_w, px_h
pixel_area_km2 = (px_w_m * px_h_m) / 1e6
flood_pixels = int((data == 2).sum())
stable_pixels = int((data == 1).sum())
return (
round(flood_pixels * pixel_area_km2, 4),
round(stable_pixels * pixel_area_km2, 4),
)
except Exception as exc:
print(f"[WARN] _parse_flood_stats: {exc}")
return None, None
+228
View File
@@ -0,0 +1,228 @@
import uuid
from datetime import datetime
from typing import Any, Dict, List, Optional
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..models import WorkflowRunORM, WorkflowStepORM
from .job_queue_service import job_queue_service
def _new_session() -> AsyncSession:
if database.AsyncSessionLocal is None:
database.init_db()
if database.AsyncSessionLocal is None:
raise RuntimeError("Database session factory is not initialized.")
return database.AsyncSessionLocal()
class WorkflowService:
"""
Lightweight workflow orchestration service backed by DB.
"""
async def create_run(
self,
workflow_name: str,
steps: List[Dict[str, Any]],
params: Optional[Dict[str, Any]] = None,
tags: Optional[Dict[str, Any]] = None,
created_by: Optional[str] = None,
db: Optional[AsyncSession] = None,
) -> str:
gen_db = db is None
if gen_db:
db = _new_session()
run_id = str(uuid.uuid4())
try:
run = WorkflowRunORM(
run_id=run_id,
workflow_name=workflow_name,
status="RUNNING",
params=params or {},
tags=tags or {},
created_by=created_by,
started_at=datetime.utcnow(),
)
db.add(run)
for step in steps:
depends_on = step.get("depends_on") or []
status = "READY" if not depends_on else "PENDING"
step_params = {
"job_type": step.get("job_type"),
"payload": step.get("payload") or {},
"task_id": step.get("task_id"),
"optional": bool(step.get("optional", False)),
}
db.add(
WorkflowStepORM(
run_id=run_id,
step_id=step.get("step_id"),
step_name=step.get("step_name") or step.get("step_id"),
status=status,
depends_on=depends_on,
params=step_params,
)
)
if gen_db:
await db.commit()
else:
await db.flush()
finally:
if gen_db:
await db.close()
await self.enqueue_ready_steps(run_id, db=None if gen_db else db)
return run_id
async def enqueue_ready_steps(self, run_id: str, db: Optional[AsyncSession] = None) -> None:
gen_db = db is None
if gen_db:
db = _new_session()
try:
result = await db.execute(
select(WorkflowStepORM).where(
WorkflowStepORM.run_id == run_id,
WorkflowStepORM.status == "READY",
)
)
ready_steps = result.scalars().all()
for step in ready_steps:
params = step.params or {}
job_type = params.get("job_type")
if not job_type:
continue
payload = params.get("payload") or {}
task_id = params.get("task_id")
await job_queue_service.create_job(
job_type,
payload=payload,
workflow_run_id=run_id,
workflow_step_id=step.step_id,
task_id=task_id,
db=db,
)
step.status = "RUNNING"
step.started_at = datetime.utcnow()
if gen_db:
await db.commit()
else:
await db.flush()
finally:
if gen_db:
await db.close()
async def mark_step_completed(
self,
run_id: str,
step_id: str,
outputs: Optional[Dict[str, Any]] = None,
db: Optional[AsyncSession] = None,
) -> None:
gen_db = db is None
if gen_db:
db = _new_session()
try:
result = await db.execute(
select(WorkflowStepORM).where(
WorkflowStepORM.run_id == run_id,
WorkflowStepORM.step_id == step_id,
)
)
step = result.scalar_one_or_none()
if not step:
return
step.status = "COMPLETED"
step.ended_at = datetime.utcnow()
if outputs:
step.outputs = outputs
await self._advance_ready_steps(run_id, db)
await self.enqueue_ready_steps(run_id, db=db)
if gen_db:
await db.commit()
else:
await db.flush()
finally:
if gen_db:
await db.close()
async def mark_step_failed(
self,
run_id: str,
step_id: str,
error: str,
db: Optional[AsyncSession] = None,
) -> None:
gen_db = db is None
if gen_db:
db = _new_session()
try:
result = await db.execute(
select(WorkflowStepORM).where(
WorkflowStepORM.run_id == run_id,
WorkflowStepORM.step_id == step_id,
)
)
step = result.scalar_one_or_none()
if not step:
return
step.status = "FAILED"
step.error = error
step.ended_at = datetime.utcnow()
await db.execute(
update(WorkflowRunORM)
.where(WorkflowRunORM.run_id == run_id)
.values(status="FAILED", ended_at=datetime.utcnow())
)
if gen_db:
await db.commit()
else:
await db.flush()
finally:
if gen_db:
await db.close()
async def _advance_ready_steps(self, run_id: str, db: AsyncSession) -> None:
result = await db.execute(
select(WorkflowStepORM).where(WorkflowStepORM.run_id == run_id)
)
steps = result.scalars().all()
completed = {s.step_id for s in steps if s.status == "COMPLETED"}
pending = [s for s in steps if s.status == "PENDING"]
for step in pending:
deps = step.depends_on or []
if all(dep in completed for dep in deps):
step.status = "READY"
# If all steps are terminal, complete run.
terminal = {"COMPLETED", "FAILED", "SKIPPED", "CANCELLED"}
if all(s.status in terminal for s in steps):
status = "COMPLETED"
if any(s.status == "FAILED" for s in steps):
status = "FAILED"
await db.execute(
update(WorkflowRunORM)
.where(WorkflowRunORM.run_id == run_id)
.values(status=status, ended_at=datetime.utcnow())
)
workflow_service = WorkflowService()
+333
View File
@@ -0,0 +1,333 @@
"""WSL 环境校验与命令执行封装。
提供:
- WslCheckResult : 单项检查结果
- WslEnvironmentReport: 完整校验报告
- check_wsl_environment(): 执行全部检查项
- run_wsl_command() : 在指定 distro 中执行命令
"""
from __future__ import annotations
import os
import shutil
import subprocess
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
# ---------------------------------------------------------------------------
# 数据结构
# ---------------------------------------------------------------------------
@dataclass
class WslCheckResult:
name: str
ok: bool
detail: str = ""
skipped: bool = False
@dataclass
class WslEnvironmentReport:
overall_ok: bool
distro: str
checks: List[WslCheckResult] = field(default_factory=list)
message: str = ""
def to_dict(self) -> Dict[str, Any]:
return {
"overall_ok": self.overall_ok,
"distro": self.distro,
"message": self.message,
"checks": [
{
"name": c.name,
"ok": c.ok,
"detail": c.detail,
"skipped": c.skipped,
}
for c in self.checks
],
}
# ---------------------------------------------------------------------------
# 低层命令执行
# ---------------------------------------------------------------------------
def _decode_subprocess_output(data: bytes) -> str:
if not data:
return ""
encodings: List[str] = []
if data.count(b"\x00") * 4 >= max(1, len(data)):
encodings.extend(["utf-16-le", "utf-16-be"])
encodings.extend(["utf-8", "gbk"])
for encoding in encodings:
try:
return data.decode(encoding).replace("\ufeff", "").replace("\x00", "").strip()
except UnicodeDecodeError:
continue
return data.decode("utf-8", errors="replace").replace("\ufeff", "").replace("\x00", "").strip()
def _find_wsl_executable() -> str:
return shutil.which("wsl.exe") or ""
def _run_windows_command(
args: List[str],
timeout: int = 30,
env: Optional[Dict[str, str]] = None,
) -> Tuple[int, str, str]:
proc_env = os.environ.copy()
if env:
proc_env.update(env)
result = subprocess.run(
args,
capture_output=True,
text=False,
timeout=timeout,
env=proc_env,
check=False,
)
return (
result.returncode,
_decode_subprocess_output(result.stdout),
_decode_subprocess_output(result.stderr),
)
def run_wsl_command(
cmd: str,
distro: Optional[str] = None,
timeout: int = 30,
env: Optional[Dict[str, str]] = None,
) -> Tuple[int, str, str]:
"""在 WSL distro 中执行 bash 命令。"""
wsl_exe = _find_wsl_executable()
if not wsl_exe:
return -2, "", "wsl.exe 未找到"
wsl_args = [wsl_exe]
if distro:
wsl_args += ["-d", distro]
wsl_args += ["bash", "-lc", cmd]
try:
return _run_windows_command(wsl_args, timeout=timeout, env=env)
except subprocess.TimeoutExpired:
return -1, "", f"命令超时({timeout}s)"
except FileNotFoundError:
return -2, "", "wsl.exe 未找到"
except Exception as exc:
return -3, "", str(exc)
def windows_path_to_wsl(win_path: str, distro: Optional[str] = None) -> str:
"""将 Windows 路径转换为 WSL 路径(调用 wslpath)。"""
if not win_path:
return ""
rc, stdout, _ = run_wsl_command(
f"wslpath -u '{win_path.replace(chr(39), '')}'",
distro=distro,
timeout=10,
)
return stdout if rc == 0 else ""
# ---------------------------------------------------------------------------
# 环境校验
# ---------------------------------------------------------------------------
def check_wsl_environment(
distro: str,
python_cmd: str = "/home/administrator/miniconda3/envs/isce2/bin/python",
stripmap_app_path: str = "", # stripmapApp.py 的 WSL 绝对路径
pipeline_script_path: str = "", # run_lt1_dinsar_pipeline.py 的 WSL 绝对路径
dem_path_win: str = "",
orbit_dir_win: str = "",
output_dir_win: str = "",
smoke_test: bool = False,
) -> WslEnvironmentReport:
"""执行全部 WSL 环境校验,返回 WslEnvironmentReport。"""
checks: List[WslCheckResult] = []
def add(name: str, ok: bool, detail: str = "", skipped: bool = False) -> None:
checks.append(WslCheckResult(name=name, ok=ok, detail=detail, skipped=skipped))
deferred_checks = [
"WSL2 支持",
"Distro 存在",
"Distro 可启动",
"bash -lc 可执行",
"Python 可执行",
"ISCE2 可 import",
"stripmapApp 存在",
"生产脚本存在",
"DEM 路径可读",
"轨道目录可读",
"输出目录可写",
"Windows→WSL 路径转换",
"Smoke Test",
]
# 1. wsl.exe 是否存在
wsl_exe = _find_wsl_executable()
wsl_installed = bool(wsl_exe)
add("WSL 已安装", wsl_installed, wsl_exe or "wsl.exe 未找到")
if not wsl_installed:
for name in deferred_checks:
add(name, False, "wsl.exe 未找到,跳过", skipped=True)
return WslEnvironmentReport(
overall_ok=False, distro=distro, checks=checks,
message="WSL 未安装,后续检查跳过",
)
# 2. WSL 服务可访问
rc, stdout, stderr = _run_windows_command([wsl_exe, "--list", "--quiet"], timeout=15)
service_ok = rc == 0
service_detail = stdout or stderr or "WSL 服务可访问"
add("WSL 服务可访问", service_ok, service_detail)
if not service_ok:
for name in deferred_checks:
add(name, False, f"WSL 服务不可访问,跳过:{service_detail}", skipped=True)
return WslEnvironmentReport(
overall_ok=False,
distro=distro,
checks=checks,
message=f"WSL 服务不可访问:{service_detail}",
)
# 3. WSL2 支持
try:
rc, raw, err = _run_windows_command([wsl_exe, "--list", "--verbose"], timeout=15)
verbose_lines = [line.strip() for line in raw.splitlines() if line.strip()]
wsl2_ok = rc == 0 and any(line.endswith(" 2") or line.endswith("\t2") for line in verbose_lines)
add("WSL2 支持", wsl2_ok, raw or err or ("检测到 WSL2 版本" if wsl2_ok else "未检测到 WSL2"))
except Exception as exc:
add("WSL2 支持", False, str(exc))
# 4. 目标 distro 存在
try:
raw = stdout
distro_exists = distro.lower() in raw.lower()
add("Distro 存在", distro_exists,
f"'{distro}' 已找到" if distro_exists else f"'{distro}' 未在列表中")
except Exception as exc:
distro_exists = False
add("Distro 存在", False, str(exc))
if not distro_exists:
for name in deferred_checks[2:]:
add(name, False, "Distro 不存在,跳过", skipped=True)
return WslEnvironmentReport(
overall_ok=False, distro=distro, checks=checks,
message=f"Distro '{distro}' 不存在",
)
# 4. Distro 可启动
rc, _, err = run_wsl_command("echo alive", distro=distro, timeout=15)
distro_ok = rc == 0
add("Distro 可启动", distro_ok, err if not distro_ok else "启动正常")
# 5. bash -lc 可执行
rc, out, err = run_wsl_command("echo bash_ok", distro=distro, timeout=10)
bash_ok = rc == 0 and "bash_ok" in out
add("bash -lc 可执行", bash_ok, err if not bash_ok else "bash 正常")
# 6. Python 可执行
rc, out, err = run_wsl_command(f"{python_cmd} --version", distro=distro, timeout=15)
py_ok = rc == 0
add("Python 可执行", py_ok, out or err)
# 7. ISCE2 可 import
rc, out, err = run_wsl_command(
f'{python_cmd} -c "import isce; print(isce.__version__)"',
distro=distro, timeout=30,
)
isce_ok = rc == 0
add("ISCE2 可 import", isce_ok, out or err)
# 8. stripmapApp.py 存在(全路径检查)
if stripmap_app_path:
rc, out, err = run_wsl_command(
f"test -f '{stripmap_app_path}' && echo found",
distro=distro, timeout=10,
)
app_ok = rc == 0 and "found" in out
add("stripmapApp 存在", app_ok,
stripmap_app_path if app_ok else f"{stripmap_app_path} 未找到")
else:
add("stripmapApp 存在", False, "ISCE2_STRIPMAP_APP 未配置", skipped=True)
# 9. 生产流水线脚本存在
if pipeline_script_path:
rc, out, err = run_wsl_command(
f"test -f '{pipeline_script_path}' && echo found",
distro=distro, timeout=10,
)
script_ok = rc == 0 and "found" in out
add("生产脚本存在", script_ok,
pipeline_script_path if script_ok else f"{pipeline_script_path} 未找到")
else:
add("生产脚本存在", False, "ISCE2_PIPELINE_SCRIPT 未配置", skipped=True)
# 10. DEM 路径可读
if dem_path_win:
wsl_dem = windows_path_to_wsl(dem_path_win, distro=distro)
rc, _, err = run_wsl_command(f"test -r '{wsl_dem}'", distro=distro, timeout=10)
add("DEM 路径可读", rc == 0, wsl_dem or err)
else:
add("DEM 路径可读", False, "ISCE2_DEM_PATH 未配置", skipped=True)
# 11. 轨道目录可读
if orbit_dir_win:
wsl_orbit = windows_path_to_wsl(orbit_dir_win, distro=distro)
rc, _, err = run_wsl_command(f"test -r '{wsl_orbit}'", distro=distro, timeout=10)
add("轨道目录可读", rc == 0, wsl_orbit or err)
else:
add("轨道目录可读", False, "ORBIT_POOL_ISCE2 未配置", skipped=True)
# 12. 输出目录可写
if output_dir_win:
wsl_out = windows_path_to_wsl(output_dir_win, distro=distro)
rc, _, err = run_wsl_command(f"test -w '{wsl_out}'", distro=distro, timeout=10)
add("输出目录可写", rc == 0, wsl_out or err)
else:
add("输出目录可写", False, "ISCE2_OUTPUT_ROOT 未配置", skipped=True)
# 13. Windows→WSL 路径转换
test_win = os.environ.get("TEMP", r"C:\Windows\Temp")
wsl_converted = windows_path_to_wsl(test_win, distro=distro)
path_ok = wsl_converted.startswith("/mnt/")
add("Windows→WSL 路径转换", path_ok,
f"{test_win} → {wsl_converted}" if wsl_converted else "转换失败")
# 14. Smoke test(可选)
if smoke_test:
rc, out, err = run_wsl_command(
f'{python_cmd} -c "import isce; from isceobj.Sensor import Lutan1; print(\'smoke_ok\')"',
distro=distro, timeout=60,
)
smoke_ok = rc == 0 and "smoke_ok" in out
add("Smoke Test", smoke_ok, out or err)
else:
add("Smoke Test", True, "已跳过(ISCE2_SMOKE_TEST_ENABLED=false)", skipped=True)
# 汇总
critical = [c for c in checks if not c.skipped]
overall_ok = all(c.ok for c in critical)
failed_names = [c.name for c in critical if not c.ok]
message = "所有检查通过" if overall_ok else f"以下检查未通过:{', '.join(failed_names)}"
return WslEnvironmentReport(
overall_ok=overall_ok,
distro=distro,
checks=checks,
message=message,
)