Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve pairing planning and statistics visibility

Browse Source
This commit is contained in:
Harmon
2026-07-07 01:04:47 +08:00
parent 2da2121829
commit 36a406ae49
22 changed files with 2085 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routers/stats.py
+221 -52
View File
@@ -11,6 +11,8 @@ from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
SOURCE_METADATA_MANAGED_FORMATS = ("LT1_ARCHIVE", "S1_ZIP")
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import distinct, func, text
from sqlalchemy.ext.asyncio import AsyncSession
@@ -51,7 +53,6 @@ from ..services.data_service import data_service
from ..services.dinsar_read_service import dinsar_read_service
from ..services.pairing_state_service import pairing_state_service
from ..services.admin_region_lookup_service import _build_region_path, _load_region_records
from ..utils import find_xml_file
from . import dependencies as _deps
from .dependencies import _get_current_user
@@ -80,6 +81,10 @@ def _ratio(numerator: int, denominator: int) -> float:
return round(float(numerator) / float(denominator), 4)
def _is_source_metadata_managed_format(value: Any) -> bool:
return str(value or "").strip().upper() in SOURCE_METADATA_MANAGED_FORMATS
def _family_label(value: Any) -> str:
text = str(value or "").strip().upper()
if text in {"LT1", "LT-1", "LUTAN", "LUTAN1"}:
@@ -102,6 +107,14 @@ def _month_from_yyyymmdd(value: Any) -> Optional[str]:
return None
def _compact_yyyymmdd(value: Any) -> Optional[str]:
text = str(value or "").strip()
digits = "".join(ch for ch in text if ch.isdigit())
if len(digits) >= 8:
return digits[:8]
return None
def _month_from_datetime(value: Any) -> Optional[str]:
if not value:
return None
@@ -594,16 +607,36 @@ async def get_statistics_dashboard(
db,
select(func.count(SourceProductAssetORM.id)).where(SourceProductAssetORM.is_active == True),
)
metadata_source_total = await _scalar_count(
db,
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.is_active == True,
SourceProductAssetORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
radar_total = await _scalar_count(db, select(func.count(RadarDataORM.id)))
metadata_asset_total = await _scalar_count(
db,
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))),
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_total = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
geometry_total = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_total = await _scalar_count(db, select(func.count(SourceMetadataDocumentORM.id)))
geometry_total = await _scalar_count(db, select(func.count(SARSceneGeometryProfileORM.id)))
geometry_ready = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SARSceneGeometryProfileORM.metadata_quality == "READY",
SARSceneGeometryProfileORM.production_readiness == "READY",
),
@@ -633,6 +666,7 @@ async def get_statistics_dashboard(
for family, source_format, parse_status, count in source_group_rows.all():
family_label = _family_label(family)
status_label = _status_label(parse_status)
format_label = str(source_format or "UNKNOWN")
count_int = _safe_int(count)
family_bucket = source_by_family_map.setdefault(
family_label,
@@ -649,7 +683,6 @@ async def get_statistics_dashboard(
family_bucket["ready_count"] += count_int
else:
family_bucket["issue_count"] += count_int
format_label = str(source_format or "UNKNOWN")
family_bucket["formats"][format_label] = family_bucket["formats"].get(format_label, 0) + count_int
source_by_format.append(
{
@@ -673,19 +706,22 @@ async def get_statistics_dashboard(
geometry_rows = await db.execute(
select(
SARSceneGeometryProfileORM.satellite_family,
SARSceneGeometryProfileORM.source_format,
SARSceneGeometryProfileORM.metadata_quality,
SARSceneGeometryProfileORM.production_readiness,
func.count(SARSceneGeometryProfileORM.id),
)
.where(SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS))
.group_by(
SARSceneGeometryProfileORM.satellite_family,
SARSceneGeometryProfileORM.source_format,
SARSceneGeometryProfileORM.metadata_quality,
SARSceneGeometryProfileORM.production_readiness,
)
.order_by(SARSceneGeometryProfileORM.satellite_family)
)
geometry_by_family_map: dict[str, dict[str, Any]] = {}
for family, metadata_quality, production_readiness, count in geometry_rows.all():
for family, source_format, metadata_quality, production_readiness, count in geometry_rows.all():
family_label = _family_label(family)
count_int = _safe_int(count)
bucket = geometry_by_family_map.setdefault(
@@ -1075,6 +1111,82 @@ async def get_statistics_dashboard(
issue_by_severity[severity_label] = issue_by_severity.get(severity_label, 0) + count_int
issue_by_code[code_label] = issue_by_code.get(code_label, 0) + count_int
missing_orbit_issue_rows = await db.execute(
select(AssetInventoryIssueORM.metadata_json)
.where(AssetInventoryIssueORM.status == "OPEN")
.where(AssetInventoryIssueORM.issue_code == "scene_missing_orbit")
)
orbit_missing_satellite_map: dict[str, dict[str, Any]] = {}
orbit_missing_top_dates: dict[tuple[str, str], int] = {}
orbit_missing_unknown_date_scene_count = 0
for (metadata_json,) in missing_orbit_issue_rows.all():
metadata = metadata_json if isinstance(metadata_json, dict) else {}
satellite = str(metadata.get("satellite") or "UNKNOWN").strip().upper() or "UNKNOWN"
ymd = _compact_yyyymmdd(metadata.get("imaging_date") or metadata.get("acquisition_start_time_utc"))
month = _month_from_yyyymmdd(ymd) if ymd else None
satellite_bucket = orbit_missing_satellite_map.setdefault(
satellite,
{
"satellite": satellite,
"affected_scene_count": 0,
"dates": defaultdict(int),
"months": defaultdict(lambda: {"affected_scene_count": 0, "dates": set()}),
"unknown_date_scene_count": 0,
},
)
satellite_bucket["affected_scene_count"] += 1
if not ymd:
satellite_bucket["unknown_date_scene_count"] += 1
orbit_missing_unknown_date_scene_count += 1
continue
satellite_bucket["dates"][ymd] += 1
orbit_missing_top_dates[(satellite, ymd)] = orbit_missing_top_dates.get((satellite, ymd), 0) + 1
if month:
month_bucket = satellite_bucket["months"][month]
month_bucket["affected_scene_count"] += 1
month_bucket["dates"].add(ymd)
orbit_missing_by_satellite: list[dict[str, Any]] = []
for satellite, bucket in sorted(orbit_missing_satellite_map.items()):
date_items = [
{"date": date_text, "affected_scene_count": _safe_int(count)}
for date_text, count in sorted(bucket["dates"].items())
]
month_items = [
{
"month": month,
"affected_scene_count": _safe_int(month_bucket["affected_scene_count"]),
"missing_orbit_date_count": len(month_bucket["dates"]),
}
for month, month_bucket in sorted(bucket["months"].items())
]
orbit_missing_by_satellite.append(
{
"satellite": satellite,
"affected_scene_count": _safe_int(bucket["affected_scene_count"]),
"missing_orbit_date_count": len(date_items),
"unknown_date_scene_count": _safe_int(bucket["unknown_date_scene_count"]),
"first_missing_date": date_items[0]["date"] if date_items else None,
"last_missing_date": date_items[-1]["date"] if date_items else None,
"months": month_items,
"dates": date_items,
}
)
orbit_missing_summary = {
"affected_scene_count": sum(item["affected_scene_count"] for item in orbit_missing_by_satellite),
"missing_orbit_date_count": sum(item["missing_orbit_date_count"] for item in orbit_missing_by_satellite),
"unknown_date_scene_count": orbit_missing_unknown_date_scene_count,
"by_satellite": orbit_missing_by_satellite,
"top_dates": [
{"satellite": satellite, "date": date_text, "affected_scene_count": _safe_int(count)}
for (satellite, date_text), count in sorted(
orbit_missing_top_dates.items(),
key=lambda item: (-item[1], item[0][0], item[0][1]),
)[:20]
],
}
inventory_state_rows = await db.execute(
select(
AssetInventoryStateORM.inventory_type,
@@ -1114,16 +1226,24 @@ async def get_statistics_dashboard(
avg_duration_seconds = round(sum(duration_seconds) / len(duration_seconds), 1) if duration_seconds else None
selected_orbit_rate = _ratio(selected_orbit_bindings, orbit_required_total)
geometry_ready_rate = _ratio(geometry_ready, source_total)
metadata_ready_rate = _ratio(metadata_asset_total, source_total)
geometry_ready_rate = _ratio(geometry_ready, geometry_total)
metadata_ready_rate = _ratio(metadata_asset_total, metadata_source_total)
result_ready_total = sum(item["ready_count"] for item in results_by_catalog)
metadata_missing_count = max(0, metadata_source_total - metadata_asset_total)
geometry_not_ready_count = max(0, geometry_total - geometry_ready)
orbit_missing_gap_count = max(0, orbit_required_total - selected_orbit_bindings)
non_orbit_open_issue_total = sum(
count
for code, count in issue_by_code.items()
if code != "scene_missing_orbit"
)
risk_count = (
max(0, source_total - metadata_asset_total)
+ max(0, source_total - geometry_ready)
+ max(0, orbit_required_total - selected_orbit_bindings)
metadata_missing_count
+ geometry_not_ready_count
+ orbit_missing_gap_count
+ result_assets_missing
+ open_issue_total
+ non_orbit_open_issue_total
)
kpis = [
@@ -1140,7 +1260,7 @@ async def get_statistics_dashboard(
"label": "元数据入库率",
"value": round(metadata_ready_rate * 100, 1),
"unit": "%",
"note": f"{metadata_asset_total}/{source_total} 景已提取 XML/元数据",
"note": f"{metadata_asset_total}/{metadata_source_total} 景已提取 XML/元数据",
"tone": "success" if metadata_ready_rate >= 0.98 else "warning",
},
{
@@ -1148,7 +1268,7 @@ async def get_statistics_dashboard(
"label": "几何画像可用率",
"value": round(geometry_ready_rate * 100, 1),
"unit": "%",
"note": f"{geometry_ready}/{source_total} 景可用于覆盖统计",
"note": f"{geometry_ready}/{geometry_total} 景可用于覆盖统计",
"tone": "success" if geometry_ready_rate >= 0.95 else "warning",
},
{
@@ -1172,7 +1292,7 @@ async def get_statistics_dashboard(
"label": "待关注项",
"value": risk_count,
"unit": "项",
"note": f"开放问题 {open_issue_total}，缺失成果资产 {result_assets_missing}",
"note": f"缺精轨 {orbit_missing_gap_count}，开放问题 {open_issue_total}，缺失成果资产 {result_assets_missing}",
"tone": "danger" if risk_count else "success",
},
]
@@ -1186,20 +1306,23 @@ async def get_statistics_dashboard(
"source_by_family": source_by_family,
"source_by_format": source_by_format,
"source_by_month": source_by_month,
"metadata_source_total": metadata_source_total,
"metadata_asset_total": metadata_asset_total,
"metadata_doc_total": metadata_doc_total,
"metadata_missing_count": metadata_missing_count,
"metadata_ready_rate": metadata_ready_rate,
"geometry_total": geometry_total,
"geometry_ready": geometry_ready,
"geometry_not_ready_count": geometry_not_ready_count,
"geometry_ready_rate": geometry_ready_rate,
"geometry_by_family": geometry_by_family,
"preview_ready": preview_ready,
"pipeline": [
{"key": "source", "label": "源资产登记", "value": source_total, "rate": 1.0},
{"key": "metadata", "label": "元数据入库", "value": metadata_asset_total, "rate": metadata_ready_rate},
{"key": "geometry", "label": "几何画像", "value": geometry_total, "rate": _ratio(geometry_total, source_total)},
{"key": "ready", "label": "可生产画像", "value": geometry_ready, "rate": geometry_ready_rate},
{"key": "preview", "label": "预览缓存", "value": preview_ready, "rate": _ratio(preview_ready, radar_total)},
{"key": "source", "label": "源资产登记", "value": source_total, "denominator": source_total, "rate": 1.0},
{"key": "metadata", "label": "元数据入库", "value": metadata_asset_total, "denominator": metadata_source_total, "rate": metadata_ready_rate},
{"key": "geometry", "label": "几何画像", "value": geometry_total, "denominator": metadata_source_total, "rate": _ratio(geometry_total, metadata_source_total)},
{"key": "ready", "label": "可生产画像", "value": geometry_ready, "denominator": geometry_total, "rate": geometry_ready_rate},
{"key": "preview", "label": "预览缓存", "value": preview_ready, "denominator": radar_total, "rate": _ratio(preview_ready, radar_total)},
],
},
"orbit": {
@@ -1208,6 +1331,8 @@ async def get_statistics_dashboard(
"orbit_required_total": orbit_required_total,
"selected_bindings": selected_orbit_bindings,
"matched_bindings": matched_orbit_bindings,
"missing_scene_count": orbit_missing_gap_count,
"missing_summary": orbit_missing_summary,
"selected_rate": selected_orbit_rate,
},
"coverage": {
@@ -1242,6 +1367,9 @@ async def get_statistics_dashboard(
"issues": {
"issue_total": issue_total,
"open_issue_total": open_issue_total,
"risk_count": risk_count,
"non_orbit_open_issue_total": non_orbit_open_issue_total,
"orbit_missing_gap_count": orbit_missing_gap_count,
"by_severity": [
{"severity": severity, "count": count}
for severity, count in sorted(issue_by_severity.items(), key=lambda kv: (-kv[1], kv[0]))
@@ -1361,12 +1489,16 @@ async def get_statistics(
"db_ready_and_cache_exists_count": 0,
"db_ready_but_cache_missing_count": 0,
}
source_xml_consistency = {
source_metadata_consistency = {
"total_records_count": 0,
"xml_detected_count": 0,
"xml_missing_count": 0,
"xml_parsed_ok_count": 0,
"xml_detected_but_unparsed_count": 0,
"metadata_document_asset_count": 0,
"metadata_document_count": 0,
"metadata_document_missing_count": 0,
"metadata_parsed_ok_count": 0,
"metadata_detected_but_unparsed_count": 0,
"geometry_ready_count": 0,
"geometry_missing_count": 0,
"geometry_not_ready_count": 0,
}
try:
@@ -1399,7 +1531,6 @@ async def get_statistics(
)
source_rows = source_rows_res.all()
source_preview_consistency["total_records_count"] = len(source_rows)
source_xml_consistency["total_records_count"] = len(source_rows)
for (
unique_id,
@@ -1415,7 +1546,6 @@ async def get_statistics(
) in source_rows:
if not file_path:
source_preview_consistency["preview_missing_count"] += 1
source_xml_consistency["xml_missing_count"] += 1
continue
cache_key = unique_id or file_path
@@ -1443,34 +1573,64 @@ async def get_statistics(
else:
source_preview_consistency["db_ready_but_cache_missing_count"] += 1
scene_dir = file_path if os.path.isdir(file_path) else os.path.dirname(file_path)
xml_path = find_xml_file(scene_dir) if scene_dir else None
has_xml = bool(xml_path and os.path.exists(xml_path))
if has_xml:
source_xml_consistency["xml_detected_count"] += 1
parsed_ok = any(
value is not None and value != ""
for value in [
scene_center_lon,
scene_center_lat,
acquisition_time_utc,
satellite_mode,
receiving_station,
product_level,
product_unique_id,
]
)
if parsed_ok:
source_xml_consistency["xml_parsed_ok_count"] += 1
else:
source_xml_consistency["xml_detected_but_unparsed_count"] += 1
else:
source_xml_consistency["xml_missing_count"] += 1
active_source_count = await _scalar_count(
db,
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.is_active == True,
SourceProductAssetORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_asset_count = await _scalar_count(
db,
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_count = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_parse_issue_count = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SourceMetadataDocumentORM.parse_status != "OK",
),
)
geometry_ready_count = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SARSceneGeometryProfileORM.metadata_quality == "READY",
SARSceneGeometryProfileORM.production_readiness == "READY",
),
)
geometry_total_count = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
source_metadata_consistency.update(
{
"total_records_count": active_source_count,
"metadata_document_asset_count": metadata_asset_count,
"metadata_document_count": metadata_doc_count,
"metadata_document_missing_count": max(0, active_source_count - metadata_asset_count),
"metadata_parsed_ok_count": max(0, metadata_doc_count - metadata_parse_issue_count),
"metadata_detected_but_unparsed_count": metadata_parse_issue_count,
"geometry_ready_count": geometry_ready_count,
"geometry_missing_count": max(0, active_source_count - geometry_total_count),
"geometry_not_ready_count": max(0, geometry_total_count - geometry_ready_count),
}
)
except Exception as e:
logger.warning("统计源数据时发生错误 (可能是表不存在): %s", e)
source_preview_consistency["error"] = str(e)
source_xml_consistency["error"] = str(e)
source_metadata_consistency["error"] = str(e)
# 4. AI 质量统计
labeled_good_count = sum(1 for record in dinsar_records if record.product.user_label == 1)
@@ -1756,7 +1916,16 @@ async def get_statistics(
"with_orbit_data_count": with_orbit_data_count,
},
"source_preview_consistency": source_preview_consistency,
"source_xml_consistency": source_xml_consistency,
"source_metadata_consistency": source_metadata_consistency,
"source_xml_consistency": {
"deprecated": True,
"replacement": "source_metadata_consistency",
"total_records_count": source_metadata_consistency.get("total_records_count", 0),
"xml_detected_count": source_metadata_consistency.get("metadata_document_asset_count", 0),
"xml_missing_count": 0,
"xml_parsed_ok_count": source_metadata_consistency.get("metadata_parsed_ok_count", 0),
"xml_detected_but_unparsed_count": 0,
},
"water_geo_consistency": water_geo_consistency,
"pairing_consistency": pairing_consistency,
"by_satellite": by_satellite,