1794 lines
70 KiB
Python
1794 lines
70 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import os
|
||
import re as _re
|
||
import time
|
||
from collections import defaultdict
|
||
from datetime import datetime
|
||
from typing import Any, Dict, Optional
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from sqlalchemy import distinct, func, text
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from sqlalchemy.future import select
|
||
|
||
try:
|
||
from shapely.geometry import Point, Polygon, box, mapping, shape
|
||
except Exception: # pragma: no cover - production dependency is optional for stats fallback
|
||
Point = None
|
||
Polygon = None
|
||
box = None
|
||
mapping = None
|
||
shape = None
|
||
|
||
from ..auth_service import ROLE_ADMIN
|
||
from ..config import settings
|
||
from ..database import get_db
|
||
from ..models import (
|
||
AssetInventoryIssueORM,
|
||
AssetInventoryStateORM,
|
||
AuthUserORM,
|
||
DinsarProductionRunORM,
|
||
DinsarTaskBatchORM,
|
||
DinsarTaskItemORM,
|
||
OrbitAssetORM,
|
||
RadarDataORM,
|
||
ResultAssetORM,
|
||
ResultIssueORM,
|
||
ResultProductORM,
|
||
SARSceneGeoORM,
|
||
SARSceneGeometryProfileORM,
|
||
SceneOrbitBindingORM,
|
||
SourceMetadataDocumentORM,
|
||
SourceProductAssetORM,
|
||
WorkflowRunORM,
|
||
)
|
||
from ..services.data_service import data_service
|
||
from ..services.dinsar_read_service import dinsar_read_service
|
||
from ..services.pairing_state_service import pairing_state_service
|
||
from ..services.admin_region_lookup_service import _build_region_path, _load_region_records
|
||
from ..utils import find_xml_file
|
||
from . import dependencies as _deps
|
||
from .dependencies import _get_current_user
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
def _safe_int(value: Any) -> int:
|
||
try:
|
||
return int(value or 0)
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def _safe_float(value: Any) -> Optional[float]:
|
||
try:
|
||
if value is None:
|
||
return None
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _ratio(numerator: int, denominator: int) -> float:
|
||
if denominator <= 0:
|
||
return 0.0
|
||
return round(float(numerator) / float(denominator), 4)
|
||
|
||
|
||
def _family_label(value: Any) -> str:
|
||
text = str(value or "").strip().upper()
|
||
if text in {"LT1", "LT-1", "LUTAN", "LUTAN1"}:
|
||
return "LT-1"
|
||
if text in {"S1", "SENTINEL1", "SENTINEL-1"}:
|
||
return "Sentinel-1"
|
||
if text in {"GF3", "GAOFEN3", "GAOFEN-3"}:
|
||
return "GF3"
|
||
return text or "未分类"
|
||
|
||
|
||
def _status_label(value: Any) -> str:
|
||
return str(value or "UNKNOWN").strip().upper() or "UNKNOWN"
|
||
|
||
|
||
def _month_from_yyyymmdd(value: Any) -> Optional[str]:
|
||
text = str(value or "").strip()
|
||
if len(text) >= 6 and text[:6].isdigit():
|
||
return f"{text[:4]}-{text[4:6]}"
|
||
return None
|
||
|
||
|
||
def _month_from_datetime(value: Any) -> Optional[str]:
|
||
if not value:
|
||
return None
|
||
try:
|
||
return value.strftime("%Y-%m")
|
||
except AttributeError:
|
||
text = str(value)
|
||
if len(text) >= 7:
|
||
return text[:7]
|
||
return None
|
||
|
||
|
||
def _percent_text(value: float) -> str:
|
||
return f"{round(value * 100, 1)}%"
|
||
|
||
|
||
def _point_bbox(lon: Any, lat: Any) -> Optional[tuple[float, float, float, float]]:
|
||
lon_value = _safe_float(lon)
|
||
lat_value = _safe_float(lat)
|
||
if lon_value is None or lat_value is None:
|
||
return None
|
||
return (lon_value, lat_value, lon_value, lat_value)
|
||
|
||
|
||
def _polygon_points(value: Any) -> list[tuple[float, float]]:
|
||
if not value:
|
||
return []
|
||
if isinstance(value, str):
|
||
try:
|
||
value = json.loads(value)
|
||
except Exception:
|
||
return []
|
||
if isinstance(value, dict):
|
||
coordinates = value.get("coordinates")
|
||
if value.get("type") == "Feature":
|
||
return _polygon_points(value.get("geometry"))
|
||
if value.get("type") == "Polygon" and coordinates:
|
||
value = coordinates[0] if coordinates else []
|
||
elif value.get("type") == "MultiPolygon" and coordinates:
|
||
value = coordinates[0][0] if coordinates and coordinates[0] else []
|
||
else:
|
||
return []
|
||
points: list[tuple[float, float]] = []
|
||
if isinstance(value, list):
|
||
for item in value:
|
||
if isinstance(item, dict):
|
||
lon = _safe_float(item.get("lon", item.get("longitude")))
|
||
lat = _safe_float(item.get("lat", item.get("latitude")))
|
||
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
||
lon = _safe_float(item[0])
|
||
lat = _safe_float(item[1])
|
||
else:
|
||
continue
|
||
if lon is not None and lat is not None:
|
||
points.append((lon, lat))
|
||
return points
|
||
|
||
|
||
def _bbox_from_polygon_or_values(
|
||
polygon_value: Any,
|
||
min_lon: Any = None,
|
||
min_lat: Any = None,
|
||
max_lon: Any = None,
|
||
max_lat: Any = None,
|
||
) -> Optional[tuple[float, float, float, float]]:
|
||
values = [_safe_float(min_lon), _safe_float(min_lat), _safe_float(max_lon), _safe_float(max_lat)]
|
||
if all(value is not None for value in values):
|
||
left, bottom, right, top = values
|
||
if left > right:
|
||
left, right = right, left
|
||
if bottom > top:
|
||
bottom, top = top, bottom
|
||
return (left, bottom, right, top)
|
||
points = _polygon_points(polygon_value)
|
||
if not points:
|
||
return None
|
||
lons = [point[0] for point in points]
|
||
lats = [point[1] for point in points]
|
||
return (min(lons), min(lats), max(lons), max(lats))
|
||
|
||
|
||
def _shape_from_polygon_or_bbox(polygon_value: Any, bbox_value: tuple[float, float, float, float]):
|
||
if Polygon is not None and shape is not None:
|
||
if polygon_value:
|
||
try:
|
||
if isinstance(polygon_value, str):
|
||
polygon_value = json.loads(polygon_value)
|
||
if isinstance(polygon_value, dict):
|
||
geom = shape(polygon_value.get("geometry") if polygon_value.get("type") == "Feature" else polygon_value)
|
||
if not geom.is_empty:
|
||
return geom
|
||
points = _polygon_points(polygon_value)
|
||
if len(points) >= 3:
|
||
geom = Polygon(points)
|
||
if geom.is_valid and not geom.is_empty:
|
||
return geom
|
||
except Exception:
|
||
pass
|
||
if box is not None:
|
||
left, bottom, right, top = bbox_value
|
||
if left != right and bottom != top:
|
||
return box(left, bottom, right, top)
|
||
return None
|
||
|
||
|
||
def _build_heatmap_grid(items: list[dict[str, Any]], *, columns: int = 48) -> dict[str, Any]:
|
||
valid_items = [
|
||
item for item in items
|
||
if item.get("bbox") is not None
|
||
]
|
||
if not valid_items:
|
||
return {
|
||
"total": len(items),
|
||
"covered_count": 0,
|
||
"cell_count": 0,
|
||
"max_count": 0,
|
||
"extent": {"min_lon": None, "min_lat": None, "max_lon": None, "max_lat": None},
|
||
"cells": [],
|
||
}
|
||
|
||
min_lon = min(item["bbox"][0] for item in valid_items)
|
||
min_lat = min(item["bbox"][1] for item in valid_items)
|
||
max_lon = max(item["bbox"][2] for item in valid_items)
|
||
max_lat = max(item["bbox"][3] for item in valid_items)
|
||
lon_span = max(max_lon - min_lon, 0.01)
|
||
lat_span = max(max_lat - min_lat, 0.01)
|
||
rows = max(16, min(40, round((columns * lat_span) / lon_span)))
|
||
cell_lon = lon_span / columns
|
||
cell_lat = lat_span / rows
|
||
buckets: dict[tuple[int, int], dict[str, Any]] = {}
|
||
|
||
def add_to_bucket(col: int, row: int, item: dict[str, Any]) -> None:
|
||
key = (col, row)
|
||
bucket = buckets.setdefault(
|
||
key,
|
||
{
|
||
"col": col,
|
||
"row": row,
|
||
"count": 0,
|
||
"families": {},
|
||
"catalogs": {},
|
||
"examples": [],
|
||
},
|
||
)
|
||
bucket["count"] += 1
|
||
family = str(item.get("family") or "").strip()
|
||
catalog = str(item.get("catalog") or "").strip()
|
||
if family:
|
||
bucket["families"][family] = bucket["families"].get(family, 0) + 1
|
||
if catalog:
|
||
bucket["catalogs"][catalog] = bucket["catalogs"].get(catalog, 0) + 1
|
||
if len(bucket["examples"]) < 4:
|
||
bucket["examples"].append({
|
||
"label": item.get("label"),
|
||
"family": family or None,
|
||
"catalog": catalog or None,
|
||
"date": item.get("date"),
|
||
})
|
||
|
||
for item in valid_items:
|
||
left, bottom, right, top = item["bbox"]
|
||
col_start = max(0, min(columns - 1, int((left - min_lon) / cell_lon)))
|
||
col_end = max(0, min(columns - 1, int((right - min_lon) / cell_lon)))
|
||
row_start = max(0, min(rows - 1, int((bottom - min_lat) / cell_lat)))
|
||
row_end = max(0, min(rows - 1, int((top - min_lat) / cell_lat)))
|
||
geom = _shape_from_polygon_or_bbox(item.get("polygon"), item["bbox"])
|
||
for col in range(col_start, col_end + 1):
|
||
for row in range(row_start, row_end + 1):
|
||
if geom is not None and box is not None:
|
||
cell = box(
|
||
min_lon + col * cell_lon,
|
||
min_lat + row * cell_lat,
|
||
min_lon + (col + 1) * cell_lon,
|
||
min_lat + (row + 1) * cell_lat,
|
||
)
|
||
try:
|
||
if not geom.intersects(cell):
|
||
continue
|
||
except Exception:
|
||
pass
|
||
add_to_bucket(col, row, item)
|
||
|
||
cells = []
|
||
for bucket in buckets.values():
|
||
col = bucket["col"]
|
||
row = bucket["row"]
|
||
dominant_family = sorted(bucket["families"].items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if bucket["families"] else None
|
||
dominant_catalog = sorted(bucket["catalogs"].items(), key=lambda kv: (-kv[1], kv[0]))[0][0] if bucket["catalogs"] else None
|
||
cells.append(
|
||
{
|
||
"col": col,
|
||
"row": row,
|
||
"count": bucket["count"],
|
||
"lon_min": round(min_lon + col * cell_lon, 6),
|
||
"lon_max": round(min_lon + (col + 1) * cell_lon, 6),
|
||
"lat_min": round(min_lat + row * cell_lat, 6),
|
||
"lat_max": round(min_lat + (row + 1) * cell_lat, 6),
|
||
"lon": round(min_lon + (col + 0.5) * cell_lon, 6),
|
||
"lat": round(min_lat + (row + 0.5) * cell_lat, 6),
|
||
"dominant_family": dominant_family,
|
||
"dominant_catalog": dominant_catalog,
|
||
"families": [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(bucket["families"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
],
|
||
"catalogs": [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(bucket["catalogs"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
],
|
||
"examples": bucket["examples"],
|
||
}
|
||
)
|
||
cells.sort(key=lambda item: (-item["count"], item["row"], item["col"]))
|
||
|
||
return {
|
||
"total": len(items),
|
||
"covered_count": len(valid_items),
|
||
"cell_count": len(cells),
|
||
"max_count": max((cell["count"] for cell in cells), default=0),
|
||
"columns": columns,
|
||
"rows": rows,
|
||
"extent": {
|
||
"min_lon": round(min_lon, 6),
|
||
"min_lat": round(min_lat, 6),
|
||
"max_lon": round(max_lon, 6),
|
||
"max_lat": round(max_lat, 6),
|
||
},
|
||
"cells": cells,
|
||
}
|
||
|
||
|
||
def _build_region_match_candidates(records: list[Any]) -> list[tuple[Any, tuple[float, float, float, float]]]:
|
||
candidates = []
|
||
for record in records:
|
||
try:
|
||
bounds = tuple(float(value) for value in record.geometry.bounds)
|
||
except Exception:
|
||
continue
|
||
if len(bounds) == 4:
|
||
candidates.append((record, bounds))
|
||
return candidates
|
||
|
||
|
||
def _match_city_region(
|
||
lon: Any,
|
||
lat: Any,
|
||
region_candidates: list[tuple[Any, tuple[float, float, float, float]]],
|
||
region_by_id: dict[str, dict[str, Any]],
|
||
) -> dict[str, Any] | None:
|
||
lon_value = _safe_float(lon)
|
||
lat_value = _safe_float(lat)
|
||
if lon_value is None or lat_value is None:
|
||
return None
|
||
try:
|
||
point = Point(lon_value, lat_value)
|
||
except Exception:
|
||
return None
|
||
|
||
matched = None
|
||
for record, bounds in region_candidates:
|
||
min_lon, min_lat, max_lon, max_lat = bounds
|
||
if lon_value < min_lon or lon_value > max_lon or lat_value < min_lat or lat_value > max_lat:
|
||
continue
|
||
try:
|
||
if record.geometry.covers(point):
|
||
matched = record
|
||
break
|
||
except Exception:
|
||
continue
|
||
if matched is None:
|
||
return None
|
||
|
||
path_names, path_tree_ids = _build_region_path(matched.tree_id, region_by_id)
|
||
city_tree_id = None
|
||
city_name = None
|
||
province_name = None
|
||
for tree_id, name in zip(path_tree_ids, path_names):
|
||
node_level = str((region_by_id.get(tree_id) or {}).get("level") or "").strip().lower()
|
||
if node_level == "province":
|
||
province_name = name
|
||
if node_level == "city":
|
||
city_tree_id = tree_id
|
||
city_name = name
|
||
break
|
||
|
||
if not city_tree_id:
|
||
level = str(getattr(matched, "level", "") or "").lower()
|
||
if level == "city":
|
||
city_tree_id = matched.tree_id
|
||
city_name = matched.name
|
||
else:
|
||
parts = str(matched.tree_id).split("-")
|
||
if len(parts) >= 3:
|
||
city_tree_id = "-".join(parts[:3])
|
||
city_name = (region_by_id.get(city_tree_id) or {}).get("name") or matched.name
|
||
|
||
if not city_tree_id:
|
||
return None
|
||
|
||
return {
|
||
"tree_id": city_tree_id,
|
||
"name": str(city_name or city_tree_id),
|
||
"province": province_name,
|
||
"matched_tree_id": matched.tree_id,
|
||
}
|
||
|
||
|
||
def _echarts_map_geometry(geometry: Any) -> dict[str, Any] | None:
|
||
if mapping is None or geometry is None or getattr(geometry, "is_empty", True):
|
||
return None
|
||
try:
|
||
simplified = geometry.simplify(0.015, preserve_topology=True)
|
||
if simplified is not None and not simplified.is_empty:
|
||
geometry = simplified
|
||
except Exception:
|
||
pass
|
||
try:
|
||
geometry_json = mapping(geometry)
|
||
except Exception:
|
||
return None
|
||
if geometry_json.get("type") not in {"Polygon", "MultiPolygon"}:
|
||
return None
|
||
coordinates = geometry_json.get("coordinates")
|
||
if not coordinates:
|
||
return None
|
||
return geometry_json
|
||
|
||
|
||
def _build_city_region_coverage(
|
||
source_points: list[dict[str, Any]],
|
||
result_points: list[dict[str, Any]],
|
||
) -> dict[str, Any]:
|
||
records, region_by_id, error = _load_region_records()
|
||
if error:
|
||
return {
|
||
"status": "unavailable",
|
||
"message": error,
|
||
"features": {"type": "FeatureCollection", "features": []},
|
||
"source": {"total": len(source_points), "matched_count": 0, "max_count": 0, "regions": []},
|
||
"results": {"total": len(result_points), "matched_count": 0, "max_count": 0, "regions": []},
|
||
}
|
||
|
||
by_tree: dict[str, dict[str, Any]] = {}
|
||
region_candidates = _build_region_match_candidates(records)
|
||
|
||
def ensure_bucket(region: dict[str, Any]) -> dict[str, Any]:
|
||
tree_id = region["tree_id"]
|
||
return by_tree.setdefault(
|
||
tree_id,
|
||
{
|
||
"tree_id": tree_id,
|
||
"name": region.get("name") or tree_id,
|
||
"province": region.get("province"),
|
||
"source_count": 0,
|
||
"result_count": 0,
|
||
"families": {},
|
||
"catalogs": {},
|
||
},
|
||
)
|
||
|
||
for item in source_points:
|
||
region = _match_city_region(item.get("lon"), item.get("lat"), region_candidates, region_by_id)
|
||
if not region:
|
||
continue
|
||
bucket = ensure_bucket(region)
|
||
bucket["source_count"] += 1
|
||
family = str(item.get("family") or "").strip()
|
||
if family:
|
||
bucket["families"][family] = bucket["families"].get(family, 0) + 1
|
||
|
||
for item in result_points:
|
||
region = _match_city_region(item.get("lon"), item.get("lat"), region_candidates, region_by_id)
|
||
if not region:
|
||
continue
|
||
bucket = ensure_bucket(region)
|
||
bucket["result_count"] += 1
|
||
catalog = str(item.get("catalog") or "").strip()
|
||
if catalog:
|
||
bucket["catalogs"][catalog] = bucket["catalogs"].get(catalog, 0) + 1
|
||
|
||
city_records = {record.tree_id: record for record in records if str(record.level or "").lower() == "city"}
|
||
features = []
|
||
for tree_id, bucket in by_tree.items():
|
||
record = city_records.get(tree_id)
|
||
if record is None:
|
||
continue
|
||
geometry_json = _echarts_map_geometry(record.geometry)
|
||
if geometry_json is None:
|
||
continue
|
||
features.append(
|
||
{
|
||
"type": "Feature",
|
||
"properties": {
|
||
"tree_id": tree_id,
|
||
"name": bucket["name"],
|
||
"province": bucket.get("province"),
|
||
"source_count": bucket["source_count"],
|
||
"result_count": bucket["result_count"],
|
||
},
|
||
"geometry": geometry_json,
|
||
}
|
||
)
|
||
try:
|
||
point = record.geometry.representative_point()
|
||
bucket["center_lon"] = float(point.x)
|
||
bucket["center_lat"] = float(point.y)
|
||
except Exception:
|
||
pass
|
||
|
||
def rows_for(kind: str) -> list[dict[str, Any]]:
|
||
count_key = "source_count" if kind == "source" else "result_count"
|
||
detail_key = "families" if kind == "source" else "catalogs"
|
||
return [
|
||
{
|
||
"tree_id": bucket["tree_id"],
|
||
"name": bucket["name"],
|
||
"province": bucket.get("province"),
|
||
"count": bucket[count_key],
|
||
"lon": bucket.get("center_lon"),
|
||
"lat": bucket.get("center_lat"),
|
||
"breakdown": [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(bucket[detail_key].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
],
|
||
}
|
||
for bucket in sorted(by_tree.values(), key=lambda item: (-item[count_key], item["name"]))
|
||
if bucket[count_key] > 0
|
||
]
|
||
|
||
source_rows = rows_for("source")
|
||
result_rows = rows_for("results")
|
||
return {
|
||
"status": "ok",
|
||
"features": {"type": "FeatureCollection", "features": features},
|
||
"source": {
|
||
"total": len(source_points),
|
||
"matched_count": sum(item["count"] for item in source_rows),
|
||
"max_count": max((item["count"] for item in source_rows), default=0),
|
||
"regions": source_rows,
|
||
},
|
||
"results": {
|
||
"total": len(result_points),
|
||
"matched_count": sum(item["count"] for item in result_rows),
|
||
"max_count": max((item["count"] for item in result_rows), default=0),
|
||
"regions": result_rows,
|
||
},
|
||
}
|
||
|
||
|
||
async def _scalar_count(db: AsyncSession, stmt) -> int:
|
||
result = await db.execute(stmt)
|
||
return _safe_int(result.scalar_one())
|
||
|
||
|
||
@router.get("/statistics/dashboard")
|
||
async def get_statistics_dashboard(
|
||
fresh: bool = False,
|
||
current_user: AuthUserORM = Depends(_get_current_user),
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""
|
||
Business-facing dashboard statistics for the production overview page.
|
||
|
||
This endpoint keeps the leadership/statistics dashboard separate from the
|
||
legacy /statistics health-consistency payload.
|
||
"""
|
||
if fresh and current_user.role != ROLE_ADMIN:
|
||
raise HTTPException(status_code=403, detail="Only admin can force refresh dashboard statistics.")
|
||
|
||
now_mono = time.monotonic()
|
||
if _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS > 0 and not fresh:
|
||
async with _deps._DASHBOARD_STATS_CACHE_LOCK:
|
||
if (
|
||
_deps._DASHBOARD_STATS_CACHE_DATA is not None
|
||
and now_mono < _deps._DASHBOARD_STATS_CACHE_EXPIRES_AT
|
||
):
|
||
return {
|
||
**_deps._DASHBOARD_STATS_CACHE_DATA,
|
||
"cache_meta": {
|
||
"enabled": True,
|
||
"hit": True,
|
||
"ttl_seconds": _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS,
|
||
"generated_at": _deps._DASHBOARD_STATS_CACHE_GENERATED_AT_UTC,
|
||
},
|
||
}
|
||
|
||
generated_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||
|
||
source_total = await _scalar_count(
|
||
db,
|
||
select(func.count(SourceProductAssetORM.id)).where(SourceProductAssetORM.is_active == True),
|
||
)
|
||
radar_total = await _scalar_count(db, select(func.count(RadarDataORM.id)))
|
||
metadata_asset_total = await _scalar_count(
|
||
db,
|
||
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))),
|
||
)
|
||
metadata_doc_total = await _scalar_count(db, select(func.count(SourceMetadataDocumentORM.id)))
|
||
geometry_total = await _scalar_count(db, select(func.count(SARSceneGeometryProfileORM.id)))
|
||
geometry_ready = await _scalar_count(
|
||
db,
|
||
select(func.count(SARSceneGeometryProfileORM.id)).where(
|
||
SARSceneGeometryProfileORM.metadata_quality == "READY",
|
||
SARSceneGeometryProfileORM.production_readiness == "READY",
|
||
),
|
||
)
|
||
preview_ready = await _scalar_count(
|
||
db,
|
||
select(func.count(RadarDataORM.id)).where(RadarDataORM.preview_cache_status == "READY"),
|
||
)
|
||
|
||
source_group_rows = await db.execute(
|
||
select(
|
||
SourceProductAssetORM.satellite_family,
|
||
SourceProductAssetORM.source_format,
|
||
SourceProductAssetORM.parse_status,
|
||
func.count(SourceProductAssetORM.id),
|
||
)
|
||
.where(SourceProductAssetORM.is_active == True)
|
||
.group_by(
|
||
SourceProductAssetORM.satellite_family,
|
||
SourceProductAssetORM.source_format,
|
||
SourceProductAssetORM.parse_status,
|
||
)
|
||
.order_by(SourceProductAssetORM.satellite_family, SourceProductAssetORM.source_format)
|
||
)
|
||
source_by_family_map: dict[str, dict[str, Any]] = {}
|
||
source_by_format: list[dict[str, Any]] = []
|
||
for family, source_format, parse_status, count in source_group_rows.all():
|
||
family_label = _family_label(family)
|
||
status_label = _status_label(parse_status)
|
||
count_int = _safe_int(count)
|
||
family_bucket = source_by_family_map.setdefault(
|
||
family_label,
|
||
{
|
||
"family": family_label,
|
||
"count": 0,
|
||
"ready_count": 0,
|
||
"issue_count": 0,
|
||
"formats": {},
|
||
},
|
||
)
|
||
family_bucket["count"] += count_int
|
||
if status_label in {"OK", "READY", "NATIVE_READY"}:
|
||
family_bucket["ready_count"] += count_int
|
||
else:
|
||
family_bucket["issue_count"] += count_int
|
||
format_label = str(source_format or "UNKNOWN")
|
||
family_bucket["formats"][format_label] = family_bucket["formats"].get(format_label, 0) + count_int
|
||
source_by_format.append(
|
||
{
|
||
"family": family_label,
|
||
"source_format": format_label,
|
||
"parse_status": status_label,
|
||
"count": count_int,
|
||
}
|
||
)
|
||
|
||
source_by_family = []
|
||
for item in source_by_family_map.values():
|
||
item["ready_rate"] = _ratio(item["ready_count"], item["count"])
|
||
item["formats"] = [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(item["formats"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
]
|
||
source_by_family.append(item)
|
||
source_by_family.sort(key=lambda row: (-row["count"], row["family"]))
|
||
|
||
geometry_rows = await db.execute(
|
||
select(
|
||
SARSceneGeometryProfileORM.satellite_family,
|
||
SARSceneGeometryProfileORM.metadata_quality,
|
||
SARSceneGeometryProfileORM.production_readiness,
|
||
func.count(SARSceneGeometryProfileORM.id),
|
||
)
|
||
.group_by(
|
||
SARSceneGeometryProfileORM.satellite_family,
|
||
SARSceneGeometryProfileORM.metadata_quality,
|
||
SARSceneGeometryProfileORM.production_readiness,
|
||
)
|
||
.order_by(SARSceneGeometryProfileORM.satellite_family)
|
||
)
|
||
geometry_by_family_map: dict[str, dict[str, Any]] = {}
|
||
for family, metadata_quality, production_readiness, count in geometry_rows.all():
|
||
family_label = _family_label(family)
|
||
count_int = _safe_int(count)
|
||
bucket = geometry_by_family_map.setdefault(
|
||
family_label,
|
||
{"family": family_label, "count": 0, "ready_count": 0, "issue_count": 0, "statuses": {}},
|
||
)
|
||
bucket["count"] += count_int
|
||
key = f"{_status_label(metadata_quality)} / {_status_label(production_readiness)}"
|
||
bucket["statuses"][key] = bucket["statuses"].get(key, 0) + count_int
|
||
if _status_label(metadata_quality) == "READY" and _status_label(production_readiness) == "READY":
|
||
bucket["ready_count"] += count_int
|
||
else:
|
||
bucket["issue_count"] += count_int
|
||
geometry_by_family = []
|
||
for item in geometry_by_family_map.values():
|
||
item["ready_rate"] = _ratio(item["ready_count"], item["count"])
|
||
item["statuses"] = [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(item["statuses"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
]
|
||
geometry_by_family.append(item)
|
||
geometry_by_family.sort(key=lambda row: (-row["count"], row["family"]))
|
||
|
||
source_month_rows = await db.execute(
|
||
select(SourceProductAssetORM.imaging_date, SourceProductAssetORM.satellite_family)
|
||
.where(SourceProductAssetORM.is_active == True)
|
||
.where(SourceProductAssetORM.imaging_date.isnot(None))
|
||
)
|
||
source_month_map: dict[str, dict[str, Any]] = {}
|
||
for imaging_date, family in source_month_rows.all():
|
||
month = _month_from_yyyymmdd(imaging_date)
|
||
if not month:
|
||
continue
|
||
family_label = _family_label(family)
|
||
bucket = source_month_map.setdefault(month, {"month": month, "total": 0, "by_family": {}})
|
||
bucket["total"] += 1
|
||
bucket["by_family"][family_label] = bucket["by_family"].get(family_label, 0) + 1
|
||
source_by_month = [source_month_map[key] for key in sorted(source_month_map)]
|
||
|
||
orbit_total = await _scalar_count(
|
||
db,
|
||
select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.is_active == True),
|
||
)
|
||
orbit_group_rows = await db.execute(
|
||
select(OrbitAssetORM.satellite_family, OrbitAssetORM.parse_status, func.count(OrbitAssetORM.id))
|
||
.where(OrbitAssetORM.is_active == True)
|
||
.group_by(OrbitAssetORM.satellite_family, OrbitAssetORM.parse_status)
|
||
.order_by(OrbitAssetORM.satellite_family)
|
||
)
|
||
orbit_by_family_map: dict[str, dict[str, Any]] = {}
|
||
for family, parse_status, count in orbit_group_rows.all():
|
||
family_label = _family_label(family)
|
||
status_label = _status_label(parse_status)
|
||
count_int = _safe_int(count)
|
||
bucket = orbit_by_family_map.setdefault(
|
||
family_label,
|
||
{"family": family_label, "count": 0, "ok_count": 0, "issue_count": 0, "statuses": {}},
|
||
)
|
||
bucket["count"] += count_int
|
||
bucket["statuses"][status_label] = bucket["statuses"].get(status_label, 0) + count_int
|
||
if status_label == "OK":
|
||
bucket["ok_count"] += count_int
|
||
else:
|
||
bucket["issue_count"] += count_int
|
||
orbit_by_family = []
|
||
for item in orbit_by_family_map.values():
|
||
item["ok_rate"] = _ratio(item["ok_count"], item["count"])
|
||
item["statuses"] = [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(item["statuses"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
]
|
||
orbit_by_family.append(item)
|
||
orbit_by_family.sort(key=lambda row: (-row["count"], row["family"]))
|
||
|
||
orbit_required_total = sum(
|
||
item["count"]
|
||
for item in source_by_family
|
||
if item["family"] in {"LT-1", "Sentinel-1"}
|
||
)
|
||
selected_orbit_bindings = await _scalar_count(
|
||
db,
|
||
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED"),
|
||
)
|
||
matched_orbit_bindings = await _scalar_count(
|
||
db,
|
||
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.match_status == "MATCHED"),
|
||
)
|
||
|
||
empty_legacy_coverage_grid = {
|
||
"total": 0,
|
||
"covered_count": 0,
|
||
"cell_count": 0,
|
||
"max_count": 0,
|
||
"extent": {"min_lon": None, "min_lat": None, "max_lon": None, "max_lat": None},
|
||
"cells": [],
|
||
}
|
||
|
||
coverage_rows = await db.execute(
|
||
select(
|
||
SARSceneGeometryProfileORM.id,
|
||
SARSceneGeometryProfileORM.satellite_family,
|
||
SARSceneGeometryProfileORM.acquisition_start_time_utc,
|
||
SARSceneGeometryProfileORM.scene_center_lon,
|
||
SARSceneGeometryProfileORM.scene_center_lat,
|
||
)
|
||
.where(
|
||
SARSceneGeometryProfileORM.scene_center_lon.isnot(None),
|
||
SARSceneGeometryProfileORM.scene_center_lat.isnot(None),
|
||
)
|
||
.order_by(SARSceneGeometryProfileORM.acquisition_start_time_utc.desc().nullslast())
|
||
)
|
||
source_region_points: list[dict[str, Any]] = []
|
||
for (
|
||
row_id,
|
||
family,
|
||
acquisition_start,
|
||
lon,
|
||
lat,
|
||
) in coverage_rows.all():
|
||
family_label = _family_label(family)
|
||
lon_float = _safe_float(lon)
|
||
lat_float = _safe_float(lat)
|
||
if lon_float is not None and lat_float is not None:
|
||
source_region_points.append(
|
||
{
|
||
"id": row_id,
|
||
"family": family_label,
|
||
"lon": lon_float,
|
||
"lat": lat_float,
|
||
"date": acquisition_start.date().isoformat() if acquisition_start else None,
|
||
}
|
||
)
|
||
source_coverage_grid = {**empty_legacy_coverage_grid, "total": len(source_region_points), "covered_count": len(source_region_points)}
|
||
|
||
result_total = await _scalar_count(db, select(func.count(ResultProductORM.id)))
|
||
result_rows = await db.execute(
|
||
select(
|
||
ResultProductORM.catalog_name,
|
||
ResultProductORM.status,
|
||
ResultProductORM.health_status,
|
||
func.count(ResultProductORM.id),
|
||
)
|
||
.group_by(ResultProductORM.catalog_name, ResultProductORM.status, ResultProductORM.health_status)
|
||
.order_by(ResultProductORM.catalog_name)
|
||
)
|
||
results_by_catalog_map: dict[str, dict[str, Any]] = {}
|
||
for catalog_name, status, health_status, count in result_rows.all():
|
||
catalog = str(catalog_name or "unknown")
|
||
count_int = _safe_int(count)
|
||
bucket = results_by_catalog_map.setdefault(
|
||
catalog,
|
||
{"catalog": catalog, "count": 0, "ready_count": 0, "issue_count": 0, "statuses": {}, "health": {}},
|
||
)
|
||
bucket["count"] += count_int
|
||
status_label = _status_label(status)
|
||
health_label = _status_label(health_status)
|
||
bucket["statuses"][status_label] = bucket["statuses"].get(status_label, 0) + count_int
|
||
bucket["health"][health_label] = bucket["health"].get(health_label, 0) + count_int
|
||
if status_label == "READY" and health_label == "OK":
|
||
bucket["ready_count"] += count_int
|
||
else:
|
||
bucket["issue_count"] += count_int
|
||
results_by_catalog = []
|
||
for item in results_by_catalog_map.values():
|
||
item["ready_rate"] = _ratio(item["ready_count"], item["count"])
|
||
item["statuses"] = [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(item["statuses"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
]
|
||
item["health"] = [
|
||
{"name": name, "count": count}
|
||
for name, count in sorted(item["health"].items(), key=lambda kv: (-kv[1], kv[0]))
|
||
]
|
||
results_by_catalog.append(item)
|
||
results_by_catalog.sort(key=lambda row: (-row["count"], row["catalog"]))
|
||
|
||
result_assets_total = await _scalar_count(db, select(func.count(ResultAssetORM.id)))
|
||
result_assets_missing = await _scalar_count(
|
||
db,
|
||
select(func.count(ResultAssetORM.id)).where(ResultAssetORM.exists_flag == False),
|
||
)
|
||
result_preview_count = await _scalar_count(
|
||
db,
|
||
select(func.count(ResultProductORM.id)).where(ResultProductORM.preview_path.isnot(None)),
|
||
)
|
||
|
||
result_month_rows = await db.execute(
|
||
select(
|
||
ResultProductORM.catalog_name,
|
||
ResultProductORM.published_at,
|
||
ResultProductORM.produced_at,
|
||
ResultProductORM.registered_at,
|
||
)
|
||
)
|
||
result_month_map: dict[str, dict[str, Any]] = {}
|
||
for catalog, published_at, produced_at, registered_at in result_month_rows.all():
|
||
month = _month_from_datetime(published_at or produced_at or registered_at)
|
||
if not month:
|
||
continue
|
||
bucket = result_month_map.setdefault(month, {"month": month, "total": 0, "by_catalog": {}})
|
||
bucket["total"] += 1
|
||
catalog_label = str(catalog or "unknown")
|
||
bucket["by_catalog"][catalog_label] = bucket["by_catalog"].get(catalog_label, 0) + 1
|
||
results_by_month = [result_month_map[key] for key in sorted(result_month_map)]
|
||
|
||
result_coverage_rows = await db.execute(
|
||
select(
|
||
ResultProductORM.id,
|
||
ResultProductORM.catalog_name,
|
||
ResultProductORM.product_type,
|
||
ResultProductORM.produced_at,
|
||
ResultProductORM.published_at,
|
||
ResultProductORM.registered_at,
|
||
ResultProductORM.min_lon,
|
||
ResultProductORM.min_lat,
|
||
ResultProductORM.max_lon,
|
||
ResultProductORM.max_lat,
|
||
ResultProductORM.coverage_polygon,
|
||
)
|
||
.where(
|
||
(ResultProductORM.coverage_polygon.isnot(None))
|
||
| (
|
||
ResultProductORM.min_lon.isnot(None)
|
||
& ResultProductORM.min_lat.isnot(None)
|
||
& ResultProductORM.max_lon.isnot(None)
|
||
& ResultProductORM.max_lat.isnot(None)
|
||
)
|
||
)
|
||
)
|
||
result_region_points: list[dict[str, Any]] = []
|
||
for (
|
||
product_id,
|
||
catalog_name,
|
||
product_type,
|
||
produced_at,
|
||
published_at,
|
||
registered_at,
|
||
min_lon_value,
|
||
min_lat_value,
|
||
max_lon_value,
|
||
max_lat_value,
|
||
coverage_polygon,
|
||
) in result_coverage_rows.all():
|
||
bbox = _bbox_from_polygon_or_values(
|
||
coverage_polygon,
|
||
min_lon_value,
|
||
min_lat_value,
|
||
max_lon_value,
|
||
max_lat_value,
|
||
)
|
||
center_lon = center_lat = None
|
||
if bbox:
|
||
center_lon = (bbox[0] + bbox[2]) / 2
|
||
center_lat = (bbox[1] + bbox[3]) / 2
|
||
result_region_points.append(
|
||
{
|
||
"id": product_id,
|
||
"catalog": str(catalog_name or product_type or "unknown"),
|
||
"lon": center_lon,
|
||
"lat": center_lat,
|
||
"date": _month_from_datetime(published_at or produced_at or registered_at),
|
||
}
|
||
)
|
||
result_coverage_grid = {**empty_legacy_coverage_grid, "total": len(result_region_points), "covered_count": len(result_region_points)}
|
||
city_region_coverage = _build_city_region_coverage(source_region_points, result_region_points)
|
||
|
||
dinsar_batch_count = await _scalar_count(db, select(func.count(DinsarTaskBatchORM.id)))
|
||
dinsar_task_count = await _scalar_count(db, select(func.count(DinsarTaskItemORM.id)))
|
||
dinsar_task_status_rows = await db.execute(
|
||
select(DinsarTaskItemORM.status, func.count(DinsarTaskItemORM.id))
|
||
.group_by(DinsarTaskItemORM.status)
|
||
.order_by(DinsarTaskItemORM.status)
|
||
)
|
||
dinsar_task_status = [
|
||
{"status": _status_label(status), "count": _safe_int(count)}
|
||
for status, count in dinsar_task_status_rows.all()
|
||
]
|
||
|
||
production_run_rows = await db.execute(
|
||
select(
|
||
DinsarProductionRunORM.run_id,
|
||
DinsarProductionRunORM.product_family,
|
||
DinsarProductionRunORM.engine_code,
|
||
DinsarProductionRunORM.status,
|
||
DinsarProductionRunORM.total_items,
|
||
DinsarProductionRunORM.completed_items,
|
||
DinsarProductionRunORM.failed_items,
|
||
DinsarProductionRunORM.started_at,
|
||
DinsarProductionRunORM.ended_at,
|
||
DinsarProductionRunORM.created_at,
|
||
DinsarProductionRunORM.latest_message,
|
||
)
|
||
.order_by(DinsarProductionRunORM.created_at.desc().nullslast(), DinsarProductionRunORM.id.desc())
|
||
)
|
||
production_status_map: dict[str, int] = {}
|
||
production_engine_map: dict[str, dict[str, Any]] = {}
|
||
recent_production_runs: list[dict[str, Any]] = []
|
||
duration_seconds: list[float] = []
|
||
production_run_count = 0
|
||
for (
|
||
run_id,
|
||
product_family,
|
||
engine_code,
|
||
status,
|
||
total_items,
|
||
completed_items,
|
||
failed_items,
|
||
started_at,
|
||
ended_at,
|
||
created_at,
|
||
latest_message,
|
||
) in production_run_rows.all():
|
||
production_run_count += 1
|
||
status_label = _status_label(status)
|
||
production_status_map[status_label] = production_status_map.get(status_label, 0) + 1
|
||
engine_label = str(engine_code or "unknown")
|
||
engine_bucket = production_engine_map.setdefault(
|
||
engine_label,
|
||
{"engine": engine_label, "count": 0, "completed": 0, "failed": 0, "running": 0},
|
||
)
|
||
engine_bucket["count"] += 1
|
||
if status_label in {"COMPLETED", "SUCCESS", "DONE"}:
|
||
engine_bucket["completed"] += 1
|
||
elif status_label in {"FAILED", "ERROR"}:
|
||
engine_bucket["failed"] += 1
|
||
elif status_label in {"RUNNING", "PENDING", "QUEUED"}:
|
||
engine_bucket["running"] += 1
|
||
if started_at and ended_at:
|
||
try:
|
||
duration_seconds.append((ended_at - started_at).total_seconds())
|
||
except Exception:
|
||
pass
|
||
if len(recent_production_runs) < 8:
|
||
recent_production_runs.append(
|
||
{
|
||
"run_id": run_id,
|
||
"product_family": product_family,
|
||
"engine_code": engine_label,
|
||
"status": status_label,
|
||
"total_items": _safe_int(total_items),
|
||
"completed_items": _safe_int(completed_items),
|
||
"failed_items": _safe_int(failed_items),
|
||
"created_at": created_at.isoformat() if created_at else None,
|
||
"started_at": started_at.isoformat() if started_at else None,
|
||
"ended_at": ended_at.isoformat() if ended_at else None,
|
||
"latest_message": latest_message,
|
||
}
|
||
)
|
||
|
||
workflow_rows = await db.execute(
|
||
select(WorkflowRunORM.workflow_name, WorkflowRunORM.status, func.count(WorkflowRunORM.id))
|
||
.group_by(WorkflowRunORM.workflow_name, WorkflowRunORM.status)
|
||
.order_by(WorkflowRunORM.workflow_name, WorkflowRunORM.status)
|
||
)
|
||
workflow_status = [
|
||
{"workflow": str(workflow or "unknown"), "status": _status_label(status), "count": _safe_int(count)}
|
||
for workflow, status, count in workflow_rows.all()
|
||
]
|
||
|
||
result_issue_rows = await db.execute(
|
||
select(ResultIssueORM.severity, ResultIssueORM.issue_code, ResultIssueORM.status, func.count(ResultIssueORM.id))
|
||
.group_by(ResultIssueORM.severity, ResultIssueORM.issue_code, ResultIssueORM.status)
|
||
.order_by(ResultIssueORM.severity, ResultIssueORM.issue_code)
|
||
)
|
||
inventory_issue_rows = await db.execute(
|
||
select(
|
||
AssetInventoryIssueORM.severity,
|
||
AssetInventoryIssueORM.issue_code,
|
||
AssetInventoryIssueORM.status,
|
||
func.count(AssetInventoryIssueORM.id),
|
||
)
|
||
.group_by(AssetInventoryIssueORM.severity, AssetInventoryIssueORM.issue_code, AssetInventoryIssueORM.status)
|
||
.order_by(AssetInventoryIssueORM.severity, AssetInventoryIssueORM.issue_code)
|
||
)
|
||
issue_total = 0
|
||
open_issue_total = 0
|
||
issue_by_severity: dict[str, int] = {}
|
||
issue_by_code: dict[str, int] = {}
|
||
for severity, issue_code, status, count in list(result_issue_rows.all()) + list(inventory_issue_rows.all()):
|
||
count_int = _safe_int(count)
|
||
status_label = _status_label(status)
|
||
severity_label = _status_label(severity)
|
||
code_label = str(issue_code or "UNKNOWN")
|
||
issue_total += count_int
|
||
if status_label == "OPEN":
|
||
open_issue_total += count_int
|
||
issue_by_severity[severity_label] = issue_by_severity.get(severity_label, 0) + count_int
|
||
issue_by_code[code_label] = issue_by_code.get(code_label, 0) + count_int
|
||
|
||
inventory_state_rows = await db.execute(
|
||
select(
|
||
AssetInventoryStateORM.inventory_type,
|
||
AssetInventoryStateORM.status,
|
||
AssetInventoryStateORM.last_seen_entry_count,
|
||
AssetInventoryStateORM.last_asset_count,
|
||
AssetInventoryStateORM.last_issue_count,
|
||
AssetInventoryStateORM.last_scan_started_at,
|
||
AssetInventoryStateORM.last_scan_finished_at,
|
||
AssetInventoryStateORM.needs_rescan,
|
||
)
|
||
.order_by(AssetInventoryStateORM.updated_at.desc().nullslast(), AssetInventoryStateORM.id.desc())
|
||
.limit(12)
|
||
)
|
||
inventory_states = [
|
||
{
|
||
"inventory_type": inventory_type,
|
||
"status": _status_label(status),
|
||
"last_seen_entry_count": _safe_int(last_seen_entry_count),
|
||
"last_asset_count": _safe_int(last_asset_count),
|
||
"last_issue_count": _safe_int(last_issue_count),
|
||
"last_scan_started_at": last_scan_started_at.isoformat() if last_scan_started_at else None,
|
||
"last_scan_finished_at": last_scan_finished_at.isoformat() if last_scan_finished_at else None,
|
||
"needs_rescan": bool(needs_rescan),
|
||
}
|
||
for (
|
||
inventory_type,
|
||
status,
|
||
last_seen_entry_count,
|
||
last_asset_count,
|
||
last_issue_count,
|
||
last_scan_started_at,
|
||
last_scan_finished_at,
|
||
needs_rescan,
|
||
) in inventory_state_rows.all()
|
||
]
|
||
|
||
avg_duration_seconds = round(sum(duration_seconds) / len(duration_seconds), 1) if duration_seconds else None
|
||
selected_orbit_rate = _ratio(selected_orbit_bindings, orbit_required_total)
|
||
geometry_ready_rate = _ratio(geometry_ready, source_total)
|
||
metadata_ready_rate = _ratio(metadata_asset_total, source_total)
|
||
result_ready_total = sum(item["ready_count"] for item in results_by_catalog)
|
||
|
||
risk_count = (
|
||
max(0, source_total - metadata_asset_total)
|
||
+ max(0, source_total - geometry_ready)
|
||
+ max(0, orbit_required_total - selected_orbit_bindings)
|
||
+ result_assets_missing
|
||
+ open_issue_total
|
||
)
|
||
|
||
kpis = [
|
||
{
|
||
"key": "source_total",
|
||
"label": "源数据资产",
|
||
"value": source_total,
|
||
"unit": "景",
|
||
"note": f"兼容台账 {radar_total} 条",
|
||
"tone": "primary",
|
||
},
|
||
{
|
||
"key": "metadata_ready",
|
||
"label": "元数据入库率",
|
||
"value": round(metadata_ready_rate * 100, 1),
|
||
"unit": "%",
|
||
"note": f"{metadata_asset_total}/{source_total} 景已提取 XML/元数据",
|
||
"tone": "success" if metadata_ready_rate >= 0.98 else "warning",
|
||
},
|
||
{
|
||
"key": "geometry_ready",
|
||
"label": "几何画像可用率",
|
||
"value": round(geometry_ready_rate * 100, 1),
|
||
"unit": "%",
|
||
"note": f"{geometry_ready}/{source_total} 景可用于覆盖统计",
|
||
"tone": "success" if geometry_ready_rate >= 0.95 else "warning",
|
||
},
|
||
{
|
||
"key": "orbit_selected",
|
||
"label": "精轨绑定率",
|
||
"value": round(selected_orbit_rate * 100, 1),
|
||
"unit": "%",
|
||
"note": f"{selected_orbit_bindings}/{orbit_required_total} 景已选中精轨",
|
||
"tone": "success" if selected_orbit_rate >= 0.95 else "warning",
|
||
},
|
||
{
|
||
"key": "result_total",
|
||
"label": "形变成果",
|
||
"value": result_total,
|
||
"unit": "项",
|
||
"note": f"健康成果 {result_ready_total} 项，预览 {result_preview_count} 项",
|
||
"tone": "primary",
|
||
},
|
||
{
|
||
"key": "risk_total",
|
||
"label": "待关注项",
|
||
"value": risk_count,
|
||
"unit": "项",
|
||
"note": f"开放问题 {open_issue_total}，缺失成果资产 {result_assets_missing}",
|
||
"tone": "danger" if risk_count else "success",
|
||
},
|
||
]
|
||
|
||
dashboard_payload = {
|
||
"generated_at": generated_at,
|
||
"kpis": kpis,
|
||
"asset": {
|
||
"source_total": source_total,
|
||
"radar_total": radar_total,
|
||
"source_by_family": source_by_family,
|
||
"source_by_format": source_by_format,
|
||
"source_by_month": source_by_month,
|
||
"metadata_asset_total": metadata_asset_total,
|
||
"metadata_doc_total": metadata_doc_total,
|
||
"metadata_ready_rate": metadata_ready_rate,
|
||
"geometry_total": geometry_total,
|
||
"geometry_ready": geometry_ready,
|
||
"geometry_ready_rate": geometry_ready_rate,
|
||
"geometry_by_family": geometry_by_family,
|
||
"preview_ready": preview_ready,
|
||
"pipeline": [
|
||
{"key": "source", "label": "源资产登记", "value": source_total, "rate": 1.0},
|
||
{"key": "metadata", "label": "元数据入库", "value": metadata_asset_total, "rate": metadata_ready_rate},
|
||
{"key": "geometry", "label": "几何画像", "value": geometry_total, "rate": _ratio(geometry_total, source_total)},
|
||
{"key": "ready", "label": "可生产画像", "value": geometry_ready, "rate": geometry_ready_rate},
|
||
{"key": "preview", "label": "预览缓存", "value": preview_ready, "rate": _ratio(preview_ready, radar_total)},
|
||
],
|
||
},
|
||
"orbit": {
|
||
"orbit_total": orbit_total,
|
||
"orbit_by_family": orbit_by_family,
|
||
"orbit_required_total": orbit_required_total,
|
||
"selected_bindings": selected_orbit_bindings,
|
||
"matched_bindings": matched_orbit_bindings,
|
||
"selected_rate": selected_orbit_rate,
|
||
},
|
||
"coverage": {
|
||
"point_total": geometry_total,
|
||
"source": source_coverage_grid,
|
||
"results": result_coverage_grid,
|
||
"city_regions": city_region_coverage,
|
||
},
|
||
"production": {
|
||
"dinsar_batch_count": dinsar_batch_count,
|
||
"dinsar_task_count": dinsar_task_count,
|
||
"dinsar_task_status": dinsar_task_status,
|
||
"run_count": production_run_count,
|
||
"run_status": [
|
||
{"status": status, "count": count}
|
||
for status, count in sorted(production_status_map.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
],
|
||
"engine_status": sorted(production_engine_map.values(), key=lambda row: (-row["count"], row["engine"])),
|
||
"avg_duration_seconds": avg_duration_seconds,
|
||
"recent_runs": recent_production_runs,
|
||
"workflow_status": workflow_status,
|
||
},
|
||
"results": {
|
||
"result_total": result_total,
|
||
"result_ready_total": result_ready_total,
|
||
"result_preview_count": result_preview_count,
|
||
"result_assets_total": result_assets_total,
|
||
"result_assets_missing": result_assets_missing,
|
||
"results_by_catalog": results_by_catalog,
|
||
"results_by_month": results_by_month,
|
||
},
|
||
"issues": {
|
||
"issue_total": issue_total,
|
||
"open_issue_total": open_issue_total,
|
||
"by_severity": [
|
||
{"severity": severity, "count": count}
|
||
for severity, count in sorted(issue_by_severity.items(), key=lambda kv: (-kv[1], kv[0]))
|
||
],
|
||
"by_code": [
|
||
{"code": code, "count": count}
|
||
for code, count in sorted(issue_by_code.items(), key=lambda kv: (-kv[1], kv[0]))[:12]
|
||
],
|
||
},
|
||
"inventory": {
|
||
"states": inventory_states,
|
||
},
|
||
"summary": {
|
||
"metadata_ready_text": _percent_text(metadata_ready_rate),
|
||
"geometry_ready_text": _percent_text(geometry_ready_rate),
|
||
"orbit_selected_text": _percent_text(selected_orbit_rate),
|
||
},
|
||
}
|
||
|
||
if _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS > 0:
|
||
async with _deps._DASHBOARD_STATS_CACHE_LOCK:
|
||
_deps._DASHBOARD_STATS_CACHE_DATA = dashboard_payload
|
||
_deps._DASHBOARD_STATS_CACHE_EXPIRES_AT = time.monotonic() + _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS
|
||
_deps._DASHBOARD_STATS_CACHE_GENERATED_AT_UTC = generated_at
|
||
|
||
return {
|
||
**dashboard_payload,
|
||
"cache_meta": {
|
||
"enabled": _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS > 0,
|
||
"hit": False,
|
||
"ttl_seconds": _deps.DASHBOARD_STATS_CACHE_TTL_SECONDS,
|
||
"generated_at": generated_at,
|
||
},
|
||
}
|
||
|
||
|
||
@router.get("/statistics")
|
||
async def get_statistics(
|
||
fresh: bool = False,
|
||
current_user: AuthUserORM = Depends(_get_current_user),
|
||
db: AsyncSession = Depends(get_db),
|
||
):
|
||
"""
|
||
获取关于Dinsar结果和源数据的统计信息。
|
||
"""
|
||
if fresh and current_user.role != ROLE_ADMIN:
|
||
raise HTTPException(status_code=403, detail="Only admin can force refresh statistics.")
|
||
|
||
now_mono = time.monotonic()
|
||
if _deps.STATS_CACHE_TTL_SECONDS > 0 and not fresh:
|
||
async with _deps._STATS_CACHE_LOCK:
|
||
if _deps._STATS_CACHE_DATA is not None and now_mono < _deps._STATS_CACHE_EXPIRES_AT:
|
||
return {
|
||
**_deps._STATS_CACHE_DATA,
|
||
"cache_meta": {
|
||
"enabled": True,
|
||
"hit": True,
|
||
"ttl_seconds": _deps.STATS_CACHE_TTL_SECONDS,
|
||
"generated_at": _deps._STATS_CACHE_GENERATED_AT_UTC,
|
||
},
|
||
}
|
||
|
||
# 1. D-InSAR 结果统计（catalog 主读模型）
|
||
dinsar_records = await dinsar_read_service.list_catalog_records(db)
|
||
dinsar_total_count = len(dinsar_records)
|
||
dinsar_cache_consistency = {
|
||
"db_marked_cached_count": 0,
|
||
"cache_file_exists_count": 0,
|
||
"db_cached_and_file_exists_count": 0,
|
||
"db_cached_but_file_missing_count": 0,
|
||
"db_uncached_but_file_exists_count": 0,
|
||
"db_uncached_and_file_missing_count": 0,
|
||
"manifest_entries_count": 0,
|
||
"manifest_missing_file_count": 0,
|
||
}
|
||
try:
|
||
for record in dinsar_records:
|
||
preview_path = str(record.product.preview_path or "").strip()
|
||
manifest_path = str(record.product.manifest_path or "").strip()
|
||
preview_exists = bool(preview_path and os.path.exists(preview_path))
|
||
fallback_exists = bool(record.image_path and os.path.exists(record.image_path))
|
||
|
||
if preview_path:
|
||
dinsar_cache_consistency["db_marked_cached_count"] += 1
|
||
if preview_exists:
|
||
dinsar_cache_consistency["cache_file_exists_count"] += 1
|
||
if preview_path and preview_exists:
|
||
dinsar_cache_consistency["db_cached_and_file_exists_count"] += 1
|
||
elif preview_path and (not preview_exists):
|
||
dinsar_cache_consistency["db_cached_but_file_missing_count"] += 1
|
||
elif fallback_exists:
|
||
dinsar_cache_consistency["db_uncached_but_file_exists_count"] += 1
|
||
else:
|
||
dinsar_cache_consistency["db_uncached_and_file_missing_count"] += 1
|
||
|
||
if manifest_path:
|
||
dinsar_cache_consistency["manifest_entries_count"] += 1
|
||
if not os.path.exists(manifest_path):
|
||
dinsar_cache_consistency["manifest_missing_file_count"] += 1
|
||
except Exception as e:
|
||
dinsar_cache_consistency["error"] = str(e)
|
||
|
||
dinsar_cached_count = dinsar_cache_consistency["db_marked_cached_count"]
|
||
|
||
# 2. 源数据统计
|
||
source_data_total_count = 0
|
||
envi_processed_count = 0
|
||
with_orbit_data_count = 0
|
||
by_satellite: Dict[str, Any] = {}
|
||
source_preview_consistency = {
|
||
"total_records_count": 0,
|
||
"geo_cache_exists_count": 0,
|
||
"raw_cache_exists_count": 0,
|
||
"preview_exists_count": 0,
|
||
"preview_missing_count": 0,
|
||
"db_ready_count": 0,
|
||
"db_ready_and_cache_exists_count": 0,
|
||
"db_ready_but_cache_missing_count": 0,
|
||
}
|
||
source_xml_consistency = {
|
||
"total_records_count": 0,
|
||
"xml_detected_count": 0,
|
||
"xml_missing_count": 0,
|
||
"xml_parsed_ok_count": 0,
|
||
"xml_detected_but_unparsed_count": 0,
|
||
}
|
||
|
||
try:
|
||
source_data_total_count_res = await db.execute(select(func.count(RadarDataORM.id)))
|
||
source_data_total_count = source_data_total_count_res.scalar_one()
|
||
|
||
if source_data_total_count > 0:
|
||
envi_processed_count_res = await db.execute(select(func.count(RadarDataORM.id)).where(RadarDataORM.is_envi_processed == True))
|
||
envi_processed_count = envi_processed_count_res.scalar_one()
|
||
|
||
with_orbit_data_count_res = await db.execute(select(func.count(RadarDataORM.id)).where(RadarDataORM.has_orbit_data == True))
|
||
with_orbit_data_count = with_orbit_data_count_res.scalar_one()
|
||
|
||
by_satellite_res = await db.execute(select(RadarDataORM.satellite, func.count(RadarDataORM.id)).group_by(RadarDataORM.satellite))
|
||
by_satellite = {sat: count for sat, count in by_satellite_res.all()}
|
||
|
||
source_rows_res = await db.execute(
|
||
select(
|
||
RadarDataORM.unique_id,
|
||
RadarDataORM.file_path,
|
||
RadarDataORM.preview_cache_status,
|
||
RadarDataORM.scene_center_lon,
|
||
RadarDataORM.scene_center_lat,
|
||
RadarDataORM.acquisition_time_utc,
|
||
RadarDataORM.satellite_mode,
|
||
RadarDataORM.receiving_station,
|
||
RadarDataORM.product_level,
|
||
RadarDataORM.product_unique_id,
|
||
)
|
||
)
|
||
source_rows = source_rows_res.all()
|
||
source_preview_consistency["total_records_count"] = len(source_rows)
|
||
source_xml_consistency["total_records_count"] = len(source_rows)
|
||
|
||
for (
|
||
unique_id,
|
||
file_path,
|
||
preview_cache_status,
|
||
scene_center_lon,
|
||
scene_center_lat,
|
||
acquisition_time_utc,
|
||
satellite_mode,
|
||
receiving_station,
|
||
product_level,
|
||
product_unique_id,
|
||
) in source_rows:
|
||
if not file_path:
|
||
source_preview_consistency["preview_missing_count"] += 1
|
||
source_xml_consistency["xml_missing_count"] += 1
|
||
continue
|
||
|
||
cache_key = unique_id or file_path
|
||
raw_cache_path = data_service.get_radar_raw_cache_path(cache_key, file_path)
|
||
geo_cache_path = data_service.get_radar_geo_cache_path(cache_key, file_path)
|
||
has_raw_cache = os.path.exists(raw_cache_path)
|
||
has_geo_cache = os.path.exists(geo_cache_path)
|
||
|
||
if has_geo_cache:
|
||
source_preview_consistency["geo_cache_exists_count"] += 1
|
||
if has_raw_cache:
|
||
source_preview_consistency["raw_cache_exists_count"] += 1
|
||
|
||
has_any_preview_cache = has_geo_cache or has_raw_cache
|
||
if has_any_preview_cache:
|
||
source_preview_consistency["preview_exists_count"] += 1
|
||
else:
|
||
source_preview_consistency["preview_missing_count"] += 1
|
||
|
||
status = (preview_cache_status or "NONE").upper()
|
||
if status == "READY":
|
||
source_preview_consistency["db_ready_count"] += 1
|
||
if has_any_preview_cache:
|
||
source_preview_consistency["db_ready_and_cache_exists_count"] += 1
|
||
else:
|
||
source_preview_consistency["db_ready_but_cache_missing_count"] += 1
|
||
|
||
scene_dir = file_path if os.path.isdir(file_path) else os.path.dirname(file_path)
|
||
xml_path = find_xml_file(scene_dir) if scene_dir else None
|
||
has_xml = bool(xml_path and os.path.exists(xml_path))
|
||
if has_xml:
|
||
source_xml_consistency["xml_detected_count"] += 1
|
||
parsed_ok = any(
|
||
value is not None and value != ""
|
||
for value in [
|
||
scene_center_lon,
|
||
scene_center_lat,
|
||
acquisition_time_utc,
|
||
satellite_mode,
|
||
receiving_station,
|
||
product_level,
|
||
product_unique_id,
|
||
]
|
||
)
|
||
if parsed_ok:
|
||
source_xml_consistency["xml_parsed_ok_count"] += 1
|
||
else:
|
||
source_xml_consistency["xml_detected_but_unparsed_count"] += 1
|
||
else:
|
||
source_xml_consistency["xml_missing_count"] += 1
|
||
|
||
except Exception as e:
|
||
logger.warning("统计源数据时发生错误 (可能是表不存在): %s", e)
|
||
source_preview_consistency["error"] = str(e)
|
||
source_xml_consistency["error"] = str(e)
|
||
|
||
# 4. AI 质量统计
|
||
labeled_good_count = sum(1 for record in dinsar_records if record.product.user_label == 1)
|
||
labeled_bad_count = sum(1 for record in dinsar_records if record.product.user_label == 0)
|
||
|
||
unlabeled_count = dinsar_total_count - labeled_good_count - labeled_bad_count
|
||
|
||
# 5. AI 预测统计
|
||
ai_good_count = sum(
|
||
1 for record in dinsar_records
|
||
if record.product.ai_score is not None and record.product.ai_score >= 0.7
|
||
)
|
||
ai_bad_count = sum(
|
||
1 for record in dinsar_records
|
||
if record.product.ai_score is not None and record.product.ai_score < 0.4
|
||
)
|
||
ai_medium_count = sum(
|
||
1 for record in dinsar_records
|
||
if record.product.ai_score is not None and 0.4 <= record.product.ai_score < 0.7
|
||
)
|
||
ai_unpredicted_count = sum(
|
||
1 for record in dinsar_records
|
||
if record.product.ai_score is None
|
||
)
|
||
|
||
# 6. IDL 处理统计（读取 runs/*.json）
|
||
idl_processing_stats: Dict[str, Any] = {
|
||
"by_workflow_success": {},
|
||
"avg_duration_by_workflow": {},
|
||
}
|
||
try:
|
||
all_runs = data_service.envi_service_list_runs_all() if hasattr(data_service, "envi_service_list_runs_all") else []
|
||
# 直接读取 runs 目录
|
||
from ..services.envi_service import list_recent_runs as _list_runs
|
||
all_runs = _list_runs(limit=500)
|
||
wf_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: {"success": 0, "failed": 0})
|
||
wf_durations: Dict[str, list] = defaultdict(list)
|
||
for run in all_runs:
|
||
wf = run.get("workflow", "unknown")
|
||
status = run.get("status", "")
|
||
dur = run.get("duration_seconds")
|
||
if status == "success":
|
||
wf_counts[wf]["success"] += 1
|
||
elif status == "failed":
|
||
wf_counts[wf]["failed"] += 1
|
||
if dur is not None:
|
||
try:
|
||
wf_durations[wf].append(float(dur))
|
||
except (TypeError, ValueError):
|
||
pass
|
||
idl_processing_stats["by_workflow_success"] = {k: dict(v) for k, v in wf_counts.items()}
|
||
idl_processing_stats["avg_duration_by_workflow"] = {
|
||
k: round(sum(v) / len(v), 1) for k, v in wf_durations.items() if v
|
||
}
|
||
except Exception as _e:
|
||
idl_processing_stats["error"] = str(_e)
|
||
|
||
# 8. 水体地理编码一致性检测
|
||
water_geo_consistency: Dict[str, Any] = {
|
||
"water_results_dir": settings.WATER_RESULTS_DIR,
|
||
"dir_scanned_count": 0,
|
||
"geo_db_exists_count": 0,
|
||
"matched_in_db_count": 0,
|
||
"unregistered_count": 0,
|
||
"registered_but_missing_count": 0,
|
||
}
|
||
try:
|
||
import re as _re2
|
||
water_dir = settings.WATER_RESULTS_DIR
|
||
_uid_re = _re2.compile(r"_(\d{7,})$")
|
||
|
||
# 从 DB 拉取所有 product_unique_id -> radar_data_id 映射
|
||
uid_rows = await db.execute(
|
||
select(RadarDataORM.product_unique_id, RadarDataORM.id)
|
||
.where(RadarDataORM.product_unique_id.isnot(None))
|
||
)
|
||
uid_to_radar_id: Dict[str, int] = {uid: rid for uid, rid in uid_rows.all() if uid}
|
||
|
||
# 从 DB 拉取所有 DONE 的 radar_data_id 集合
|
||
done_rows = await db.execute(
|
||
select(SARSceneGeoORM.radar_data_id, SARSceneGeoORM.geo_path)
|
||
.where(SARSceneGeoORM.status == "DONE")
|
||
)
|
||
done_radar_ids: Dict[int, str] = {rid: gp for rid, gp in done_rows.all()}
|
||
|
||
if os.path.isdir(water_dir):
|
||
for entry in os.scandir(water_dir):
|
||
if not entry.is_dir() or not entry.name.startswith("scene_"):
|
||
continue
|
||
water_geo_consistency["dir_scanned_count"] += 1
|
||
|
||
# 检查目录内是否有 *_geo_db 文件
|
||
geo_db_path = None
|
||
for f in os.scandir(entry.path):
|
||
if f.name.endswith("_geo_db") and not f.name.endswith(".hdr") and not f.name.endswith(".sml"):
|
||
geo_db_path = f.path
|
||
break
|
||
if not geo_db_path:
|
||
continue
|
||
water_geo_consistency["geo_db_exists_count"] += 1
|
||
|
||
# 解析 product_unique_id
|
||
m = _uid_re.search(entry.name)
|
||
if not m:
|
||
continue
|
||
uid = m.group(1).lstrip("0") or m.group(1)
|
||
radar_id = uid_to_radar_id.get(m.group(1)) or uid_to_radar_id.get(uid)
|
||
if not radar_id:
|
||
continue
|
||
water_geo_consistency["matched_in_db_count"] += 1
|
||
|
||
if radar_id not in done_radar_ids:
|
||
water_geo_consistency["unregistered_count"] += 1
|
||
|
||
# 反向检查：DB DONE 但 geo_db 文件不存在
|
||
for radar_id, geo_path in done_radar_ids.items():
|
||
if geo_path and not os.path.exists(geo_path):
|
||
water_geo_consistency["registered_but_missing_count"] += 1
|
||
|
||
except Exception as _e:
|
||
water_geo_consistency["error"] = str(_e)
|
||
|
||
# 7. D-InSAR 结果按月统计（从 name 字段解析主影像日期）
|
||
dinsar_by_month: list = []
|
||
try:
|
||
_month_counts: Dict[str, int] = defaultdict(int)
|
||
_date_re = _re.compile(r"(\d{8})")
|
||
for record in dinsar_records:
|
||
name = (
|
||
record.product.task_alias
|
||
or record.product.display_name
|
||
or record.product.task_name
|
||
or record.display_name
|
||
)
|
||
if not name:
|
||
continue
|
||
dates = _date_re.findall(name)
|
||
if not dates:
|
||
continue
|
||
master_date = dates[0]
|
||
_month_counts[f"{master_date[:4]}-{master_date[4:6]}"] += 1
|
||
dinsar_by_month = [
|
||
{"month": k, "count": v}
|
||
for k, v in sorted(_month_counts.items())
|
||
]
|
||
except Exception as _e:
|
||
dinsar_by_month = []
|
||
|
||
pairing_consistency: Dict[str, Any] = {
|
||
"metric_cache_count": 0,
|
||
"network_run_count": 0,
|
||
"network_edge_count": 0,
|
||
"dirty_scene_count": 0,
|
||
"cache_status": None,
|
||
"needs_rebuild": None,
|
||
"duplicate_reverse_pair_count": 0,
|
||
"invalid_orientation_count": 0,
|
||
"network_edge_orphan_count": 0,
|
||
"task_orphan_count": 0,
|
||
"result_trace_missing_count": 0,
|
||
"result_trace_orphan_count": 0,
|
||
"result_trace_pair_mismatch_count": 0,
|
||
}
|
||
try:
|
||
pairing_status = await pairing_state_service.get_pairing_system_status(db)
|
||
pairing_consistency["metric_cache_count"] = int(pairing_status.get("pair_count") or 0)
|
||
pairing_consistency["network_run_count"] = int(pairing_status.get("network_run_count") or 0)
|
||
pairing_consistency["network_edge_count"] = int(pairing_status.get("network_edge_count") or 0)
|
||
pairing_consistency["dirty_scene_count"] = int(pairing_status.get("dirty_scene_count") or 0)
|
||
pairing_consistency["cache_status"] = pairing_status.get("status")
|
||
pairing_consistency["needs_rebuild"] = bool(pairing_status.get("needs_rebuild"))
|
||
pairing_consistency["duplicate_reverse_pair_count"] = int(
|
||
pairing_status.get("duplicate_reverse_pair_count") or 0
|
||
)
|
||
pairing_consistency["network_edge_orphan_count"] = int(
|
||
pairing_status.get("orphan_edge_count") or 0
|
||
)
|
||
|
||
invalid_orientation_result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT COUNT(*)
|
||
FROM pairing_metric_cache
|
||
WHERE
|
||
master_imaging_date IS NULL
|
||
OR slave_imaging_date IS NULL
|
||
OR master_scene_uid IS NULL
|
||
OR slave_scene_uid IS NULL
|
||
OR master_imaging_date > slave_imaging_date
|
||
OR (
|
||
master_imaging_date = slave_imaging_date
|
||
AND (
|
||
master_scene_uid > slave_scene_uid
|
||
OR (
|
||
master_scene_uid = slave_scene_uid
|
||
AND master_scene_ref_id > slave_scene_ref_id
|
||
)
|
||
)
|
||
)
|
||
"""
|
||
)
|
||
)
|
||
pairing_consistency["invalid_orientation_count"] = int(
|
||
invalid_orientation_result.scalar_one() or 0
|
||
)
|
||
|
||
result_trace_missing_result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT COUNT(*)
|
||
FROM result_products
|
||
WHERE catalog_name = 'dinsar'
|
||
AND (
|
||
COALESCE(pair_uid, '') = ''
|
||
OR COALESCE(network_run_id, '') = ''
|
||
OR network_edge_id IS NULL
|
||
OR COALESCE(policy_version, '') = ''
|
||
)
|
||
"""
|
||
)
|
||
)
|
||
pairing_consistency["result_trace_missing_count"] = int(
|
||
result_trace_missing_result.scalar_one() or 0
|
||
)
|
||
|
||
result_trace_orphan_result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT COUNT(*)
|
||
FROM result_products rp
|
||
LEFT JOIN pairing_network_runs pnr
|
||
ON pnr.network_run_id = rp.network_run_id
|
||
LEFT JOIN pairing_network_edges pne
|
||
ON pne.id = rp.network_edge_id
|
||
AND pne.network_run_ref_id = pnr.id
|
||
WHERE rp.catalog_name = 'dinsar'
|
||
AND COALESCE(rp.pair_uid, '') <> ''
|
||
AND COALESCE(rp.network_run_id, '') <> ''
|
||
AND rp.network_edge_id IS NOT NULL
|
||
AND (pnr.id IS NULL OR pne.id IS NULL)
|
||
"""
|
||
)
|
||
)
|
||
pairing_consistency["result_trace_orphan_count"] = int(
|
||
result_trace_orphan_result.scalar_one() or 0
|
||
)
|
||
|
||
result_trace_pair_mismatch_result = await db.execute(
|
||
text(
|
||
"""
|
||
SELECT COUNT(*)
|
||
FROM result_products rp
|
||
JOIN pairing_network_runs pnr
|
||
ON pnr.network_run_id = rp.network_run_id
|
||
JOIN pairing_network_edges pne
|
||
ON pne.id = rp.network_edge_id
|
||
AND pne.network_run_ref_id = pnr.id
|
||
JOIN pairing_metric_cache pmc
|
||
ON pmc.id = pne.metric_cache_ref_id
|
||
WHERE rp.catalog_name = 'dinsar'
|
||
AND COALESCE(rp.pair_uid, '') <> ''
|
||
AND COALESCE(pmc.pair_uid, '') <> ''
|
||
AND rp.pair_uid <> pmc.pair_uid
|
||
"""
|
||
)
|
||
)
|
||
pairing_consistency["result_trace_pair_mismatch_count"] = int(
|
||
result_trace_pair_mismatch_result.scalar_one() or 0
|
||
)
|
||
except Exception as _e:
|
||
pairing_consistency["error"] = str(_e)
|
||
|
||
stats_payload = {
|
||
"dinsar_results_overview": {
|
||
"total_count": dinsar_total_count,
|
||
"cached_count": dinsar_cached_count,
|
||
"uncached_count": dinsar_total_count - dinsar_cached_count,
|
||
},
|
||
"dinsar_cache_consistency": dinsar_cache_consistency,
|
||
"source_data_overview": {
|
||
"total_count": source_data_total_count,
|
||
"envi_processed_count": envi_processed_count,
|
||
"with_orbit_data_count": with_orbit_data_count,
|
||
},
|
||
"source_preview_consistency": source_preview_consistency,
|
||
"source_xml_consistency": source_xml_consistency,
|
||
"water_geo_consistency": water_geo_consistency,
|
||
"pairing_consistency": pairing_consistency,
|
||
"by_satellite": by_satellite,
|
||
"idl_processing_stats": idl_processing_stats,
|
||
"dinsar_by_month": dinsar_by_month,
|
||
"ai_quality_overview": {
|
||
"good_count": labeled_good_count,
|
||
"bad_count": labeled_bad_count,
|
||
"unlabeled_count": unlabeled_count
|
||
},
|
||
"ai_prediction_overview": {
|
||
"good_count": ai_good_count,
|
||
"bad_count": ai_bad_count,
|
||
"medium_count": ai_medium_count,
|
||
"unpredicted_count": ai_unpredicted_count
|
||
}
|
||
}
|
||
|
||
generated_at = datetime.utcnow().isoformat() + "Z"
|
||
if _deps.STATS_CACHE_TTL_SECONDS > 0:
|
||
async with _deps._STATS_CACHE_LOCK:
|
||
_deps._STATS_CACHE_DATA = stats_payload
|
||
_deps._STATS_CACHE_EXPIRES_AT = time.monotonic() + _deps.STATS_CACHE_TTL_SECONDS
|
||
_deps._STATS_CACHE_GENERATED_AT_UTC = generated_at
|
||
|
||
return {
|
||
**stats_payload,
|
||
"cache_meta": {
|
||
"enabled": _deps.STATS_CACHE_TTL_SECONDS > 0,
|
||
"hit": False,
|
||
"ttl_seconds": _deps.STATS_CACHE_TTL_SECONDS,
|
||
"generated_at": generated_at,
|
||
},
|
||
}
|