from __future__ import annotations import asyncio import gzip import hashlib import json import math import mimetypes import os import re import struct from datetime import datetime from pathlib import Path from typing import Any, Optional from geoalchemy2.shape import from_shape from shapely.geometry import Polygon from sqlalchemy import String, cast, delete, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from ..config import settings from ..models import ResultAssetORM, ResultCatalogStateORM, ResultIssueORM, ResultProductORM from .admin_region_lookup_service import lookup_admin_region_for_point from .landsar_sbas_service import landsar_sbas_service from .sbas_insar_production_service import sbas_insar_production_service SBAS_INSAR_CATALOG_NAME = "sbas_insar" JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" _READY_STATUSES = { "PRODUCTS_READY", "MONITOR_POINTS_READY", "WORKFLOW_COMPLETED", "LANDSAR_SBAS_COMPLETED", "LANDSAR_SBAS_PARTIAL", } _REQUIRED_ASSET_ROLES = {"primary_geotiff", "quality_geotiff"} _LANDSAR_REQUIRED_ASSET_ROLES = {"primary_geotiff"} _WGS84_GEOGCS_WKT = ( 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563]],' 'PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]' ) _CORE_ASSETS = ( ("run_manifest", "Run manifest", "run_manifest.json", True, False), ("stack_manifest", "Stack manifest", "stack_manifest.json", True, False), ("workflow_summary", "Workflow summary", "workflow_summary.json", False, False), ("product_summary", "Product summary", "product_summary.json", False, False), ("quality_summary", "Quality summary", "quality_summary.json", False, False), ("monitor_points_summary", "Monitor points summary", "monitor_points_summary.json", False, False), ( "point_vector_summary", "LOS point-vector summary", "publish/vectors/los_rate_points_summary.json", False, False, ), ( "point_vector_geojson_gz", "LOS point-vector GeoJSON.gz", "publish/vectors/los_rate_points.geojson.gz", False, False, ), ( "primary_geocoded_preview", "LOS velocity preview, toward radar positive", "publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png", False, True, ), ( "quality_geocoded_preview", "LOS velocity sigma preview", "publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png", False, False, ), ( "primary_geotiff", "LOS velocity GeoTIFF, toward radar positive", "publish/geotiff/los_rate_toward_m_per_year.tif", True, True, ), ( "alternate_geotiff", "LOS velocity GeoTIFF, away from radar positive", "publish/geotiff/los_rate_away_m_per_year.tif", False, False, ), ( "quality_geotiff", "LOS velocity sigma GeoTIFF", "publish/geotiff/los_sigma_m_per_year.tif", True, False, ), ( "primary_rgb_geotiff", "LOS velocity RGB GeoTIFF", "publish/geotiff/los_rate_toward_m_per_year.hls.geo_rgb.tif", False, False, ), ( "quality_rgb_geotiff", "LOS velocity sigma RGB GeoTIFF", "publish/geotiff/los_sigma_m_per_year.cc.geo_rgb.tif", False, False, ), ("gamma_phase_rate", "Gamma phase-rate GeoTIFF", "publish/geotiff/ts_rate_rad_per_year.tif", False, False), ("gamma_sigma_rate", "Gamma sigma-rate GeoTIFF", "publish/geotiff/sigma_rate_rad_per_year.tif", False, False), ("height_correction", "Height correction GeoTIFF", "publish/geotiff/hgt_correction_m.tif", False, False), ) _LANDSAR_CORE_ASSETS = ( ("run_manifest", "Run manifest", "run_manifest.json", True, False), ("stack_manifest", "Task/Input_Data manifest", "stack_manifest.json", True, False), ("workflow_summary", "LandSAR SBAS workflow summary", "workflow_summary.json", True, False), ("product_summary", "LandSAR SBAS product summary", "product_summary.json", False, False), ("quality_summary", "LandSAR SBAS quality summary", "quality_summary.json", False, False), ("command_manifest", "LandSAR SBAS command manifest", "landsar_command_manifest.json", False, False), ("native_console_log", "LandSAR SBAS console log", "native_logs", False, False), ("primary_preview", "LandSAR LOS preview", "publish/landsar/preview.png", False, True), ("primary_geotiff", "LandSAR LOS time-series GeoTIFF", "publish/landsar/los_timeseries.tif", True, True), ("secondary_geotiff", "LandSAR post-raster GeoTIFF", "publish/landsar/post_raster.tif", False, False), ) _EXPERT_GAMMA_CORE_ASSETS = ( ("run_manifest", "Run manifest", "run_manifest.json", True, False), ("stack_manifest", "Stack manifest", "stack_manifest.json", True, False), ("workflow_summary", "Workflow summary", "workflow_summary.json", False, False), ("monitor_points_summary", "Monitor points summary", "monitor_points_summary.json", False, False), ( "unwrapped_phase_summary", "Expert Gamma geocoded unwrapped phase summary", "publish/geotiff/unwrapped/unwrapped_phase_summary.json", False, False, ), ( "unwrapped_phase_radar_colorbar", "Expert Gamma rmg.cm unwrapped phase radar colorbar", "publish/geotiff/unwrapped/unwrapped_phase_rmg_colorbar.png", False, False, ), ( "point_vector_summary", "Expert Gamma LOS point-vector summary", "publish/vectors/los_rate_points_summary.json", False, False, ), ( "point_vector_geojson_gz", "Expert Gamma LOS point-vector GeoJSON.gz", "publish/vectors/los_rate_points.geojson.gz", False, False, ), ( "primary_geocoded_preview", "Expert Gamma geo_los_def_rate RGB PNG preview", "publish/geotiff/geo_los_def_rate_rgb_preview.png", False, True, ), ( "primary_rate_color_preview", "Expert Gamma pure geo_los_def_rate hls.cm PNG preview", "publish/geotiff/geo_los_def_rate_pure_hls_preview.png", False, False, ), ( "primary_geotiff", "Expert Gamma geo_los_def_rate GeoTIFF", "publish/geotiff/geo_los_def_rate.tif", True, True, ), ( "primary_rgb_geotiff", "Expert Gamma geo_los_def_rate RGB GeoTIFF", "publish/geotiff/geo_los_def_rate_rgb.tif", False, False, ), ( "primary_colorbar", "Expert Gamma hls.cm deformation-rate colorbar", "publish/geotiff/geo_los_def_rate_hls_colorbar.png", False, False, ), ( "monitor_points", "Expert Gamma disp_prt_2d point time series", "publish/points/disp_point.txt", False, False, ), ( "monitor_point_items", "Expert Gamma disp_prt_2d column definitions", "publish/points/items.txt", False, False, ), ( "monitor_point_selection", "Expert Gamma disp_prt_2d selected radar points", "publish/points/disp_point_sel.txt", False, False, ), ( "monitor_point_selection_metadata", "Expert Gamma monitor point selection metadata", "publish/points/disp_point_selection.json", False, False, ), ) def _utcnow() -> datetime: return datetime.utcnow() def _normalize_path(path: str | os.PathLike[str]) -> str: return os.path.normpath(os.path.abspath(os.fspath(path))) def _read_json(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8") as fp: payload = json.load(fp) return payload if isinstance(payload, dict) else {} def _safe_read_json(path: Path) -> dict[str, Any]: if not path.is_file(): return {} try: return _read_json(path) except Exception: return {} def _safe_float(value: Any) -> Optional[float]: try: parsed = float(value) except (TypeError, ValueError): return None if parsed != parsed: return None return parsed def _safe_int(value: Any) -> Optional[int]: try: return int(float(value)) except (TypeError, ValueError): return None def _read_gamma_key_values(path: Path) -> dict[str, str]: values: dict[str, str] = {} if not path.is_file(): return values for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): if ":" not in line: continue key, raw_value = line.split(":", 1) key = key.strip() value = raw_value.strip().split()[0] if raw_value.strip() else "" if key: values[key] = value return values def _read_gamma_int_param(path: Path, key: str) -> Optional[int]: return _safe_int(_read_gamma_key_values(path).get(key)) def _haversine_m(lon1: float, lat1: float, lon2: float, lat2: float) -> float: radius_m = 6371008.8 phi1 = math.radians(lat1) phi2 = math.radians(lat2) dphi = math.radians(lat2 - lat1) dlambda = math.radians(lon2 - lon1) a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 return 2 * radius_m * math.asin(min(1.0, math.sqrt(a))) def _parse_datetime(value: Any) -> Optional[datetime]: text = str(value or "").strip() if not text: return None if text.endswith("Z"): text = text[:-1] + "+00:00" try: return datetime.fromisoformat(text).replace(tzinfo=None) except ValueError: return None def _stable_digest(*parts: Any, length: int = 20) -> str: payload = "||".join(str(part or "") for part in parts) return hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:length] def _asset_format(path: str) -> Optional[str]: lowered = path.lower() if lowered.endswith(".geojson.gz"): return "geojson.gz" ext = Path(path).suffix.lower() return { ".bmp": "bmp", ".csv": "csv", ".geo": "gamma_binary", ".gz": "gzip", ".json": "json", ".log": "log", ".png": "png", ".sh": "shell", ".tif": "geotiff", ".tiff": "geotiff", ".txt": "text", }.get(ext) def _media_type(path: str) -> Optional[str]: lowered = path.lower() if lowered.endswith(".geojson.gz"): return "application/gzip" ext = Path(path).suffix.lower() explicit = { ".bmp": "image/bmp", ".csv": "text/csv", ".gz": "application/gzip", ".json": "application/json", ".log": "text/plain", ".png": "image/png", ".sh": "text/x-shellscript", ".tif": "image/tiff", ".tiff": "image/tiff", ".txt": "text/plain", } return explicit.get(ext) or mimetypes.guess_type(path)[0] def _build_rgb_geotiff_preview(source: Path, target: Path) -> Optional[str]: if target.is_file(): return str(target) if not source.is_file(): return None try: import numpy as np import rasterio from PIL import Image from rasterio.enums import Resampling except Exception: return None try: with rasterio.open(source) as src: scale = max(src.width / 1600, src.height / 1600, 1) out_w = max(1, int(src.width / scale)) out_h = max(1, int(src.height / scale)) mask = src.dataset_mask(out_shape=(out_h, out_w)) > 0 if src.count >= 3: data = src.read( [1, 2, 3], out_shape=(3, out_h, out_w), resampling=Resampling.bilinear, ) rgb = np.moveaxis(data, 0, -1) if rgb.dtype != np.uint8: rgb = np.clip(rgb, 0, 255).astype("uint8") else: data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.bilinear).astype("float32") finite = np.isfinite(data) valid = finite & mask if np.any(valid): p2, p98 = np.nanpercentile(data[valid], [2, 98]) if not np.isfinite(p2) or not np.isfinite(p98) or p98 <= p2: p2 = float(np.nanmin(data[valid])) p98 = float(np.nanmax(data[valid])) norm = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1) else: norm = np.zeros_like(data, dtype="float32") gray = np.where(np.isfinite(norm), norm * 255, 0).astype("uint8") rgb = np.dstack([gray, gray, gray]) alpha = np.where(mask, 255, 0).astype("uint8") image = Image.fromarray(np.dstack([rgb, alpha]), mode="RGBA") resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") image.thumbnail((1600, 1600), resampling) target.parent.mkdir(parents=True, exist_ok=True) image.save(target, "PNG", optimize=True) return str(target) except Exception: return None def _find_gamma_hls_colormap() -> Optional[Path]: candidates = [ Path(r"\\wsl.localhost\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps\hls.cm"), Path(r"\\wsl$\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps\hls.cm"), ] for candidate in candidates: if candidate.is_file(): return candidate return None def _find_gamma_colormap(name: str) -> Optional[Path]: safe_name = Path(str(name or "")).name if not safe_name: return None candidates = [ Path(r"\\wsl.localhost\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps") / safe_name, Path(r"\\wsl$\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps") / safe_name, ] for candidate in candidates: if candidate.is_file(): return candidate return None def _read_gamma_colormap(path: Path) -> list[tuple[int, int, int]]: colors: list[tuple[int, int, int]] = [] if not path.is_file(): return colors for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): parts = line.strip().split() if len(parts) < 3: continue try: red, green, blue = (max(0, min(255, int(float(value)))) for value in parts[:3]) except ValueError: continue colors.append((red, green, blue)) return colors def _build_gamma_colormap_colorbar( target: Path, *, colormap_name: str, min_value: float, max_value: float, unit: str, title: str, ) -> Optional[str]: if target.is_file(): return str(target) source = _find_gamma_colormap(colormap_name) colors = _read_gamma_colormap(source) if source else [] if not colors: return None try: import numpy as np from PIL import Image, ImageDraw, ImageFont width = 920 bar_height = 34 label_height = 78 margin_x = 48 margin_top = 14 target.parent.mkdir(parents=True, exist_ok=True) image = Image.new("RGBA", (width, bar_height + label_height), (255, 255, 255, 0)) draw = ImageDraw.Draw(image) gradient_width = width - margin_x * 2 color_array = np.asarray(colors, dtype=np.uint8) for x in range(gradient_width): idx = int(round((x / max(1, gradient_width - 1)) * (len(color_array) - 1))) draw.line( [(margin_x + x, margin_top), (margin_x + x, margin_top + bar_height)], fill=tuple(int(value) for value in color_array[idx]) + (255,), ) draw.rectangle( [margin_x, margin_top, margin_x + gradient_width, margin_top + bar_height], outline=(15, 23, 42, 180), width=1, ) try: font = ImageFont.truetype("arial.ttf", 14) small_font = ImageFont.truetype("arial.ttf", 12) except Exception: font = ImageFont.load_default() small_font = font ticks = [ (min_value, f"{min_value:g}"), (0.0, "0"), (max_value, f"{max_value:g}"), ] for value, label in ticks: ratio = (value - min_value) / max(max_value - min_value, 1e-6) x = margin_x + int(round(ratio * gradient_width)) draw.line([(x, margin_top + bar_height), (x, margin_top + bar_height + 7)], fill=(15, 23, 42, 220), width=1) text = f"{label} {unit}".strip() bbox = draw.textbbox((0, 0), text, font=font) draw.text((x - (bbox[2] - bbox[0]) / 2, margin_top + bar_height + 10), text, fill=(15, 23, 42, 255), font=font) source_label = f"{title} / {colormap_name}" source_bbox = draw.textbbox((0, 0), source_label, font=small_font) draw.text( (width - margin_x - (source_bbox[2] - source_bbox[0]), margin_top + bar_height + 34), source_label, fill=(71, 85, 105, 255), font=small_font, ) image.save(target, "PNG", optimize=True) return str(target) except Exception: return None def _build_gamma_hls_colorbar(target: Path, *, min_mm_year: float = -80.0, max_mm_year: float = 80.0) -> Optional[str]: if target.is_file(): return str(target) source = _find_gamma_hls_colormap() colors = _read_gamma_colormap(source) if source else [] if not colors: return None try: import numpy as np from PIL import Image, ImageDraw, ImageFont width = 900 bar_height = 34 label_height = 58 margin_x = 46 margin_top = 14 target.parent.mkdir(parents=True, exist_ok=True) image = Image.new("RGBA", (width, bar_height + label_height), (255, 255, 255, 0)) draw = ImageDraw.Draw(image) gradient_width = width - margin_x * 2 color_array = np.asarray(colors, dtype=np.uint8) for x in range(gradient_width): idx = int(round((x / max(1, gradient_width - 1)) * (len(color_array) - 1))) draw.line( [(margin_x + x, margin_top), (margin_x + x, margin_top + bar_height)], fill=tuple(int(value) for value in color_array[idx]) + (255,), ) draw.rectangle( [margin_x, margin_top, margin_x + gradient_width, margin_top + bar_height], outline=(15, 23, 42, 180), width=1, ) ticks = [ (min_mm_year, f"{min_mm_year:g}"), (0.0, "0"), (max_mm_year, f"{max_mm_year:g}"), ] try: font = ImageFont.truetype("arial.ttf", 14) small_font = ImageFont.truetype("arial.ttf", 12) except Exception: font = ImageFont.load_default() small_font = font for value, label in ticks: ratio = (value - min_mm_year) / max(max_mm_year - min_mm_year, 1e-6) x = margin_x + int(round(ratio * gradient_width)) draw.line([(x, margin_top + bar_height), (x, margin_top + bar_height + 7)], fill=(15, 23, 42, 220), width=1) text = f"{label} mm/yr" bbox = draw.textbbox((0, 0), text, font=font) draw.text((x - (bbox[2] - bbox[0]) / 2, margin_top + bar_height + 10), text, fill=(15, 23, 42, 255), font=font) source_label = "Gamma hls.cm" source_bbox = draw.textbbox((0, 0), source_label, font=small_font) draw.text( (width - margin_x - (source_bbox[2] - source_bbox[0]), margin_top + bar_height + 33), source_label, fill=(71, 85, 105, 255), font=small_font, ) image.save(target, "PNG", optimize=True) return str(target) except Exception: return None def _build_gamma_hls_rate_preview( source: Path, target: Path, *, coverage_source: Optional[Path] = None, min_native: float = -0.08, max_native: float = 0.08, ) -> Optional[str]: if target.is_file(): return str(target) if not source.is_file(): return None colormap_path = _find_gamma_hls_colormap() colors = _read_gamma_colormap(colormap_path) if colormap_path else [] if not colors: return None try: import numpy as np import rasterio from PIL import Image from rasterio.enums import Resampling except Exception: return None try: with rasterio.open(source) as src: scale = max(src.width / 1600, src.height / 1600, 1) out_w = max(1, int(src.width / scale)) out_h = max(1, int(src.height / scale)) data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.nearest).astype("float32") coverage = np.ones((out_h, out_w), dtype=bool) if coverage_source and coverage_source.is_file(): with rasterio.open(coverage_source) as coverage_src: if coverage_src.width == src.width and coverage_src.height == src.height and coverage_src.count >= 3: coverage_rgb = coverage_src.read( [1, 2, 3], out_shape=(3, out_h, out_w), resampling=Resampling.nearest, ) coverage = np.any(coverage_rgb != 0, axis=0) valid = np.isfinite(data) & coverage & (data != 0.0) ratio = np.clip((data - min_native) / max(max_native - min_native, 1e-12), 0.0, 1.0) color_array = np.asarray(colors, dtype=np.uint8) indices = np.rint(ratio * (len(color_array) - 1)).astype(np.int32) rgb = color_array[np.clip(indices, 0, len(color_array) - 1)] alpha = np.where(valid, 255, 0).astype(np.uint8) image = Image.fromarray(np.dstack([rgb, alpha]), mode="RGBA") resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") image.thumbnail((1600, 1600), resampling) target.parent.mkdir(parents=True, exist_ok=True) image.save(target, "PNG", optimize=True) return str(target) except Exception: return None def _parse_expert_disp_point_table(disp_point_path: Path) -> list[dict[str, Any]]: if not disp_point_path.is_file(): return [] import csv rows = list(csv.reader(disp_point_path.read_text(encoding="utf-8", errors="ignore").splitlines())) if len(rows) < 2: return [] header = [str(value or "").strip() for value in rows[0]] dates = [value for value in header[5:] if value] points: list[dict[str, Any]] = [] for index, row in enumerate(rows[1:], start=1): if len(row) < 5: continue values = [str(value or "").strip() for value in row] try: img_x = int(float(values[0])) img_y = int(float(values[1])) except ValueError: continue displacements: list[dict[str, Any]] = [] for date_text, value_text in zip(dates, values[5:]): try: displacement = float(value_text) except ValueError: continue date_clean = date_text.strip() if len(date_clean) == 8 and date_clean.isdigit(): date_iso = f"{date_clean[0:4]}-{date_clean[4:6]}-{date_clean[6:8]}" else: date_iso = date_clean displacements.append({"date": date_iso, "displacement_mm": displacement}) points.append( { "point_id": f"expert_point_{index:03d}", "img_x": img_x, "img_y": img_y, "height_m": _safe_float(values[2]) if len(values) > 2 else None, "deformation_rate_mm_per_year": _safe_float(values[3]) if len(values) > 3 else None, "stdev_residual_phase_rad": _safe_float(values[4]) if len(values) > 4 else None, "displacements": displacements, } ) return points def _read_expert_sbas_dates(run_dir: Path) -> list[str]: rmli_tab = run_dir / "sbas" / "RMLI_tab" dates: list[str] = [] if rmli_tab.is_file(): for line in rmli_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): parts = line.split() if not parts: continue match = re.search(r"(\d{8})", Path(parts[0]).name) if match: raw = match.group(1) dates.append(f"{raw[0:4]}-{raw[4:6]}-{raw[6:8]}") if dates: return dates points = _parse_expert_disp_point_table(run_dir / "publish" / "points" / "disp_point.txt") for point in points: displacements = point.get("displacements") or [] if displacements: return [str(item.get("date") or "") for item in displacements if item.get("date")] return [] def _read_radar_float32(path: Path, *, width: int, img_x: int, img_y: int) -> Optional[float]: if width <= 0 or img_x < 0 or img_y < 0 or not path.is_file(): return None offset = (img_y * width + img_x) * 4 try: if offset < 0 or offset + 4 > path.stat().st_size: return None with path.open("rb") as fp: fp.seek(offset) payload = fp.read(4) if len(payload) != 4: return None value = struct.unpack(">f", payload)[0] return value if math.isfinite(value) else None except Exception: return None def _read_geo_rate_window( *, source_path: Path, coverage_path: Path, center_row: int, center_col: int, radius: int, width: int, height: int, ) -> tuple[Any, Any, int, int]: import numpy as np import rasterio row0 = max(0, center_row - radius) row1 = min(height - 1, center_row + radius) col0 = max(0, center_col - radius) col1 = min(width - 1, center_col + radius) if row0 > row1 or col0 > col1: return None, None, row0, col0 window = rasterio.windows.Window(col0, row0, col1 - col0 + 1, row1 - row0 + 1) with rasterio.open(source_path) as src: rate = src.read(1, window=window) coverage = None if coverage_path.is_file(): with rasterio.open(coverage_path) as cov: if cov.width == width and cov.height == height and cov.count >= 3: rgb = cov.read([1, 2, 3], window=window) coverage = np.any(rgb != 0, axis=0) return rate, coverage, row0, col0 def _query_expert_gamma_point_timeseries(run_dir: Path, *, lon: float, lat: float) -> dict[str, Any]: source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" lt_path = run_dir / "dem" / f"{_safe_read_json(run_dir / 'run_manifest.json').get('reference_date') or ''}.lt_fine" if not lt_path.is_file(): candidates = sorted((run_dir / "dem").glob("*.lt_fine")) lt_path = candidates[0] if candidates else lt_path mli_par = run_dir / "sbas" / "mli.ave.par" disp_tab = run_dir / "sbas" / "disp.TS_tab" if not source_path.is_file(): raise FileNotFoundError("geo_los_def_rate.tif is missing") if not lt_path.is_file(): raise FileNotFoundError("Gamma lookup table *.lt_fine is missing") if not mli_par.is_file(): raise FileNotFoundError("mli.ave.par is missing") if not disp_tab.is_file(): raise FileNotFoundError("disp.TS_tab is missing") try: import numpy as np import rasterio except Exception as exc: raise RuntimeError(f"point_query_dependency_unavailable: {exc}") from exc with rasterio.open(source_path) as src: width = int(src.width) height = int(src.height) if lon < src.bounds.left or lon > src.bounds.right or lat < src.bounds.bottom or lat > src.bounds.top: raise ValueError("requested WGS84 coordinate is outside product geocoded bounds") center_row, center_col = src.index(lon, lat) transform = src.transform valid_choice: Optional[dict[str, Any]] = None for radius in [0, 1, 2, 4, 8, 16, 32, 64, 128, 256]: rate, coverage, row0, col0 = _read_geo_rate_window( source_path=source_path, coverage_path=coverage_path, center_row=int(center_row), center_col=int(center_col), radius=radius, width=width, height=height, ) if rate is None: continue valid = np.isfinite(rate) & (rate != 0.0) if coverage is not None: valid &= coverage if not np.any(valid): continue rows, cols = np.where(valid) abs_rows = rows + row0 abs_cols = cols + col0 distances = (abs_rows - int(center_row)) ** 2 + (abs_cols - int(center_col)) ** 2 best_index = int(np.argmin(distances)) matched_row = int(abs_rows[best_index]) matched_col = int(abs_cols[best_index]) matched_lon, matched_lat = rasterio.transform.xy(transform, matched_row, matched_col, offset="center") rate_value = float(rate[rows[best_index], cols[best_index]]) valid_choice = { "geo_row": matched_row, "geo_col": matched_col, "lon": float(matched_lon), "lat": float(matched_lat), "los_rate_mm_per_year": rate_value * 1000.0, "source_native_m_per_year": rate_value, "search_radius_pixels": radius, "distance_m": _haversine_m(lon, lat, float(matched_lon), float(matched_lat)), } break if valid_choice is None: raise ValueError("no valid deformation pixel found near requested WGS84 coordinate") lt_offset = (int(valid_choice["geo_row"]) * width + int(valid_choice["geo_col"])) * 8 if lt_offset < 0 or lt_offset + 8 > lt_path.stat().st_size: raise ValueError("matched geocoded pixel is outside lookup table") with lt_path.open("rb") as fp: fp.seek(lt_offset) range_value, azimuth_value = struct.unpack(">ff", fp.read(8)) if not math.isfinite(range_value) or not math.isfinite(azimuth_value): raise ValueError("lookup table returned invalid radar coordinates") radar_width = _read_gamma_int_param(mli_par, "range_samples") or _read_gamma_int_param(mli_par, "width") or 0 radar_lines = _read_gamma_int_param(mli_par, "azimuth_lines") or _read_gamma_int_param(mli_par, "nlines") or 0 img_x = int(round(range_value)) img_y = int(round(azimuth_value)) if radar_width <= 0 or radar_lines <= 0: raise ValueError("invalid radar image dimensions") img_x = min(max(0, img_x), radar_width - 1) img_y = min(max(0, img_y), radar_lines - 1) raw_disp_paths = [ Path(_wsl_path_to_windows(line.strip())) for line in disp_tab.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip() ] dates = _read_expert_sbas_dates(run_dir) displacements: list[dict[str, Any]] = [] for index, disp_path in enumerate(raw_disp_paths): value_m = _read_radar_float32(disp_path, width=radar_width, img_x=img_x, img_y=img_y) date = dates[index] if index < len(dates) else f"epoch_{index + 1:03d}" if value_m is None: displacements.append({"date": date, "displacement_mm": None}) else: displacements.append({"date": date, "displacement_mm": value_m * 1000.0}) if not any(item.get("displacement_mm") is not None for item in displacements): raise ValueError("matched radar pixel has no readable displacement values") return { "schema": "insar.gamma-sbas-point-query/v1", "source_tool": "disp.TS_tab_radar_pixel_sample", "query": {"lon": lon, "lat": lat}, "matched": { **valid_choice, "used_nearest": int(valid_choice["geo_row"]) != int(center_row) or int(valid_choice["geo_col"]) != int(center_col), "input_geo_row": int(center_row), "input_geo_col": int(center_col), "radar_range": float(range_value), "radar_azimuth": float(azimuth_value), "img_x": img_x, "img_y": img_y, "radar_width": radar_width, "radar_lines": radar_lines, }, "unit": "mm", "rate_unit": "mm/yr", "displacement_count": len(displacements), "displacements": displacements, } def _locate_radar_points_in_geocoded_product(run_dir: Path, points: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: targets: list[tuple[str, float, float]] = [] for point in points: point_id = str(point.get("point_id") or "").strip() img_x = _safe_float(point.get("img_x")) img_y = _safe_float(point.get("img_y")) if point_id and img_x is not None and img_y is not None: targets.append((point_id, img_x, img_y)) if not targets: return {} source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" manifest = _safe_read_json(run_dir / "run_manifest.json") lt_path = run_dir / "dem" / f"{manifest.get('reference_date') or ''}.lt_fine" if not lt_path.is_file(): candidates = sorted((run_dir / "dem").glob("*.lt_fine")) lt_path = candidates[0] if candidates else lt_path if not source_path.is_file() or not lt_path.is_file(): return {} try: import numpy as np import rasterio from rasterio.windows import Window except Exception: return {} try: with rasterio.open(source_path) as src: width = int(src.width) height = int(src.height) transform = src.transform expected_size = width * height * 2 * 4 if lt_path.stat().st_size < expected_size: return {} lookup = np.memmap(lt_path, dtype=">f4", mode="r", shape=(height, width, 2)) coarse_step = 16 coarse_best: dict[str, dict[str, Any]] = { point_id: {"distance2": math.inf, "row": None, "col": None} for point_id, _, _ in targets } for row0 in range(0, height, coarse_step * 128): row1 = min(height, row0 + coarse_step * 128) sampled = lookup[row0:row1:coarse_step, 0:width:coarse_step, :] if sampled.size == 0: continue range_chunk = np.asarray(sampled[:, :, 0], dtype=np.float32) azimuth_chunk = np.asarray(sampled[:, :, 1], dtype=np.float32) valid = np.isfinite(range_chunk) & np.isfinite(azimuth_chunk) & (range_chunk >= 0.0) & (azimuth_chunk >= 0.0) if not np.any(valid): continue for point_id, img_x, img_y in targets: distances = (range_chunk - np.float32(img_x)) ** 2 + (azimuth_chunk - np.float32(img_y)) ** 2 distances = np.where(valid, distances, np.float32(np.inf)) flat_index = int(np.argmin(distances)) value = float(distances.flat[flat_index]) if value >= coarse_best[point_id]["distance2"]: continue local_row, local_col = np.unravel_index(flat_index, distances.shape) coarse_best[point_id] = { "distance2": value, "row": int(row0 + local_row * coarse_step), "col": int(local_col * coarse_step), } results: dict[str, dict[str, Any]] = {} coverage_src = None if coverage_path.is_file(): try: candidate = rasterio.open(coverage_path) if candidate.width == width and candidate.height == height and candidate.count >= 3: coverage_src = candidate else: candidate.close() except Exception: coverage_src = None try: refine_radius = coarse_step * 12 for point_id, img_x, img_y in targets: seed = coarse_best.get(point_id) or {} seed_row = seed.get("row") seed_col = seed.get("col") if seed_row is None or seed_col is None: continue row0 = max(0, int(seed_row) - refine_radius) row1 = min(height, int(seed_row) + refine_radius + 1) col0 = max(0, int(seed_col) - refine_radius) col1 = min(width, int(seed_col) + refine_radius + 1) local = lookup[row0:row1, col0:col1, :] range_local = np.asarray(local[:, :, 0], dtype=np.float32) azimuth_local = np.asarray(local[:, :, 1], dtype=np.float32) valid = np.isfinite(range_local) & np.isfinite(azimuth_local) & (range_local >= 0.0) & (azimuth_local >= 0.0) if not np.any(valid): continue distances = (range_local - np.float32(img_x)) ** 2 + (azimuth_local - np.float32(img_y)) ** 2 distances = np.where(valid, distances, np.float32(np.inf)) flat_index = int(np.argmin(distances)) value = float(distances.flat[flat_index]) if not math.isfinite(value): continue local_row, local_col = np.unravel_index(flat_index, distances.shape) row = int(row0 + local_row) col = int(col0 + local_col) lon, lat = rasterio.transform.xy(transform, row, col, offset="center") rate_mm_per_year = None coverage_valid = None try: rate_window = src.read(1, window=Window(col, row, 1, 1)) rate_value = float(rate_window[0, 0]) if math.isfinite(rate_value): rate_mm_per_year = rate_value * 1000.0 except Exception: rate_mm_per_year = None if coverage_src is not None: try: rgb = coverage_src.read([1, 2, 3], window=Window(col, row, 1, 1)) coverage_valid = bool(np.any(rgb != 0)) except Exception: coverage_valid = None results[point_id] = { "lon": float(lon), "lat": float(lat), "geo_row": row, "geo_col": col, "geo_match_distance_px": math.sqrt(value), "geo_los_rate_mm_per_year": rate_mm_per_year, "geo_coverage_valid": coverage_valid, "geo_source": "gamma_lt_fine_nearest_inverse", } finally: if coverage_src is not None: coverage_src.close() del lookup return results except Exception: return {} def _file_summary(path: Path) -> dict[str, Any]: return { "path": str(path), "exists": path.is_file(), "size_bytes": path.stat().st_size if path.is_file() else None, } def _build_expert_gamma_primary_geotiff_stats(path: Path) -> dict[str, Any]: stats = { **_file_summary(path), "schema": "insar.gamma-sbas-expert-primary-geotiff-stats/v1", "source": "expert_gamma_geo_los_def_rate", "unit": "mm/yr", "native_unit": "m/yr", "scale_to_unit": 1000.0, "zero_is_valid": False, "validity_rule": "expert_rgb_coverage_finite_nonzero_values", } if not path.is_file(): return stats try: import numpy as np import rasterio from rasterio.windows import Window except Exception as exc: stats["error"] = f"raster_stats_dependency_unavailable: {exc}" return stats try: coverage_path = path.with_name("geo_los_def_rate_rgb.tif") coverage_mask_source = str(coverage_path) if coverage_path.is_file() else None valid_count = 0 zero_count = 0 nonzero_count = 0 sample_count = 0 value_sum = 0.0 value_sumsq = 0.0 value_min: Optional[float] = None value_max: Optional[float] = None percentile_chunks: list[Any] = [] max_percentile_samples = 5_000_000 tile_size = 1024 with rasterio.open(path) as src: coverage_src = None if coverage_path.is_file(): try: candidate = rasterio.open(coverage_path) if candidate.width == src.width and candidate.height == src.height and candidate.count >= 3: coverage_src = candidate else: candidate.close() except Exception: coverage_src = None total_count = int(src.width * src.height) stats.update( { "width": int(src.width), "height": int(src.height), "band_count": int(src.count), "dtype": str(src.dtypes[0]) if src.dtypes else None, "crs": str(src.crs) if src.crs else None, "nodata": _safe_float(src.nodata), "metadata_nodata_applied": _safe_float(src.nodata) == 0.0, "coverage_mask_source": coverage_mask_source if coverage_src is not None else None, "bounds": { "left": _safe_float(src.bounds.left), "bottom": _safe_float(src.bounds.bottom), "right": _safe_float(src.bounds.right), "top": _safe_float(src.bounds.top), }, "total_count": total_count, } ) for row_off in range(0, src.height, tile_size): for col_off in range(0, src.width, tile_size): window = Window( col_off=col_off, row_off=row_off, width=min(tile_size, src.width - col_off), height=min(tile_size, src.height - row_off), ) data = src.read(1, window=window, masked=False) values = np.asarray(data, dtype=np.float64).reshape(-1) if values.size == 0: continue finite = np.isfinite(values) if coverage_src is not None: coverage_rgb = coverage_src.read([1, 2, 3], window=window, masked=False) coverage = np.any(coverage_rgb != 0, axis=0).reshape(-1) else: coverage = np.ones(values.shape, dtype=bool) covered_values = values[finite & coverage] zero_count += int(np.count_nonzero(covered_values == 0.0)) values = values[finite & coverage & (values != 0.0)] if values.size == 0: continue scaled = values * 1000.0 valid_count += int(scaled.size) nonzero_count += int(np.count_nonzero(values != 0.0)) value_sum += float(scaled.sum(dtype=np.float64)) value_sumsq += float(np.square(scaled, dtype=np.float64).sum(dtype=np.float64)) chunk_min = float(scaled.min()) chunk_max = float(scaled.max()) value_min = chunk_min if value_min is None else min(value_min, chunk_min) value_max = chunk_max if value_max is None else max(value_max, chunk_max) if sample_count < max_percentile_samples: remaining = max_percentile_samples - sample_count if scaled.size <= remaining: sample = scaled else: step = max(1, int(np.ceil(scaled.size / remaining))) sample = scaled[::step][:remaining] percentile_chunks.append(sample.astype(np.float64, copy=True)) sample_count += int(sample.size) if coverage_src is not None: coverage_src.close() stats["valid_count"] = valid_count stats["zero_count"] = zero_count stats["nonzero_count"] = nonzero_count stats["valid_ratio"] = (valid_count / stats["total_count"]) if stats.get("total_count") else None if valid_count <= 0: return stats mean = value_sum / valid_count variance = max((value_sumsq / valid_count) - (mean * mean), 0.0) stats.update( { "min": value_min, "max": value_max, "mean": mean, "stddev": float(variance ** 0.5), "sample_count": sample_count, "percentiles_sampled": sample_count < valid_count, } ) if percentile_chunks: percentile_values = np.concatenate(percentile_chunks) p01, p05, median, p95, p99 = np.percentile(percentile_values, [1, 5, 50, 95, 99]) stats.update( { "p01": float(p01), "p05": float(p05), "median": float(median), "p95": float(p95), "p99": float(p99), } ) return stats except Exception as exc: stats["error"] = f"raster_stats_failed: {exc}" return stats def _build_expert_gamma_quality_summary(run_dir: Path) -> dict[str, Any]: primary_stats = _build_expert_gamma_primary_geotiff_stats( run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" ) return { "schema": "insar.gamma-sbas-derived-quality-summary/v1", "source": "derived_from_expert_gamma_outputs", "note": "Catalog/UI inspection stats derived from expert geo_los_def_rate.tif; quality_summary.json is not required for the expert workflow.", "primary_geotiff": primary_stats, } def _pixel_center(transform: tuple[float, float, float, float, float, float], row: int, col: int) -> tuple[float, float]: a, b, c, d, e, f = transform x = c + (col + 0.5) * a + (row + 0.5) * b y = f + (col + 0.5) * d + (row + 0.5) * e return float(x), float(y) def _normalize_crs_label(value: Any) -> Optional[str]: text = str(value or "").strip() if not text: return None upper = text.upper() if "EPSG" in upper and "4326" in upper: return "EPSG:4326" if "WGS 84" in upper or "WGS_1984" in upper: return "EPSG:4326" return text[:240] def _build_expert_gamma_point_vector(run_dir: Path, *, summary_context: Optional[dict[str, Any]] = None) -> dict[str, Any]: source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" output_path = run_dir / "publish" / "vectors" / "los_rate_points.geojson.gz" summary_path = run_dir / "publish" / "vectors" / "los_rate_points_summary.json" previous_summary = _safe_read_json(summary_path) context = summary_context or {} stack_dates = context.get("stack_dates") or [] admin_region = context.get("admin_region") if isinstance(context.get("admin_region"), dict) else {} admin_names = admin_region.get("names") if isinstance(admin_region.get("names"), dict) else {} fields = [ "run_id", "row", "col", "lon", "lat", "los_rate_mm_per_year", "source_native_m_per_year", "date_start", "date_end", "reference_date", "admin_province", "admin_city", ] summary: dict[str, Any] = { "schema": "insar.gamma-sbas-expert-point-vector-summary/v1", "generated_at": previous_summary.get("generated_at") or _utcnow().isoformat(timespec="seconds") + "Z", "ready": False, "feature_count": 0, "output_geojson_gz": str(output_path), "output_size_bytes": output_path.stat().st_size if output_path.is_file() else 0, "fields": fields, "source_geotiffs": { "geo_los_def_rate": str(source_path), "coverage_mask": str(coverage_path) if coverage_path.is_file() else None, }, "unit": "mm/yr", "native_unit": "m/yr", "scale_to_unit": 1000.0, "zero_is_valid": False, "validity_rule": "expert_rgb_coverage_finite_nonzero_values", "date_start": context.get("date_start") or (stack_dates[0] if stack_dates else None), "date_end": context.get("date_end") or (stack_dates[-1] if stack_dates else None), "reference_date": context.get("reference_date"), "admin_region": { "province": admin_names.get("province") or admin_region.get("province"), "city": admin_names.get("city") or admin_region.get("city"), }, "los_convention": context.get("los_sign_convention") or "Gamma expert geo_los_def_rate output; sign follows the expert workflow.", "frontend_policy": "download_only; do not render full point GeoJSON in browser", } if not source_path.is_file(): _write_json_if_changed(summary_path, {**summary, "error": "source_geotiff_missing"}) return summary if output_path.is_file() and summary_path.is_file(): input_paths = [source_path] if coverage_path.is_file(): input_paths.append(coverage_path) output_mtime = output_path.stat().st_mtime if ( previous_summary.get("validity_rule") == "expert_rgb_coverage_finite_nonzero_values" and previous_summary.get("zero_is_valid") is False and all(output_mtime >= item.stat().st_mtime for item in input_paths) ): return previous_summary try: import numpy as np import rasterio from rasterio.windows import Window except Exception as exc: summary["error"] = f"point_vector_dependency_unavailable: {exc}" _write_json_if_changed(summary_path, summary) return summary try: output_path.parent.mkdir(parents=True, exist_ok=True) feature_count = 0 zero_count = 0 tile_size = 512 with rasterio.open(source_path) as src: coverage_src = None if coverage_path.is_file(): candidate = rasterio.open(coverage_path) if candidate.width == src.width and candidate.height == src.height and candidate.count >= 3: coverage_src = candidate else: candidate.close() transform = tuple(float(value) for value in src.transform.to_gdal()) # Convert GDAL geotransform (c, a, b, f, d, e) to affine tuple used by _pixel_center. transform = (transform[1], transform[2], transform[0], transform[4], transform[5], transform[3]) nodata = _safe_float(src.nodata) summary.update( { "width": int(src.width), "height": int(src.height), "crs": _normalize_crs_label(str(src.crs) if src.crs else None), "nodata": nodata, "metadata_nodata_applied": nodata == 0.0, "coverage_mask_source": str(coverage_path) if coverage_src is not None else None, "total_count": int(src.width * src.height), } ) with gzip.open(output_path, "wt", encoding="utf-8", compresslevel=6) as handle: handle.write('{"type":"FeatureCollection","features":[\n') first = True for row_off in range(0, src.height, tile_size): for col_off in range(0, src.width, tile_size): window = Window( col_off=col_off, row_off=row_off, width=min(tile_size, src.width - col_off), height=min(tile_size, src.height - row_off), ) data = src.read(1, window=window, masked=False).astype("float64", copy=False) finite = np.isfinite(data) coverage = np.ones(data.shape, dtype=bool) if coverage_src is not None: rgb = coverage_src.read([1, 2, 3], window=window, masked=False) coverage = np.any(rgb != 0, axis=0) covered = finite & coverage zero_count += int(np.count_nonzero(data[covered] == 0.0)) valid = covered & (data != 0.0) rows, cols = np.where(valid) for local_row, local_col in zip(rows.tolist(), cols.tolist()): row = int(row_off + local_row) col = int(col_off + local_col) lon, lat = _pixel_center(transform, row, col) native_value = float(data[local_row, local_col]) properties = { "run_id": context.get("run_id") or run_dir.name, "row": row, "col": col, "lon": lon, "lat": lat, "los_rate_mm_per_year": native_value * 1000.0, "source_native_m_per_year": native_value, "date_start": summary.get("date_start"), "date_end": summary.get("date_end"), "reference_date": summary.get("reference_date"), "admin_province": summary["admin_region"].get("province"), "admin_city": summary["admin_region"].get("city"), } feature = { "type": "Feature", "geometry": {"type": "Point", "coordinates": [lon, lat]}, "properties": properties, } if not first: handle.write(",\n") handle.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":"))) first = False feature_count += 1 handle.write("\n]}\n") if coverage_src is not None: coverage_src.close() summary.update( { "ready": output_path.is_file() and output_path.stat().st_size > 0, "feature_count": feature_count, "zero_count": zero_count, "output_size_bytes": output_path.stat().st_size if output_path.is_file() else 0, } ) _write_json_if_changed(summary_path, summary) return summary except Exception as exc: summary["error"] = f"point_vector_export_failed: {exc}" _write_json_if_changed(summary_path, summary) return summary def _write_text_if_changed(path: Path, text: str) -> bool: if path.is_file(): try: if path.read_text(encoding="utf-8", errors="ignore") == text: return False except Exception: pass path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") return True def _write_json_if_changed(path: Path, payload: dict[str, Any]) -> bool: text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n" return _write_text_if_changed(path, text) _MONITOR_POINT_SELECTION_DEFINITIONS = [ ( "toward_high_rate_low_sigma", "趋近雷达高形变低残差点", "rate > 0，且绝对速率位于高分位，残差低，用于检查明显正向形变区域。", ), ( "away_high_rate_low_sigma", "远离雷达高形变低残差点", "rate < 0，且绝对速率位于高分位，残差低，用于检查明显负向形变区域。", ), ( "high_abs_rate_low_sigma", "高绝对速率低残差点", "不区分正负，优先选择绝对速率高且残差低的有效点。", ), ( "stable_low_sigma", "近零低残差代表点", "绝对速率位于低分位且残差低，用于对照相对稳定区域。", ), ( "center_valid", "覆盖区中心有效点", "从有效像元中选取最接近雷达网格中心的点，用于空间位置对照。", ), ] def _read_expert_monitor_point_selection(points_dir: Path) -> dict[tuple[int, int], dict[str, Any]]: selection_path = points_dir / "disp_point_sel.txt" rows: list[dict[str, Any]] = [] if selection_path.is_file(): for index, line in enumerate(selection_path.read_text(encoding="utf-8", errors="ignore").splitlines()): parts = line.strip().split() if len(parts) < 2: continue try: img_x = int(float(parts[0])) img_y = int(float(parts[1])) except ValueError: continue key, label, description = ( _MONITOR_POINT_SELECTION_DEFINITIONS[index] if index < len(_MONITOR_POINT_SELECTION_DEFINITIONS) else ( f"extra_representative_{index + 1:03d}", f"补充代表点 {index + 1}", "自动选点数量超过内置策略说明时的补充代表点。", ) ) rows.append( { "selection_rank": index + 1, "selection_key": key, "selection_label": label, "selection_description": description, "img_x": img_x, "img_y": img_y, } ) payload = { "schema": "insar.gamma-sbas-expert-monitor-point-selection/v1", "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", "source": "disp_point_sel.txt", "selection_count": len(rows), "strategy": "auto_representative_points", "strategy_note": "自动选取趋近/远离雷达高形变、绝对高形变、近零稳定和中心有效点；时序仍由 Gamma disp_prt_2d 输出。", "points": rows, } if rows: _write_json_if_changed(points_dir / "disp_point_selection.json", payload) return {(int(item["img_x"]), int(item["img_y"])): item for item in rows} def _wsl_path_to_windows(path: str) -> str: text = str(path or "").strip() if text.startswith("/mnt/") and len(text) > 6 and text[6] == "/": drive = text[5].upper() return f"{drive}:{text[6:]}".replace("/", "\\") return text def _build_expert_unwrapped_phase_radar_browse( run_dir: Path, source_paths: list[Path], *, width: int, lines: int, ) -> dict[str, dict[str, Any]]: unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" browse_by_pair: dict[str, dict[str, Any]] = {} if not source_paths or width <= 0 or lines <= 0: return browse_by_pair try: import numpy as np from PIL import Image except Exception: return browse_by_pair colormap_path = _find_gamma_colormap("rmg.cm") colors = _read_gamma_colormap(colormap_path) if colormap_path else [] if not colors: return browse_by_pair color_array = np.asarray(colors, dtype=np.uint8) unwrapped_dir.mkdir(parents=True, exist_ok=True) _build_gamma_colormap_colorbar( unwrapped_dir / "unwrapped_phase_rmg_colorbar.png", colormap_name="rmg.cm", min_value=-6.28, max_value=6.28, unit="rad", title="Gamma unwrapped phase browse", ) for source in source_paths: pair_id = source.name.replace(".unw.atmsub_1", "") output_bmp = unwrapped_dir / f"{source.name}.rdc_rmg.bmp" output_png = unwrapped_dir / f"{source.name}.rdc_rmg_preview.png" item = { "pair_id": pair_id, "radar_coordinates": True, "browse_command": "rasdt_pwr mli.ave 1 - 1 1 -6.28 6.28 1 rmg.cm ... 1.0 0.35 24", "colormap": "Gamma rmg.cm", "display_range_rad": [-6.28, 6.28], "bmp": _file_summary(output_bmp), "preview": _file_summary(output_png), "ready": False, } if not source.is_file(): item["error"] = "source_missing" browse_by_pair[pair_id] = item continue if ( output_png.is_file() and output_png.stat().st_mtime >= source.stat().st_mtime and (not output_bmp.is_file() or output_bmp.stat().st_mtime >= source.stat().st_mtime) ): item["bmp"] = _file_summary(output_bmp) item["preview"] = _file_summary(output_png) item["ready"] = output_png.stat().st_size > 0 browse_by_pair[pair_id] = item continue try: data = np.fromfile(source, dtype=">f4", count=int(width) * int(lines)).reshape((int(lines), int(width))) valid = np.isfinite(data) & (data != 0.0) wrapped = ((data + 6.28) % 12.56) - 6.28 ratio = np.clip((wrapped + 6.28) / 12.56, 0.0, 1.0) indices = np.rint(ratio * (len(color_array) - 1)).astype(np.int32) rgb = color_array[np.clip(indices, 0, len(color_array) - 1)] rgb = np.where(valid[..., None], rgb, 0).astype(np.uint8) image = Image.fromarray(rgb, mode="RGB") image.save(output_bmp, "BMP") preview = image.copy() resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") preview.thumbnail((1600, 1600), resampling) preview.save(output_png, "PNG", optimize=True) item.update( { "bmp": _file_summary(output_bmp), "preview": _file_summary(output_png), "valid_count": int(np.count_nonzero(valid)), "ready": output_png.is_file() and output_png.stat().st_size > 0, } ) except Exception as exc: item["error"] = str(exc) browse_by_pair[pair_id] = item return browse_by_pair def _build_expert_unwrapped_phase_derivatives(run_dir: Path) -> dict[str, Any]: unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" summary_path = unwrapped_dir / "unwrapped_phase_summary.json" final_tab = run_dir / "sbas" / "final_unw_tab" manifest = _safe_read_json(run_dir / "run_manifest.json") stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") stack = stack_manifest.get("stack") if isinstance(stack_manifest.get("stack"), dict) else {} reference_date = str( manifest.get("reference_date") or stack.get("reference_date") or (manifest.get("coregistration") or {}).get("reference_date") or "" ).strip() dem_par = run_dir / "dem" / f"{reference_date}_seg.dem_par" if reference_date else next(iter(sorted((run_dir / "dem").glob("*_seg.dem_par"))), run_dir / "dem" / "_missing_seg.dem_par") lookup = run_dir / "dem" / f"{reference_date}.lt_fine" if reference_date else next(iter(sorted((run_dir / "dem").glob("*.lt_fine"))), run_dir / "dem" / "_missing.lt_fine") mli_par = run_dir / "sbas" / "mli.ave.par" previous = _safe_read_json(summary_path) source_paths: list[Path] = [] if final_tab.is_file(): for line in final_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): text = line.strip().split() if text: source_paths.append(Path(_wsl_path_to_windows(text[0]))) summary: dict[str, Any] = { "schema": "insar.gamma-sbas-expert-unwrapped-phase-summary/v1", "generated_at": previous.get("generated_at") or _utcnow().isoformat(timespec="seconds") + "Z", "source_stage": "final_unw_tab", "source_tab": _file_summary(final_tab), "source_count": len(source_paths), "ready": False, "products": [], "note": "Geocoded GeoTIFF derivatives from the final unwrapped phase files consumed by the expert Gamma SBAS inversion.", } if not source_paths: _write_json_if_changed(summary_path, summary) return summary outputs = [ unwrapped_dir / f"{path.name}.geo.tif" for path in source_paths ] radar_outputs = [ unwrapped_dir / f"{path.name}.rdc_rmg_preview.png" for path in source_paths ] radar_colorbar = unwrapped_dir / "unwrapped_phase_rmg_colorbar.png" previous_products = previous.get("products") if isinstance(previous.get("products"), list) else [] previous_has_radar_browse = ( len(previous_products) == len(source_paths) and all((item.get("radar_browse") or {}).get("ready") for item in previous_products if isinstance(item, dict)) ) if ( summary_path.is_file() and all(output.is_file() for output in outputs) and all(output.is_file() for output in radar_outputs) and radar_colorbar.is_file() and previous_has_radar_browse and all( output.stat().st_mtime >= source.stat().st_mtime for source, output in zip(source_paths, outputs) if source.is_file() ) and all( output.stat().st_mtime >= source.stat().st_mtime for source, output in zip(source_paths, radar_outputs) if source.is_file() ) ): return previous try: import numpy as np import rasterio from PIL import Image from rasterio.transform import from_origin except Exception as exc: summary["error"] = f"unwrapped_phase_dependency_unavailable: {exc}" _write_json_if_changed(summary_path, summary) return summary def read_gamma_param(path: Path, key: str) -> Optional[str]: if not path.is_file(): return None for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): parts = line.split() if parts and parts[0].rstrip(":") == key.rstrip(":") and len(parts) > 1: return parts[1] return None try: rdc_width = _safe_int(read_gamma_param(mli_par, "range_samples")) rdc_lines = _safe_int(read_gamma_param(mli_par, "azimuth_lines")) dem_width = _safe_int(read_gamma_param(dem_par, "width")) dem_lines = _safe_int(read_gamma_param(dem_par, "nlines")) corner_lon = _safe_float(read_gamma_param(dem_par, "corner_lon")) corner_lat = _safe_float(read_gamma_param(dem_par, "corner_lat")) post_lon = _safe_float(read_gamma_param(dem_par, "post_lon")) post_lat = _safe_float(read_gamma_param(dem_par, "post_lat")) if None in (rdc_width, rdc_lines, dem_width, dem_lines, corner_lon, corner_lat, post_lon, post_lat): raise RuntimeError("required Gamma geometry parameters are missing") if not lookup.is_file(): raise FileNotFoundError(str(lookup)) unwrapped_dir.mkdir(parents=True, exist_ok=True) radar_browse_by_pair = _build_expert_unwrapped_phase_radar_browse( run_dir, source_paths, width=int(rdc_width), lines=int(rdc_lines), ) lut = np.fromfile(lookup, dtype=">c8").reshape((int(dem_lines), int(dem_width))) rng = np.rint(lut.real).astype(np.int32) az = np.rint(lut.imag).astype(np.int32) valid_lut = ( np.isfinite(lut.real) & np.isfinite(lut.imag) & (rng >= 0) & (rng < int(rdc_width)) & (az >= 0) & (az < int(rdc_lines)) ) del lut transform = from_origin(float(corner_lon), float(corner_lat), abs(float(post_lon)), abs(float(post_lat))) products: list[dict[str, Any]] = [] preview_limit = 6 for source in source_paths: if not source.is_file(): products.append({"source": str(source), "ready": False, "error": "source_missing"}) continue pair_id = source.name.replace(".unw.atmsub_1", "") output_tif = unwrapped_dir / f"{source.name}.geo.tif" output_preview = unwrapped_dir / f"{source.name}.geo_preview.png" data = np.fromfile(source, dtype=">f4", count=int(rdc_width) * int(rdc_lines)).reshape((int(rdc_lines), int(rdc_width))) geo = np.full((int(dem_lines), int(dem_width)), np.nan, dtype=np.float32) geo[valid_lut] = data[az[valid_lut], rng[valid_lut]] finite_nonzero = np.isfinite(geo) & (geo != 0.0) with rasterio.open( output_tif, "w", driver="GTiff", width=int(dem_width), height=int(dem_lines), count=1, dtype="float32", crs=_WGS84_GEOGCS_WKT, transform=transform, nodata=np.nan, compress="deflate", ) as dst: dst.write(geo, 1) if np.any(finite_nonzero) and len(products) < preview_limit: valid_values = geo[finite_nonzero] p02, p98 = np.nanpercentile(valid_values, [2, 98]) if not np.isfinite(p02) or not np.isfinite(p98) or p98 <= p02: p02 = float(np.nanmin(valid_values)) p98 = float(np.nanmax(valid_values)) norm = np.clip((geo - p02) / max(p98 - p02, 1e-6), 0, 1) gray = np.where(finite_nonzero, norm * 255, 0).astype("uint8") alpha = np.where(finite_nonzero, 255, 0).astype("uint8") rgba = np.dstack([gray, gray, gray, alpha]) image = Image.fromarray(rgba, mode="RGBA") resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") image.thumbnail((1600, 1600), resampling) image.save(output_preview, "PNG", optimize=True) products.append( { "pair_id": pair_id, "source": str(source), "geotiff": _file_summary(output_tif), "preview": _file_summary(output_preview), "radar_browse": radar_browse_by_pair.get(pair_id) or {}, "valid_count": int(np.count_nonzero(finite_nonzero)), "unit": "rad", "description": "Final model-corrected unwrapped phase used by Gamma SBAS inversion, geocoded for inspection.", "ready": output_tif.is_file() and output_tif.stat().st_size > 0, } ) del data, geo, finite_nonzero summary.update( { "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", "ready": bool(products) and all(item.get("ready") for item in products), "products": products, } ) _write_json_if_changed(summary_path, summary) return summary except Exception as exc: summary["error"] = f"unwrapped_phase_export_failed: {exc}" _write_json_if_changed(summary_path, summary) return summary def _write_expert_monitor_point_derivatives(run_dir: Path) -> dict[str, Any]: points_dir = run_dir / "publish" / "points" disp_point_path = points_dir / "disp_point.txt" items_path = points_dir / "items.txt" selection_by_xy = _read_expert_monitor_point_selection(points_dir) points = _parse_expert_disp_point_table(disp_point_path) geo_locations = _locate_radar_points_in_geocoded_product(run_dir, points) out_dir = run_dir / "publish" / "monitor_points" summary_path = run_dir / "monitor_points_summary.json" monitor_outputs: list[dict[str, Any]] = [] if points: out_dir.mkdir(parents=True, exist_ok=True) expected_names = set() for index in range(1, len(points) + 1): point_id = f"expert_point_{index:03d}" expected_names.update( { f"{point_id}_timeseries.csv", f"{point_id}_timeseries.png", f"{point_id}_metadata.json", } ) for old_path in out_dir.iterdir(): if old_path.is_file() and old_path.name not in expected_names: try: old_path.unlink() except Exception: pass for point in points: point_id = str(point["point_id"]) csv_path = out_dir / f"{point_id}_timeseries.csv" png_path = out_dir / f"{point_id}_timeseries.png" metadata_path = out_dir / f"{point_id}_metadata.json" displacements = point.get("displacements") or [] csv_lines = ["date,displacement_mm"] csv_lines.extend(f"{item['date']},{item['displacement_mm']:.6f}" for item in displacements) curve_needs_refresh = _write_text_if_changed(csv_path, "\n".join(csv_lines) + "\n") selection = selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}) geo_location = geo_locations.get(point_id) or {} metadata = { "schema": "insar.gamma-sbas-expert-monitor-point/v1", "point_id": point_id, "source_tool": "disp_prt_2d", "selection_rank": selection.get("selection_rank"), "selection_key": selection.get("selection_key"), "selection_label": selection.get("selection_label"), "selection_description": selection.get("selection_description"), "img_x": point.get("img_x"), "img_y": point.get("img_y"), "lon": geo_location.get("lon"), "lat": geo_location.get("lat"), "geo_row": geo_location.get("geo_row"), "geo_col": geo_location.get("geo_col"), "geo_match_distance_px": geo_location.get("geo_match_distance_px"), "geo_los_rate_mm_per_year": geo_location.get("geo_los_rate_mm_per_year"), "geo_coverage_valid": geo_location.get("geo_coverage_valid"), "geo_source": geo_location.get("geo_source"), "height_m": point.get("height_m"), "deformation_rate_mm_per_year": point.get("deformation_rate_mm_per_year"), "stdev_residual_phase_rad": point.get("stdev_residual_phase_rad"), "displacement_count": len(displacements), "displacements": displacements, "source_files": { "items": str(items_path), "disp_point": str(disp_point_path), }, } _write_json_if_changed(metadata_path, metadata) try: import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.dates import DateFormatter if curve_needs_refresh or not png_path.is_file(): x_values = [datetime.fromisoformat(str(item["date"])) for item in displacements] x_labels = [str(item["date"]) for item in displacements] y_values = [float(item["displacement_mm"]) for item in displacements] fig_width = max(8.0, len(x_values) * 1.2) fig, ax = plt.subplots(figsize=(fig_width, 4.5), dpi=150) ax.plot(x_values, y_values, marker="o", markersize=5, linewidth=1.8, color="#1d4ed8") ax.axhline(0, color="#94a3b8", linewidth=0.8) ax.set_title( f"Gamma SBAS {point_id} rate={point.get('deformation_rate_mm_per_year') or 0:.3f} mm/yr", fontsize=10, ) ax.set_xlabel("Date") ax.set_ylabel("Displacement (mm)") ax.grid(True, alpha=0.25) ax.set_xticks(x_values) ax.set_xticklabels(x_labels, rotation=35, ha="right") ax.xaxis.set_major_formatter(DateFormatter("%Y-%m-%d")) fig.tight_layout() fig.savefig(png_path) plt.close(fig) except Exception: if not png_path.is_file(): png_path.write_bytes(b"") monitor_outputs.append( { "point_id": point_id, "metadata": metadata, "files": { "png": _file_summary(png_path), "csv": _file_summary(csv_path), "metadata": _file_summary(metadata_path), }, } ) summary = { "schema": "insar.gamma-sbas-expert-monitor-points-summary/v1", "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", "mode": "expert_disp_prt_2d", "source_tool": "disp_prt_2d", "source_files": { "items": _file_summary(items_path), "disp_point": _file_summary(disp_point_path), "selection": _file_summary(points_dir / "disp_point_selection.json"), }, "monitor_points": [ { "point_id": point.get("point_id"), "selection_rank": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_rank"), "selection_key": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_key"), "selection_label": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_label"), "selection_description": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_description"), "img_x": point.get("img_x"), "img_y": point.get("img_y"), "lon": (geo_locations.get(str(point.get("point_id"))) or {}).get("lon"), "lat": (geo_locations.get(str(point.get("point_id"))) or {}).get("lat"), "geo_row": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_row"), "geo_col": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_col"), "geo_match_distance_px": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_match_distance_px"), "geo_los_rate_mm_per_year": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_los_rate_mm_per_year"), "geo_coverage_valid": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_coverage_valid"), "geo_source": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_source"), "height_m": point.get("height_m"), "deformation_rate_mm_per_year": point.get("deformation_rate_mm_per_year"), "stdev_residual_phase_rad": point.get("stdev_residual_phase_rad"), "displacement_count": len(point.get("displacements") or []), "displacements": point.get("displacements") or [], } for point in points ], "monitor_outputs": monitor_outputs, "ready": bool(monitor_outputs) and all( (item.get("files") or {}).get("png", {}).get("exists") and (item.get("files") or {}).get("csv", {}).get("exists") and (item.get("files") or {}).get("metadata", {}).get("exists") for item in monitor_outputs ), } if points: _write_json_if_changed(summary_path, summary) return summary def _bbox_polygon( min_lon: Optional[float], min_lat: Optional[float], max_lon: Optional[float], max_lat: Optional[float], ): if None in (min_lon, min_lat, max_lon, max_lat): return None if min_lon == max_lon or min_lat == max_lat: return None return Polygon( [ (min_lon, min_lat), (max_lon, min_lat), (max_lon, max_lat), (min_lon, max_lat), (min_lon, min_lat), ] ) def _stack_dates_from_manifest(stack_manifest: dict[str, Any], manifest: dict[str, Any], stack: dict[str, Any]) -> list[str]: values: list[str] = [] for source in ( stack_manifest.get("dates"), stack.get("dates"), manifest.get("dates"), [scene.get("date") for scene in stack_manifest.get("scenes") or [] if isinstance(scene, dict)], [scene.get("date") for scene in manifest.get("scenes") or [] if isinstance(scene, dict)], ): if not isinstance(source, list): continue for item in source: text = str(item or "").strip() if text: values.append(text) return sorted(dict.fromkeys(values)) class SbasInsarCatalogService: def get_run_root(self) -> str: root = Path(settings.GAMMA_SBAS_WORK_ROOT or Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production") run_root = root / "runs" run_root.mkdir(parents=True, exist_ok=True) return _normalize_path(run_root) def get_run_roots(self) -> list[str]: roots = [self.get_run_root()] try: landsar_root = landsar_sbas_service.configured_run_root() if landsar_root not in roots: roots.append(landsar_root) except Exception: pass return roots def _iter_run_manifest_paths(self, run_root: Optional[str] = None) -> list[str]: roots = [run_root] if run_root else self.get_run_roots() manifest_paths: list[str] = [] for raw_root in roots: root = Path(raw_root) if not root.is_dir(): continue manifest_paths.extend( _normalize_path(path) for path in sorted(root.glob("*/run_manifest.json")) if self._is_publish_ready(path.parent, _safe_read_json(path)) ) return sorted(dict.fromkeys(manifest_paths)) @staticmethod def _is_landsar_manifest(manifest: dict[str, Any]) -> bool: return str(manifest.get("processor_code") or "").strip().lower() == "landsar_sbas" @staticmethod def _is_expert_gamma_manifest(manifest: dict[str, Any], run_dir: Optional[Path] = None) -> bool: execution_mode = str(manifest.get("execution_mode") or "").strip().lower() if execution_mode == "expert_manifest_script_workflow": return True if run_dir is not None and (run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif").is_file(): return True return False def _asset_definitions_for_manifest(self, manifest: dict[str, Any], run_dir: Optional[Path] = None): if self._is_landsar_manifest(manifest): return _LANDSAR_CORE_ASSETS if self._is_expert_gamma_manifest(manifest, run_dir): return _EXPERT_GAMMA_CORE_ASSETS return _CORE_ASSETS def _required_roles_for_manifest(self, manifest: dict[str, Any], run_dir: Optional[Path] = None) -> set[str]: if self._is_expert_gamma_manifest(manifest, run_dir): return {"primary_geotiff"} return _LANDSAR_REQUIRED_ASSET_ROLES if self._is_landsar_manifest(manifest) else _REQUIRED_ASSET_ROLES def _is_publish_ready(self, run_dir: Path, manifest: dict[str, Any]) -> bool: status = str(manifest.get("status") or "").strip().upper() asset_defs = self._asset_definitions_for_manifest(manifest, run_dir) required_roles = self._required_roles_for_manifest(manifest, run_dir) required_outputs_ready = all( (run_dir / relative_path).is_file() for role, _name, relative_path, is_required, _is_primary in asset_defs if role in required_roles and is_required ) return status in _READY_STATUSES or required_outputs_ready def _tree_fingerprint(self, manifest_paths: list[str]) -> str: records: list[dict[str, Any]] = [] for raw_path in manifest_paths: manifest_path = Path(raw_path) run_dir = manifest_path.parent manifest_payload = _safe_read_json(manifest_path) tracked_paths = [ manifest_path, run_dir / "monitor_points_summary.json", ] if not self._is_expert_gamma_manifest(manifest_payload, run_dir): tracked_paths.extend( [ run_dir / "product_summary.json", run_dir / "quality_summary.json", ] ) else: tracked_paths.extend( [ run_dir / "diff_dir" / "bprep_file.png", run_dir / "diff_dir" / "mean.cc_mask.bmp", run_dir / "sbas" / "final_unw_tab", ] ) diff_dir = run_dir / "diff_dir" if diff_dir.is_dir(): tracked_paths.extend(sorted(diff_dir.glob("*.adf.unw.bmp"))) tracked_paths.extend( run_dir / relative_path for _role, _name, relative_path, _required, _primary in self._asset_definitions_for_manifest( manifest_payload, run_dir ) if not (run_dir / relative_path).is_dir() ) for path in tracked_paths: if not path.exists(): continue stat = path.stat() records.append( { "path": str(path.relative_to(run_dir)).replace("\\", "/"), "run": run_dir.name, "size": stat.st_size, "mtime_ns": stat.st_mtime_ns, } ) encoded = json.dumps(records, sort_keys=True, ensure_ascii=True) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() async def _get_or_create_catalog_state(self, db: AsyncSession, *, storage_root: str) -> ResultCatalogStateORM: result = await db.execute( select(ResultCatalogStateORM).where(ResultCatalogStateORM.catalog_name == SBAS_INSAR_CATALOG_NAME) ) state = result.scalar_one_or_none() if state is None: state = ResultCatalogStateORM( catalog_name=SBAS_INSAR_CATALOG_NAME, product_family="timeseries", storage_root=storage_root, status="READY", needs_rebuild=False, ) db.add(state) await db.flush() elif state.storage_root != storage_root: state.storage_root = storage_root if state.product_family != "timeseries": state.product_family = "timeseries" return state def _asset_row( self, run_dir: Path, *, role: str, name: str, relative_path: str, is_required: bool, is_primary: bool, ) -> ResultAssetORM: absolute_path = run_dir / relative_path exists = absolute_path.is_file() return ResultAssetORM( asset_role=role[:32], asset_name=name, relative_path=relative_path.replace("\\", "/"), absolute_path=_normalize_path(absolute_path), format=_asset_format(relative_path), media_type=_media_type(relative_path), is_required=is_required, is_primary=is_primary, exists_flag=exists, file_size=absolute_path.stat().st_size if exists else None, srid=4326 if ( (relative_path.lower().endswith((".tif", ".tiff")) and "/geotiff/" in relative_path) or (relative_path.lower().endswith((".tif", ".tiff")) and "/landsar/" in relative_path) or relative_path.lower().endswith(".geojson.gz") ) else None, ) def _monitor_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]: monitor_dir = run_dir / "publish" / "monitor_points" if not monitor_dir.is_dir(): return [] rows: list[ResultAssetORM] = [] for path in sorted(monitor_dir.iterdir()): if not path.is_file(): continue suffix = path.suffix.lower() if suffix not in {".png", ".csv", ".json"}: continue role = { ".png": "monitor_point_curve", ".csv": "monitor_point_csv", ".json": "monitor_point_metadata", }[suffix] relative_path = str(path.relative_to(run_dir)).replace("\\", "/") rows.append( self._asset_row( run_dir, role=role, name=path.name, relative_path=relative_path, is_required=False, is_primary=False, ) ) return rows def _unwrapped_phase_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]: unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" if not unwrapped_dir.is_dir(): return [] rows: list[ResultAssetORM] = [] for path in sorted(unwrapped_dir.iterdir()): if not path.is_file(): continue lowered = path.name.lower() if lowered in {"unwrapped_phase_summary.json", "unwrapped_phase_rmg_colorbar.png"}: continue if lowered.endswith(".rdc_rmg_preview.png"): role = "unwrapped_phase_radar_preview" name = f"Radar-coordinate rmg unwrapped phase preview {path.name}" elif lowered.endswith(".rdc_rmg.bmp"): role = "unwrapped_phase_radar_bmp" name = f"Gamma radar-coordinate rmg unwrapped phase BMP {path.name}" elif lowered.endswith((".tif", ".tiff")): role = "unwrapped_phase_geotiff" name = f"Geocoded unwrapped phase {path.name}" elif lowered.endswith(".png"): role = "unwrapped_phase_preview" name = f"Unwrapped phase preview {path.name}" else: continue relative_path = str(path.relative_to(run_dir)).replace("\\", "/") rows.append( self._asset_row( run_dir, role=role, name=name, relative_path=relative_path, is_required=False, is_primary=False, ) ) return rows def _gamma_intermediate_qc_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]: rows: list[ResultAssetORM] = [] static_assets = ( ( "gamma_qc_baseline_plot", "Gamma baseline network plot", run_dir / "diff_dir" / "bprep_file.png", ), ( "gamma_qc_mean_coherence", "Gamma mean coherence mask", run_dir / "diff_dir" / "mean.cc_mask.bmp", ), ) for role, name, path in static_assets: if not path.is_file(): continue rows.append( self._asset_row( run_dir, role=role, name=f"{name} {path.name}", relative_path=str(path.relative_to(run_dir)).replace("\\", "/"), is_required=False, is_primary=False, ) ) diff_dir = run_dir / "diff_dir" final_tab = run_dir / "sbas" / "final_unw_tab" pair_ids: list[str] = [] if final_tab.is_file(): try: for line in final_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): raw_path = line.strip().split()[0] if line.strip() else "" if not raw_path: continue name = Path(_wsl_path_to_windows(raw_path)).name pair_id = name.replace(".unw.atmsub_1", "").replace(".unw", "") if pair_id and pair_id not in pair_ids: pair_ids.append(pair_id) except OSError: pair_ids = [] unwrapped_paths: list[Path] = [] for pair_id in pair_ids: path = diff_dir / f"{pair_id}.adf.unw.bmp" if path.is_file(): unwrapped_paths.append(path) if not unwrapped_paths and diff_dir.is_dir(): unwrapped_paths = sorted(diff_dir.glob("*.adf.unw.bmp")) if len(unwrapped_paths) > 3: last_index = len(unwrapped_paths) - 1 indexes = sorted({round(index * last_index / 2) for index in range(3)}) unwrapped_paths = [unwrapped_paths[index] for index in indexes] for path in unwrapped_paths: rows.append( self._asset_row( run_dir, role="gamma_qc_unwrapped_phase", name=f"Gamma representative filtered unwrapped phase {path.name}", relative_path=str(path.relative_to(run_dir)).replace("\\", "/"), is_required=False, is_primary=False, ) ) return rows def _build_product(self, manifest_path: str) -> ResultProductORM: manifest_file = Path(manifest_path) run_dir = manifest_file.parent manifest = _read_json(manifest_file) if not self._is_publish_ready(run_dir, manifest): raise ValueError(f"run is not publish-ready: {manifest.get('status') or 'UNKNOWN'}") if self._is_landsar_manifest(manifest): return self._build_landsar_product(run_dir, manifest_file, manifest) detail = sbas_insar_production_service.get_run_detail(run_dir.name) coverage = detail.get("geographic_coverage") or {} stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") monitor_summary = _safe_read_json(run_dir / "monitor_points_summary.json") workflow_summary = _safe_read_json(run_dir / "workflow_summary.json") is_expert_gamma = self._is_expert_gamma_manifest(manifest, run_dir) product_summary = {} if is_expert_gamma else _safe_read_json(run_dir / "product_summary.json") quality_summary = ( _build_expert_gamma_quality_summary(run_dir) if is_expert_gamma else _safe_read_json(run_dir / "quality_summary.json") ) point_vector_summary = _safe_read_json(run_dir / "publish" / "vectors" / "los_rate_points_summary.json") asset_definitions = self._asset_definitions_for_manifest(manifest, run_dir) if is_expert_gamma: _build_rgb_geotiff_preview( run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif", run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb_preview.png", ) _build_gamma_hls_rate_preview( run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif", run_dir / "publish" / "geotiff" / "geo_los_def_rate_pure_hls_preview.png", coverage_source=run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif", min_native=-0.08, max_native=0.08, ) _build_gamma_hls_colorbar( run_dir / "publish" / "geotiff" / "geo_los_def_rate_hls_colorbar.png", min_mm_year=-80.0, max_mm_year=80.0, ) point_vector_context = { "run_id": str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name, "reference_date": str( manifest.get("reference_date") or (stack_manifest.get("stack") or {}).get("reference_date") or "" ).strip() or None, "stack_dates": _stack_dates_from_manifest(stack_manifest, manifest, stack_manifest.get("stack") or {}), "los_sign_convention": "Gamma expert geo_los_def_rate output; sign and unit semantics follow the expert workflow.", "admin_region": coverage.get("admin_region"), } point_vector_context["date_start"] = ( point_vector_context["stack_dates"][0] if point_vector_context["stack_dates"] else None ) point_vector_context["date_end"] = ( point_vector_context["stack_dates"][-1] if point_vector_context["stack_dates"] else None ) point_vector_summary = _build_expert_gamma_point_vector( run_dir, summary_context=point_vector_context, ) unwrapped_phase_summary = _build_expert_unwrapped_phase_derivatives(run_dir) monitor_summary = _write_expert_monitor_point_derivatives(run_dir) or monitor_summary else: unwrapped_phase_summary = {} bbox = coverage.get("bbox") or {} min_lon = _safe_float(bbox.get("min_lon")) min_lat = _safe_float(bbox.get("min_lat")) max_lon = _safe_float(bbox.get("max_lon")) max_lat = _safe_float(bbox.get("max_lat")) poly = _bbox_polygon(min_lon, min_lat, max_lon, max_lat) run_id = str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name stack = stack_manifest.get("stack") or manifest.get("stack") or {} stack_id = str(manifest.get("stack_id") or stack_manifest.get("stack_id") or stack.get("stack_id") or "").strip() stack_dates = _stack_dates_from_manifest(stack_manifest, manifest, stack) reference_date = str( manifest.get("reference_date") or stack.get("reference_date") or (manifest.get("coregistration") or {}).get("reference_date") or "" ).strip() or None display_name = stack_id or f"Gamma SBAS {run_id}" product_id = str(manifest.get("product_id") or "").strip() or f"gamma_sbas_{run_id}" if len(product_id) > 64: product_id = f"gamma_sbas_{_stable_digest(product_id, run_dir, length=32)}" assets: list[ResultAssetORM] = [ self._asset_row( run_dir, role=role, name=name, relative_path=relative_path, is_required=is_required, is_primary=is_primary, ) for role, name, relative_path, is_required, is_primary in asset_definitions ] assets.extend(self._monitor_asset_rows(run_dir)) if is_expert_gamma: assets.extend(self._unwrapped_phase_asset_rows(run_dir)) assets.extend(self._gamma_intermediate_qc_asset_rows(run_dir)) preview_asset = next((asset for asset in assets if asset.asset_role == "primary_geocoded_preview" and asset.exists_flag), None) primary_asset = next((asset for asset in assets if asset.asset_role == "primary_geotiff" and asset.exists_flag), None) missing_required = [asset for asset in assets if asset.is_required and not asset.exists_flag] default_los_product = product_summary.get("default_los_product") los_sign_convention = product_summary.get("los_sign_convention") if is_expert_gamma: default_los_product = default_los_product or "geo_los_def_rate" los_sign_convention = ( los_sign_convention or "Gamma expert geo_los_def_rate output; sign and unit semantics follow the expert workflow." ) color_policy = { "schema": "insar.gamma-sbas-color-policy/v1", "source": "expert_gamma_command", "browse_command": "rasdt_pwr los_def_rate ... -0.08 0.08 0 hls.cm ... 24", "colormap": "Gamma hls.cm", "data_range_native": [-0.08, 0.08], "display_range_mm_per_year": [-80.0, 80.0], "note": "The RGB browse GeoTIFF is generated by Gamma with hls.cm. Treat it as the expert browse standard unless the project defines a separate cartographic standard.", } else: default_los_product = default_los_product or "los_rate_toward_m_per_year" los_sign_convention = los_sign_convention or "toward radar positive; away from radar negative" color_policy = product_summary.get("color_policy") produced_at = ( _parse_datetime(monitor_summary.get("generated_at")) or _parse_datetime(product_summary.get("generated_at")) or _parse_datetime(workflow_summary.get("generated_at")) or _parse_datetime(manifest.get("updated_at")) or _parse_datetime(manifest.get("created_at")) ) center = coverage.get("center") or {} admin_region = coverage.get("admin_region") or lookup_admin_region_for_point(center.get("lon"), center.get("lat")) scene_count = ( _safe_int(manifest.get("scene_count")) or len(stack_manifest.get("scenes") or []) or len(stack_dates) ) summary_json = { "schema": "insar.gamma-sbas-result-catalog-summary/v1", "run_id": run_id, "stack_id": stack_id or None, "stack": stack, "reference_date": reference_date, "stack_dates": stack_dates, "stack_size": len(stack_dates), "date_start": stack_dates[0] if stack_dates else None, "date_end": stack_dates[-1] if stack_dates else None, "scene_count": scene_count, "pair_count": _safe_int(manifest.get("pair_count")), "status": manifest.get("status"), "next_stage": manifest.get("next_stage"), "los_sign_convention": los_sign_convention, "default_los_product": default_los_product, "color_policy": color_policy, "center": center or None, "admin_region": admin_region, "geographic_coverage": coverage, "quality": quality_summary, "monitor_points": monitor_summary, "point_vector": point_vector_summary, "unwrapped_phase": unwrapped_phase_summary, "workflow": { "status": ((manifest.get("workflow") or {}).get("status")), "summary": ((manifest.get("workflow") or {}).get("summary")) or workflow_summary, }, "source_run_dir": str(run_dir), } product = ResultProductORM( product_id=product_id, catalog_name=SBAS_INSAR_CATALOG_NAME, product_family="timeseries", product_type="sbas_insar", display_name=display_name, task_name="Gamma SBAS-InSAR", task_alias=run_id, stack_key=stack_id or run_id, run_key=run_id, profile_code=str(stack.get("relative_orbit") or manifest.get("relative_orbit") or "").strip() or None, engine_code="gamma", engine_version=str((manifest.get("engine") or {}).get("version") or "").strip() or None, package_schema=str(manifest.get("schema") or "").strip() or "insar.gamma-sbas-run/v1", package_layout="gamma_sbas_expert_workflow_run", processor_code="gamma_ipta_sbas", runtime_id=settings.GAMMA_SBAS_RUNTIME_ID, status="READY" if not missing_required else "INCOMPLETE", health_status="OK" if not missing_required else "WARN", publish_dir=_normalize_path(run_dir / "publish"), manifest_path=_normalize_path(manifest_file), source_primary_path=primary_asset.absolute_path if primary_asset else None, native_output_dir=_normalize_path(run_dir), preview_path=preview_asset.absolute_path if preview_asset else None, primary_asset_path=primary_asset.absolute_path if primary_asset else None, summary_json=summary_json, tags_json={ "sensor": stack.get("satellite") or manifest.get("platform"), "orbit_direction": stack.get("orbit_direction") or manifest.get("direction"), "product": "Gamma SBAS", "workflow_mode": "expert_document" if is_expert_gamma else "legacy_gamma", "admin_region": (admin_region or {}).get("display_name") if isinstance(admin_region, dict) else None, }, min_lon=min_lon, min_lat=min_lat, max_lon=max_lon, max_lat=max_lat, geom=from_shape(poly, srid=4326) if poly is not None else None, coverage_polygon=(coverage.get("geojson") or coverage.get("scene_footprints_geojson")), produced_at=produced_at, published_at=produced_at, ) for asset in assets: product.assets.append(asset) if asset.is_required and not asset.exists_flag: product.issues.append( ResultIssueORM( asset=asset, issue_code="MISSING_REQUIRED_ASSET", severity="ERROR", status="OPEN", scope="file", message=f"Required SBAS asset is missing: {asset.relative_path}", ) ) if not preview_asset: product.issues.append( ResultIssueORM( issue_code="MISSING_PREVIEW", severity="WARN", status="OPEN", scope="product", message="Primary geocoded preview PNG is missing.", ) ) if poly is None: product.issues.append( ResultIssueORM( issue_code="MISSING_COVERAGE", severity="WARN", status="OPEN", scope="product", message="No valid EPSG:4326 geographic coverage bbox was found.", ) ) return product def _build_landsar_product(self, run_dir: Path, manifest_file: Path, manifest: dict[str, Any]) -> ResultProductORM: try: detail = landsar_sbas_service.get_run_detail(run_dir.name) except Exception: detail = {} coverage = detail.get("geographic_coverage") or manifest.get("geographic_coverage") or {} stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") product_summary = _safe_read_json(run_dir / "product_summary.json") quality_summary = _safe_read_json(run_dir / "quality_summary.json") workflow_summary = _safe_read_json(run_dir / "workflow_summary.json") bbox = coverage.get("bbox") or {} min_lon = _safe_float(bbox.get("min_lon")) min_lat = _safe_float(bbox.get("min_lat")) max_lon = _safe_float(bbox.get("max_lon")) max_lat = _safe_float(bbox.get("max_lat")) poly = _bbox_polygon(min_lon, min_lat, max_lon, max_lat) run_id = str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name stack_id = str(manifest.get("stack_id") or run_id).strip() stack_dates = _stack_dates_from_manifest(stack_manifest, manifest, {}) if not stack_dates: stack_dates = [str(item or "").strip() for item in manifest.get("dates") or [] if str(item or "").strip()] display_name = str(manifest.get("run_label") or f"LandSAR SBAS {run_id}").strip() product_id = str(manifest.get("product_id") or "").strip() or f"landsar_sbas_{run_id}" if len(product_id) > 64: product_id = f"landsar_sbas_{_stable_digest(product_id, run_dir, length=32)}" assets: list[ResultAssetORM] = [ self._asset_row( run_dir, role=role, name=name, relative_path=relative_path, is_required=is_required, is_primary=is_primary, ) for role, name, relative_path, is_required, is_primary in _LANDSAR_CORE_ASSETS if not (run_dir / relative_path).is_dir() ] native_logs_dir = run_dir / "native_logs" if native_logs_dir.is_dir(): for path in sorted(native_logs_dir.rglob("*")): if not path.is_file(): continue relative_path = str(path.relative_to(run_dir)).replace("\\", "/") assets.append( self._asset_row( run_dir, role="native_log" if path.suffix.lower() == ".log" else "native_parameter", name=path.name, relative_path=relative_path, is_required=False, is_primary=False, ) ) task_publish_root = run_dir / "publish" / "landsar" if task_publish_root.is_dir(): for path in sorted(task_publish_root.rglob("*")): if not path.is_file(): continue relative_path = str(path.relative_to(run_dir)).replace("\\", "/") if relative_path in {item.relative_path for item in assets}: continue role = "landsar_task_geotiff" if path.suffix.lower() in {".tif", ".tiff"} else "landsar_task_asset" assets.append( self._asset_row( run_dir, role=role, name=path.name, relative_path=relative_path, is_required=False, is_primary=False, ) ) preview_asset = next((asset for asset in assets if asset.asset_role == "primary_preview" and asset.exists_flag), None) primary_asset = next((asset for asset in assets if asset.asset_role == "primary_geotiff" and asset.exists_flag), None) missing_required = [asset for asset in assets if asset.is_required and not asset.exists_flag] produced_at = ( _parse_datetime(workflow_summary.get("ended_at")) or _parse_datetime(product_summary.get("generated_at")) or _parse_datetime(manifest.get("ended_at")) or _parse_datetime(manifest.get("created_at")) ) center = coverage.get("center") or {} admin_region = coverage.get("admin_region") or lookup_admin_region_for_point(center.get("lon"), center.get("lat")) scene_count = _safe_int(manifest.get("scene_count")) or len(stack_dates) task_count = _safe_int(manifest.get("task_count")) summary_json = { "schema": "insar.landsar-sbas-result-catalog-summary/v1", "run_id": run_id, "stack_id": stack_id or None, "reference_date": stack_dates[0] if stack_dates else None, "stack_dates": stack_dates, "stack_size": len(stack_dates), "date_start": manifest.get("date_start") or (stack_dates[0] if stack_dates else None), "date_end": manifest.get("date_end") or (stack_dates[-1] if stack_dates else None), "scene_count": scene_count, "task_count": task_count, "pair_count": _safe_int(manifest.get("pair_count")), "status": manifest.get("status"), "next_stage": manifest.get("next_stage"), "los_sign_convention": product_summary.get("los_sign_convention") or "LandSAR LOS output; semantics pending algorithm confirmation.", "default_los_product": product_summary.get("default_los_product") or "los_timeseries", "center": center or None, "admin_region": admin_region, "geographic_coverage": coverage, "quality": quality_summary, "workflow": workflow_summary, "source_run_dir": str(run_dir), "output_semantics_note": product_summary.get("output_semantics_note"), } product = ResultProductORM( product_id=product_id, catalog_name=SBAS_INSAR_CATALOG_NAME, product_family="timeseries", product_type="sbas_insar", display_name=display_name, task_name="LandSAR SBAS-InSAR", task_alias=run_id, stack_key=stack_id or run_id, run_key=run_id, profile_code="lt1_landsar_sbas", engine_code="landsar", engine_version=None, package_schema=str(manifest.get("schema") or "").strip() or "insar.landsar-sbas-run/v1", package_layout="landsar_sbas_console_run", processor_code="landsar_sbas", runtime_id="landsar_console", status="READY" if not missing_required else "INCOMPLETE", health_status="OK" if not missing_required else "WARN", publish_dir=_normalize_path(run_dir / "publish"), manifest_path=_normalize_path(manifest_file), source_primary_path=primary_asset.absolute_path if primary_asset else None, native_output_dir=_normalize_path(run_dir), preview_path=preview_asset.absolute_path if preview_asset else None, primary_asset_path=primary_asset.absolute_path if primary_asset else None, summary_json=summary_json, tags_json={ "sensor": "LT1", "product": "LandSAR SBAS", "processor_code": "landsar_sbas", "admin_region": (admin_region or {}).get("display_name") if isinstance(admin_region, dict) else None, }, min_lon=min_lon, min_lat=min_lat, max_lon=max_lon, max_lat=max_lat, geom=from_shape(poly, srid=4326) if poly is not None else None, coverage_polygon=(coverage.get("geojson") or coverage.get("scene_footprints_geojson")), produced_at=produced_at, published_at=produced_at, ) for asset in assets: product.assets.append(asset) if asset.is_required and not asset.exists_flag: product.issues.append( ResultIssueORM( asset=asset, issue_code="MISSING_REQUIRED_ASSET", severity="ERROR", status="OPEN", scope="file", message=f"Required LandSAR SBAS asset is missing: {asset.relative_path}", ) ) if not preview_asset: product.issues.append( ResultIssueORM( issue_code="MISSING_PREVIEW", severity="WARN", status="OPEN", scope="product", message="LandSAR SBAS preview PNG is missing.", ) ) if poly is None: product.issues.append( ResultIssueORM( issue_code="MISSING_COVERAGE", severity="WARN", status="OPEN", scope="product", message="No valid EPSG:4326 geographic coverage bbox was found.", ) ) return product async def rebuild_catalog(self, db: AsyncSession, *, full_rebuild: bool = True) -> dict[str, Any]: run_root = self.get_run_root() run_roots = self.get_run_roots() manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) state.status = "REBUILDING" state.needs_rebuild = False state.last_message = "SBAS catalog rebuild in progress" await db.commit() if full_rebuild: await db.execute(delete(ResultProductORM).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME)) await db.commit() registered = 0 failed = 0 issue_count = 0 details: list[dict[str, Any]] = [] for manifest_path in manifest_paths: try: product = await asyncio.to_thread(self._build_product, manifest_path) product_issue_count = len(product.issues) product_id = product.product_id product_status = product.status db.add(product) await db.flush() await db.commit() registered += 1 issue_count += product_issue_count details.append( { "manifest_path": manifest_path, "product_id": product_id, "status": product_status, "issues": product_issue_count, } ) except Exception as exc: await db.rollback() failed += 1 issue_count += 1 details.append({"manifest_path": manifest_path, "status": "error", "message": str(exc)}) await db.commit() db_count_result = await db.execute( select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) ) db_count = int(db_count_result.scalar_one() or 0) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) state.manifest_count = len(manifest_paths) state.manifest_fingerprint = fingerprint state.db_count = db_count state.issue_count = issue_count state.needs_rebuild = False state.status = "READY" if failed == 0 else "WARN" now = _utcnow() state.last_full_rebuild_at = now state.last_incremental_scan_at = now state.last_message = ( f"SBAS catalog rebuild finished: runs={len(manifest_paths)}, " f"registered={registered}, failed={failed}, issues={issue_count}" ) await db.commit() return { "catalog_name": SBAS_INSAR_CATALOG_NAME, "storage_root": run_root, "storage_roots": run_roots, "run_count": len(manifest_paths), "manifest_count": len(manifest_paths), "manifest_fingerprint": fingerprint, "registered": registered, "failed": failed, "issue_count": issue_count, "details": details, } async def list_products( self, db: AsyncSession, *, limit: int = 100, offset: int = 0, status: Optional[str] = None, query: Optional[str] = None, admin_region: Optional[str] = None, ) -> dict[str, Any]: safe_limit = max(1, min(int(limit or 100), 500)) safe_offset = max(0, int(offset or 0)) stmt = select(ResultProductORM).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) count_stmt = select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) if status: stmt = stmt.where(ResultProductORM.status == status) count_stmt = count_stmt.where(ResultProductORM.status == status) if query: like_value = f"%{query.strip()}%" predicate = or_( ResultProductORM.display_name.ilike(like_value), ResultProductORM.product_id.ilike(like_value), ResultProductORM.run_key.ilike(like_value), ResultProductORM.stack_key.ilike(like_value), ) stmt = stmt.where(predicate) count_stmt = count_stmt.where(predicate) if admin_region: like_value = f"%{admin_region.strip()}%" predicate = or_( cast(ResultProductORM.summary_json, String).ilike(like_value), cast(ResultProductORM.tags_json, String).ilike(like_value), ) stmt = stmt.where(predicate) count_stmt = count_stmt.where(predicate) total_result = await db.execute(count_stmt) total = int(total_result.scalar_one() or 0) result = await db.execute( stmt.order_by(ResultProductORM.published_at.desc().nullslast(), ResultProductORM.id.desc()) .offset(safe_offset) .limit(safe_limit) ) items: list[dict[str, Any]] = [] for product in result.scalars().all(): summary = product.summary_json or {} items.append( { "id": product.id, "product_id": product.product_id, "display_name": product.display_name, "run_key": product.run_key, "stack_key": product.stack_key, "engine_code": product.engine_code, "processor_code": product.processor_code, "runtime_id": product.runtime_id, "status": product.status, "health_status": product.health_status, "preview_path": product.preview_path, "primary_asset_path": product.primary_asset_path, "reference_date": summary.get("reference_date"), "date_start": summary.get("date_start"), "date_end": summary.get("date_end"), "stack_dates": summary.get("stack_dates") or [], "stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []), "scene_count": summary.get("scene_count"), "pair_count": summary.get("pair_count"), "los_sign_convention": summary.get("los_sign_convention"), "color_policy": summary.get("color_policy"), "center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")), "admin_region": summary.get("admin_region") or ((summary.get("geographic_coverage") or {}).get("admin_region")), "min_lon": product.min_lon, "min_lat": product.min_lat, "max_lon": product.max_lon, "max_lat": product.max_lat, "published_at": product.published_at, } ) return { "items": items, "total": total, "limit": safe_limit, "offset": safe_offset, "has_more": safe_offset + len(items) < total, } async def get_product_detail(self, db: AsyncSession, *, product_db_id: int) -> Optional[dict[str, Any]]: result = await db.execute(select(ResultProductORM).where(ResultProductORM.id == product_db_id)) product = result.scalar_one_or_none() if product is None or product.catalog_name != SBAS_INSAR_CATALOG_NAME: return None assets_result = await db.execute( select(ResultAssetORM) .where(ResultAssetORM.product_ref_id == product.id) .order_by(ResultAssetORM.is_primary.desc(), ResultAssetORM.asset_role.asc(), ResultAssetORM.id.asc()) ) issues_result = await db.execute( select(ResultIssueORM) .where(ResultIssueORM.product_ref_id == product.id) .order_by(ResultIssueORM.severity.asc(), ResultIssueORM.id.asc()) ) summary = product.summary_json or {} return { "id": product.id, "product_id": product.product_id, "catalog_name": product.catalog_name, "product_type": product.product_type, "display_name": product.display_name, "run_key": product.run_key, "run_id": summary.get("run_id") or product.run_key, "stack_key": product.stack_key, "profile_code": product.profile_code, "engine_code": product.engine_code, "engine_version": product.engine_version, "package_schema": product.package_schema, "package_layout": product.package_layout, "processor_code": product.processor_code, "runtime_id": product.runtime_id, "status": product.status, "health_status": product.health_status, "publish_dir": product.publish_dir, "manifest_path": product.manifest_path, "source_primary_path": product.source_primary_path, "native_output_dir": product.native_output_dir, "preview_path": product.preview_path, "primary_asset_path": product.primary_asset_path, "reference_date": summary.get("reference_date"), "date_start": summary.get("date_start"), "date_end": summary.get("date_end"), "stack_dates": summary.get("stack_dates") or [], "stack_size": summary.get("stack_size") or len(summary.get("stack_dates") or []), "scene_count": summary.get("scene_count"), "pair_count": summary.get("pair_count"), "los_sign_convention": summary.get("los_sign_convention"), "default_los_product": summary.get("default_los_product"), "color_policy": summary.get("color_policy"), "quality": summary.get("quality"), "monitor_points": summary.get("monitor_points"), "point_vector": summary.get("point_vector"), "unwrapped_phase": summary.get("unwrapped_phase"), "workflow": summary.get("workflow"), "geographic_coverage": summary.get("geographic_coverage"), "center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")), "admin_region": summary.get("admin_region") or ((summary.get("geographic_coverage") or {}).get("admin_region")), "coverage_polygon": product.coverage_polygon, "min_lon": product.min_lon, "min_lat": product.min_lat, "max_lon": product.max_lon, "max_lat": product.max_lat, "produced_at": product.produced_at, "published_at": product.published_at, "registered_at": product.registered_at, "updated_at": product.updated_at, "assets": [ { "id": asset.id, "asset_role": asset.asset_role, "asset_name": asset.asset_name, "relative_path": asset.relative_path, "absolute_path": asset.absolute_path, "format": asset.format, "media_type": asset.media_type, "is_required": asset.is_required, "is_primary": asset.is_primary, "exists_flag": asset.exists_flag, "file_size": asset.file_size, "srid": asset.srid, } for asset in assets_result.scalars().all() ], "issues": [ { "id": issue.id, "issue_code": issue.issue_code, "severity": issue.severity, "status": issue.status, "scope": issue.scope, "message": issue.message, "detected_at": issue.detected_at, } for issue in issues_result.scalars().all() ], } async def query_point_timeseries( self, db: AsyncSession, *, product_db_id: int, lon: float, lat: float, ) -> Optional[dict[str, Any]]: result = await db.execute(select(ResultProductORM).where(ResultProductORM.id == product_db_id)) product = result.scalar_one_or_none() if product is None or product.catalog_name != SBAS_INSAR_CATALOG_NAME: return None manifest_path = str(product.manifest_path or "").strip() run_dir = Path(manifest_path).parent if manifest_path else Path(str(product.native_output_dir or "")) if not run_dir.is_dir(): raise FileNotFoundError("SBAS run directory not found") manifest = _safe_read_json(run_dir / "run_manifest.json") if not self._is_expert_gamma_manifest(manifest, run_dir): raise ValueError("point time-series query is only available for expert Gamma SBAS products") return _query_expert_gamma_point_timeseries(run_dir, lon=lon, lat=lat) async def get_asset(self, db: AsyncSession, *, product_db_id: int, asset_id: int) -> Optional[ResultAssetORM]: result = await db.execute( select(ResultAssetORM) .join(ResultProductORM, ResultProductORM.id == ResultAssetORM.product_ref_id) .where( ResultProductORM.id == product_db_id, ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME, ResultAssetORM.id == asset_id, ) ) return result.scalar_one_or_none() async def get_catalog_status(self, db: AsyncSession) -> dict[str, Any]: run_root = self.get_run_root() run_roots = self.get_run_roots() manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) db_count_result = await db.execute( select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) ) db_count = int(db_count_result.scalar_one() or 0) needs_rebuild = ( state.manifest_count != len(manifest_paths) or state.db_count != db_count or state.manifest_fingerprint != fingerprint ) state.manifest_count = len(manifest_paths) state.db_count = db_count state.needs_rebuild = needs_rebuild state.last_incremental_scan_at = _utcnow() state.status = "WARN" if needs_rebuild else "READY" state.last_message = ( f"SBAS catalog rebuild required: runs={len(manifest_paths)}, db={db_count}" if needs_rebuild else "SBAS catalog is in sync" ) await db.commit() return { "catalog_name": state.catalog_name, "product_family": state.product_family, "storage_root": state.storage_root, "storage_roots": run_roots, "status": state.status, "needs_rebuild": state.needs_rebuild, "run_count": len(manifest_paths), "manifest_count": state.manifest_count, "manifest_fingerprint": state.manifest_fingerprint, "current_manifest_fingerprint": fingerprint, "db_count": db_count, "issue_count": state.issue_count, "last_message": state.last_message, "last_boot_check_at": state.last_boot_check_at, "last_full_rebuild_at": state.last_full_rebuild_at, "last_incremental_scan_at": state.last_incremental_scan_at, } async def bootstrap_catalog_on_startup_clean(self) -> dict[str, Any]: from ..database import AsyncSessionLocal if AsyncSessionLocal is None: raise RuntimeError("Database session factory is not initialized.") async with AsyncSessionLocal() as db: run_root = self.get_run_root() run_roots = self.get_run_roots() manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) db_count_result = await db.execute( select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) ) db_count = int(db_count_result.scalar_one() or 0) needs_rebuild = ( state.manifest_count != len(manifest_paths) or state.db_count != db_count or state.manifest_fingerprint != fingerprint ) state.last_boot_check_at = _utcnow() await db.commit() rebuilt = False result: dict[str, Any] = {} if needs_rebuild and settings.RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP: result = await self.rebuild_catalog(db, full_rebuild=True) rebuilt = True db_count = int(result.get("registered") or db_count) else: state.manifest_count = len(manifest_paths) state.db_count = db_count state.manifest_fingerprint = fingerprint if not needs_rebuild else state.manifest_fingerprint state.needs_rebuild = needs_rebuild state.status = "WARN" if needs_rebuild else "READY" state.last_message = "SBAS boot check complete" await db.commit() return { "storage_root": run_root, "storage_roots": run_roots, "manifest_count": len(manifest_paths), "current_manifest_fingerprint": fingerprint, "indexed_manifest_fingerprint": state.manifest_fingerprint, "db_count": db_count, "needs_rebuild": needs_rebuild and not rebuilt, "rebuilt": rebuilt, "queued": False, "registered": result.get("registered"), "failed": result.get("failed"), } sbas_insar_catalog_service = SbasInsarCatalogService()