Add LandSAR cluster worker deployment

This commit is contained in:
2026-06-24 14:10:58 +08:00
parent 71c524967c
commit da09ba05cb
88 changed files with 9300 additions and 3011 deletions
+282
View File
@@ -0,0 +1,282 @@
"""Backfill XML refRow/refColumn corner mappings for archive-managed radar records."""
from __future__ import annotations
import argparse
import asyncio
import gzip
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
from geoalchemy2.shape import from_shape # noqa: E402
from shapely.geometry import Polygon # noqa: E402
from sqlalchemy import or_, select # noqa: E402
from sqlalchemy.ext.asyncio import AsyncSession # noqa: E402
from backend.app import database # noqa: E402
from backend.app.config import settings # noqa: E402
from backend.app.models import RadarDataORM, SourceMetadataDocumentORM, SourceProductAssetORM # noqa: E402
from backend.app.services.asset_inventory_service import _parse_radar_xml_metadata_bytes # noqa: E402
from backend.app.utils import build_corner_pixel_mapping # noqa: E402
SOURCE_FORMATS = {"LT1_ARCHIVE"}
DOCUMENT_TYPES = {"LT1_META"}
def _metadata_dict(value: Any) -> Dict[str, Any]:
return dict(value or {}) if isinstance(value, dict) else {}
def _normalize_families(values: Optional[Iterable[str]]) -> Sequence[str]:
families = []
for value in values or ["LT1"]:
for item in str(value or "").split(","):
family = item.strip().upper()
if family and family not in families:
families.append(family)
supported = {"LT1"}
invalid = [item for item in families if item not in supported]
if invalid:
raise SystemExit(f"Unsupported family: {', '.join(invalid)}")
return families or ["LT1"]
def _decode_document(doc: SourceMetadataDocumentORM) -> Optional[bytes]:
payload = bytes(doc.content_bytes or b"")
if not payload:
return None
if str(doc.content_encoding or "").lower() == "gzip":
return gzip.decompress(payload)
return payload
def _mapping_from_xml_document(doc: SourceMetadataDocumentORM) -> Tuple[Optional[Dict[str, Any]], Optional[Any]]:
payload = _decode_document(doc)
if not payload:
return None, None
coverage_polygon, xml_meta = _parse_radar_xml_metadata_bytes(payload)
mapping = xml_meta.get("corner_pixel_mapping") if isinstance(xml_meta, dict) else None
return mapping if isinstance(mapping, dict) else None, coverage_polygon
def _mapping_from_metadata(metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]:
existing = metadata.get("corner_pixel_mapping")
if isinstance(existing, dict):
return existing
corners = metadata.get("corner_details")
if isinstance(corners, dict):
mapping = build_corner_pixel_mapping(corners)
if isinstance(mapping, dict):
return mapping
return None
def _bbox(points: Any) -> Optional[Tuple[float, float, float, float]]:
try:
ring = [(float(item[0]), float(item[1])) for item in points or []]
except (TypeError, ValueError, IndexError):
return None
if len(ring) < 3:
return None
lons = [item[0] for item in ring]
lats = [item[1] for item in ring]
return min(lons), min(lats), max(lons), max(lats)
async def _first_metadata_document(
db: AsyncSession,
*,
source_asset_id: Optional[int],
radar_data_id: Optional[int],
) -> Optional[SourceMetadataDocumentORM]:
clauses = []
if source_asset_id:
clauses.append(SourceMetadataDocumentORM.source_asset_id == int(source_asset_id))
if radar_data_id:
clauses.append(SourceMetadataDocumentORM.radar_data_id == int(radar_data_id))
if not clauses:
return None
stmt = (
select(SourceMetadataDocumentORM)
.where(or_(*clauses))
.where(SourceMetadataDocumentORM.document_type.in_(sorted(DOCUMENT_TYPES)))
.order_by(SourceMetadataDocumentORM.id.asc())
.limit(1)
)
return (await db.execute(stmt)).scalar_one_or_none()
async def _load_source_assets(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[SourceProductAssetORM]:
stmt = (
select(SourceProductAssetORM)
.where(SourceProductAssetORM.satellite_family.in_(families))
.where(SourceProductAssetORM.source_format.in_(sorted(SOURCE_FORMATS)))
.where(SourceProductAssetORM.is_active.is_(True))
.order_by(SourceProductAssetORM.id.asc())
)
if limit > 0:
stmt = stmt.limit(limit)
return list((await db.execute(stmt)).scalars().all())
async def _load_radar_records(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[RadarDataORM]:
stmt = (
select(RadarDataORM)
.where(RadarDataORM.satellite_family.in_(families))
.where(RadarDataORM.source_format.in_(sorted(SOURCE_FORMATS)))
.order_by(RadarDataORM.id.asc())
)
if limit > 0:
stmt = stmt.limit(limit)
return list((await db.execute(stmt)).scalars().all())
async def backfill(args: argparse.Namespace) -> Dict[str, int]:
families = _normalize_families(args.family)
database.init_db(settings.DATABASE_URL)
stats = {
"source_seen": 0,
"source_updated": 0,
"source_existing": 0,
"source_missing_mapping": 0,
"radar_seen": 0,
"radar_updated": 0,
"radar_existing": 0,
"radar_missing_mapping": 0,
"cache_invalidated": 0,
"document_parse_failed": 0,
}
async with database.AsyncSessionLocal() as db:
source_assets = await _load_source_assets(db, families, int(args.limit or 0))
source_mappings: Dict[int, Dict[str, Any]] = {}
source_polygons: Dict[int, Any] = {}
now = datetime.utcnow()
for asset in source_assets:
stats["source_seen"] += 1
metadata = _metadata_dict(asset.metadata_json)
mapping = _mapping_from_metadata(metadata)
coverage_polygon = metadata.get("coverage_polygon")
if mapping:
stats["source_existing"] += 1
else:
doc = await _first_metadata_document(db, source_asset_id=asset.id, radar_data_id=None)
if doc:
try:
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
except Exception:
stats["document_parse_failed"] += 1
mapping = None
if not mapping:
stats["source_missing_mapping"] += 1
continue
metadata["corner_pixel_mapping"] = mapping
if coverage_polygon:
metadata["coverage_polygon"] = coverage_polygon
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
asset.metadata_json = metadata
asset.updated_at = now
db.add(asset)
stats["source_updated"] += 1
if mapping and asset.id is not None:
source_mappings[int(asset.id)] = mapping
if coverage_polygon:
source_polygons[int(asset.id)] = coverage_polygon
radar_records = await _load_radar_records(db, families, int(args.limit or 0))
for radar in radar_records:
stats["radar_seen"] += 1
metadata = _metadata_dict(radar.metadata_json)
mapping = _mapping_from_metadata(metadata)
coverage_polygon = radar.coverage_polygon or metadata.get("coverage_polygon")
if mapping:
stats["radar_existing"] += 1
else:
source_id = int(radar.source_product_ref_id or 0)
mapping = source_mappings.get(source_id)
if source_id and source_id in source_polygons:
coverage_polygon = coverage_polygon or source_polygons[source_id]
if not mapping:
doc = await _first_metadata_document(
db,
source_asset_id=radar.source_product_ref_id,
radar_data_id=radar.id,
)
if doc:
try:
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
except Exception:
stats["document_parse_failed"] += 1
mapping = None
if not mapping:
stats["radar_missing_mapping"] += 1
continue
metadata["corner_pixel_mapping"] = mapping
if coverage_polygon:
metadata["coverage_polygon"] = coverage_polygon
radar.coverage_polygon = coverage_polygon
bbox = _bbox(coverage_polygon)
if bbox:
radar.min_lon, radar.min_lat, radar.max_lon, radar.max_lat = bbox
try:
polygon = Polygon(coverage_polygon)
if not polygon.is_valid:
polygon = polygon.buffer(0)
if not polygon.is_empty:
radar.geom = from_shape(polygon, srid=4326)
except Exception:
pass
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
radar.metadata_json = metadata
db.add(radar)
stats["radar_updated"] += 1
if args.invalidate_preview and mapping:
radar.preview_cache_status = "NONE"
radar.preview_cache_version = None
radar.preview_cache_path = None
radar.preview_cache_error = "corner_pixel_mapping_backfilled"
radar.preview_cache_updated_at = now
db.add(radar)
stats["cache_invalidated"] += 1
if args.apply:
await db.commit()
else:
await db.rollback()
return stats
def main() -> int:
parser = argparse.ArgumentParser(description="Backfill corner_pixel_mapping into source/radar metadata_json.")
parser.add_argument("--apply", action="store_true", help="commit database changes")
parser.add_argument("--family", action="append", help="family to process: LT1 or GF3; defaults to LT1")
parser.add_argument("--limit", type=int, default=0, help="maximum source/radar rows per table; 0 means all")
parser.add_argument(
"--no-invalidate-preview",
dest="invalidate_preview",
action="store_false",
help="do not mark old geocorrected WebP caches stale",
)
parser.set_defaults(invalidate_preview=True)
args = parser.parse_args()
stats = asyncio.run(backfill(args))
print(json.dumps({"apply": bool(args.apply), **stats}, ensure_ascii=False, indent=2, default=str))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+54
View File
@@ -5,6 +5,7 @@ import argparse
import asyncio
import json
import sys
from datetime import datetime
from pathlib import Path
from typing import Iterable, List, Optional
@@ -34,6 +35,55 @@ def _normalize_families(values: Optional[Iterable[str]]) -> List[str]:
async def _run(args: argparse.Namespace) -> dict:
families = _normalize_families(args.family)
database.init_db(settings.DATABASE_URL)
def _print_progress(payload: dict) -> None:
stamp = datetime.now().strftime("%H:%M:%S")
event = payload.get("event")
if event == "planned":
print(
"[{}] planned records_seen={} candidates={} skipped_ready={} families={}".format(
stamp,
payload.get("records_seen", 0),
payload.get("candidate_count", 0),
payload.get("skipped_ready", 0),
",".join(payload.get("families") or []),
),
flush=True,
)
return
if event == "item":
print(
(
"[{}] {}/{} ready={} failed={} missing={} raw_failed={} "
"skipped_ready={} status={} product={}"
).format(
stamp,
payload.get("processed", 0),
payload.get("total", 0),
payload.get("ready", 0),
payload.get("failed", 0),
payload.get("missing_source", 0),
payload.get("raw_failed", 0),
payload.get("skipped_ready", 0),
payload.get("status") or "",
payload.get("product_name") or "",
),
flush=True,
)
return
if event == "completed":
print(
"[{}] completed ready={} cached={} skipped_ready={} failed={} missing={}".format(
stamp,
payload.get("ready", 0),
payload.get("cached", 0),
payload.get("skipped_ready", 0),
payload.get("failed", 0),
payload.get("missing_source", 0),
),
flush=True,
)
summary = await asset_inventory_service.build_archive_preview_caches(
families=families,
limit=args.limit,
@@ -41,6 +91,8 @@ async def _run(args: argparse.Namespace) -> dict:
apply=bool(args.apply),
progress_start=0,
progress_end=100,
progress_callback=_print_progress if args.progress else None,
progress_interval=args.progress_interval,
)
return {"apply": bool(args.apply), **summary}
@@ -51,6 +103,8 @@ def main() -> int:
parser.add_argument("--family", action="append", help="family to process: LT1, S1, or comma-separated values")
parser.add_argument("--limit", type=int, default=0, help="maximum rows to build; 0 means all pending rows")
parser.add_argument("--force", action="store_true", help="rebuild even if preview_cache_status is READY")
parser.add_argument("--progress", action="store_true", help="print command-line progress while building")
parser.add_argument("--progress-interval", type=int, default=10, help="print every N processed candidates")
args = parser.parse_args()
payload = asyncio.run(_run(args))
+1 -1
View File
@@ -399,7 +399,7 @@ async def _build_archive_previews(
summary["items"].append(item)
continue
source_corner_mapping = DataService.get_radar_source_corner_mapping(record.file_path)
source_corner_mapping = DataService.get_radar_record_corner_mapping(record)
ok_geo, geo_error = image_service.create_geocorrected_radar_cached_image(
preview_source,
geo_cache_path,
+5
View File
@@ -0,0 +1,5 @@
@echo off
setlocal
cd /d "%~dp0\.."
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0start_landsar_cluster_worker.ps1"
pause
+116
View File
@@ -0,0 +1,116 @@
param(
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path,
[string]$PythonPath = "",
[string]$WorkerId = "",
[switch]$Background
)
$ErrorActionPreference = "Stop"
function Read-DotEnvValue {
param(
[string]$Path,
[string]$Name
)
if (-not (Test-Path -LiteralPath $Path)) {
return ""
}
$line = Get-Content -LiteralPath $Path |
Where-Object { $_ -match "^\s*$([regex]::Escape($Name))\s*=" } |
Select-Object -Last 1
if (-not $line) {
return ""
}
$value = ($line -split "=", 2)[1].Trim()
if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) {
$value = $value.Substring(1, $value.Length - 2)
}
return $value
}
function Resolve-PythonPath {
param(
[string]$ExplicitPath,
[string]$EnvPath
)
$candidates = @(
$ExplicitPath,
$EnvPath,
"C:\ProgramData\anaconda3\envs\InSAR\python.exe",
"python.exe"
) | Where-Object { $_ -and $_.Trim() } | Select-Object -Unique
foreach ($candidate in $candidates) {
if (Test-Path -LiteralPath $candidate) {
return (Resolve-Path -LiteralPath $candidate).Path
}
$command = Get-Command $candidate -ErrorAction SilentlyContinue
if ($command) {
return $command.Source
}
}
throw "Python interpreter not found. Set PYTHON_PATH in .env or pass -PythonPath."
}
$RepoRoot = (Resolve-Path -LiteralPath $RepoRoot).Path
$envPath = Join-Path $RepoRoot ".env"
$templatePath = Join-Path $RepoRoot "config\landsar_cluster_worker.env.example"
$workerScript = Join-Path $RepoRoot "run_landsar_cluster_worker.py"
if (-not (Test-Path -LiteralPath $workerScript)) {
throw "Worker script not found: $workerScript"
}
if (-not (Test-Path -LiteralPath $envPath)) {
if (Test-Path -LiteralPath $templatePath) {
Copy-Item -LiteralPath $templatePath -Destination $envPath
throw ".env was created from config\landsar_cluster_worker.env.example. Review paths and credentials, then run this launcher again."
}
throw ".env not found: $envPath"
}
$dotenvPythonPath = Read-DotEnvValue -Path $envPath -Name "PYTHON_PATH"
$python = Resolve-PythonPath -ExplicitPath $PythonPath -EnvPath $dotenvPythonPath
$allowedTypes = Read-DotEnvValue -Path $envPath -Name "JOB_WORKER_ALLOWED_TYPES"
if (-not $allowedTypes) {
$env:JOB_WORKER_ALLOWED_TYPES = "LANDSAR_CLUSTER_ITEM"
}
$concurrency = Read-DotEnvValue -Path $envPath -Name "JOB_WORKER_CONCURRENCY"
if (-not $concurrency) {
$env:JOB_WORKER_CONCURRENCY = "1"
}
if ($WorkerId) {
$env:LANDSAR_CLUSTER_WORKER_ID = $WorkerId
}
$logDir = Join-Path $RepoRoot "logs\landsar_cluster_worker"
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$stdoutLog = Join-Path $logDir "worker_$timestamp.log"
$stderrLog = Join-Path $logDir "worker_$timestamp.err.log"
Write-Host "LandSAR cluster worker launcher"
Write-Host "RepoRoot: $RepoRoot"
Write-Host "Python: $python"
Write-Host "Env: $envPath"
Write-Host "Log: $stdoutLog"
Write-Host "Mode: $(if ($Background) { 'background' } else { 'foreground' })"
Set-Location $RepoRoot
if ($Background) {
$process = Start-Process `
-FilePath $python `
-ArgumentList @($workerScript) `
-WorkingDirectory $RepoRoot `
-RedirectStandardOutput $stdoutLog `
-RedirectStandardError $stderrLog `
-WindowStyle Hidden `
-PassThru
Write-Host "Started background worker. PID=$($process.Id)"
return
}
& $python $workerScript 2>&1 | Tee-Object -FilePath $stdoutLog -Append
exit $LASTEXITCODE
@@ -0,0 +1,138 @@
param(
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path,
[string]$PostgresDataDir = "D:\PostgreSQLData",
[string]$PostgresServiceName = "postgresql-x64-17",
[string]$FirewallRuleName = "InSAR PostgreSQL 5432 LandSAR Cluster",
[int]$PostgresPort = 5432
)
$ErrorActionPreference = "Stop"
function Read-DotEnvValue {
param(
[string]$Path,
[string]$Name
)
if (-not (Test-Path -LiteralPath $Path)) {
return ""
}
$line = Get-Content -LiteralPath $Path |
Where-Object { $_ -match "^\s*$([regex]::Escape($Name))\s*=" } |
Select-Object -Last 1
if (-not $line) {
return ""
}
$value = ($line -split "=", 2)[1].Trim()
if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) {
$value = $value.Substring(1, $value.Length - 2)
}
return $value
}
function Normalize-WorkerAddress {
param([string]$Raw)
$value = $Raw.Trim()
if (-not $value) {
return $null
}
if ($value -match "/") {
$parts = $value -split "/", 2
$ip = $parts[0]
$prefix = [int]$parts[1]
if ($prefix -lt 0 -or $prefix -gt 32) {
throw "Invalid CIDR prefix: $value"
}
} else {
$ip = $value
$prefix = 32
}
$parsed = $null
if (-not [System.Net.IPAddress]::TryParse($ip, [ref]$parsed)) {
throw "Invalid IP address: $value"
}
if ($parsed.AddressFamily -ne [System.Net.Sockets.AddressFamily]::InterNetwork) {
throw "Only IPv4 addresses are supported for LandSAR cluster workers: $value"
}
return "$ip/$prefix"
}
$envPath = Join-Path $RepoRoot ".env"
$allowedRaw = Read-DotEnvValue -Path $envPath -Name "LANDSAR_CLUSTER_ALLOWED_WORKER_IPS"
if (-not $allowedRaw) {
throw "LANDSAR_CLUSTER_ALLOWED_WORKER_IPS is empty. Set it in .env, for example: LANDSAR_CLUSTER_ALLOWED_WORKER_IPS=192.168.1.6"
}
$allowed = @()
foreach ($part in ($allowedRaw -split "[,;]")) {
$normalized = Normalize-WorkerAddress $part
if ($normalized -and ($allowed -notcontains $normalized)) {
$allowed += $normalized
}
}
if (-not $allowed) {
throw "No valid LandSAR cluster worker IPs found in LANDSAR_CLUSTER_ALLOWED_WORKER_IPS."
}
$hbaPath = Join-Path $PostgresDataDir "pg_hba.conf"
if (-not (Test-Path -LiteralPath $hbaPath)) {
throw "pg_hba.conf not found: $hbaPath"
}
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
Copy-Item -LiteralPath $hbaPath -Destination "$hbaPath.bak_$timestamp"
$begin = "# BEGIN InSAR LandSAR cluster workers"
$end = "# END InSAR LandSAR cluster workers"
$content = Get-Content -LiteralPath $hbaPath
$newContent = New-Object System.Collections.Generic.List[string]
$inside = $false
foreach ($line in $content) {
if ($line -eq $begin) {
$inside = $true
continue
}
if ($line -eq $end) {
$inside = $false
continue
}
if (-not $inside) {
$newContent.Add($line)
}
}
$newContent.Add("")
$newContent.Add($begin)
foreach ($address in $allowed) {
$newContent.Add(("host insar_management all {0,-20} scram-sha-256" -f $address))
}
$newContent.Add($end)
Set-Content -LiteralPath $hbaPath -Value $newContent -Encoding ASCII
$pgCtl = "C:\Program Files\PostgreSQL\17\bin\pg_ctl.exe"
if (Test-Path -LiteralPath $pgCtl) {
& $pgCtl reload -D $PostgresDataDir | Out-Host
} else {
Restart-Service -Name $PostgresServiceName
}
$remoteAddresses = $allowed | ForEach-Object { ($_ -split "/", 2)[0] }
$rule = Get-NetFirewallRule -DisplayName $FirewallRuleName -ErrorAction SilentlyContinue
if (-not $rule) {
New-NetFirewallRule `
-DisplayName $FirewallRuleName `
-Direction Inbound `
-Action Allow `
-Protocol TCP `
-LocalPort $PostgresPort `
-RemoteAddress $remoteAddresses `
-Profile Any | Out-Null
} else {
$rule | Set-NetFirewallRule -Enabled True -Direction Inbound -Action Allow -Profile Any
$rule | Get-NetFirewallAddressFilter | Set-NetFirewallAddressFilter -RemoteAddress $remoteAddresses
$rule | Get-NetFirewallPortFilter | Set-NetFirewallPortFilter -Protocol TCP -LocalPort $PostgresPort
}
Write-Host "LandSAR cluster network access synced."
Write-Host ("Allowed workers: " + ($allowed -join ", "))
Write-Host "PostgreSQL hba: $hbaPath"
Write-Host "Firewall rule: $FirewallRuleName"