Add LandSAR cluster worker deployment
This commit is contained in:
@@ -0,0 +1,282 @@
|
||||
"""Backfill XML refRow/refColumn corner mappings for archive-managed radar records."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, Optional, Sequence, Tuple
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from geoalchemy2.shape import from_shape # noqa: E402
|
||||
from shapely.geometry import Polygon # noqa: E402
|
||||
from sqlalchemy import or_, select # noqa: E402
|
||||
from sqlalchemy.ext.asyncio import AsyncSession # noqa: E402
|
||||
|
||||
from backend.app import database # noqa: E402
|
||||
from backend.app.config import settings # noqa: E402
|
||||
from backend.app.models import RadarDataORM, SourceMetadataDocumentORM, SourceProductAssetORM # noqa: E402
|
||||
from backend.app.services.asset_inventory_service import _parse_radar_xml_metadata_bytes # noqa: E402
|
||||
from backend.app.utils import build_corner_pixel_mapping # noqa: E402
|
||||
|
||||
|
||||
SOURCE_FORMATS = {"LT1_ARCHIVE"}
|
||||
DOCUMENT_TYPES = {"LT1_META"}
|
||||
|
||||
|
||||
def _metadata_dict(value: Any) -> Dict[str, Any]:
|
||||
return dict(value or {}) if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _normalize_families(values: Optional[Iterable[str]]) -> Sequence[str]:
|
||||
families = []
|
||||
for value in values or ["LT1"]:
|
||||
for item in str(value or "").split(","):
|
||||
family = item.strip().upper()
|
||||
if family and family not in families:
|
||||
families.append(family)
|
||||
supported = {"LT1"}
|
||||
invalid = [item for item in families if item not in supported]
|
||||
if invalid:
|
||||
raise SystemExit(f"Unsupported family: {', '.join(invalid)}")
|
||||
return families or ["LT1"]
|
||||
|
||||
|
||||
def _decode_document(doc: SourceMetadataDocumentORM) -> Optional[bytes]:
|
||||
payload = bytes(doc.content_bytes or b"")
|
||||
if not payload:
|
||||
return None
|
||||
if str(doc.content_encoding or "").lower() == "gzip":
|
||||
return gzip.decompress(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _mapping_from_xml_document(doc: SourceMetadataDocumentORM) -> Tuple[Optional[Dict[str, Any]], Optional[Any]]:
|
||||
payload = _decode_document(doc)
|
||||
if not payload:
|
||||
return None, None
|
||||
coverage_polygon, xml_meta = _parse_radar_xml_metadata_bytes(payload)
|
||||
mapping = xml_meta.get("corner_pixel_mapping") if isinstance(xml_meta, dict) else None
|
||||
return mapping if isinstance(mapping, dict) else None, coverage_polygon
|
||||
|
||||
|
||||
def _mapping_from_metadata(metadata: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
existing = metadata.get("corner_pixel_mapping")
|
||||
if isinstance(existing, dict):
|
||||
return existing
|
||||
corners = metadata.get("corner_details")
|
||||
if isinstance(corners, dict):
|
||||
mapping = build_corner_pixel_mapping(corners)
|
||||
if isinstance(mapping, dict):
|
||||
return mapping
|
||||
return None
|
||||
|
||||
|
||||
def _bbox(points: Any) -> Optional[Tuple[float, float, float, float]]:
|
||||
try:
|
||||
ring = [(float(item[0]), float(item[1])) for item in points or []]
|
||||
except (TypeError, ValueError, IndexError):
|
||||
return None
|
||||
if len(ring) < 3:
|
||||
return None
|
||||
lons = [item[0] for item in ring]
|
||||
lats = [item[1] for item in ring]
|
||||
return min(lons), min(lats), max(lons), max(lats)
|
||||
|
||||
|
||||
async def _first_metadata_document(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source_asset_id: Optional[int],
|
||||
radar_data_id: Optional[int],
|
||||
) -> Optional[SourceMetadataDocumentORM]:
|
||||
clauses = []
|
||||
if source_asset_id:
|
||||
clauses.append(SourceMetadataDocumentORM.source_asset_id == int(source_asset_id))
|
||||
if radar_data_id:
|
||||
clauses.append(SourceMetadataDocumentORM.radar_data_id == int(radar_data_id))
|
||||
if not clauses:
|
||||
return None
|
||||
stmt = (
|
||||
select(SourceMetadataDocumentORM)
|
||||
.where(or_(*clauses))
|
||||
.where(SourceMetadataDocumentORM.document_type.in_(sorted(DOCUMENT_TYPES)))
|
||||
.order_by(SourceMetadataDocumentORM.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return (await db.execute(stmt)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def _load_source_assets(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[SourceProductAssetORM]:
|
||||
stmt = (
|
||||
select(SourceProductAssetORM)
|
||||
.where(SourceProductAssetORM.satellite_family.in_(families))
|
||||
.where(SourceProductAssetORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
||||
.where(SourceProductAssetORM.is_active.is_(True))
|
||||
.order_by(SourceProductAssetORM.id.asc())
|
||||
)
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def _load_radar_records(db: AsyncSession, families: Sequence[str], limit: int) -> Sequence[RadarDataORM]:
|
||||
stmt = (
|
||||
select(RadarDataORM)
|
||||
.where(RadarDataORM.satellite_family.in_(families))
|
||||
.where(RadarDataORM.source_format.in_(sorted(SOURCE_FORMATS)))
|
||||
.order_by(RadarDataORM.id.asc())
|
||||
)
|
||||
if limit > 0:
|
||||
stmt = stmt.limit(limit)
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def backfill(args: argparse.Namespace) -> Dict[str, int]:
|
||||
families = _normalize_families(args.family)
|
||||
database.init_db(settings.DATABASE_URL)
|
||||
stats = {
|
||||
"source_seen": 0,
|
||||
"source_updated": 0,
|
||||
"source_existing": 0,
|
||||
"source_missing_mapping": 0,
|
||||
"radar_seen": 0,
|
||||
"radar_updated": 0,
|
||||
"radar_existing": 0,
|
||||
"radar_missing_mapping": 0,
|
||||
"cache_invalidated": 0,
|
||||
"document_parse_failed": 0,
|
||||
}
|
||||
|
||||
async with database.AsyncSessionLocal() as db:
|
||||
source_assets = await _load_source_assets(db, families, int(args.limit or 0))
|
||||
source_mappings: Dict[int, Dict[str, Any]] = {}
|
||||
source_polygons: Dict[int, Any] = {}
|
||||
now = datetime.utcnow()
|
||||
|
||||
for asset in source_assets:
|
||||
stats["source_seen"] += 1
|
||||
metadata = _metadata_dict(asset.metadata_json)
|
||||
mapping = _mapping_from_metadata(metadata)
|
||||
coverage_polygon = metadata.get("coverage_polygon")
|
||||
if mapping:
|
||||
stats["source_existing"] += 1
|
||||
else:
|
||||
doc = await _first_metadata_document(db, source_asset_id=asset.id, radar_data_id=None)
|
||||
if doc:
|
||||
try:
|
||||
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
||||
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
||||
except Exception:
|
||||
stats["document_parse_failed"] += 1
|
||||
mapping = None
|
||||
if not mapping:
|
||||
stats["source_missing_mapping"] += 1
|
||||
continue
|
||||
metadata["corner_pixel_mapping"] = mapping
|
||||
if coverage_polygon:
|
||||
metadata["coverage_polygon"] = coverage_polygon
|
||||
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
||||
asset.metadata_json = metadata
|
||||
asset.updated_at = now
|
||||
db.add(asset)
|
||||
stats["source_updated"] += 1
|
||||
|
||||
if mapping and asset.id is not None:
|
||||
source_mappings[int(asset.id)] = mapping
|
||||
if coverage_polygon:
|
||||
source_polygons[int(asset.id)] = coverage_polygon
|
||||
|
||||
radar_records = await _load_radar_records(db, families, int(args.limit or 0))
|
||||
for radar in radar_records:
|
||||
stats["radar_seen"] += 1
|
||||
metadata = _metadata_dict(radar.metadata_json)
|
||||
mapping = _mapping_from_metadata(metadata)
|
||||
coverage_polygon = radar.coverage_polygon or metadata.get("coverage_polygon")
|
||||
if mapping:
|
||||
stats["radar_existing"] += 1
|
||||
else:
|
||||
source_id = int(radar.source_product_ref_id or 0)
|
||||
mapping = source_mappings.get(source_id)
|
||||
if source_id and source_id in source_polygons:
|
||||
coverage_polygon = coverage_polygon or source_polygons[source_id]
|
||||
if not mapping:
|
||||
doc = await _first_metadata_document(
|
||||
db,
|
||||
source_asset_id=radar.source_product_ref_id,
|
||||
radar_data_id=radar.id,
|
||||
)
|
||||
if doc:
|
||||
try:
|
||||
mapping, coverage_polygon_from_doc = _mapping_from_xml_document(doc)
|
||||
coverage_polygon = coverage_polygon_from_doc or coverage_polygon
|
||||
except Exception:
|
||||
stats["document_parse_failed"] += 1
|
||||
mapping = None
|
||||
if not mapping:
|
||||
stats["radar_missing_mapping"] += 1
|
||||
continue
|
||||
|
||||
metadata["corner_pixel_mapping"] = mapping
|
||||
if coverage_polygon:
|
||||
metadata["coverage_polygon"] = coverage_polygon
|
||||
radar.coverage_polygon = coverage_polygon
|
||||
bbox = _bbox(coverage_polygon)
|
||||
if bbox:
|
||||
radar.min_lon, radar.min_lat, radar.max_lon, radar.max_lat = bbox
|
||||
try:
|
||||
polygon = Polygon(coverage_polygon)
|
||||
if not polygon.is_valid:
|
||||
polygon = polygon.buffer(0)
|
||||
if not polygon.is_empty:
|
||||
radar.geom = from_shape(polygon, srid=4326)
|
||||
except Exception:
|
||||
pass
|
||||
metadata["corner_pixel_mapping_backfilled_at"] = now.isoformat()
|
||||
radar.metadata_json = metadata
|
||||
db.add(radar)
|
||||
stats["radar_updated"] += 1
|
||||
|
||||
if args.invalidate_preview and mapping:
|
||||
radar.preview_cache_status = "NONE"
|
||||
radar.preview_cache_version = None
|
||||
radar.preview_cache_path = None
|
||||
radar.preview_cache_error = "corner_pixel_mapping_backfilled"
|
||||
radar.preview_cache_updated_at = now
|
||||
db.add(radar)
|
||||
stats["cache_invalidated"] += 1
|
||||
|
||||
if args.apply:
|
||||
await db.commit()
|
||||
else:
|
||||
await db.rollback()
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Backfill corner_pixel_mapping into source/radar metadata_json.")
|
||||
parser.add_argument("--apply", action="store_true", help="commit database changes")
|
||||
parser.add_argument("--family", action="append", help="family to process: LT1 or GF3; defaults to LT1")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum source/radar rows per table; 0 means all")
|
||||
parser.add_argument(
|
||||
"--no-invalidate-preview",
|
||||
dest="invalidate_preview",
|
||||
action="store_false",
|
||||
help="do not mark old geocorrected WebP caches stale",
|
||||
)
|
||||
parser.set_defaults(invalidate_preview=True)
|
||||
args = parser.parse_args()
|
||||
stats = asyncio.run(backfill(args))
|
||||
print(json.dumps({"apply": bool(args.apply), **stats}, ensure_ascii=False, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -5,6 +5,7 @@ import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Iterable, List, Optional
|
||||
|
||||
@@ -34,6 +35,55 @@ def _normalize_families(values: Optional[Iterable[str]]) -> List[str]:
|
||||
async def _run(args: argparse.Namespace) -> dict:
|
||||
families = _normalize_families(args.family)
|
||||
database.init_db(settings.DATABASE_URL)
|
||||
|
||||
def _print_progress(payload: dict) -> None:
|
||||
stamp = datetime.now().strftime("%H:%M:%S")
|
||||
event = payload.get("event")
|
||||
if event == "planned":
|
||||
print(
|
||||
"[{}] planned records_seen={} candidates={} skipped_ready={} families={}".format(
|
||||
stamp,
|
||||
payload.get("records_seen", 0),
|
||||
payload.get("candidate_count", 0),
|
||||
payload.get("skipped_ready", 0),
|
||||
",".join(payload.get("families") or []),
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
if event == "item":
|
||||
print(
|
||||
(
|
||||
"[{}] {}/{} ready={} failed={} missing={} raw_failed={} "
|
||||
"skipped_ready={} status={} product={}"
|
||||
).format(
|
||||
stamp,
|
||||
payload.get("processed", 0),
|
||||
payload.get("total", 0),
|
||||
payload.get("ready", 0),
|
||||
payload.get("failed", 0),
|
||||
payload.get("missing_source", 0),
|
||||
payload.get("raw_failed", 0),
|
||||
payload.get("skipped_ready", 0),
|
||||
payload.get("status") or "",
|
||||
payload.get("product_name") or "",
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
if event == "completed":
|
||||
print(
|
||||
"[{}] completed ready={} cached={} skipped_ready={} failed={} missing={}".format(
|
||||
stamp,
|
||||
payload.get("ready", 0),
|
||||
payload.get("cached", 0),
|
||||
payload.get("skipped_ready", 0),
|
||||
payload.get("failed", 0),
|
||||
payload.get("missing_source", 0),
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
summary = await asset_inventory_service.build_archive_preview_caches(
|
||||
families=families,
|
||||
limit=args.limit,
|
||||
@@ -41,6 +91,8 @@ async def _run(args: argparse.Namespace) -> dict:
|
||||
apply=bool(args.apply),
|
||||
progress_start=0,
|
||||
progress_end=100,
|
||||
progress_callback=_print_progress if args.progress else None,
|
||||
progress_interval=args.progress_interval,
|
||||
)
|
||||
return {"apply": bool(args.apply), **summary}
|
||||
|
||||
@@ -51,6 +103,8 @@ def main() -> int:
|
||||
parser.add_argument("--family", action="append", help="family to process: LT1, S1, or comma-separated values")
|
||||
parser.add_argument("--limit", type=int, default=0, help="maximum rows to build; 0 means all pending rows")
|
||||
parser.add_argument("--force", action="store_true", help="rebuild even if preview_cache_status is READY")
|
||||
parser.add_argument("--progress", action="store_true", help="print command-line progress while building")
|
||||
parser.add_argument("--progress-interval", type=int, default=10, help="print every N processed candidates")
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = asyncio.run(_run(args))
|
||||
|
||||
@@ -399,7 +399,7 @@ async def _build_archive_previews(
|
||||
summary["items"].append(item)
|
||||
continue
|
||||
|
||||
source_corner_mapping = DataService.get_radar_source_corner_mapping(record.file_path)
|
||||
source_corner_mapping = DataService.get_radar_record_corner_mapping(record)
|
||||
ok_geo, geo_error = image_service.create_geocorrected_radar_cached_image(
|
||||
preview_source,
|
||||
geo_cache_path,
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0\.."
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0start_landsar_cluster_worker.ps1"
|
||||
pause
|
||||
@@ -0,0 +1,116 @@
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path,
|
||||
[string]$PythonPath = "",
|
||||
[string]$WorkerId = "",
|
||||
[switch]$Background
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Read-DotEnvValue {
|
||||
param(
|
||||
[string]$Path,
|
||||
[string]$Name
|
||||
)
|
||||
if (-not (Test-Path -LiteralPath $Path)) {
|
||||
return ""
|
||||
}
|
||||
$line = Get-Content -LiteralPath $Path |
|
||||
Where-Object { $_ -match "^\s*$([regex]::Escape($Name))\s*=" } |
|
||||
Select-Object -Last 1
|
||||
if (-not $line) {
|
||||
return ""
|
||||
}
|
||||
$value = ($line -split "=", 2)[1].Trim()
|
||||
if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) {
|
||||
$value = $value.Substring(1, $value.Length - 2)
|
||||
}
|
||||
return $value
|
||||
}
|
||||
|
||||
function Resolve-PythonPath {
|
||||
param(
|
||||
[string]$ExplicitPath,
|
||||
[string]$EnvPath
|
||||
)
|
||||
$candidates = @(
|
||||
$ExplicitPath,
|
||||
$EnvPath,
|
||||
"C:\ProgramData\anaconda3\envs\InSAR\python.exe",
|
||||
"python.exe"
|
||||
) | Where-Object { $_ -and $_.Trim() } | Select-Object -Unique
|
||||
|
||||
foreach ($candidate in $candidates) {
|
||||
if (Test-Path -LiteralPath $candidate) {
|
||||
return (Resolve-Path -LiteralPath $candidate).Path
|
||||
}
|
||||
$command = Get-Command $candidate -ErrorAction SilentlyContinue
|
||||
if ($command) {
|
||||
return $command.Source
|
||||
}
|
||||
}
|
||||
throw "Python interpreter not found. Set PYTHON_PATH in .env or pass -PythonPath."
|
||||
}
|
||||
|
||||
$RepoRoot = (Resolve-Path -LiteralPath $RepoRoot).Path
|
||||
$envPath = Join-Path $RepoRoot ".env"
|
||||
$templatePath = Join-Path $RepoRoot "config\landsar_cluster_worker.env.example"
|
||||
$workerScript = Join-Path $RepoRoot "run_landsar_cluster_worker.py"
|
||||
|
||||
if (-not (Test-Path -LiteralPath $workerScript)) {
|
||||
throw "Worker script not found: $workerScript"
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $envPath)) {
|
||||
if (Test-Path -LiteralPath $templatePath) {
|
||||
Copy-Item -LiteralPath $templatePath -Destination $envPath
|
||||
throw ".env was created from config\landsar_cluster_worker.env.example. Review paths and credentials, then run this launcher again."
|
||||
}
|
||||
throw ".env not found: $envPath"
|
||||
}
|
||||
|
||||
$dotenvPythonPath = Read-DotEnvValue -Path $envPath -Name "PYTHON_PATH"
|
||||
$python = Resolve-PythonPath -ExplicitPath $PythonPath -EnvPath $dotenvPythonPath
|
||||
|
||||
$allowedTypes = Read-DotEnvValue -Path $envPath -Name "JOB_WORKER_ALLOWED_TYPES"
|
||||
if (-not $allowedTypes) {
|
||||
$env:JOB_WORKER_ALLOWED_TYPES = "LANDSAR_CLUSTER_ITEM"
|
||||
}
|
||||
$concurrency = Read-DotEnvValue -Path $envPath -Name "JOB_WORKER_CONCURRENCY"
|
||||
if (-not $concurrency) {
|
||||
$env:JOB_WORKER_CONCURRENCY = "1"
|
||||
}
|
||||
if ($WorkerId) {
|
||||
$env:LANDSAR_CLUSTER_WORKER_ID = $WorkerId
|
||||
}
|
||||
|
||||
$logDir = Join-Path $RepoRoot "logs\landsar_cluster_worker"
|
||||
New-Item -ItemType Directory -Force -Path $logDir | Out-Null
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
$stdoutLog = Join-Path $logDir "worker_$timestamp.log"
|
||||
$stderrLog = Join-Path $logDir "worker_$timestamp.err.log"
|
||||
|
||||
Write-Host "LandSAR cluster worker launcher"
|
||||
Write-Host "RepoRoot: $RepoRoot"
|
||||
Write-Host "Python: $python"
|
||||
Write-Host "Env: $envPath"
|
||||
Write-Host "Log: $stdoutLog"
|
||||
Write-Host "Mode: $(if ($Background) { 'background' } else { 'foreground' })"
|
||||
|
||||
Set-Location $RepoRoot
|
||||
|
||||
if ($Background) {
|
||||
$process = Start-Process `
|
||||
-FilePath $python `
|
||||
-ArgumentList @($workerScript) `
|
||||
-WorkingDirectory $RepoRoot `
|
||||
-RedirectStandardOutput $stdoutLog `
|
||||
-RedirectStandardError $stderrLog `
|
||||
-WindowStyle Hidden `
|
||||
-PassThru
|
||||
Write-Host "Started background worker. PID=$($process.Id)"
|
||||
return
|
||||
}
|
||||
|
||||
& $python $workerScript 2>&1 | Tee-Object -FilePath $stdoutLog -Append
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,138 @@
|
||||
param(
|
||||
[string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path,
|
||||
[string]$PostgresDataDir = "D:\PostgreSQLData",
|
||||
[string]$PostgresServiceName = "postgresql-x64-17",
|
||||
[string]$FirewallRuleName = "InSAR PostgreSQL 5432 LandSAR Cluster",
|
||||
[int]$PostgresPort = 5432
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Read-DotEnvValue {
|
||||
param(
|
||||
[string]$Path,
|
||||
[string]$Name
|
||||
)
|
||||
if (-not (Test-Path -LiteralPath $Path)) {
|
||||
return ""
|
||||
}
|
||||
$line = Get-Content -LiteralPath $Path |
|
||||
Where-Object { $_ -match "^\s*$([regex]::Escape($Name))\s*=" } |
|
||||
Select-Object -Last 1
|
||||
if (-not $line) {
|
||||
return ""
|
||||
}
|
||||
$value = ($line -split "=", 2)[1].Trim()
|
||||
if (($value.StartsWith('"') -and $value.EndsWith('"')) -or ($value.StartsWith("'") -and $value.EndsWith("'"))) {
|
||||
$value = $value.Substring(1, $value.Length - 2)
|
||||
}
|
||||
return $value
|
||||
}
|
||||
|
||||
function Normalize-WorkerAddress {
|
||||
param([string]$Raw)
|
||||
$value = $Raw.Trim()
|
||||
if (-not $value) {
|
||||
return $null
|
||||
}
|
||||
if ($value -match "/") {
|
||||
$parts = $value -split "/", 2
|
||||
$ip = $parts[0]
|
||||
$prefix = [int]$parts[1]
|
||||
if ($prefix -lt 0 -or $prefix -gt 32) {
|
||||
throw "Invalid CIDR prefix: $value"
|
||||
}
|
||||
} else {
|
||||
$ip = $value
|
||||
$prefix = 32
|
||||
}
|
||||
$parsed = $null
|
||||
if (-not [System.Net.IPAddress]::TryParse($ip, [ref]$parsed)) {
|
||||
throw "Invalid IP address: $value"
|
||||
}
|
||||
if ($parsed.AddressFamily -ne [System.Net.Sockets.AddressFamily]::InterNetwork) {
|
||||
throw "Only IPv4 addresses are supported for LandSAR cluster workers: $value"
|
||||
}
|
||||
return "$ip/$prefix"
|
||||
}
|
||||
|
||||
$envPath = Join-Path $RepoRoot ".env"
|
||||
$allowedRaw = Read-DotEnvValue -Path $envPath -Name "LANDSAR_CLUSTER_ALLOWED_WORKER_IPS"
|
||||
if (-not $allowedRaw) {
|
||||
throw "LANDSAR_CLUSTER_ALLOWED_WORKER_IPS is empty. Set it in .env, for example: LANDSAR_CLUSTER_ALLOWED_WORKER_IPS=192.168.1.6"
|
||||
}
|
||||
|
||||
$allowed = @()
|
||||
foreach ($part in ($allowedRaw -split "[,;]")) {
|
||||
$normalized = Normalize-WorkerAddress $part
|
||||
if ($normalized -and ($allowed -notcontains $normalized)) {
|
||||
$allowed += $normalized
|
||||
}
|
||||
}
|
||||
if (-not $allowed) {
|
||||
throw "No valid LandSAR cluster worker IPs found in LANDSAR_CLUSTER_ALLOWED_WORKER_IPS."
|
||||
}
|
||||
|
||||
$hbaPath = Join-Path $PostgresDataDir "pg_hba.conf"
|
||||
if (-not (Test-Path -LiteralPath $hbaPath)) {
|
||||
throw "pg_hba.conf not found: $hbaPath"
|
||||
}
|
||||
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
Copy-Item -LiteralPath $hbaPath -Destination "$hbaPath.bak_$timestamp"
|
||||
|
||||
$begin = "# BEGIN InSAR LandSAR cluster workers"
|
||||
$end = "# END InSAR LandSAR cluster workers"
|
||||
$content = Get-Content -LiteralPath $hbaPath
|
||||
$newContent = New-Object System.Collections.Generic.List[string]
|
||||
$inside = $false
|
||||
foreach ($line in $content) {
|
||||
if ($line -eq $begin) {
|
||||
$inside = $true
|
||||
continue
|
||||
}
|
||||
if ($line -eq $end) {
|
||||
$inside = $false
|
||||
continue
|
||||
}
|
||||
if (-not $inside) {
|
||||
$newContent.Add($line)
|
||||
}
|
||||
}
|
||||
|
||||
$newContent.Add("")
|
||||
$newContent.Add($begin)
|
||||
foreach ($address in $allowed) {
|
||||
$newContent.Add(("host insar_management all {0,-20} scram-sha-256" -f $address))
|
||||
}
|
||||
$newContent.Add($end)
|
||||
Set-Content -LiteralPath $hbaPath -Value $newContent -Encoding ASCII
|
||||
|
||||
$pgCtl = "C:\Program Files\PostgreSQL\17\bin\pg_ctl.exe"
|
||||
if (Test-Path -LiteralPath $pgCtl) {
|
||||
& $pgCtl reload -D $PostgresDataDir | Out-Host
|
||||
} else {
|
||||
Restart-Service -Name $PostgresServiceName
|
||||
}
|
||||
|
||||
$remoteAddresses = $allowed | ForEach-Object { ($_ -split "/", 2)[0] }
|
||||
$rule = Get-NetFirewallRule -DisplayName $FirewallRuleName -ErrorAction SilentlyContinue
|
||||
if (-not $rule) {
|
||||
New-NetFirewallRule `
|
||||
-DisplayName $FirewallRuleName `
|
||||
-Direction Inbound `
|
||||
-Action Allow `
|
||||
-Protocol TCP `
|
||||
-LocalPort $PostgresPort `
|
||||
-RemoteAddress $remoteAddresses `
|
||||
-Profile Any | Out-Null
|
||||
} else {
|
||||
$rule | Set-NetFirewallRule -Enabled True -Direction Inbound -Action Allow -Profile Any
|
||||
$rule | Get-NetFirewallAddressFilter | Set-NetFirewallAddressFilter -RemoteAddress $remoteAddresses
|
||||
$rule | Get-NetFirewallPortFilter | Set-NetFirewallPortFilter -Protocol TCP -LocalPort $PostgresPort
|
||||
}
|
||||
|
||||
Write-Host "LandSAR cluster network access synced."
|
||||
Write-Host ("Allowed workers: " + ($allowed -join ", "))
|
||||
Write-Host "PostgreSQL hba: $hbaPath"
|
||||
Write-Host "Firewall rule: $FirewallRuleName"
|
||||
Reference in New Issue
Block a user