Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden LandSAR cluster transfer e2e

Browse Source
This commit is contained in:
Harmon
2026-06-27 01:20:06 +08:00
parent 40178ad1a4
commit 19ae3ec37f
7 changed files with 629 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/cluster_transport.py
+387 -83
View File
@@ -6,16 +6,19 @@ data from the main server and push results back via HTTP.
"""
from __future__ import annotations
import asyncio
import io
import json
import os
import queue
import shutil
import tempfile
import threading
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from typing import TYPE_CHECKING
from typing import Any, Iterable, Iterator, TYPE_CHECKING
if TYPE_CHECKING:
from ..models.orm import DinsarProductionRunItemORM, DinsarProductionRunORM
@@ -46,6 +49,22 @@ def _cluster_request_headers() -> dict[str, str]:
return {"X-Cluster-Token": token}
def normalize_cluster_relative_path(relative_path: Any) -> str:
text = str(relative_path or "").strip().replace("\\", "/")
if not text or text.startswith("/") or os.path.splitdrive(text)[0]:
raise ValueError(f"Unsafe cluster relative path: {relative_path}")
parts = []
for part in text.split("/"):
if not part or part == ".":
continue
if part == ".." or os.path.splitdrive(part)[0]:
raise ValueError(f"Unsafe cluster relative path: {relative_path}")
parts.append(part)
if not parts:
raise ValueError(f"Unsafe cluster relative path: {relative_path}")
return "/".join(parts)
def safe_extract_zip(zf: zipfile.ZipFile, target_dir: str) -> None:
"""Extract a zip after verifying all members stay inside target_dir."""
target_root = os.path.abspath(target_dir)
@@ -69,7 +88,7 @@ def safe_extract_zip(zf: zipfile.ZipFile, target_dir: str) -> None:
def _zip_directory_contents(source_dir: str, zip_path: str) -> None:
source_root = os.path.abspath(source_dir)
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as zf:
for current, _, files in os.walk(source_root):
for name in files:
path = os.path.join(current, name)
@@ -77,6 +96,268 @@ def _zip_directory_contents(source_dir: str, zip_path: str) -> None:
zf.write(path, arcname)
def _build_multipart_form_data(
*,
fields: dict[str, str],
files: list[tuple[str, str, str, bytes]],
boundary: str,
) -> bytes:
body = io.BytesIO()
def _write_field(
name: str,
filename: str,
content_type: str,
data: bytes,
) -> None:
body.write(f"--{boundary}\r\n".encode("utf-8"))
if filename:
body.write(
f'Content-Disposition: form-data; name="{name}"; '
f'filename="{filename}"\r\n'.encode("utf-8")
)
body.write(f"Content-Type: {content_type}\r\n".encode("utf-8"))
else:
body.write(
f'Content-Disposition: form-data; name="{name}"\r\n'.encode(
"utf-8"
)
)
body.write(b"\r\n")
body.write(data)
body.write(b"\r\n")
for name, value in fields.items():
_write_field(name, "", "text/plain", str(value or "").encode("utf-8"))
for name, filename, content_type, data in files:
_write_field(name, filename, content_type, data)
body.write(f"--{boundary}--\r\n".encode("utf-8"))
return body.getvalue()
def _is_lt1_source_file(name: str) -> bool:
lower_name = str(name or "").lower()
return lower_name.startswith("lt1") and lower_name.endswith(
(".xml", ".tif", ".tiff", ".jpg", ".jpeg", ".rpc")
)
def _extract_date_from_name(name: str) -> str:
import re
match = re.search(r"((?:19|20)\d{6})", str(name or ""))
return match.group(1) if match else ""
def _has_landsar_input_pair(input_data_dir: str) -> bool:
if not os.path.isdir(input_data_dir):
return False
by_date: dict[str, set[str]] = {}
for entry in os.scandir(input_data_dir):
if not entry.is_file():
continue
lower_name = entry.name.lower()
if not lower_name.endswith((".xml", ".tif", ".tiff")):
continue
if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"):
continue
if lower_name.endswith(".xml") and "_slc" not in lower_name:
continue
date_text = _extract_date_from_name(entry.name)
if not date_text:
continue
kinds = by_date.setdefault(date_text, set())
if lower_name.endswith(".xml"):
kinds.add("xml")
elif lower_name.endswith((".tif", ".tiff")):
kinds.add("tif")
return sum(1 for kinds in by_date.values() if {"xml", "tif"} <= kinds) >= 2
def _has_direct_landsar_source_file(directory: str) -> bool:
if not os.path.isdir(directory):
return False
try:
for entry in os.scandir(directory):
if entry.is_file() and _is_lt1_source_file(entry.name):
return True
except OSError:
return False
return False
def _iter_tree_files(root_dir: str, relative_root: str = "") -> Iterator[tuple[str, str]]:
if not os.path.isdir(root_dir):
return
root_abs = os.path.abspath(root_dir)
for current, _, files in os.walk(root_abs):
for name in sorted(files):
path = os.path.join(current, name)
rel = os.path.relpath(path, root_abs)
if relative_root:
rel = os.path.join(relative_root, rel)
yield path, rel
def _iter_direct_files(root_dir: str, relative_root: str = "") -> Iterator[tuple[str, str]]:
if not os.path.isdir(root_dir):
return
for entry in sorted(os.scandir(root_dir), key=lambda item: item.name.lower()):
if not entry.is_file():
continue
rel = os.path.join(relative_root, entry.name) if relative_root else entry.name
yield entry.path, rel
def iter_cluster_input_package_files(source_task_dir: str) -> Iterator[tuple[str, str]]:
"""Yield the minimal LandSAR input package file set.
A Task_Pool item can contain both raw LT-1 ``master/slave`` files and a
derived ``Input_Data`` directory. Shipping both copies is wasteful for
remote workers. Prefer a valid LandSAR ``Input_Data`` pair; otherwise ship
only direct raw files under ``master`` and ``slave``. This keeps the
extracted layout compatible with LandSAR validation while avoiding nested
duplicate scene copies.
"""
source_dir = os.path.abspath(source_task_dir)
if not os.path.isdir(source_dir):
return
pair_meta = os.path.join(source_dir, ".dinsar_pair.json")
if os.path.isfile(pair_meta):
yield pair_meta, ".dinsar_pair.json"
input_data_dir = os.path.join(source_dir, "Input_Data")
if _has_landsar_input_pair(input_data_dir):
yield from _iter_tree_files(input_data_dir, "Input_Data")
else:
master_dir = os.path.join(source_dir, "master")
slave_dir = os.path.join(source_dir, "slave")
if _has_direct_landsar_source_file(master_dir) and _has_direct_landsar_source_file(slave_dir):
yield from _iter_direct_files(master_dir, "master")
yield from _iter_direct_files(slave_dir, "slave")
else:
yield from _iter_tree_files(source_dir)
return
orbit_dir = os.path.join(source_dir, "orbit")
if os.path.isdir(orbit_dir):
yield from _iter_tree_files(orbit_dir, "orbit")
def build_cluster_input_manifest(source_task_dir: str) -> dict[str, Any]:
files = []
total_bytes = 0
for abs_path, rel_path in iter_cluster_input_package_files(source_task_dir):
if not os.path.isfile(abs_path):
continue
normalized_rel = normalize_cluster_relative_path(rel_path)
stat = os.stat(abs_path)
size = int(stat.st_size)
total_bytes += size
files.append(
{
"relative_path": normalized_rel,
"size": size,
"mtime": float(stat.st_mtime),
}
)
return {
"task_name": os.path.basename(os.path.normpath(source_task_dir)),
"file_count": len(files),
"total_bytes": total_bytes,
"files": files,
}
class _QueueZipWriter:
def __init__(self, output_queue: "queue.Queue[object]") -> None:
self._queue = output_queue
self._closed = False
def write(self, data: bytes) -> int:
if self._closed:
raise BrokenPipeError("zip stream is closed")
chunk = bytes(data)
if not chunk:
return 0
while not self._closed:
try:
self._queue.put(chunk, timeout=1)
return len(chunk)
except queue.Full:
continue
raise BrokenPipeError("zip stream is closed")
def flush(self) -> None:
return None
def close(self) -> None:
self._closed = True
def stream_zip_files(
files: Iterable[tuple[str, str]],
*,
top_level_dir: str,
) -> Iterator[bytes]:
"""Stream a zip archive for *files* without materializing it first."""
output_queue: "queue.Queue[object]" = queue.Queue(maxsize=8)
sentinel = object()
writer = _QueueZipWriter(output_queue)
safe_top = str(top_level_dir or "Task").strip().strip("/\\") or "Task"
def _put_control(item: object) -> None:
while True:
try:
output_queue.put(item, timeout=1)
return
except queue.Full:
if writer._closed:
return
def _producer() -> None:
try:
with zipfile.ZipFile(
writer,
mode="w",
compression=zipfile.ZIP_STORED,
allowZip64=True,
) as zf:
for abs_path, rel_path in files:
if writer._closed:
raise BrokenPipeError("zip stream is closed")
if not os.path.isfile(abs_path):
continue
arcname = os.path.join(safe_top, rel_path).replace("\\", "/")
zf.write(abs_path, arcname)
except Exception as exc:
if not writer._closed:
_put_control(exc)
finally:
_put_control(sentinel)
producer = threading.Thread(
target=_producer,
name="cluster-input-zip-stream",
daemon=True,
)
producer.start()
try:
while True:
item = output_queue.get()
if item is sentinel:
break
if isinstance(item, Exception):
raise item
yield item # type: ignore[misc]
finally:
writer.close()
def _resolve_cluster_server_url() -> str:
"""Return the main-server HTTP base URL for cluster data transport.
@@ -142,58 +423,92 @@ async def materialize_cluster_input(
local_task_dir: str,
task_id: str,
) -> None:
"""Download and extract the input data for a cluster item.
Calls ``GET /api/cluster/input-package/{item_id}`` on the main
server, retrieves a zip containing the Task_Pool directory tree,
and extracts it so that *local_task_dir* exists locally.
"""
"""Download the input data for a cluster item file by file."""
from .task_service import task_service
server_url = _resolve_cluster_server_url()
download_url = f"{server_url}/api/cluster/input-package/{item.id}"
manifest_url = f"{server_url}/api/cluster/input-manifest/{item.id}"
parent_dir = os.path.dirname(local_task_dir)
task_name = os.path.basename(local_task_dir)
package_task_name = os.path.basename(str(item.source_task_dir or "")) or task_name
staging_dir = os.path.join(parent_dir, f".{task_name}.download")
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Downloading input data from {download_url} ...",
f"[cluster] Fetching input manifest from {manifest_url} ...",
)
tmp_zip = os.path.join(
tempfile.mkdtemp(prefix=f"cluster_input_{item.id}_"),
f"{package_task_name}.zip",
)
try:
def _request_json(url: str) -> dict[str, Any]:
req = urllib.request.Request(
download_url,
url,
headers=_cluster_request_headers(),
method="GET",
)
with urllib.request.urlopen(req, timeout=_cluster_transfer_timeout()) as resp:
with open(tmp_zip, "wb") as fh:
return json.loads(resp.read().decode("utf-8"))
def _download_file(relative_path: str, expected_size: int) -> None:
normalized_rel = normalize_cluster_relative_path(relative_path)
file_url = (
f"{server_url}/api/cluster/input-file/{item.id}"
f"?relative_path={urllib.parse.quote(normalized_rel, safe='')}"
)
target_path = os.path.join(staging_dir, *normalized_rel.split("/"))
tmp_path = f"{target_path}.part"
os.makedirs(os.path.dirname(target_path), exist_ok=True)
req = urllib.request.Request(
file_url,
headers=_cluster_request_headers(),
method="GET",
)
with urllib.request.urlopen(req, timeout=_cluster_transfer_timeout()) as resp:
with open(tmp_path, "wb") as fh:
shutil.copyfileobj(resp, fh, 8 * 1024 * 1024)
os.makedirs(parent_dir, exist_ok=True)
if os.path.isdir(local_task_dir):
shutil.rmtree(local_task_dir)
with zipfile.ZipFile(tmp_zip, "r") as zf:
safe_extract_zip(zf, parent_dir)
extracted_task_dir = os.path.join(parent_dir, package_task_name)
if (
os.path.isdir(extracted_task_dir)
and os.path.normcase(os.path.abspath(extracted_task_dir))
!= os.path.normcase(os.path.abspath(local_task_dir))
):
os.replace(extracted_task_dir, local_task_dir)
if not os.path.isdir(local_task_dir):
actual_size = os.path.getsize(tmp_path)
if expected_size >= 0 and actual_size != expected_size:
raise RuntimeError(
f"Extraction did not create expected directory: {local_task_dir}"
f"Downloaded file size mismatch for {normalized_rel}: "
f"expected={expected_size} actual={actual_size}"
)
os.replace(tmp_path, target_path)
try:
manifest = await asyncio.to_thread(_request_json, manifest_url)
files = list(manifest.get("files") or [])
if not files:
raise RuntimeError(f"Input manifest contains no files: {manifest_url}")
total_bytes = int(manifest.get("total_bytes") or 0)
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Downloading input files: count={len(files)} bytes={total_bytes}",
)
if os.path.isdir(staging_dir):
await asyncio.to_thread(shutil.rmtree, staging_dir)
os.makedirs(staging_dir, exist_ok=True)
for index, file_info in enumerate(files, start=1):
rel_path = normalize_cluster_relative_path(file_info.get("relative_path"))
expected_size = int(file_info.get("size") or -1)
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Downloading input file {index}/{len(files)}: "
f"{rel_path} ({expected_size} bytes)",
)
await asyncio.to_thread(_download_file, rel_path, expected_size)
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Downloaded input file {index}/{len(files)}: {rel_path}",
)
if os.path.isdir(local_task_dir):
await asyncio.to_thread(shutil.rmtree, local_task_dir)
os.makedirs(parent_dir, exist_ok=True)
os.replace(staging_dir, local_task_dir)
await task_service.add_log(
task_id,
@@ -216,9 +531,8 @@ async def materialize_cluster_input(
) from exc
finally:
try:
tmp_root = os.path.dirname(tmp_zip)
if os.path.isdir(tmp_root):
shutil.rmtree(tmp_root)
if os.path.isdir(staging_dir):
shutil.rmtree(staging_dir)
except Exception:
pass
@@ -247,51 +561,32 @@ async def upload_cluster_result(
tmp_root = tempfile.mkdtemp(prefix=f"cluster_result_{item.id}_")
tmp_zip = os.path.join(tmp_root, "result.zip")
try:
def _package_and_upload() -> dict:
_zip_directory_contents(managed_run_dir, tmp_zip)
await task_service.add_log(
task_id,
"INFO",
"[cluster] Uploading results to main server ...",
)
boundary = "----ClusterUploadBoundary"
body = io.BytesIO()
def _write_field(name, filename, content_type, data):
body.write(f"--{boundary}\r\n".encode("utf-8"))
if filename:
body.write(
f'Content-Disposition: form-data; name="{name}"; '
f'filename="{filename}"\r\n'.encode("utf-8")
)
body.write(f"Content-Type: {content_type}\r\n".encode("utf-8"))
else:
body.write(
f'Content-Disposition: form-data; name="{name}"\r\n'.encode(
"utf-8"
)
)
body.write(b"\r\n")
body.write(data)
body.write(b"\r\n")
_write_field("run_id", "", "text/plain", (run.run_id or "").encode("utf-8"))
_write_field("run_key", "", "text/plain", str(run_key or "").encode("utf-8"))
with open(tmp_zip, "rb") as fh:
_write_field(
"result_zip",
os.path.basename(tmp_zip),
"application/zip",
fh.read(),
body = _build_multipart_form_data(
fields={
"run_id": str(run.run_id or ""),
"run_key": str(run_key or ""),
},
files=[
(
"result_zip",
os.path.basename(tmp_zip),
"application/zip",
fh.read(),
),
],
boundary=boundary,
)
body.write(f"--{boundary}--\r\n".encode("utf-8"))
upload_url = f"{server_url}/api/cluster/upload-result/{item.id}"
req = urllib.request.Request(
upload_url,
data=body.getvalue(),
data=body,
headers={
"Content-Type": f"multipart/form-data; boundary={boundary}",
**_cluster_request_headers(),
@@ -300,15 +595,24 @@ async def upload_cluster_result(
)
with urllib.request.urlopen(req, timeout=_cluster_transfer_timeout()) as resp:
result = json.loads(resp.read().decode("utf-8"))
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Results uploaded: "
f"registered={result.get('registered', False)} "
f"processed={result.get('processed', 0)}",
)
return bool(result.get("registered", False))
return json.loads(resp.read().decode("utf-8"))
try:
await task_service.add_log(
task_id,
"INFO",
"[cluster] Uploading results to main server ...",
)
result = await asyncio.to_thread(_package_and_upload)
await task_service.add_log(
task_id,
"INFO",
f"[cluster] Results uploaded: "
f"registered={result.get('registered', False)} "
f"processed={result.get('processed', 0)}",
)
return bool(result.get("registered", False))
except urllib.error.HTTPError as exc:
body_text = ""