Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden LandSAR cluster transfer e2e

Browse Source
This commit is contained in:
Harmon
2026-06-27 01:20:06 +08:00
parent 40178ad1a4
commit 19ae3ec37f
7 changed files with 629 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_cluster_transport.py
+128
View File
@@ -3,20 +3,63 @@ import os
import tempfile
import unittest
import zipfile
from io import BytesIO
from types import SimpleNamespace
from unittest import mock
from multipart import parse_form
from backend.app.services.cluster_transport import (
_build_multipart_form_data,
_zip_directory_contents,
build_cluster_input_manifest,
iter_cluster_input_package_files,
normalize_cluster_relative_path,
resolve_cluster_local_run_dir,
resolve_cluster_local_task_dir,
safe_extract_zip,
stream_zip_files,
)
from backend.app.services.dinsar_completion_files import repair_managed_completion_files
from backend.app.services.dinsar_naming import RUN_META_FILENAME
class ClusterTransportTests(unittest.TestCase):
def test_build_multipart_form_data_is_parseable(self):
boundary = "----ClusterUploadBoundary"
body = _build_multipart_form_data(
fields={"run_id": "run-1", "run_key": "key-1"},
files=[("result_zip", "result.zip", "application/zip", b"zip-bytes")],
boundary=boundary,
)
fields = {}
files = {}
def on_field(field):
fields[field.field_name.decode("utf-8")] = field.value.decode("utf-8")
def on_file(file):
files[file.field_name.decode("utf-8")] = {
"file_name": file.file_name.decode("utf-8"),
"size": file.size,
"content": file.file_object.getvalue(),
}
parse_form(
{
"Content-Type": f"multipart/form-data; boundary={boundary}".encode("utf-8"),
"Content-Length": str(len(body)).encode("utf-8"),
},
BytesIO(body),
on_field,
on_file,
)
self.assertEqual(fields, {"run_id": "run-1", "run_key": "key-1"})
self.assertEqual(files["result_zip"]["file_name"], "result.zip")
self.assertEqual(files["result_zip"]["size"], len(b"zip-bytes"))
self.assertEqual(files["result_zip"]["content"], b"zip-bytes")
def test_zip_directory_contents_excludes_top_level_run_dir(self):
with tempfile.TemporaryDirectory() as root:
run_dir = os.path.join(root, "run_abc")
@@ -40,6 +83,91 @@ class ClusterTransportTests(unittest.TestCase):
self.assertIn("native/raw.txt", names)
self.assertFalse(any(name.startswith("run_abc/") for name in names))
def test_cluster_input_package_prefers_input_data(self):
with tempfile.TemporaryDirectory() as root:
task_dir = os.path.join(root, "Task_20260101_20260113")
input_dir = os.path.join(task_dir, "Input_Data")
master_dir = os.path.join(task_dir, "master")
slave_dir = os.path.join(task_dir, "slave")
orbit_dir = os.path.join(task_dir, "orbit")
os.makedirs(input_dir)
os.makedirs(master_dir)
os.makedirs(slave_dir)
os.makedirs(orbit_dir)
with open(os.path.join(task_dir, ".dinsar_pair.json"), "w", encoding="utf-8") as fp:
json.dump({"pair_key": "pair"}, fp)
for date_text in ("20260101", "20260113"):
base = f"LT1A_MONO_TEST_{date_text}_SLC"
with open(os.path.join(input_dir, f"{base}.xml"), "wb") as fp:
fp.write(b"xml")
with open(os.path.join(input_dir, f"{base}.tiff"), "wb") as fp:
fp.write(b"tif")
with open(os.path.join(master_dir, "LT1A_raw_20260101.tiff"), "wb") as fp:
fp.write(b"raw-master")
with open(os.path.join(slave_dir, "LT1A_raw_20260113.tiff"), "wb") as fp:
fp.write(b"raw-slave")
with open(os.path.join(orbit_dir, "orbit.txt"), "wb") as fp:
fp.write(b"orbit")
names = {rel.replace("\\", "/") for _, rel in iter_cluster_input_package_files(task_dir)}
self.assertIn(".dinsar_pair.json", names)
self.assertIn("Input_Data/LT1A_MONO_TEST_20260101_SLC.xml", names)
self.assertIn("Input_Data/LT1A_MONO_TEST_20260113_SLC.tiff", names)
self.assertIn("orbit/orbit.txt", names)
self.assertFalse(any(name.startswith("master/") for name in names))
self.assertFalse(any(name.startswith("slave/") for name in names))
def test_cluster_input_manifest_reports_files_and_size(self):
with tempfile.TemporaryDirectory() as root:
task_dir = os.path.join(root, "Task_20260101_20260113")
input_dir = os.path.join(task_dir, "Input_Data")
os.makedirs(input_dir)
total = 0
for date_text, payload in (("20260101", b"xml1"), ("20260113", b"xml2")):
base = f"LT1A_MONO_TEST_{date_text}_SLC"
for suffix, content in ((".xml", payload), (".tiff", payload * 2)):
path = os.path.join(input_dir, f"{base}{suffix}")
with open(path, "wb") as fp:
fp.write(content)
total += len(content)
manifest = build_cluster_input_manifest(task_dir)
self.assertEqual(manifest["task_name"], "Task_20260101_20260113")
self.assertEqual(manifest["file_count"], 4)
self.assertEqual(manifest["total_bytes"], total)
self.assertTrue(
all("\\" not in item["relative_path"] for item in manifest["files"])
)
def test_normalize_cluster_relative_path_rejects_escape(self):
for value in ("../a.txt", "a/../../b.txt", "/abs.txt", r"C:\abs.txt"):
with self.subTest(value=value):
with self.assertRaises(ValueError):
normalize_cluster_relative_path(value)
self.assertEqual(
normalize_cluster_relative_path(r"Input_Data\scene.tif"),
"Input_Data/scene.tif",
)
def test_stream_zip_files_preserves_task_top_level(self):
with tempfile.TemporaryDirectory() as root:
payload_path = os.path.join(root, "payload.txt")
with open(payload_path, "wb") as fp:
fp.write(b"payload")
zip_bytes = b"".join(
stream_zip_files([(payload_path, "Input_Data/payload.txt")], top_level_dir="Task_A")
)
zip_path = os.path.join(root, "streamed.zip")
with open(zip_path, "wb") as fp:
fp.write(zip_bytes)
with zipfile.ZipFile(zip_path, "r") as zf:
self.assertEqual(zf.read("Task_A/Input_Data/payload.txt"), b"payload")
def test_safe_extract_zip_rejects_path_escape(self):
with tempfile.TemporaryDirectory() as root:
zip_path = os.path.join(root, "unsafe.zip")