Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve orbit corruption diagnostics

Browse Source
This commit is contained in:
Harmon
2026-04-16 00:06:46 +08:00
parent 428802eac0
commit 5af296cc2d
4 changed files with 276 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py
+87 -28
View File
@@ -23,6 +23,38 @@ class StateVector:
vz: float
def _guess_text_encoding(input_txt: Path) -> str:
with input_txt.open("rb") as handle:
sample = handle.read(4096)
if sample.startswith(b"\xff\xfe"):
return "utf-16-le"
if sample.startswith(b"\xfe\xff"):
return "utf-16-be"
# Handle UTF-16-style wide-char text without BOM.
if sample and sample.count(b"\x00") * 4 >= len(sample):
even_nuls = sample[0::2].count(b"\x00")
odd_nuls = sample[1::2].count(b"\x00")
return "utf-16-le" if odd_nuls >= even_nuls else "utf-16-be"
return "utf-8"
def _preview_line(value: str, limit: int = 200) -> str:
text = value.replace("\x00", "").replace("\t", " ").strip()
if len(text) > limit:
text = text[:limit] + "..."
return text or "<empty>"
def _parse_error(input_txt: Path, line_no: int, message: str, raw_line: str = "") -> ValueError:
detail = f"{message} (file={input_txt}, line={line_no})"
if raw_line:
detail += f". snippet={_preview_line(raw_line)!r}"
return ValueError(detail)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Convert LT-1 GpsData .txt orbit files to ISCE2 LUTAN1 orbit XML."
@@ -98,40 +130,67 @@ def find_text(root: ET.Element, path: str) -> str:
def parse_orbit_file(input_txt: Path) -> list[StateVector]:
vectors: list[StateVector] = []
with input_txt.open("r", encoding="utf-8") as handle:
for line in handle:
encoding = _guess_text_encoding(input_txt)
with input_txt.open("rb") as handle:
for line_no, raw_line in enumerate(handle, start=1):
line = raw_line.decode(encoding, errors="replace").replace("\ufeff", "")
if not line.strip() or line.startswith("#"):
continue
if "\x00" in line:
nul_count = line.count("\x00")
raise _parse_error(
input_txt,
line_no,
(
"Orbit TXT contains NUL bytes and appears truncated or corrupted"
f" (nul_count={nul_count})"
),
raw_line=line,
)
parts = line.split()
if len(parts) < 12:
continue
sec_float = float(parts[5])
sec_int = int(sec_float)
microsecond = int(round((sec_float - sec_int) * 1_000_000))
timestamp = datetime(
int(parts[0]),
int(parts[1]),
int(parts[2]),
int(parts[3]),
int(parts[4]),
sec_int,
microsecond,
)
vectors.append(
StateVector(
time=timestamp,
x=float(parts[6]),
y=float(parts[7]),
z=float(parts[8]),
vx=float(parts[9]),
vy=float(parts[10]),
vz=float(parts[11]),
raise _parse_error(
input_txt,
line_no,
f"Malformed orbit record: expected at least 12 columns, got {len(parts)}",
raw_line=line,
)
)
try:
sec_float = float(parts[5])
sec_int = int(sec_float)
microsecond = int(round((sec_float - sec_int) * 1_000_000))
timestamp = datetime(
int(parts[0]),
int(parts[1]),
int(parts[2]),
int(parts[3]),
int(parts[4]),
sec_int,
microsecond,
)
vectors.append(
StateVector(
time=timestamp,
x=float(parts[6]),
y=float(parts[7]),
z=float(parts[8]),
vx=float(parts[9]),
vy=float(parts[10]),
vz=float(parts[11]),
)
)
except ValueError as exc:
raise _parse_error(
input_txt,
line_no,
f"Malformed orbit record: {exc}",
raw_line=line,
) from exc
if not vectors:
raise ValueError(f"No orbit records parsed from {input_txt}")
backend/app/routers/orbit.py
+3
View File
@@ -161,12 +161,15 @@ async def get_orbit_status(
"duplicate_count": source_stats.duplicate_count,
"errors": source_stats.errors,
"quarantine_path": source_gap_summary["quarantine_path"],
"sample_limit": source_gap_summary["sample_limit"],
"source_without_isce2_count": source_gap_summary["source_without_isce2_count"],
"source_without_envi_count": source_gap_summary["source_without_envi_count"],
"envi_without_source_count": source_gap_summary["envi_without_source_count"],
"isce2_without_source_count": source_gap_summary["isce2_without_source_count"],
"suspect_bad_count": source_gap_summary["suspect_bad_count"],
"suspect_bad_samples": source_gap_summary["suspect_bad_samples"],
"bad_source_sample_count": source_gap_summary["bad_source_sample_count"],
"bad_source_samples": source_gap_summary["bad_source_samples"],
"source_without_envi_samples": source_gap_summary["source_without_envi_samples"],
"envi_without_source_samples": source_gap_summary["envi_without_source_samples"],
"isce2_without_source_samples": source_gap_summary["isce2_without_source_samples"],
backend/app/services/orbit_converter.py
+101 -20
View File
@@ -161,25 +161,55 @@ def _inspect_orbit_txt_health(path: str) -> Dict[str, Any]:
"contains_nul_bytes": False,
"nul_byte_count": 0,
"tail_has_nul_bytes": False,
"first_nul_offset": -1,
}
try:
nul_byte_count = 0
tail = b""
offset = 0
first_nul_offset = -1
with open(path, "rb") as stream:
while True:
chunk = stream.read(1024 * 1024)
if not chunk:
break
if first_nul_offset < 0:
chunk_nul_index = chunk.find(b"\x00")
if chunk_nul_index >= 0:
first_nul_offset = offset + chunk_nul_index
nul_byte_count += chunk.count(b"\x00")
tail = (tail + chunk)[-4096:]
offset += len(chunk)
info["nul_byte_count"] = nul_byte_count
info["contains_nul_bytes"] = nul_byte_count > 0
info["tail_has_nul_bytes"] = b"\x00" in tail
info["first_nul_offset"] = first_nul_offset
except OSError as exc:
info["read_error"] = str(exc)
return info
def _truncate_message(text: str, limit: int = 1200) -> str:
value = str(text or "").replace("\x00", "[NUL]").strip()
if len(value) <= limit:
return value
return value[:limit] + "...<truncated>"
def _compact_error_message(text: str, limit: int = 1200) -> str:
value = str(text or "").replace("\x00", "[NUL]").strip()
if not value:
return ""
lines = [line.strip() for line in value.splitlines() if line.strip()]
if len(lines) > 1:
for line in reversed(lines):
if re.match(r"^[A-Za-z_][\w.]*:\s", line):
return _truncate_message(line, limit=limit)
return _truncate_message(value, limit=limit)
def _convert_to_isce2_xml_file(txt_path: str, xml_path: str) -> None:
if "isce2" not in _CONVERTERS:
raise KeyError("未注册 isce2 轨道转换器")
@@ -209,16 +239,57 @@ def _build_invalid_source_record(
record: Dict[str, Any] = {
"name": stem,
"source": txt_path,
"error": str(error),
"error": _compact_error_message(error),
}
if envi_path:
record["envi_path"] = envi_path
if isce2_path:
record["isce2_path"] = isce2_path
record.update(_inspect_orbit_txt_health(txt_path))
record["has_corruption_signal"] = _has_orbit_corruption_signal(record)
return record
def _has_orbit_corruption_signal(item: Dict[str, Any]) -> bool:
first_nul_offset = item.get("first_nul_offset", -1)
return bool(
item.get("read_error")
or item.get("contains_nul_bytes")
or item.get("tail_has_nul_bytes")
or int(item.get("nul_byte_count") or 0) > 0
or (
isinstance(first_nul_offset, (int, float))
and int(first_nul_offset) >= 0
)
)
def _build_source_gap_sample(
stem: str,
source_entry: Optional[Dict[str, Any]] = None,
envi_entry: Optional[Dict[str, Any]] = None,
isce2_entry: Optional[Dict[str, Any]] = None,
inspect_source: bool = False,
) -> Dict[str, Any]:
source_path = (source_entry or {}).get("path", "")
sample: Dict[str, Any] = {
"name": stem,
"source": source_path,
"source_path": source_path,
"envi_path": (envi_entry or {}).get("path", ""),
"isce2_path": (isce2_entry or {}).get("path", ""),
"has_corruption_signal": False,
}
if inspect_source and source_path:
sample.update(_inspect_orbit_txt_health(source_path))
sample["has_corruption_signal"] = _has_orbit_corruption_signal(sample)
if sample.get("read_error"):
sample["error"] = f"Source health scan failed: {sample['read_error']}"
elif sample["has_corruption_signal"]:
sample["error"] = "Source TXT contains NUL bytes and appears corrupted"
return sample
def validate_orbit_source(txt_path: str, stem: str = "", scratch_root: str = "") -> Dict[str, Any]:
name = stem or os.path.splitext(os.path.basename(txt_path))[0]
try:
@@ -259,28 +330,42 @@ def summarize_source_orbit_gaps(
source_without_envi = sorted(source_stems - envi_stems)
envi_without_source = sorted(envi_stems - source_stems)
isce2_without_source = sorted(isce2_stems - source_stems)
sample_limit = 20
def _make_samples(stems: List[str]) -> List[Dict[str, str]]:
samples: List[Dict[str, str]] = []
for stem in stems[:20]:
def _make_samples(
stems: List[str],
inspect_source: bool = False,
) -> List[Dict[str, Any]]:
samples: List[Dict[str, Any]] = []
for stem in stems[:sample_limit]:
samples.append(
{
"name": stem,
"source_path": source_files.get(stem, {}).get("path", ""),
"envi_path": envi_files.get(stem, {}).get("path", ""),
"isce2_path": isce2_files.get(stem, {}).get("path", ""),
}
_build_source_gap_sample(
stem,
source_files.get(stem),
envi_files.get(stem),
isce2_files.get(stem),
inspect_source=inspect_source,
)
)
return samples
suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
bad_source_samples = [
item for item in suspect_bad_samples
if item.get("has_corruption_signal")
]
return {
"sample_limit": sample_limit,
"quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
"source_without_isce2_count": len(source_without_isce2),
"source_without_envi_count": len(source_without_envi),
"envi_without_source_count": len(envi_without_source),
"isce2_without_source_count": len(isce2_without_source),
"suspect_bad_count": len(source_without_isce2),
"suspect_bad_samples": _make_samples(source_without_isce2),
"suspect_bad_samples": suspect_bad_samples,
"bad_source_sample_count": len(bad_source_samples),
"bad_source_samples": bad_source_samples,
"source_without_envi_samples": _make_samples(source_without_envi),
"envi_without_source_samples": _make_samples(envi_without_source),
"isce2_without_source_samples": _make_samples(isce2_without_source),
@@ -558,7 +643,8 @@ def _convert_to_isce2_xml(
timeout=60,
)
if result.returncode != 0:
raise RuntimeError(result.stderr or result.stdout or "Orbit conversion failed without output")
detail = result.stderr or result.stdout or "Orbit conversion failed without output"
raise RuntimeError(_compact_error_message(detail))
# ---------------------------------------------------------------------------
@@ -857,14 +943,9 @@ def repair_orbit_pools(
_convert_to_isce2_xml_file(txt_path, xml_path)
repaired_from_envi.append(stem)
except Exception as exc:
repair_errors.append(
{
"name": stem,
"source": txt_path,
"target": xml_path,
"error": str(exc),
}
)
item = _build_invalid_source_record(stem, txt_path, exc, envi_path=txt_path, isce2_path=xml_path)
item["target"] = xml_path
repair_errors.append(item)
after = check_orbit_consistency(envi_pool, isce2_pool)
return {