Improve orbit corruption diagnostics

2026-04-16 00:06:46 +08:00
parent 428802eac0
commit 5af296cc2d
4 changed files with 276 additions and 72 deletions
@@ -23,6 +23,38 @@ class StateVector:
    vz: float


+def _guess_text_encoding(input_txt: Path) -> str:
+    with input_txt.open("rb") as handle:
+        sample = handle.read(4096)
+
+    if sample.startswith(b"\xff\xfe"):
+        return "utf-16-le"
+    if sample.startswith(b"\xfe\xff"):
+        return "utf-16-be"
+
+    # Handle UTF-16-style wide-char text without BOM.
+    if sample and sample.count(b"\x00") * 4 >= len(sample):
+        even_nuls = sample[0::2].count(b"\x00")
+        odd_nuls = sample[1::2].count(b"\x00")
+        return "utf-16-le" if odd_nuls >= even_nuls else "utf-16-be"
+
+    return "utf-8"
+
+
+def _preview_line(value: str, limit: int = 200) -> str:
+    text = value.replace("\x00", "").replace("\t", " ").strip()
+    if len(text) > limit:
+        text = text[:limit] + "..."
+    return text or "<empty>"
+
+
+def _parse_error(input_txt: Path, line_no: int, message: str, raw_line: str = "") -> ValueError:
+    detail = f"{message} (file={input_txt}, line={line_no})"
+    if raw_line:
+        detail += f". snippet={_preview_line(raw_line)!r}"
+    return ValueError(detail)
+
+
 def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Convert LT-1 GpsData .txt orbit files to ISCE2 LUTAN1 orbit XML."
@@ -98,15 +130,35 @@ def find_text(root: ET.Element, path: str) -> str:
 def parse_orbit_file(input_txt: Path) -> list[StateVector]:
    vectors: list[StateVector] = []

-    with input_txt.open("r", encoding="utf-8") as handle:
-        for line in handle:
+    encoding = _guess_text_encoding(input_txt)
+    with input_txt.open("rb") as handle:
+        for line_no, raw_line in enumerate(handle, start=1):
+            line = raw_line.decode(encoding, errors="replace").replace("\ufeff", "")
            if not line.strip() or line.startswith("#"):
                continue

+            if "\x00" in line:
+                nul_count = line.count("\x00")
+                raise _parse_error(
+                    input_txt,
+                    line_no,
+                    (
+                        "Orbit TXT contains NUL bytes and appears truncated or corrupted"
+                        f" (nul_count={nul_count})"
+                    ),
+                    raw_line=line,
+                )
+
            parts = line.split()
            if len(parts) < 12:
-                continue
+                raise _parse_error(
+                    input_txt,
+                    line_no,
+                    f"Malformed orbit record: expected at least 12 columns, got {len(parts)}",
+                    raw_line=line,
+                )

+            try:
                sec_float = float(parts[5])
                sec_int = int(sec_float)
                microsecond = int(round((sec_float - sec_int) * 1_000_000))
@@ -132,6 +184,13 @@ def parse_orbit_file(input_txt: Path) -> list[StateVector]:
                        vz=float(parts[11]),
                    )
                )
+            except ValueError as exc:
+                raise _parse_error(
+                    input_txt,
+                    line_no,
+                    f"Malformed orbit record: {exc}",
+                    raw_line=line,
+                ) from exc

    if not vectors:
        raise ValueError(f"No orbit records parsed from {input_txt}")
@@ -161,12 +161,15 @@ async def get_orbit_status(
            "duplicate_count": source_stats.duplicate_count,
            "errors": source_stats.errors,
            "quarantine_path": source_gap_summary["quarantine_path"],
+            "sample_limit": source_gap_summary["sample_limit"],
            "source_without_isce2_count": source_gap_summary["source_without_isce2_count"],
            "source_without_envi_count": source_gap_summary["source_without_envi_count"],
            "envi_without_source_count": source_gap_summary["envi_without_source_count"],
            "isce2_without_source_count": source_gap_summary["isce2_without_source_count"],
            "suspect_bad_count": source_gap_summary["suspect_bad_count"],
            "suspect_bad_samples": source_gap_summary["suspect_bad_samples"],
+            "bad_source_sample_count": source_gap_summary["bad_source_sample_count"],
+            "bad_source_samples": source_gap_summary["bad_source_samples"],
            "source_without_envi_samples": source_gap_summary["source_without_envi_samples"],
            "envi_without_source_samples": source_gap_summary["envi_without_source_samples"],
            "isce2_without_source_samples": source_gap_summary["isce2_without_source_samples"],
@@ -161,25 +161,55 @@ def _inspect_orbit_txt_health(path: str) -> Dict[str, Any]:
        "contains_nul_bytes": False,
        "nul_byte_count": 0,
        "tail_has_nul_bytes": False,
+        "first_nul_offset": -1,
    }
    try:
        nul_byte_count = 0
        tail = b""
+        offset = 0
+        first_nul_offset = -1
        with open(path, "rb") as stream:
            while True:
                chunk = stream.read(1024 * 1024)
                if not chunk:
                    break
+                if first_nul_offset < 0:
+                    chunk_nul_index = chunk.find(b"\x00")
+                    if chunk_nul_index >= 0:
+                        first_nul_offset = offset + chunk_nul_index
                nul_byte_count += chunk.count(b"\x00")
                tail = (tail + chunk)[-4096:]
+                offset += len(chunk)
        info["nul_byte_count"] = nul_byte_count
        info["contains_nul_bytes"] = nul_byte_count > 0
        info["tail_has_nul_bytes"] = b"\x00" in tail
+        info["first_nul_offset"] = first_nul_offset
    except OSError as exc:
        info["read_error"] = str(exc)
    return info


+def _truncate_message(text: str, limit: int = 1200) -> str:
+    value = str(text or "").replace("\x00", "[NUL]").strip()
+    if len(value) <= limit:
+        return value
+    return value[:limit] + "...<truncated>"
+
+
+def _compact_error_message(text: str, limit: int = 1200) -> str:
+    value = str(text or "").replace("\x00", "[NUL]").strip()
+    if not value:
+        return ""
+
+    lines = [line.strip() for line in value.splitlines() if line.strip()]
+    if len(lines) > 1:
+        for line in reversed(lines):
+            if re.match(r"^[A-Za-z_][\w.]*:\s", line):
+                return _truncate_message(line, limit=limit)
+
+    return _truncate_message(value, limit=limit)
+
+
 def _convert_to_isce2_xml_file(txt_path: str, xml_path: str) -> None:
    if "isce2" not in _CONVERTERS:
        raise KeyError("未注册 isce2 轨道转换器")
@@ -209,16 +239,57 @@ def _build_invalid_source_record(
    record: Dict[str, Any] = {
        "name": stem,
        "source": txt_path,
-        "error": str(error),
+        "error": _compact_error_message(error),
    }
    if envi_path:
        record["envi_path"] = envi_path
    if isce2_path:
        record["isce2_path"] = isce2_path
    record.update(_inspect_orbit_txt_health(txt_path))
+    record["has_corruption_signal"] = _has_orbit_corruption_signal(record)
    return record


+def _has_orbit_corruption_signal(item: Dict[str, Any]) -> bool:
+    first_nul_offset = item.get("first_nul_offset", -1)
+    return bool(
+        item.get("read_error")
+        or item.get("contains_nul_bytes")
+        or item.get("tail_has_nul_bytes")
+        or int(item.get("nul_byte_count") or 0) > 0
+        or (
+            isinstance(first_nul_offset, (int, float))
+            and int(first_nul_offset) >= 0
+        )
+    )
+
+
+def _build_source_gap_sample(
+    stem: str,
+    source_entry: Optional[Dict[str, Any]] = None,
+    envi_entry: Optional[Dict[str, Any]] = None,
+    isce2_entry: Optional[Dict[str, Any]] = None,
+    inspect_source: bool = False,
+) -> Dict[str, Any]:
+    source_path = (source_entry or {}).get("path", "")
+    sample: Dict[str, Any] = {
+        "name": stem,
+        "source": source_path,
+        "source_path": source_path,
+        "envi_path": (envi_entry or {}).get("path", ""),
+        "isce2_path": (isce2_entry or {}).get("path", ""),
+        "has_corruption_signal": False,
+    }
+    if inspect_source and source_path:
+        sample.update(_inspect_orbit_txt_health(source_path))
+        sample["has_corruption_signal"] = _has_orbit_corruption_signal(sample)
+        if sample.get("read_error"):
+            sample["error"] = f"Source health scan failed: {sample['read_error']}"
+        elif sample["has_corruption_signal"]:
+            sample["error"] = "Source TXT contains NUL bytes and appears corrupted"
+    return sample
+
+
 def validate_orbit_source(txt_path: str, stem: str = "", scratch_root: str = "") -> Dict[str, Any]:
    name = stem or os.path.splitext(os.path.basename(txt_path))[0]
    try:
@@ -259,28 +330,42 @@ def summarize_source_orbit_gaps(
    source_without_envi = sorted(source_stems - envi_stems)
    envi_without_source = sorted(envi_stems - source_stems)
    isce2_without_source = sorted(isce2_stems - source_stems)
+    sample_limit = 20

-    def _make_samples(stems: List[str]) -> List[Dict[str, str]]:
-        samples: List[Dict[str, str]] = []
-        for stem in stems[:20]:
+    def _make_samples(
+        stems: List[str],
+        inspect_source: bool = False,
+    ) -> List[Dict[str, Any]]:
+        samples: List[Dict[str, Any]] = []
+        for stem in stems[:sample_limit]:
            samples.append(
-                {
-                    "name": stem,
-                    "source_path": source_files.get(stem, {}).get("path", ""),
-                    "envi_path": envi_files.get(stem, {}).get("path", ""),
-                    "isce2_path": isce2_files.get(stem, {}).get("path", ""),
-                }
+                _build_source_gap_sample(
+                    stem,
+                    source_files.get(stem),
+                    envi_files.get(stem),
+                    isce2_files.get(stem),
+                    inspect_source=inspect_source,
+                )
            )
        return samples

+    suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True)
+    bad_source_samples = [
+        item for item in suspect_bad_samples
+        if item.get("has_corruption_signal")
+    ]
+
    return {
+        "sample_limit": sample_limit,
        "quarantine_path": _default_quarantine_root(source_dir, quarantine_root),
        "source_without_isce2_count": len(source_without_isce2),
        "source_without_envi_count": len(source_without_envi),
        "envi_without_source_count": len(envi_without_source),
        "isce2_without_source_count": len(isce2_without_source),
        "suspect_bad_count": len(source_without_isce2),
-        "suspect_bad_samples": _make_samples(source_without_isce2),
+        "suspect_bad_samples": suspect_bad_samples,
+        "bad_source_sample_count": len(bad_source_samples),
+        "bad_source_samples": bad_source_samples,
        "source_without_envi_samples": _make_samples(source_without_envi),
        "envi_without_source_samples": _make_samples(envi_without_source),
        "isce2_without_source_samples": _make_samples(isce2_without_source),
@@ -558,7 +643,8 @@ def _convert_to_isce2_xml(
        timeout=60,
    )
    if result.returncode != 0:
-        raise RuntimeError(result.stderr or result.stdout or "Orbit conversion failed without output")
+        detail = result.stderr or result.stdout or "Orbit conversion failed without output"
+        raise RuntimeError(_compact_error_message(detail))


 # ---------------------------------------------------------------------------
@@ -857,14 +943,9 @@ def repair_orbit_pools(
                _convert_to_isce2_xml_file(txt_path, xml_path)
                repaired_from_envi.append(stem)
            except Exception as exc:
-                repair_errors.append(
-                    {
-                        "name": stem,
-                        "source": txt_path,
-                        "target": xml_path,
-                        "error": str(exc),
-                    }
-                )
+                item = _build_invalid_source_record(stem, txt_path, exc, envi_path=txt_path, isce2_path=xml_path)
+                item["target"] = xml_path
+                repair_errors.append(item)

    after = check_orbit_consistency(envi_pool, isce2_pool)
    return {
@@ -34,6 +34,67 @@ const formatMismatchList = (items, formatter) => (
  asArray(items).map(formatter).join('; ')
 );

+const formatIntegerText = (value) => {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed.toLocaleString() : '0';
+};
+
+const getOrbitSourcePath = (item) => item?.source || item?.source_path || '';
+
+const hasOrbitCorruptionSignal = (item) => Boolean(
+  item && (
+    item.has_corruption_signal ||
+    item.contains_nul_bytes ||
+    toNumber(item.nul_byte_count) > 0 ||
+    item.tail_has_nul_bytes ||
+    Number(item.first_nul_offset) >= 0 ||
+    item.read_error
+  )
+);
+
+const renderOrbitSourceIssueDetails = (item, en, formatPathText) => {
+  const sourcePath = getOrbitSourcePath(item);
+  const nulCount = toNumber(item?.nul_byte_count);
+  const firstNulOffset = Number(item?.first_nul_offset);
+  const hasNulDetails =
+    item?.contains_nul_bytes ||
+    nulCount > 0 ||
+    item?.tail_has_nul_bytes ||
+    firstNulOffset >= 0;
+
+  return (
+    <>
+      {sourcePath && (
+        <div style={{ color: '#64748b' }}>
+          {en ? 'Source: ' : '源文件：'}{formatPathText(sourcePath)}
+        </div>
+      )}
+      {item?.envi_path && (
+        <div style={{ color: '#64748b' }}>
+          {en ? 'ENVI: ' : 'ENVI：'}{formatPathText(item.envi_path)}
+        </div>
+      )}
+      {item?.isce2_path && (
+        <div style={{ color: '#64748b' }}>
+          {en ? 'ISCE2: ' : 'ISCE2：'}{formatPathText(item.isce2_path)}
+        </div>
+      )}
+      {hasNulDetails && (
+        <div style={{ color: '#64748b' }}>
+          {en ? 'NUL bytes: ' : 'NUL 字节：'}{formatIntegerText(nulCount)}
+          {firstNulOffset >= 0 ? `${en ? ', first offset: ' : '，首个偏移：'}${formatIntegerText(firstNulOffset)}` : ''}
+          {item?.tail_has_nul_bytes ? (en ? ' (tail contains NUL)' : '（文件尾含 NUL）') : ''}
+        </div>
+      )}
+      {item?.read_error && (
+        <div style={{ color: '#64748b' }}>
+          {en ? 'Health scan error: ' : '文件健康扫描异常：'}{item.read_error}
+        </div>
+      )}
+    </>
+  );
+};
+
 const formatSourceRootRole = (role, en = false) => {
  switch (role) {
    case 'radar_source':
@@ -302,6 +363,12 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
  const orbitEnviWithoutSourceCount = toNumber(orbitSource.envi_without_source_count);
  const orbitIsce2WithoutSourceCount = toNumber(orbitSource.isce2_without_source_count);
  const orbitQuarantinePath = orbitSource.quarantine_path || orbitStatus?.source_gaps?.quarantine_path;
+  const orbitBadSourceSamples = asArray(orbitSource.bad_source_samples).filter(hasOrbitCorruptionSignal);
+  const orbitSuspectBadSamples = asArray(orbitSource.suspect_bad_samples);
+  const orbitSuspectWithoutCorruptionSamples = orbitSuspectBadSamples.filter(
+    (item) => !orbitBadSourceSamples.some((badItem) => badItem.name === item.name)
+  );
+  const orbitBadSourceSampleCount = toNumber(orbitSource.bad_source_sample_count || orbitBadSourceSamples.length);
  const orbitOverallHealthy = Boolean(
    orbitStatus &&
    orbitMismatchCount === 0 &&
@@ -951,6 +1018,13 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
                    : `疑似坏源 TXT（源文件存在但 ISCE2 XML 缺失）：${orbitSuspectBadCount}`}
                </div>
              )}
+              {orbitBadSourceSampleCount > 0 && (
+                <div className="health-card-note error">
+                  {en
+                    ? `Sampled source TXT with corruption signals: ${orbitBadSourceSampleCount}`
+                    : `抽样检测到损坏信号的源 TXT：${orbitBadSourceSampleCount}`}
+                </div>
+              )}
              {orbitDatabase.sample_missing_in_envi?.length > 0 && (
                <div className="health-card-note error">
                  {en ? 'DB expected but ENVI pool missing: ' : '数据库期望但 ENVI 池缺失：'}
@@ -963,19 +1037,16 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
                  {orbitDatabase.sample_missing_in_isce2.slice(0, 5).join(', ')}
                </div>
              )}
-              {(orbitSource.suspect_bad_samples || []).slice(0, 5).map((item) => (
+              {orbitBadSourceSamples.slice(0, 5).map((item) => (
+                <div key={`orbit-bad-source-${item.name}`} className="health-card-note error">
+                  {item.name} - {item.error || (en ? 'Corruption signal detected' : '检测到损坏信号')}
+                  {renderOrbitSourceIssueDetails(item, en, formatPathText)}
+                </div>
+              ))}
+              {orbitSuspectWithoutCorruptionSamples.slice(0, 5).map((item) => (
                <div key={`orbit-suspect-bad-${item.name}`} className="health-card-note warn">
                  {item.name}
-                  {item.source_path && (
-                    <div style={{ color: '#64748b' }}>
-                      {en ? 'Source: ' : '源文件：'}{formatPathText(item.source_path)}
-                    </div>
-                  )}
-                  {item.envi_path && (
-                    <div style={{ color: '#64748b' }}>
-                      {en ? 'ENVI: ' : 'ENVI：'}{formatPathText(item.envi_path)}
-                    </div>
-                  )}
+                  {renderOrbitSourceIssueDetails(item, en, formatPathText)}
                </div>
              ))}
              {(orbitConsistency.mismatches || []).slice(0, 5).map((item) => (
@@ -1102,13 +1173,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
                          {(orbitSyncResult.confirmed_bad || []).slice(0, 5).map((item, index) => (
                            <div key={`orbit-quarantine-bad-${index}`} className="health-card-note error">
                              {item.name} - {item.error}
-                              <div style={{ color: '#64748b' }}>
-                                {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)}
-                              </div>
-                              <div style={{ color: '#64748b' }}>
-                                {en ? 'NUL bytes: ' : 'NUL 字节：'}{toNumber(item.nul_byte_count)}
-                                {item.tail_has_nul_bytes ? (en ? ' (tail contains NUL)' : '（文件尾含 NUL）') : ''}
-                              </div>
+                              {renderOrbitSourceIssueDetails(item, en, formatPathText)}
                              {item.quarantined_source && (
                                <div style={{ color: '#64748b' }}>
                                  {en ? 'Moved source to: ' : '源文件已移至：'}{formatPathText(item.quarantined_source)}
@@ -1163,9 +1228,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
                          {(orbitSyncResult.repair_errors || []).slice(0, 3).map((item, index) => (
                            <div key={`orbit-repair-error-${index}`} className="health-card-note error">
                              {item.name} - {item.error}
-                              <div style={{ color: '#64748b' }}>
-                                {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)}
-                              </div>
+                              {renderOrbitSourceIssueDetails(item, en, formatPathText)}
                            </div>
                          ))}
                          {(orbitSyncResult.sync_result?.source?.errors || []).slice(0, 3).map((item, index) => (
@@ -1176,9 +1239,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
                          {(orbitSyncResult.sync_result?.invalid_sources || []).slice(0, 5).map((item, index) => (
                            <div key={`orbit-repair-invalid-${index}`} className="health-card-note error">
                              {item.name} - {item.error}
-                              <div style={{ color: '#64748b' }}>
-                                {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)}
-                              </div>
+                              {renderOrbitSourceIssueDetails(item, en, formatPathText)}
                            </div>
                          ))}
                          {(orbitSyncResult.sync_result?.isce2?.errors || []).slice(0, 3).map((item, index) => (