diff --git a/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py b/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py index b793e34..bdaf27a 100644 --- a/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py +++ b/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py @@ -23,6 +23,38 @@ class StateVector: vz: float +def _guess_text_encoding(input_txt: Path) -> str: + with input_txt.open("rb") as handle: + sample = handle.read(4096) + + if sample.startswith(b"\xff\xfe"): + return "utf-16-le" + if sample.startswith(b"\xfe\xff"): + return "utf-16-be" + + # Handle UTF-16-style wide-char text without BOM. + if sample and sample.count(b"\x00") * 4 >= len(sample): + even_nuls = sample[0::2].count(b"\x00") + odd_nuls = sample[1::2].count(b"\x00") + return "utf-16-le" if odd_nuls >= even_nuls else "utf-16-be" + + return "utf-8" + + +def _preview_line(value: str, limit: int = 200) -> str: + text = value.replace("\x00", "").replace("\t", " ").strip() + if len(text) > limit: + text = text[:limit] + "..." + return text or "" + + +def _parse_error(input_txt: Path, line_no: int, message: str, raw_line: str = "") -> ValueError: + detail = f"{message} (file={input_txt}, line={line_no})" + if raw_line: + detail += f". snippet={_preview_line(raw_line)!r}" + return ValueError(detail) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Convert LT-1 GpsData .txt orbit files to ISCE2 LUTAN1 orbit XML." @@ -98,40 +130,67 @@ def find_text(root: ET.Element, path: str) -> str: def parse_orbit_file(input_txt: Path) -> list[StateVector]: vectors: list[StateVector] = [] - with input_txt.open("r", encoding="utf-8") as handle: - for line in handle: + encoding = _guess_text_encoding(input_txt) + with input_txt.open("rb") as handle: + for line_no, raw_line in enumerate(handle, start=1): + line = raw_line.decode(encoding, errors="replace").replace("\ufeff", "") if not line.strip() or line.startswith("#"): continue + if "\x00" in line: + nul_count = line.count("\x00") + raise _parse_error( + input_txt, + line_no, + ( + "Orbit TXT contains NUL bytes and appears truncated or corrupted" + f" (nul_count={nul_count})" + ), + raw_line=line, + ) + parts = line.split() if len(parts) < 12: - continue - - sec_float = float(parts[5]) - sec_int = int(sec_float) - microsecond = int(round((sec_float - sec_int) * 1_000_000)) - - timestamp = datetime( - int(parts[0]), - int(parts[1]), - int(parts[2]), - int(parts[3]), - int(parts[4]), - sec_int, - microsecond, - ) - - vectors.append( - StateVector( - time=timestamp, - x=float(parts[6]), - y=float(parts[7]), - z=float(parts[8]), - vx=float(parts[9]), - vy=float(parts[10]), - vz=float(parts[11]), + raise _parse_error( + input_txt, + line_no, + f"Malformed orbit record: expected at least 12 columns, got {len(parts)}", + raw_line=line, ) - ) + + try: + sec_float = float(parts[5]) + sec_int = int(sec_float) + microsecond = int(round((sec_float - sec_int) * 1_000_000)) + + timestamp = datetime( + int(parts[0]), + int(parts[1]), + int(parts[2]), + int(parts[3]), + int(parts[4]), + sec_int, + microsecond, + ) + + vectors.append( + StateVector( + time=timestamp, + x=float(parts[6]), + y=float(parts[7]), + z=float(parts[8]), + vx=float(parts[9]), + vy=float(parts[10]), + vz=float(parts[11]), + ) + ) + except ValueError as exc: + raise _parse_error( + input_txt, + line_no, + f"Malformed orbit record: {exc}", + raw_line=line, + ) from exc if not vectors: raise ValueError(f"No orbit records parsed from {input_txt}") diff --git a/backend/app/routers/orbit.py b/backend/app/routers/orbit.py index 3301a5f..9d190b4 100644 --- a/backend/app/routers/orbit.py +++ b/backend/app/routers/orbit.py @@ -161,12 +161,15 @@ async def get_orbit_status( "duplicate_count": source_stats.duplicate_count, "errors": source_stats.errors, "quarantine_path": source_gap_summary["quarantine_path"], + "sample_limit": source_gap_summary["sample_limit"], "source_without_isce2_count": source_gap_summary["source_without_isce2_count"], "source_without_envi_count": source_gap_summary["source_without_envi_count"], "envi_without_source_count": source_gap_summary["envi_without_source_count"], "isce2_without_source_count": source_gap_summary["isce2_without_source_count"], "suspect_bad_count": source_gap_summary["suspect_bad_count"], "suspect_bad_samples": source_gap_summary["suspect_bad_samples"], + "bad_source_sample_count": source_gap_summary["bad_source_sample_count"], + "bad_source_samples": source_gap_summary["bad_source_samples"], "source_without_envi_samples": source_gap_summary["source_without_envi_samples"], "envi_without_source_samples": source_gap_summary["envi_without_source_samples"], "isce2_without_source_samples": source_gap_summary["isce2_without_source_samples"], diff --git a/backend/app/services/orbit_converter.py b/backend/app/services/orbit_converter.py index e6eef8f..1d01ffb 100644 --- a/backend/app/services/orbit_converter.py +++ b/backend/app/services/orbit_converter.py @@ -161,25 +161,55 @@ def _inspect_orbit_txt_health(path: str) -> Dict[str, Any]: "contains_nul_bytes": False, "nul_byte_count": 0, "tail_has_nul_bytes": False, + "first_nul_offset": -1, } try: nul_byte_count = 0 tail = b"" + offset = 0 + first_nul_offset = -1 with open(path, "rb") as stream: while True: chunk = stream.read(1024 * 1024) if not chunk: break + if first_nul_offset < 0: + chunk_nul_index = chunk.find(b"\x00") + if chunk_nul_index >= 0: + first_nul_offset = offset + chunk_nul_index nul_byte_count += chunk.count(b"\x00") tail = (tail + chunk)[-4096:] + offset += len(chunk) info["nul_byte_count"] = nul_byte_count info["contains_nul_bytes"] = nul_byte_count > 0 info["tail_has_nul_bytes"] = b"\x00" in tail + info["first_nul_offset"] = first_nul_offset except OSError as exc: info["read_error"] = str(exc) return info +def _truncate_message(text: str, limit: int = 1200) -> str: + value = str(text or "").replace("\x00", "[NUL]").strip() + if len(value) <= limit: + return value + return value[:limit] + "..." + + +def _compact_error_message(text: str, limit: int = 1200) -> str: + value = str(text or "").replace("\x00", "[NUL]").strip() + if not value: + return "" + + lines = [line.strip() for line in value.splitlines() if line.strip()] + if len(lines) > 1: + for line in reversed(lines): + if re.match(r"^[A-Za-z_][\w.]*:\s", line): + return _truncate_message(line, limit=limit) + + return _truncate_message(value, limit=limit) + + def _convert_to_isce2_xml_file(txt_path: str, xml_path: str) -> None: if "isce2" not in _CONVERTERS: raise KeyError("未注册 isce2 轨道转换器") @@ -209,16 +239,57 @@ def _build_invalid_source_record( record: Dict[str, Any] = { "name": stem, "source": txt_path, - "error": str(error), + "error": _compact_error_message(error), } if envi_path: record["envi_path"] = envi_path if isce2_path: record["isce2_path"] = isce2_path record.update(_inspect_orbit_txt_health(txt_path)) + record["has_corruption_signal"] = _has_orbit_corruption_signal(record) return record +def _has_orbit_corruption_signal(item: Dict[str, Any]) -> bool: + first_nul_offset = item.get("first_nul_offset", -1) + return bool( + item.get("read_error") + or item.get("contains_nul_bytes") + or item.get("tail_has_nul_bytes") + or int(item.get("nul_byte_count") or 0) > 0 + or ( + isinstance(first_nul_offset, (int, float)) + and int(first_nul_offset) >= 0 + ) + ) + + +def _build_source_gap_sample( + stem: str, + source_entry: Optional[Dict[str, Any]] = None, + envi_entry: Optional[Dict[str, Any]] = None, + isce2_entry: Optional[Dict[str, Any]] = None, + inspect_source: bool = False, +) -> Dict[str, Any]: + source_path = (source_entry or {}).get("path", "") + sample: Dict[str, Any] = { + "name": stem, + "source": source_path, + "source_path": source_path, + "envi_path": (envi_entry or {}).get("path", ""), + "isce2_path": (isce2_entry or {}).get("path", ""), + "has_corruption_signal": False, + } + if inspect_source and source_path: + sample.update(_inspect_orbit_txt_health(source_path)) + sample["has_corruption_signal"] = _has_orbit_corruption_signal(sample) + if sample.get("read_error"): + sample["error"] = f"Source health scan failed: {sample['read_error']}" + elif sample["has_corruption_signal"]: + sample["error"] = "Source TXT contains NUL bytes and appears corrupted" + return sample + + def validate_orbit_source(txt_path: str, stem: str = "", scratch_root: str = "") -> Dict[str, Any]: name = stem or os.path.splitext(os.path.basename(txt_path))[0] try: @@ -259,28 +330,42 @@ def summarize_source_orbit_gaps( source_without_envi = sorted(source_stems - envi_stems) envi_without_source = sorted(envi_stems - source_stems) isce2_without_source = sorted(isce2_stems - source_stems) + sample_limit = 20 - def _make_samples(stems: List[str]) -> List[Dict[str, str]]: - samples: List[Dict[str, str]] = [] - for stem in stems[:20]: + def _make_samples( + stems: List[str], + inspect_source: bool = False, + ) -> List[Dict[str, Any]]: + samples: List[Dict[str, Any]] = [] + for stem in stems[:sample_limit]: samples.append( - { - "name": stem, - "source_path": source_files.get(stem, {}).get("path", ""), - "envi_path": envi_files.get(stem, {}).get("path", ""), - "isce2_path": isce2_files.get(stem, {}).get("path", ""), - } + _build_source_gap_sample( + stem, + source_files.get(stem), + envi_files.get(stem), + isce2_files.get(stem), + inspect_source=inspect_source, + ) ) return samples + suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) + bad_source_samples = [ + item for item in suspect_bad_samples + if item.get("has_corruption_signal") + ] + return { + "sample_limit": sample_limit, "quarantine_path": _default_quarantine_root(source_dir, quarantine_root), "source_without_isce2_count": len(source_without_isce2), "source_without_envi_count": len(source_without_envi), "envi_without_source_count": len(envi_without_source), "isce2_without_source_count": len(isce2_without_source), "suspect_bad_count": len(source_without_isce2), - "suspect_bad_samples": _make_samples(source_without_isce2), + "suspect_bad_samples": suspect_bad_samples, + "bad_source_sample_count": len(bad_source_samples), + "bad_source_samples": bad_source_samples, "source_without_envi_samples": _make_samples(source_without_envi), "envi_without_source_samples": _make_samples(envi_without_source), "isce2_without_source_samples": _make_samples(isce2_without_source), @@ -558,7 +643,8 @@ def _convert_to_isce2_xml( timeout=60, ) if result.returncode != 0: - raise RuntimeError(result.stderr or result.stdout or "Orbit conversion failed without output") + detail = result.stderr or result.stdout or "Orbit conversion failed without output" + raise RuntimeError(_compact_error_message(detail)) # --------------------------------------------------------------------------- @@ -857,14 +943,9 @@ def repair_orbit_pools( _convert_to_isce2_xml_file(txt_path, xml_path) repaired_from_envi.append(stem) except Exception as exc: - repair_errors.append( - { - "name": stem, - "source": txt_path, - "target": xml_path, - "error": str(exc), - } - ) + item = _build_invalid_source_record(stem, txt_path, exc, envi_path=txt_path, isce2_path=xml_path) + item["target"] = xml_path + repair_errors.append(item) after = check_orbit_consistency(envi_pool, isce2_pool) return { diff --git a/frontend/src/HealthCheckPanel.jsx b/frontend/src/HealthCheckPanel.jsx index aca6e24..b7b6927 100644 --- a/frontend/src/HealthCheckPanel.jsx +++ b/frontend/src/HealthCheckPanel.jsx @@ -34,6 +34,67 @@ const formatMismatchList = (items, formatter) => ( asArray(items).map(formatter).join('; ') ); +const formatIntegerText = (value) => { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed.toLocaleString() : '0'; +}; + +const getOrbitSourcePath = (item) => item?.source || item?.source_path || ''; + +const hasOrbitCorruptionSignal = (item) => Boolean( + item && ( + item.has_corruption_signal || + item.contains_nul_bytes || + toNumber(item.nul_byte_count) > 0 || + item.tail_has_nul_bytes || + Number(item.first_nul_offset) >= 0 || + item.read_error + ) +); + +const renderOrbitSourceIssueDetails = (item, en, formatPathText) => { + const sourcePath = getOrbitSourcePath(item); + const nulCount = toNumber(item?.nul_byte_count); + const firstNulOffset = Number(item?.first_nul_offset); + const hasNulDetails = + item?.contains_nul_bytes || + nulCount > 0 || + item?.tail_has_nul_bytes || + firstNulOffset >= 0; + + return ( + <> + {sourcePath && ( +
+ {en ? 'Source: ' : '源文件：'}{formatPathText(sourcePath)} +
+ )} + {item?.envi_path && ( +
+ {en ? 'ENVI: ' : 'ENVI：'}{formatPathText(item.envi_path)} +
+ )} + {item?.isce2_path && ( +
+ {en ? 'ISCE2: ' : 'ISCE2：'}{formatPathText(item.isce2_path)} +
+ )} + {hasNulDetails && ( +
+ {en ? 'NUL bytes: ' : 'NUL 字节：'}{formatIntegerText(nulCount)} + {firstNulOffset >= 0 ? `${en ? ', first offset: ' : '，首个偏移：'}${formatIntegerText(firstNulOffset)}` : ''} + {item?.tail_has_nul_bytes ? (en ? ' (tail contains NUL)' : '（文件尾含 NUL）') : ''} +
+ )} + {item?.read_error && ( +
+ {en ? 'Health scan error: ' : '文件健康扫描异常：'}{item.read_error} +
+ )} + + ); +}; + const formatSourceRootRole = (role, en = false) => { switch (role) { case 'radar_source': @@ -302,6 +363,12 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { const orbitEnviWithoutSourceCount = toNumber(orbitSource.envi_without_source_count); const orbitIsce2WithoutSourceCount = toNumber(orbitSource.isce2_without_source_count); const orbitQuarantinePath = orbitSource.quarantine_path || orbitStatus?.source_gaps?.quarantine_path; + const orbitBadSourceSamples = asArray(orbitSource.bad_source_samples).filter(hasOrbitCorruptionSignal); + const orbitSuspectBadSamples = asArray(orbitSource.suspect_bad_samples); + const orbitSuspectWithoutCorruptionSamples = orbitSuspectBadSamples.filter( + (item) => !orbitBadSourceSamples.some((badItem) => badItem.name === item.name) + ); + const orbitBadSourceSampleCount = toNumber(orbitSource.bad_source_sample_count || orbitBadSourceSamples.length); const orbitOverallHealthy = Boolean( orbitStatus && orbitMismatchCount === 0 && @@ -951,6 +1018,13 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { : `疑似坏源 TXT（源文件存在但 ISCE2 XML 缺失）：${orbitSuspectBadCount}`} )} + {orbitBadSourceSampleCount > 0 && ( +
+ {en + ? `Sampled source TXT with corruption signals: ${orbitBadSourceSampleCount}` + : `抽样检测到损坏信号的源 TXT：${orbitBadSourceSampleCount}`} +
+ )} {orbitDatabase.sample_missing_in_envi?.length > 0 && (
{en ? 'DB expected but ENVI pool missing: ' : '数据库期望但 ENVI 池缺失：'} @@ -963,19 +1037,16 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {orbitDatabase.sample_missing_in_isce2.slice(0, 5).join(', ')}
)} - {(orbitSource.suspect_bad_samples || []).slice(0, 5).map((item) => ( + {orbitBadSourceSamples.slice(0, 5).map((item) => ( +
+ {item.name} - {item.error || (en ? 'Corruption signal detected' : '检测到损坏信号')} + {renderOrbitSourceIssueDetails(item, en, formatPathText)} +
+ ))} + {orbitSuspectWithoutCorruptionSamples.slice(0, 5).map((item) => (
{item.name} - {item.source_path && ( -
- {en ? 'Source: ' : '源文件：'}{formatPathText(item.source_path)} -
- )} - {item.envi_path && ( -
- {en ? 'ENVI: ' : 'ENVI：'}{formatPathText(item.envi_path)} -
- )} + {renderOrbitSourceIssueDetails(item, en, formatPathText)}
))} {(orbitConsistency.mismatches || []).slice(0, 5).map((item) => ( @@ -1102,13 +1173,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {(orbitSyncResult.confirmed_bad || []).slice(0, 5).map((item, index) => (
{item.name} - {item.error} -
- {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)} -
-
- {en ? 'NUL bytes: ' : 'NUL 字节：'}{toNumber(item.nul_byte_count)} - {item.tail_has_nul_bytes ? (en ? ' (tail contains NUL)' : '（文件尾含 NUL）') : ''} -
+ {renderOrbitSourceIssueDetails(item, en, formatPathText)} {item.quarantined_source && (
{en ? 'Moved source to: ' : '源文件已移至：'}{formatPathText(item.quarantined_source)} @@ -1163,9 +1228,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {(orbitSyncResult.repair_errors || []).slice(0, 3).map((item, index) => (
{item.name} - {item.error} -
- {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)} -
+ {renderOrbitSourceIssueDetails(item, en, formatPathText)}
))} {(orbitSyncResult.sync_result?.source?.errors || []).slice(0, 3).map((item, index) => ( @@ -1176,9 +1239,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {(orbitSyncResult.sync_result?.invalid_sources || []).slice(0, 5).map((item, index) => (
{item.name} - {item.error} -
- {en ? 'Source: ' : '源文件：'}{formatPathText(item.source)} -
+ {renderOrbitSourceIssueDetails(item, en, formatPathText)}
))} {(orbitSyncResult.sync_result?.isce2?.errors || []).slice(0, 3).map((item, index) => (