diff --git a/docs/PYINT_GAMMA_LT1_INTEGRATION_NOTES_20260430.md b/docs/PYINT_GAMMA_LT1_INTEGRATION_NOTES_20260430.md new file mode 100644 index 0000000..d6eb1cf --- /dev/null +++ b/docs/PYINT_GAMMA_LT1_INTEGRATION_NOTES_20260430.md @@ -0,0 +1,204 @@ +# PyINT/Gamma LT-1 集成维护记录 + +更新时间：2026-04-30 + +## 1. 背景 + +本轮工作目标是评估并吸收专家提供的 `D:\Code\PyINT_LT1` 中 LT-1 / Gamma 相关修复，继续推进当前项目内 `PyINT / Gamma` 的 D-InSAR 生产能力。 + +已有排查结论见： + +- `docs/PYINT_GAMMA_AB_FINDINGS_20260420.md` +- `docs/PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md` + +之前 A/B 排查已经确认：当前 PyINT/Gamma 的关键问题主要在 LT-1 导入、DEM 几何和 `coreg` 前置链路，不是解缠、地理编码或导出问题。并且旧的 rescue/fallback 曾经掩盖真实 `coreg` 失败，导致流程成功但产物全 0。 + +## 2. 专家代码判断 + +专家目录： + +`D:\Code\PyINT_LT1` + +该目录不是完整 PyINT 替代仓库，而是一组 LT-1 专用脚本补丁，主要包含： + +- `down2slc_LT1.py` +- `down2slc_cat_LT1.py` +- `down2slc_LT1_all.py` +- `coreg_gamma.py` +- `coreg_gamma_all.py` +- `generate_rdc_dem.py` +- `makedem_pyint.py` +- `makedem.py` +- `diff/unwrap/geocode` 相关脚本 + +可吸收价值： + +- 修正 LT-1 文件名日期提取偏移。 +- 修正 LT-1 批处理错误日志名。 +- 改善 `coreg_gamma.py` 中 master/slave 路径、RSLC 路径和 masterDate 选择。 +- 增加 master 日期本身的 RSLC 生成处理。 +- 对多景拼接场景更明确地区分单片 SLC、update SLC 和最终拼接 SLC。 + +不能整包覆盖的原因： + +- 专家版多个脚本大量使用裸 `os.system()`，部分命令失败不会立即中断。 +- 当前项目已经收紧失败语义，要求 Gamma 核心命令失败即失败，避免再次出现“假成功、全 0 结果”。 +- 专家版引用 `LT1_precision_orbit.py`，但该文件不在 `D:\Code\PyINT_LT1` 中。 +- 当前项目已经有系统级精轨桥接 `backend/app/pyint_pipeline/apply_lt1_precise_orbit.py`，不应退回到未纳管的外部精轨脚本。 +- 当前 WSL 环境没有 `csh/tcsh`，不能让 Python 运行链硬依赖 `LT1_import_SLC_from_zipfiles1` 这类 csh 脚本。 + +## 3. 本轮改动范围 + +本轮只修改 Gamma/PyINT vendored 脚本，不修改 ISCE2 或 ENVI D-InSAR 生产链。 + +改动文件： + +- `third_party/PyINT/pyint/down2slc_LT1.py` +- `third_party/PyINT/pyint/down2slc_cat_LT1.py` +- `third_party/PyINT/pyint/down2slc_LT1_all.py` +- `third_party/PyINT/pyint/coreg_gamma.py` +- `third_party/PyINT/pyint/generate_rdc_dem.py` + +## 4. 已落实策略 + +### 4.1 LT-1 日期解析 + +旧实现使用固定切片 `file0[41:48]`，会把 `20241206` 截成 `0241206`。 + +本轮改为从文件名中正则提取 `20\d{6}`，同时兼容 `.tar.gz` 和 `.tiff` 输入。 + +### 4.2 LT-1 单景导入 + +保留当前 Python 直接调用 `par_LT1_SLC` 的方式，不强制依赖 `csh`。 + +如果运行环境中存在 `par_LT1_SLC_YSLi`： + +- 额外生成 `.slc.update` / `.slc.update.par` +- 将 update 参数文件中的 `state_vector_*` 合回主 `.slc.par` +- 同时对 `.slc.par` 和 `.slc.update.par` 执行当前项目的精轨桥接 + +如果运行环境中不存在 `par_LT1_SLC_YSLi`： + +- 继续使用 `par_LT1_SLC` 的输出 +- 打印 warning +- 仍执行当前项目的精轨桥接 + +当前 WSL/Gamma 环境检查结果： + +- `par_LT1_SLC` 可用 +- `SLC_cat_list.py` 可用 +- `par_LT1_SLC_YSLi` 不可用 +- `csh/tcsh` 不可用 + +因此 `par_LT1_SLC_YSLi` 必须保持可选，不能作为硬依赖。 + +### 4.3 LT-1 多景拼接导入 + +多景场景改为逐景直接调用 Gamma 导入命令，并明确产出： + +- `_.slc` +- `_.slc.par` +- 可选 `_.slc.update` +- 可选 `_.slc.update.par` + +拼接时： + +- 如果所有分片都有 update SLC，则优先用 update SLC tab 调用 `SLC_cat_list.py` +- 如果 update SLC 不可用，则回退到普通 SLC tab，并打印 warning +- 最终 `.slc.par` 仍会再执行一次当前项目的精轨桥接 + +### 4.4 错误日志 + +`down2slc_LT1_all.py` 的错误日志从： + +`down2slc_sen_all.err` + +修正为： + +`down2slc_LT1_all.err` + +这能让运行失败时的阶段日志更准确。 + +### 4.5 coreg / DEM masterDate 兜底 + +`coreg_gamma.py` 和 `generate_rdc_dem.py` 增加 masterDate 实际存在性检查： + +- 如果模板中的 `masterDate` 在 `SLC/` 下存在，则照常使用。 +- 如果不存在，则选择最接近的已有 SLC 日期，并打印提示。 + +`coreg_gamma.py` 同时增加 master 日期本身处理： + +- 如果 `Mdate == Sdate`，直接将 master SLC 复制为 master RSLC。 +- 生成 master RSLC 的多视幅度。 +- 不再进入 slave coreg 流程。 + +## 5. 保留的安全边界 + +本轮没有恢复旧 rescue/fallback。 + +必须继续保留： + +- Gamma 核心命令失败即失败。 +- `run_lt1_pyint_pipeline.py` 的全 0 二进制产物检查。 +- 当前项目的 LT-1 精轨桥接和运行摘要记录。 +- `Task_*` 输入资产适配层。 + +这几个边界是防止 PyINT/Gamma 再次产出“看似成功但科学结果无效”的关键。 + +## 6. 已完成验证 + +已完成静态验证： + +```bash +python3 -m py_compile \ + third_party/PyINT/pyint/down2slc_LT1.py \ + third_party/PyINT/pyint/down2slc_cat_LT1.py \ + third_party/PyINT/pyint/down2slc_LT1_all.py \ + third_party/PyINT/pyint/coreg_gamma.py \ + third_party/PyINT/pyint/generate_rdc_dem.py +``` + +已完成 diff 检查： + +```bash +git diff --check +``` + +已确认本轮 Git 改动只包含 Gamma/PyINT 文件和本维护文档，不涉及 ISCE2 / ENVI 生产链。 + +## 7. 后续验证建议 + +建议用历史失败样例做 A/B： + +- `D:\Task_Pool\DInSAR\Task_260416_Gamma_PyINT\Task_20230602_20230720` + +重点看： + +- `down2slc_LT1_all.err` +- `coreg_gamma_all.err` +- `SLC//orbit_bridge_summary.json` +- `pyint_run_summary.json` +- `diff_filt` / `cor` / `unw` / `geo_unw` 的 all-zero 检查结果 + +判断标准： + +- 不能靠 rescue 跳过失败。 +- `coreg` 若失败，应明确失败并留下阶段日志。 +- 若流程成功，关键二进制产物不能全 0。 +- 精轨桥接 summary 应覆盖 master/slave 日期。 + +## 8. 建议提交 + +建议和本轮代码一起提交： + +```powershell +git add docs/PYINT_GAMMA_LT1_INTEGRATION_NOTES_20260430.md ` + third_party/PyINT/pyint/coreg_gamma.py ` + third_party/PyINT/pyint/down2slc_LT1.py ` + third_party/PyINT/pyint/down2slc_LT1_all.py ` + third_party/PyINT/pyint/down2slc_cat_LT1.py ` + third_party/PyINT/pyint/generate_rdc_dem.py + +git commit -m "fix(pyint): integrate LT1 gamma import and coreg fixes" +``` + diff --git a/third_party/PyINT/pyint/coreg_gamma.py b/third_party/PyINT/pyint/coreg_gamma.py index 0b0fae8..99f07fd 100644 --- a/third_party/PyINT/pyint/coreg_gamma.py +++ b/third_party/PyINT/pyint/coreg_gamma.py @@ -9,10 +9,30 @@ import numpy as np import os import sys import argparse +import re from pyint import _utils as ut +def _resolve_existing_master_date(slc_dir, requested_date): + if not os.path.isdir(slc_dir): + return requested_date + existing_dates = sorted( + item for item in os.listdir(slc_dir) + if re.match(r'^\d{8}$', item) and os.path.isdir(os.path.join(slc_dir, item)) + ) + if not existing_dates: + return requested_date + if requested_date in existing_dates: + return requested_date + if not requested_date or not re.match(r'^\d{8}$', str(requested_date)): + resolved = existing_dates[0] + else: + resolved = min(existing_dates, key=lambda item: abs(int(item) - int(requested_date))) + print('masterDate %s not found; using existing SLC date: %s' % (requested_date, resolved)) + return resolved + + def _run_or_raise(call_str, stage): rc = os.system(call_str) if rc != 0: @@ -75,7 +95,7 @@ def main(argv): templateDict=ut.update_template(templateFile) rlks = templateDict['range_looks'] azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] + Mdate = _resolve_existing_master_date(slcDir, templateDict['masterDate']) IFGPair = Mdate + '-' + Sdate demDir = scratchDir + '/' + projectName + '/DEM' @@ -97,6 +117,16 @@ def main(argv): Srslc0 = SrslcDir + "/" + Sdate + ".rslc0" SrslcPar0 = SrslcDir + "/" + Sdate + ".rslc0.par" + if Mdate == Sdate: + Mslc0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + MslcPar0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + ut.copy_file(Mslc0, Srslc) + ut.copy_file(MslcPar0, SrslcPar) + call_str = 'multi_look ' + Srslc + ' ' + SrslcPar + ' ' + Sramp + ' ' + SrampPar + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'master_multi_look_rslc') + print('Master date: copy SLC to RSLC done.') + sys.exit(0) + ##################################################### ## copy all of the master files into slave folder for parallel processing remove_file = [] @@ -230,7 +260,7 @@ def main(argv): for path in ( lt0, lt1, mli0, diff0, offs0, snr0, offsets0, coffs0, coffsets0, off, offs, snr, offsets, coffs, coffsets, Srslc0, SrslcPar0, - Mslc, Mamp, HGTSIM, + Mslc, MslcPar, Mamp, MampPar, HGTSIM, ): _safe_remove(path) diff --git a/third_party/PyINT/pyint/down2slc_LT1.py b/third_party/PyINT/pyint/down2slc_LT1.py index 3c1f69f..2217c3b 100644 --- a/third_party/PyINT/pyint/down2slc_LT1.py +++ b/third_party/PyINT/pyint/down2slc_LT1.py @@ -23,8 +23,10 @@ from pyint import _utils as ut def get_LT1_date(raw_file): file0 = os.path.basename(raw_file) - date = file0[41:48] - return date + match = re.search(r'(20\d{6})', file0) + if match: + return match.group(1) + return '' def get_satellite(raw_file): if 'LT1A_MONO_' in raw_file: @@ -70,6 +72,48 @@ def _run_checked(command, cwd=None): return result +def _format_state_vector_line(line): + stripped = line.strip() + parts = stripped.split() + if not parts: + return '' + label = parts[0] + if label.startswith('state_vector_position_') and len(parts) >= 4: + return '%s %14.4f %14.4f %14.4f m m m' % ( + label, + float(parts[1]), + float(parts[2]), + float(parts[3]), + ) + if label.startswith('state_vector_velocity_') and len(parts) >= 4: + return '%s %13.5f %13.5f %13.5f m/s m/s m/s' % ( + label, + float(parts[1]), + float(parts[2]), + float(parts[3]), + ) + return stripped + + +def _replace_state_vectors_from_update(slc_par_path, update_par_path): + if not os.path.isfile(update_par_path): + return + with open(slc_par_path, 'r', encoding='utf-8', errors='ignore') as fp: + base_lines = fp.read().splitlines() + with open(update_par_path, 'r', encoding='utf-8', errors='ignore') as fp: + update_state_lines = [ + _format_state_vector_line(line) + for line in fp.read().splitlines() + if 'state_vector' in line + ] + if not update_state_lines: + return + merged = [line for line in base_lines if 'state_vector' not in line] + merged.extend(update_state_lines) + with open(slc_par_path, 'w', encoding='utf-8') as fp: + fp.write('\n'.join(merged) + '\n') + + def _cleanup_paths(paths): for path in paths: if not path: @@ -184,15 +228,32 @@ def main(argv): input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(zipfile_ref, work_dir) slc_path = work_dir + '/' + date + '.slc' slc_par_path = work_dir + '/' + date + '.slc.par' + update_path = work_dir + '/' + date + '.slc.update' + update_par_path = work_dir + '/' + date + '.slc.update.par' _run_checked( ['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path], cwd=work_dir, ) if not os.path.isfile(slc_path) or not os.path.isfile(slc_par_path): raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) + + bridge_targets = [slc_par_path] + ysli_command = shutil.which('par_LT1_SLC_YSLi') + if ysli_command: + _run_checked( + [ysli_command, input_tiff, input_xml, update_path, update_par_path, '0'], + cwd=work_dir, + ) + if not os.path.isfile(update_path) or not os.path.isfile(update_par_path): + raise RuntimeError('LT-1 import produced no update SLC outputs for date: ' + date) + _replace_state_vectors_from_update(slc_par_path, update_par_path) + bridge_targets.append(update_par_path) + else: + print('WARNING: par_LT1_SLC_YSLi is not available; using par_LT1_SLC outputs only.') + bridge_result = orbit_bridge.apply_precise_orbit( date, - [slc_par_path], + bridge_targets, work_dir=work_dir, operation_tag='lt1_import', ) @@ -215,10 +276,9 @@ def main(argv): if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab - if os.system(call_str) != 0: - raise RuntimeError('Failed to write SLC tab for date: ' + date) + with open(SLC_Tab, 'w') as fp: + for kk in range(len(SLC_list)): + fp.write(SLC_list[kk] + ' ' + SLC_par_list[kk] + '\n') with open(work_dir + '/down2slc.dat', 'w') as f: f.write('ok\n') print("Down to SLC for %s is done! " % date) diff --git a/third_party/PyINT/pyint/down2slc_LT1_all.py b/third_party/PyINT/pyint/down2slc_LT1_all.py index 09e167f..87f8353 100644 --- a/third_party/PyINT/pyint/down2slc_LT1_all.py +++ b/third_party/PyINT/pyint/down2slc_LT1_all.py @@ -136,7 +136,7 @@ def main(argv): for k0 in date_list: print(k0) - err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + err_txt = scratchDir + '/' + projectName + '/down2slc_LT1_all.err' if os.path.isfile(err_txt): os.remove(err_txt) data_para = [] diff --git a/third_party/PyINT/pyint/down2slc_cat_LT1.py b/third_party/PyINT/pyint/down2slc_cat_LT1.py index 04e02fd..0dbb067 100644 --- a/third_party/PyINT/pyint/down2slc_cat_LT1.py +++ b/third_party/PyINT/pyint/down2slc_cat_LT1.py @@ -14,6 +14,8 @@ import time import glob import argparse import re +import shutil +import tarfile from pyint import _orbit_bridge as orbit_bridge from pyint import _utils as ut @@ -21,8 +23,10 @@ from pyint import _utils as ut def get_LT1_date(raw_file): file0 = os.path.basename(raw_file) - date = file0[41:48] - return date + match = re.search(r'(20\d{6})', file0) + if match: + return match.group(1) + return '' def get_satellite(raw_file): if 'LT1A_MONO_' in raw_file: @@ -48,6 +52,168 @@ def write_input_list(list_path, paths): f.write(path + '\n') +def _strip_lt1_extension(path): + name = os.path.basename(path) + if name.lower().endswith('.tar.gz'): + return name[:-7] + if name.lower().endswith('.tiff'): + return name[:-5] + return os.path.splitext(name)[0] + + +def _get_product_id(path): + parts = _strip_lt1_extension(path).split('_') + if parts: + return parts[-1] + return '' + + +def _resolve_lt1_input_scene(raw_path, work_dir, product_id): + cleanup_paths = [] + raw_lower = raw_path.lower() + if raw_lower.endswith('.tiff'): + input_xml = re.sub(r'\.tiff$', '.meta.xml', raw_path, flags=re.IGNORECASE) + if not os.path.isfile(input_xml): + raise FileNotFoundError('LT-1 meta xml does not exist: ' + input_xml) + return raw_path, input_xml, cleanup_paths + + if raw_lower.endswith('.tar.gz'): + temp_dir = os.path.join(work_dir, 'tmp_data_dir_' + str(product_id or 'scene')) + if os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + os.makedirs(temp_dir, exist_ok=True) + cleanup_paths.append(temp_dir) + with tarfile.open(raw_path, 'r:gz') as archive: + member_names = archive.getnames() + tiff_members = [name for name in member_names if name.lower().endswith('.tiff')] + xml_members = [name for name in member_names if name.lower().endswith('meta.xml')] + if not tiff_members or not xml_members: + raise RuntimeError('LT-1 archive is missing .tiff or meta.xml: ' + raw_path) + tiff_member = tiff_members[0] + xml_member = xml_members[0] + archive.extract(tiff_member, path=temp_dir) + archive.extract(xml_member, path=temp_dir) + return os.path.join(temp_dir, tiff_member), os.path.join(temp_dir, xml_member), cleanup_paths + + raise RuntimeError('Unsupported LT-1 input, only .tiff or .tar.gz are supported: ' + raw_path) + + +def _cleanup_paths(paths): + for path in paths: + if not path: + continue + try: + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.isfile(path): + os.remove(path) + except OSError: + continue + + +def _format_state_vector_line(line): + stripped = line.strip() + parts = stripped.split() + if not parts: + return '' + label = parts[0] + if label.startswith('state_vector_position_') and len(parts) >= 4: + return '%s %14.4f %14.4f %14.4f m m m' % ( + label, + float(parts[1]), + float(parts[2]), + float(parts[3]), + ) + if label.startswith('state_vector_velocity_') and len(parts) >= 4: + return '%s %13.5f %13.5f %13.5f m/s m/s m/s' % ( + label, + float(parts[1]), + float(parts[2]), + float(parts[3]), + ) + return stripped + + +def _replace_state_vectors_from_update(slc_par_path, update_par_path): + if not os.path.isfile(update_par_path): + return + with open(slc_par_path, 'r', encoding='utf-8', errors='ignore') as fp: + base_lines = fp.read().splitlines() + with open(update_par_path, 'r', encoding='utf-8', errors='ignore') as fp: + update_state_lines = [ + _format_state_vector_line(line) + for line in fp.read().splitlines() + if 'state_vector' in line + ] + if not update_state_lines: + return + merged = [line for line in base_lines if 'state_vector' not in line] + merged.extend(update_state_lines) + with open(slc_par_path, 'w', encoding='utf-8') as fp: + fp.write('\n'.join(merged) + '\n') + + +def _run_checked(command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError( + 'Command failed (%s): %s%s' % ( + result.returncode, + ' '.join(command), + ('\n' + detail) if detail else '', + ) + ) + return result + + +def _import_lt1_scene(raw_path, date, work_dir, product_id): + cleanup_paths = [] + try: + input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(raw_path, work_dir, product_id) + product_suffix = str(product_id or _get_product_id(raw_path) or 'scene') + prefix = date + '_' + product_suffix + slc_path = work_dir + '/' + prefix + '.slc' + slc_par_path = work_dir + '/' + prefix + '.slc.par' + update_path = work_dir + '/' + prefix + '.slc.update' + update_par_path = work_dir + '/' + prefix + '.slc.update.par' + _run_checked( + ['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path], + cwd=work_dir, + ) + required = [slc_path, slc_par_path] + ysli_command = shutil.which('par_LT1_SLC_YSLi') + if ysli_command: + _run_checked( + [ysli_command, input_tiff, input_xml, update_path, update_par_path, '0'], + cwd=work_dir, + ) + required.extend([update_path, update_par_path]) + _replace_state_vectors_from_update(slc_par_path, update_par_path) + else: + update_path = '' + update_par_path = '' + print('WARNING: par_LT1_SLC_YSLi is not available; using par_LT1_SLC outputs only.') + + missing = [path for path in required if not os.path.isfile(path)] + if missing: + raise RuntimeError('LT-1 import is missing outputs: ' + ', '.join(missing)) + return { + 'slc': slc_path, + 'slc_par': slc_par_path, + 'update': update_path, + 'update_par': update_par_path, + } + finally: + _cleanup_paths(cleanup_paths) + + def cmdLineParse(): parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ formatter_class=argparse.RawTextHelpFormatter,\ @@ -113,35 +279,18 @@ def main(argv): file_num=len(raw_files) + imported_items = [] for kk in range(file_num): zipfile_ref=str(raw_files[kk]) outfile_name=zipfile_ref.split('/')[-1] print(outfile_name) - before_slc = set(glob.glob(work_dir + '/*.slc')) - before_slc_par = set(glob.glob(work_dir + '/*.slc.par')) - before_update = set(glob.glob(work_dir + '/*.slc.update')) - before_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) - call_str = "echo " + zipfile_ref + " >date" - if os.system(call_str) != 0: - raise RuntimeError('Failed to materialize LT-1 input list for date: ' + date) - call_str = 'LT1_import_SLC_from_zipfiles1 date 0 ' - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('LT1_import_SLC_from_zipfiles1 failed for date %s with rc=%s' % (date, rc)) - after_slc = set(glob.glob(work_dir + '/*.slc')) - after_slc_par = set(glob.glob(work_dir + '/*.slc.par')) - after_update = set(glob.glob(work_dir + '/*.slc.update')) - after_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) - new_slc = sorted(after_slc - before_slc) - new_slc_par = sorted(after_slc_par - before_slc_par) - new_update = sorted(after_update - before_update) - new_update_par = sorted(after_update_par - before_update_par) - if not new_slc or not new_slc_par: - raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) - if len(new_update) != len(new_update_par): - raise RuntimeError('LT-1 import produced mismatched update SLC outputs for date: ' + date) - bridge_targets = sorted((after_slc_par - before_slc_par) | (after_update_par - before_update_par)) + product_id = _get_product_id(zipfile_ref) or str(kk + 1) + imported = _import_lt1_scene(zipfile_ref, date, work_dir, product_id) + imported_items.append(imported) + bridge_targets = [imported['slc_par']] + if imported.get('update_par'): + bridge_targets.append(imported['update_par']) if bridge_targets: bridge_result = orbit_bridge.apply_precise_orbit( date, @@ -157,10 +306,10 @@ def main(argv): SLC_Tab = work_dir + '/' + date+'_SLC_Tab' SLC_Tab_update = work_dir + '/' + date+'_update_SLC_Tab' - SLC_update_list = sorted(glob.glob(work_dir + '/*.slc.update')) - SLC_update_par_list = sorted(glob.glob(work_dir + '/*.slc.update.par')) - SLC_list = sorted(glob.glob(work_dir + '/*.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) + SLC_list = [item['slc'] for item in imported_items] + SLC_par_list = [item['slc_par'] for item in imported_items] + SLC_update_list = [item['update'] for item in imported_items if item.get('update')] + SLC_update_par_list = [item['update_par'] for item in imported_items if item.get('update_par')] if len(SLC_list) == 0 or len(SLC_par_list) == 0: raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) if len(SLC_list) != len(SLC_par_list): @@ -171,16 +320,21 @@ def main(argv): if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab - if os.system(call_str) != 0: - raise RuntimeError('Failed to write SLC tab for date: ' + date) - call_str = 'echo ' + SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + ' >> ' + SLC_Tab_update - if os.system(call_str) != 0: - raise RuntimeError('Failed to write update SLC tab for date: ' + date) - call_str = 'SLC_cat_list.py ' + SLC_Tab_update + ' ' + date + '.slc ' + date + '.slc.par ' - if os.system(call_str) != 0: - raise RuntimeError('SLC_cat_list.py failed for date: ' + date) + with open(SLC_Tab, 'w') as fp: + for kk in range(len(SLC_list)): + fp.write(SLC_list[kk] + ' ' + SLC_par_list[kk] + '\n') + cat_tab = SLC_Tab + if len(SLC_update_list) == len(SLC_list): + with open(SLC_Tab_update, 'w') as fp: + for kk in range(len(SLC_update_list)): + fp.write(SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + '\n') + cat_tab = SLC_Tab_update + else: + print('WARNING: update SLC outputs are unavailable; concatenating par_LT1_SLC outputs.') + _run_checked( + ['SLC_cat_list.py', cat_tab, date + '.slc', date + '.slc.par'], + cwd=work_dir, + ) if not os.path.isfile(work_dir + '/' + date + '.slc.par'): raise RuntimeError('Final LT-1 concatenated SLC parameter file is missing for date: ' + date) final_bridge_result = orbit_bridge.apply_precise_orbit( diff --git a/third_party/PyINT/pyint/generate_rdc_dem.py b/third_party/PyINT/pyint/generate_rdc_dem.py index 12f16bd..614c4cc 100644 --- a/third_party/PyINT/pyint/generate_rdc_dem.py +++ b/third_party/PyINT/pyint/generate_rdc_dem.py @@ -9,9 +9,29 @@ import numpy as np import os import sys import argparse +import re from pyint import _utils as ut + +def _resolve_existing_master_date(slc_dir, requested_date): + if not os.path.isdir(slc_dir): + return requested_date + existing_dates = sorted( + item for item in os.listdir(slc_dir) + if re.match(r'^\d{8}$', item) and os.path.isdir(os.path.join(slc_dir, item)) + ) + if not existing_dates: + return requested_date + if requested_date in existing_dates: + return requested_date + if not requested_date or not re.match(r'^\d{8}$', str(requested_date)): + resolved = existing_dates[0] + else: + resolved = min(existing_dates, key=lambda item: abs(int(item) - int(requested_date))) + print('masterDate %s not found; using existing SLC date: %s' % (requested_date, resolved)) + return resolved + def cmdLineParse(): parser = argparse.ArgumentParser(description='Generate radar-coordinates based DEM.',\ formatter_class=argparse.RawTextHelpFormatter,\ @@ -46,12 +66,13 @@ def main(argv): templateFile = templateDir + "/" + projectName + ".template" templateDict=ut.update_template(templateFile) - Mdate = templateDict['masterDate'] + Mdate = templateDict['masterDate'] DEMDir = os.getenv('DEMDIR') processDir = scratchDir + '/' + projectName + "/ifgrams" - slcDir = scratchDir + '/' + projectName + "/SLC" + slcDir = scratchDir + '/' + projectName + "/SLC" + Mdate = _resolve_existing_master_date(slcDir, Mdate) rlks = templateDict['range_looks'] azlks = templateDict['azimuth_looks']