Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pyint): integrate LT1 gamma import and coreg fixes

Browse Source
This commit is contained in:
Harmon
2026-04-30 13:54:49 +08:00
parent 6513ff1623
commit 3474e25839
6 changed files with 521 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/PyINT/pyint/coreg_gamma.py
Vendored
+32 -2
View File
@@ -9,10 +9,30 @@ import numpy as np
import os
import sys
import argparse
import re
from pyint import _utils as ut
def _resolve_existing_master_date(slc_dir, requested_date):
if not os.path.isdir(slc_dir):
return requested_date
existing_dates = sorted(
item for item in os.listdir(slc_dir)
if re.match(r'^\d{8}$', item) and os.path.isdir(os.path.join(slc_dir, item))
)
if not existing_dates:
return requested_date
if requested_date in existing_dates:
return requested_date
if not requested_date or not re.match(r'^\d{8}$', str(requested_date)):
resolved = existing_dates[0]
else:
resolved = min(existing_dates, key=lambda item: abs(int(item) - int(requested_date)))
print('masterDate %s not found; using existing SLC date: %s' % (requested_date, resolved))
return resolved
def _run_or_raise(call_str, stage):
rc = os.system(call_str)
if rc != 0:
@@ -75,7 +95,7 @@ def main(argv):
templateDict=ut.update_template(templateFile)
rlks = templateDict['range_looks']
azlks = templateDict['azimuth_looks']
Mdate = templateDict['masterDate']
Mdate = _resolve_existing_master_date(slcDir, templateDict['masterDate'])
IFGPair = Mdate + '-' + Sdate
demDir = scratchDir + '/' + projectName + '/DEM'
@@ -97,6 +117,16 @@ def main(argv):
Srslc0 = SrslcDir + "/" + Sdate + ".rslc0"
SrslcPar0 = SrslcDir + "/" + Sdate + ".rslc0.par"
if Mdate == Sdate:
Mslc0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc'
MslcPar0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par'
ut.copy_file(Mslc0, Srslc)
ut.copy_file(MslcPar0, SrslcPar)
call_str = 'multi_look ' + Srslc + ' ' + SrslcPar + ' ' + Sramp + ' ' + SrampPar + ' ' + rlks + ' ' + azlks
_run_or_raise(call_str, 'master_multi_look_rslc')
print('Master date: copy SLC to RSLC done.')
sys.exit(0)
#####################################################
## copy all of the master files into slave folder for parallel processing
remove_file = []
@@ -230,7 +260,7 @@ def main(argv):
for path in (
lt0, lt1, mli0, diff0, offs0, snr0, offsets0, coffs0, coffsets0,
off, offs, snr, offsets, coffs, coffsets, Srslc0, SrslcPar0,
Mslc, Mamp, HGTSIM,
Mslc, MslcPar, Mamp, MampPar, HGTSIM,
):
_safe_remove(path)
third_party/PyINT/pyint/down2slc_LT1.py
Vendored
+67 -7
View File
@@ -23,8 +23,10 @@ from pyint import _utils as ut
def get_LT1_date(raw_file):
file0 = os.path.basename(raw_file)
date = file0[41:48]
return date
match = re.search(r'(20\d{6})', file0)
if match:
return match.group(1)
return ''
def get_satellite(raw_file):
if 'LT1A_MONO_' in raw_file:
@@ -70,6 +72,48 @@ def _run_checked(command, cwd=None):
return result
def _format_state_vector_line(line):
stripped = line.strip()
parts = stripped.split()
if not parts:
return ''
label = parts[0]
if label.startswith('state_vector_position_') and len(parts) >= 4:
return '%s %14.4f %14.4f %14.4f m m m' % (
label,
float(parts[1]),
float(parts[2]),
float(parts[3]),
)
if label.startswith('state_vector_velocity_') and len(parts) >= 4:
return '%s %13.5f %13.5f %13.5f m/s m/s m/s' % (
label,
float(parts[1]),
float(parts[2]),
float(parts[3]),
)
return stripped
def _replace_state_vectors_from_update(slc_par_path, update_par_path):
if not os.path.isfile(update_par_path):
return
with open(slc_par_path, 'r', encoding='utf-8', errors='ignore') as fp:
base_lines = fp.read().splitlines()
with open(update_par_path, 'r', encoding='utf-8', errors='ignore') as fp:
update_state_lines = [
_format_state_vector_line(line)
for line in fp.read().splitlines()
if 'state_vector' in line
]
if not update_state_lines:
return
merged = [line for line in base_lines if 'state_vector' not in line]
merged.extend(update_state_lines)
with open(slc_par_path, 'w', encoding='utf-8') as fp:
fp.write('\n'.join(merged) + '\n')
def _cleanup_paths(paths):
for path in paths:
if not path:
@@ -184,15 +228,32 @@ def main(argv):
input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(zipfile_ref, work_dir)
slc_path = work_dir + '/' + date + '.slc'
slc_par_path = work_dir + '/' + date + '.slc.par'
update_path = work_dir + '/' + date + '.slc.update'
update_par_path = work_dir + '/' + date + '.slc.update.par'
_run_checked(
['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path],
cwd=work_dir,
)
if not os.path.isfile(slc_path) or not os.path.isfile(slc_par_path):
raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date)
bridge_targets = [slc_par_path]
ysli_command = shutil.which('par_LT1_SLC_YSLi')
if ysli_command:
_run_checked(
[ysli_command, input_tiff, input_xml, update_path, update_par_path, '0'],
cwd=work_dir,
)
if not os.path.isfile(update_path) or not os.path.isfile(update_par_path):
raise RuntimeError('LT-1 import produced no update SLC outputs for date: ' + date)
_replace_state_vectors_from_update(slc_par_path, update_par_path)
bridge_targets.append(update_par_path)
else:
print('WARNING: par_LT1_SLC_YSLi is not available; using par_LT1_SLC outputs only.')
bridge_result = orbit_bridge.apply_precise_orbit(
date,
[slc_par_path],
bridge_targets,
work_dir=work_dir,
operation_tag='lt1_import',
)
@@ -215,10 +276,9 @@ def main(argv):
if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab)
for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab
if os.system(call_str) != 0:
raise RuntimeError('Failed to write SLC tab for date: ' + date)
with open(SLC_Tab, 'w') as fp:
for kk in range(len(SLC_list)):
fp.write(SLC_list[kk] + ' ' + SLC_par_list[kk] + '\n')
with open(work_dir + '/down2slc.dat', 'w') as f:
f.write('ok\n')
print("Down to SLC for %s is done! " % date)
third_party/PyINT/pyint/down2slc_LT1_all.py
Vendored
+1 -1
View File
@@ -136,7 +136,7 @@ def main(argv):
for k0 in date_list:
print(k0)
err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err'
err_txt = scratchDir + '/' + projectName + '/down2slc_LT1_all.err'
if os.path.isfile(err_txt): os.remove(err_txt)
data_para = []
third_party/PyINT/pyint/down2slc_cat_LT1.py
Vendored
+194 -40
View File
@@ -14,6 +14,8 @@ import time
import glob
import argparse
import re
import shutil
import tarfile
from pyint import _orbit_bridge as orbit_bridge
from pyint import _utils as ut
@@ -21,8 +23,10 @@ from pyint import _utils as ut
def get_LT1_date(raw_file):
file0 = os.path.basename(raw_file)
date = file0[41:48]
return date
match = re.search(r'(20\d{6})', file0)
if match:
return match.group(1)
return ''
def get_satellite(raw_file):
if 'LT1A_MONO_' in raw_file:
@@ -48,6 +52,168 @@ def write_input_list(list_path, paths):
f.write(path + '\n')
def _strip_lt1_extension(path):
name = os.path.basename(path)
if name.lower().endswith('.tar.gz'):
return name[:-7]
if name.lower().endswith('.tiff'):
return name[:-5]
return os.path.splitext(name)[0]
def _get_product_id(path):
parts = _strip_lt1_extension(path).split('_')
if parts:
return parts[-1]
return ''
def _resolve_lt1_input_scene(raw_path, work_dir, product_id):
cleanup_paths = []
raw_lower = raw_path.lower()
if raw_lower.endswith('.tiff'):
input_xml = re.sub(r'\.tiff$', '.meta.xml', raw_path, flags=re.IGNORECASE)
if not os.path.isfile(input_xml):
raise FileNotFoundError('LT-1 meta xml does not exist: ' + input_xml)
return raw_path, input_xml, cleanup_paths
if raw_lower.endswith('.tar.gz'):
temp_dir = os.path.join(work_dir, 'tmp_data_dir_' + str(product_id or 'scene'))
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
os.makedirs(temp_dir, exist_ok=True)
cleanup_paths.append(temp_dir)
with tarfile.open(raw_path, 'r:gz') as archive:
member_names = archive.getnames()
tiff_members = [name for name in member_names if name.lower().endswith('.tiff')]
xml_members = [name for name in member_names if name.lower().endswith('meta.xml')]
if not tiff_members or not xml_members:
raise RuntimeError('LT-1 archive is missing .tiff or meta.xml: ' + raw_path)
tiff_member = tiff_members[0]
xml_member = xml_members[0]
archive.extract(tiff_member, path=temp_dir)
archive.extract(xml_member, path=temp_dir)
return os.path.join(temp_dir, tiff_member), os.path.join(temp_dir, xml_member), cleanup_paths
raise RuntimeError('Unsupported LT-1 input, only .tiff or .tar.gz are supported: ' + raw_path)
def _cleanup_paths(paths):
for path in paths:
if not path:
continue
try:
if os.path.isdir(path):
shutil.rmtree(path)
elif os.path.isfile(path):
os.remove(path)
except OSError:
continue
def _format_state_vector_line(line):
stripped = line.strip()
parts = stripped.split()
if not parts:
return ''
label = parts[0]
if label.startswith('state_vector_position_') and len(parts) >= 4:
return '%s %14.4f %14.4f %14.4f m m m' % (
label,
float(parts[1]),
float(parts[2]),
float(parts[3]),
)
if label.startswith('state_vector_velocity_') and len(parts) >= 4:
return '%s %13.5f %13.5f %13.5f m/s m/s m/s' % (
label,
float(parts[1]),
float(parts[2]),
float(parts[3]),
)
return stripped
def _replace_state_vectors_from_update(slc_par_path, update_par_path):
if not os.path.isfile(update_par_path):
return
with open(slc_par_path, 'r', encoding='utf-8', errors='ignore') as fp:
base_lines = fp.read().splitlines()
with open(update_par_path, 'r', encoding='utf-8', errors='ignore') as fp:
update_state_lines = [
_format_state_vector_line(line)
for line in fp.read().splitlines()
if 'state_vector' in line
]
if not update_state_lines:
return
merged = [line for line in base_lines if 'state_vector' not in line]
merged.extend(update_state_lines)
with open(slc_par_path, 'w', encoding='utf-8') as fp:
fp.write('\n'.join(merged) + '\n')
def _run_checked(command, cwd=None):
result = subprocess.run(
command,
cwd=cwd,
text=True,
capture_output=True,
check=False,
)
if result.returncode != 0:
detail = (result.stderr or result.stdout or "").strip()
raise RuntimeError(
'Command failed (%s): %s%s' % (
result.returncode,
' '.join(command),
('\n' + detail) if detail else '',
)
)
return result
def _import_lt1_scene(raw_path, date, work_dir, product_id):
cleanup_paths = []
try:
input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(raw_path, work_dir, product_id)
product_suffix = str(product_id or _get_product_id(raw_path) or 'scene')
prefix = date + '_' + product_suffix
slc_path = work_dir + '/' + prefix + '.slc'
slc_par_path = work_dir + '/' + prefix + '.slc.par'
update_path = work_dir + '/' + prefix + '.slc.update'
update_par_path = work_dir + '/' + prefix + '.slc.update.par'
_run_checked(
['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path],
cwd=work_dir,
)
required = [slc_path, slc_par_path]
ysli_command = shutil.which('par_LT1_SLC_YSLi')
if ysli_command:
_run_checked(
[ysli_command, input_tiff, input_xml, update_path, update_par_path, '0'],
cwd=work_dir,
)
required.extend([update_path, update_par_path])
_replace_state_vectors_from_update(slc_par_path, update_par_path)
else:
update_path = ''
update_par_path = ''
print('WARNING: par_LT1_SLC_YSLi is not available; using par_LT1_SLC outputs only.')
missing = [path for path in required if not os.path.isfile(path)]
if missing:
raise RuntimeError('LT-1 import is missing outputs: ' + ', '.join(missing))
return {
'slc': slc_path,
'slc_par': slc_par_path,
'update': update_path,
'update_par': update_par_path,
}
finally:
_cleanup_paths(cleanup_paths)
def cmdLineParse():
parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\
formatter_class=argparse.RawTextHelpFormatter,\
@@ -113,35 +279,18 @@ def main(argv):
file_num=len(raw_files)
imported_items = []
for kk in range(file_num):
zipfile_ref=str(raw_files[kk])
outfile_name=zipfile_ref.split('/')[-1]
print(outfile_name)
before_slc = set(glob.glob(work_dir + '/*.slc'))
before_slc_par = set(glob.glob(work_dir + '/*.slc.par'))
before_update = set(glob.glob(work_dir + '/*.slc.update'))
before_update_par = set(glob.glob(work_dir + '/*.slc.update.par'))
call_str = "echo " + zipfile_ref + " >date"
if os.system(call_str) != 0:
raise RuntimeError('Failed to materialize LT-1 input list for date: ' + date)
call_str = 'LT1_import_SLC_from_zipfiles1 date 0 '
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('LT1_import_SLC_from_zipfiles1 failed for date %s with rc=%s' % (date, rc))
after_slc = set(glob.glob(work_dir + '/*.slc'))
after_slc_par = set(glob.glob(work_dir + '/*.slc.par'))
after_update = set(glob.glob(work_dir + '/*.slc.update'))
after_update_par = set(glob.glob(work_dir + '/*.slc.update.par'))
new_slc = sorted(after_slc - before_slc)
new_slc_par = sorted(after_slc_par - before_slc_par)
new_update = sorted(after_update - before_update)
new_update_par = sorted(after_update_par - before_update_par)
if not new_slc or not new_slc_par:
raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date)
if len(new_update) != len(new_update_par):
raise RuntimeError('LT-1 import produced mismatched update SLC outputs for date: ' + date)
bridge_targets = sorted((after_slc_par - before_slc_par) | (after_update_par - before_update_par))
product_id = _get_product_id(zipfile_ref) or str(kk + 1)
imported = _import_lt1_scene(zipfile_ref, date, work_dir, product_id)
imported_items.append(imported)
bridge_targets = [imported['slc_par']]
if imported.get('update_par'):
bridge_targets.append(imported['update_par'])
if bridge_targets:
bridge_result = orbit_bridge.apply_precise_orbit(
date,
@@ -157,10 +306,10 @@ def main(argv):
SLC_Tab = work_dir + '/' + date+'_SLC_Tab'
SLC_Tab_update = work_dir + '/' + date+'_update_SLC_Tab'
SLC_update_list = sorted(glob.glob(work_dir + '/*.slc.update'))
SLC_update_par_list = sorted(glob.glob(work_dir + '/*.slc.update.par'))
SLC_list = sorted(glob.glob(work_dir + '/*.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par'))
SLC_list = [item['slc'] for item in imported_items]
SLC_par_list = [item['slc_par'] for item in imported_items]
SLC_update_list = [item['update'] for item in imported_items if item.get('update')]
SLC_update_par_list = [item['update_par'] for item in imported_items if item.get('update_par')]
if len(SLC_list) == 0 or len(SLC_par_list) == 0:
raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date)
if len(SLC_list) != len(SLC_par_list):
@@ -171,16 +320,21 @@ def main(argv):
if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab)
for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab
if os.system(call_str) != 0:
raise RuntimeError('Failed to write SLC tab for date: ' + date)
call_str = 'echo ' + SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + ' >> ' + SLC_Tab_update
if os.system(call_str) != 0:
raise RuntimeError('Failed to write update SLC tab for date: ' + date)
call_str = 'SLC_cat_list.py ' + SLC_Tab_update + ' ' + date + '.slc ' + date + '.slc.par '
if os.system(call_str) != 0:
raise RuntimeError('SLC_cat_list.py failed for date: ' + date)
with open(SLC_Tab, 'w') as fp:
for kk in range(len(SLC_list)):
fp.write(SLC_list[kk] + ' ' + SLC_par_list[kk] + '\n')
cat_tab = SLC_Tab
if len(SLC_update_list) == len(SLC_list):
with open(SLC_Tab_update, 'w') as fp:
for kk in range(len(SLC_update_list)):
fp.write(SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + '\n')
cat_tab = SLC_Tab_update
else:
print('WARNING: update SLC outputs are unavailable; concatenating par_LT1_SLC outputs.')
_run_checked(
['SLC_cat_list.py', cat_tab, date + '.slc', date + '.slc.par'],
cwd=work_dir,
)
if not os.path.isfile(work_dir + '/' + date + '.slc.par'):
raise RuntimeError('Final LT-1 concatenated SLC parameter file is missing for date: ' + date)
final_bridge_result = orbit_bridge.apply_precise_orbit(
third_party/PyINT/pyint/generate_rdc_dem.py
Vendored
+23 -2
View File
@@ -9,9 +9,29 @@ import numpy as np
import os
import sys
import argparse
import re
from pyint import _utils as ut
def _resolve_existing_master_date(slc_dir, requested_date):
if not os.path.isdir(slc_dir):
return requested_date
existing_dates = sorted(
item for item in os.listdir(slc_dir)
if re.match(r'^\d{8}$', item) and os.path.isdir(os.path.join(slc_dir, item))
)
if not existing_dates:
return requested_date
if requested_date in existing_dates:
return requested_date
if not requested_date or not re.match(r'^\d{8}$', str(requested_date)):
resolved = existing_dates[0]
else:
resolved = min(existing_dates, key=lambda item: abs(int(item) - int(requested_date)))
print('masterDate %s not found; using existing SLC date: %s' % (requested_date, resolved))
return resolved
def cmdLineParse():
parser = argparse.ArgumentParser(description='Generate radar-coordinates based DEM.',\
formatter_class=argparse.RawTextHelpFormatter,\
@@ -46,12 +66,13 @@ def main(argv):
templateFile = templateDir + "/" + projectName + ".template"
templateDict=ut.update_template(templateFile)
Mdate = templateDict['masterDate']
Mdate = templateDict['masterDate']
DEMDir = os.getenv('DEMDIR')
processDir = scratchDir + '/' + projectName + "/ifgrams"
slcDir = scratchDir + '/' + projectName + "/SLC"
slcDir = scratchDir + '/' + projectName + "/SLC"
Mdate = _resolve_existing_master_date(slcDir, Mdate)
rlks = templateDict['range_looks']
azlks = templateDict['azimuth_looks']