Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Sentinel-1 asset management and PyINT pipeline support

Browse Source
This commit is contained in:
Harmon
2026-05-14 03:02:56 +08:00
parent 508ca5641a
commit 729c5c2a7f
61 changed files with 7800 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/PyINT/pyint/coreg_gamma_all.py
Vendored
+5 -2
View File
@@ -158,8 +158,11 @@ def main(argv):
if not os.path.isdir(rslcDir): os.mkdir(rslcDir)
ensure_master_rslc(projectName)
if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py'
else: cmd_command = 'coreg_gamma.py'
template_satellite = str(ut.update_template(templateDir + "/" + projectName + ".template").get('satelite', '') or '')
if str(template_satellite).startswith('S1') or 'S1' in projectName:
cmd_command = 'coreg_s1_gamma.py'
else:
cmd_command = 'coreg_gamma.py'
err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err'
if os.path.isfile(err_txt): os.remove(err_txt)
third_party/PyINT/pyint/coreg_s1_gamma.py
Vendored
+2 -2
View File
@@ -231,9 +231,9 @@ def main(argv):
# if os.path.isfile(HGTSIM): os.remove(HGTSIM)
else:
print('The SLC has already coregister Done')
sys.exit(1)
sys.exit(0)
print("Coregister TOP SLC image to the reference TOPS image is done !!")
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/down2slc_cat_sen.py
Vendored
+73 -26
View File
@@ -14,6 +14,7 @@ import time
import glob
import argparse
import csv
from pathlib import Path
from pyint import _utils as ut
@@ -29,10 +30,11 @@ def get_s1_time(raw_file):
return times
def get_satellite(raw_file):
if 'S1A_IW_SLC_' in raw_file:
s0 = 'A'
name = os.path.basename(str(raw_file))
if name.startswith('S1') and len(name) >= 3:
s0 = name[2]
else:
s0 = 'B'
s0 = 'A'
return s0
@@ -86,8 +88,8 @@ def main(argv):
#if not os.path.isdir(opod_dir):
# os.mkdir(opod_dir)
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir
os.system(call_str)
if opod_dir and not os.path.isdir(opod_dir):
os.makedirs(opod_dir, exist_ok=True)
if not os.path.isdir(slc_dir):
os.mkdir(slc_dir)
@@ -99,9 +101,6 @@ def main(argv):
t_date = 't_' + date
call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date
os.system(call_str)
start_swath = templateDict['start_swath']
end_swath = templateDict['end_swath']
@@ -119,13 +118,23 @@ def main(argv):
# k_swath = '-'
k_swath = ut.get_sardata_swath(start_swath,end_swath)
raw_files = ut.read_txt2list(t_date)
raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
if not raw_files:
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
with open(t_date, 'w', encoding='utf-8') as f:
for raw_file in raw_files:
f.write(str(raw_file) + '\n')
satellite = get_satellite(str(raw_files[0]))
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip')
raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE')
file_num=len(raw_files)
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
os.system(call_str)
if raw_files and os.path.isdir(str(raw_files[0])):
call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
else:
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
for kk in range(file_num):
zipfile_ref=str(raw_files[kk])
@@ -135,22 +144,39 @@ def main(argv):
#os.system(call_str)
#call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 '
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
os.system(call_str)
os.chdir(work_dir)
call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *"
os.system(call_str)
call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *"
os.system(call_str)
call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *"
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
os.chdir(work_dir)
rename_rules = [
('vv.iw1.slc', 'iw1_' + str(kk) + '.slc'),
('vv.iw2.slc', 'iw2_' + str(kk) + '.slc'),
('vv.iw3.slc', 'iw3_' + str(kk) + '.slc'),
('vv.SLC_tab', 'SLC_tab'),
('vv.slc', 'slc'),
('tops_par', 'TOPS_par'),
]
for src_token, dst_token in rename_rules:
for candidate in list(Path(work_dir).iterdir()):
if src_token not in candidate.name:
continue
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
if target != candidate and not target.exists():
candidate.rename(target)
SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk)
SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par'))
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 concatenated SLC generation did not produce expected IW products for ' + date)
cat_str = 'touch slc_list slc_par_list top_par_list'
os.system(call_str)
os.system(cat_str)
list_num=len(SLC_list)
for tt in range(list_num):
call_str = 'echo ' + SLC_list[tt] + ' >> slc_list'
@@ -160,7 +186,9 @@ def main(argv):
call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list'
os.system(call_str)
call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('paste SLC tab failed with rc=' + str(rc))
call_str = 'rm slc_list slc_par_list top_par_list'
os.system(call_str)
SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc'))
@@ -188,23 +216,42 @@ def main(argv):
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
#call_str = 'rm *iw* *vv.SLC_tab'
#os.system(call_str)
if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab)
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 concatenated final SLC products are incomplete for ' + date)
for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('failed to append concatenated SLC tab with rc=' + str(rc))
BURST = SLC_par_list[kk].replace('slc.par','burst.par')
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
call_str = "echo 'SLC has already down' >down2slc.dat"
os.system(call_str)
TSLC = work_dir + '/' + date + '.slc'
TSLCPar = work_dir + '/' + date + '.slc.par'
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
print("Down to SLC for %s is done! " % date)
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/down2slc_sen.py
Vendored
+61 -28
View File
@@ -13,6 +13,7 @@ import getopt
import time
import glob
import argparse
from pathlib import Path
from pyint import _utils as ut
@@ -23,10 +24,11 @@ def get_s1_date(raw_file):
return date
def get_satellite(raw_file):
if 'S1A_IW_SLC_' in raw_file:
s0 = 'A'
name = os.path.basename(str(raw_file))
if name.startswith('S1') and len(name) >= 3:
s0 = name[2]
else:
s0 = 'B'
s0 = 'A'
return s0
@@ -88,16 +90,13 @@ def main(argv):
# os.mkdir(opod_dir)
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir
os.system(call_str)
if opod_dir and not os.path.isdir(opod_dir):
os.makedirs(opod_dir, exist_ok=True)
os.chdir(work_dir)
t_date = 't_' + date
call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date
os.system(call_str)
start_swath = templateDict['start_swath']
end_swath = templateDict['end_swath']
@@ -115,53 +114,87 @@ def main(argv):
# k_swath = '-'
k_swath = ut.get_sardata_swath(start_swath,end_swath)
raw_files = ut.read_txt2list(t_date)
raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
if not raw_files:
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
with open(t_date, 'w', encoding='utf-8') as f:
for raw_file in raw_files:
f.write(str(raw_file) + '\n')
satellite = get_satellite(str(raw_files[0]))
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
zipfile_ref=str(raw_files[0])
outfile_name=zipfile_ref.split('/')[-1].split('.')[0]
burst_number_table_ref=outfile_name + '.BURST_tab'
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
os.system(call_str)
if raw_files and os.path.isdir(str(raw_files[0])):
call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
else:
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
# call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
os.chdir(work_dir)
call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *"
#call_str = "rename vv.slc.iw1 IW1.slc * "
os.system(call_str)
call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *"
#call_str = "rename vv.slc.iw2 IW2.slc * "
os.system(call_str)
call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *"
#call_str = "rename vv.slc.iw3 IW3.slc * "
os.system(call_str)
call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par "
os.system(call_str)
rename_rules = [
('vv.iw1.slc', 'IW1.slc'),
('vv.iw2.slc', 'IW2.slc'),
('vv.iw3.slc', 'IW3.slc'),
('vv.SLC_tab', 'SLC_tab'),
('vv.slc', 'slc'),
('tops_par', 'TOPS_par'),
]
for src_token, dst_token in rename_rules:
for candidate in list(Path(work_dir).iterdir()):
if src_token not in candidate.name:
continue
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
if target != candidate and not target.exists():
candidate.rename(target)
SLC_Tab = work_dir + '/' + date+'_SLC_Tab'
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 SLC generation did not produce expected IW SLC products for ' + date)
if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab)
for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('failed to append SLC tab with rc=' + str(rc))
BURST = SLC_par_list[kk].replace('slc.par','burst.par')
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str)
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
call_str = "echo 'SLC has already down' >down2slc.dat"
os.system(call_str)
TSLC = work_dir + '/' + date + '.slc'
TSLCPar = work_dir + '/' + date + '.slc.par'
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
print("Down to SLC for %s is done! " % date)
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/down2slc_sen_all.py
Vendored
+25 -4
View File
@@ -32,9 +32,27 @@ def work(data0):
stderr = p.stderr
if type(stderr) == bytes:
aa=stderr.decode("utf-8")
aa=stderr.decode("utf-8", errors="replace")
else:
aa = stderr
if type(stdout) == bytes:
bb=stdout.decode("utf-8", errors="replace")
else:
bb = stdout
if p.returncode != 0:
detail_parts = []
if bb:
detail_parts.append(bb)
if aa:
detail_parts.append(aa)
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
with open(err_txt, 'a') as f:
f.write(str0)
if detail:
f.write(detail)
f.write('\n')
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
if aa:
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
@@ -83,7 +101,7 @@ def main(argv):
projectDir = scratchDir + '/' + projectName
downDir = scratchDir + '/' + projectName + "/DOWNLOAD"
slcDir = scratchDir + '/' + projectName + "/SLC"
raw_file_list = glob.glob(downDir + '/S1*.zip')
raw_file_list = glob.glob(downDir + '/S1*.zip') + glob.glob(downDir + '/S1*.SAFE')
templateDir = os.getenv('TEMPLATEDIR')
templateFile = templateDir + "/" + projectName + ".template"
templateDict=ut.update_template(templateFile)
@@ -139,12 +157,15 @@ def main(argv):
k00 = 0
if k00==0:
data_para.append(data0)
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
failures = [str(item) for item in results if isinstance(item, Exception)]
if failures:
raise RuntimeError('\n\n'.join(failures))
os.chdir(downDir)
print("Down to SLC for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time())
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/extract_s1_bursts.py
Vendored
+1 -1
View File
@@ -291,7 +291,7 @@ def main(argv):
iw3 = Sdate + '.IW3.slc*'
call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3
os.system(call_str)
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/extract_s1_bursts_all.py
Vendored
+31 -4
View File
@@ -26,9 +26,27 @@ def work(data0):
stderr = p.stderr
if type(stderr) == bytes:
aa=stderr.decode("utf-8")
aa=stderr.decode("utf-8", errors="replace")
else:
aa = stderr
if type(stdout) == bytes:
bb=stdout.decode("utf-8", errors="replace")
else:
bb = stdout
if p.returncode != 0:
detail_parts = []
if bb:
detail_parts.append(bb)
if aa:
detail_parts.append(aa)
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
with open(err_file, 'a') as f:
f.write(str0)
if detail:
f.write(detail)
f.write('\n')
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
if aa:
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
@@ -75,22 +93,31 @@ def main(argv):
projectDir = scratchDir + '/' + projectName
slcDir = scratchDir + '/' + projectName + '/SLC'
slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))]
templateDir = os.getenv('TEMPLATEDIR')
templateFile = templateDir + "/" + projectName + ".template"
templateDict=ut.update_template(templateFile)
master_date = templateDict['masterDate']
err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err'
if os.path.isfile(err_txt): os.remove(err_txt)
data_para = []
for i in range(len(slc_list)):
if slc_list[i] == master_date:
continue
cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]]
data0 = [cmd0,err_txt]
data_para.append(data0)
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
failures = [str(item) for item in results if isinstance(item, Exception)]
if failures:
raise RuntimeError('\n\n'.join(failures))
print("Extract TOPS bursts for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time())
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/phase2los.py
Vendored
+1 -2
View File
@@ -83,7 +83,7 @@ def main(argv):
wavelength = 0.0312283810417
elif satelite == 'TSX':
wavelength = 0.03106657823461874
elif satelite == 'S1A':
elif str(satelite).startswith('S1'):
wavelength = 0.0554657647
elif satelite == 'ALOS2':
wavelength = 0.2424525
@@ -106,4 +106,3 @@ def main(argv):
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/phase2los_all.py
Vendored
+1 -1
View File
@@ -105,7 +105,7 @@ def main(argv):
print("Generate differential interferograms for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time())
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])
third_party/PyINT/pyint/pyintApp.py
Vendored
+1 -1
View File
@@ -210,7 +210,7 @@ def main(argv):
if templateDict['raw2slc_all'] == '1': # only for S1 data now
print('Start to convert downloaded-raw data into SLC ...')
print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel']))
if satelite=='S1A':
if str(satelite).startswith('S1'):
call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel']
_run_or_raise(call_str, 'raw2slc_s1')
elif satelite=='ALOS':
third_party/PyINT/pyint/slc_sen_cat.py
Vendored
+1 -1
View File
@@ -159,7 +159,7 @@ def main(argv):
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str)
print("Down to SLC for %s is done! " % date)
sys.exit(1)
sys.exit(0)
if __name__ == '__main__':
main(sys.argv[:])