Files
insar-management-system-v2/third_party/PyINT/pyint/_utils_old.py
T

624 lines
21 KiB
Python

############################################################
# Program is part of PyINT V2.1 #
# Copyright 2017-2019 Yunmeng Cao #
# Contact: ymcmrs@gmail.com #
############################################################
#from datetime import datetime
import datetime
import urllib.request
import os
import numpy as np
import random
import h5py
from pathlib import Path
import linecache
import time
import glob
from tqdm import tqdm
from concurrent.futures import ProcessPoolExecutor, as_completed
####################### update template #############################
def update_template(template_file):
templateDict = {}
######### download using SSARA ######
#templateDict['sensor'] = 'sentinel-1'
#templateDict['track'] = '14'
#templateDict['frame'] = '75'
templateDict['start_time'] = '1989-01-01'
templateDict['end_time'] = '2189-01-01'
####### basic parameters for interferometry ######
templateDict['start_swath'] = '1'
templateDict['end_swath'] = '3'
templateDict['start_burst'] = '1'
templateDict['end_burst'] = '20'
templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m
templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table
templateDict['Igram_Spsflg'] = '1' # Range spectral filtering
templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering
templateDict['rwin4cor'] = '256' # range window length for coregistration
templateDict['azwin4cor'] = '256' # azimuth window length for coregistration
templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters
templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters
templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR
templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both]
templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant
templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave
templateDict['Igram_Cor_awin'] = '5' # used for cc_wave
templateDict['Igram_Cor_Win'] = '5' # used for adf
templateDict['adf_alpha'] = '0.4' # used for adf
######## sim phase ##################
templateDict['Igram_Flag_TDM'] = 'N'
templateDict['Simphase_rpos'] = '-'
templateDict['Simphase_azpos'] = '-'
templateDict['Simphase_rwin'] = '256'
templateDict['Simphase_azwin'] = '256'
templateDict['Simphase_thresh'] = '-'
#### unwrap phase ###########
templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;]
templateDict['unwrap_patr'] = '1'
templateDict['unwrap_pataz'] = '1'
templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap
#### geocode #########
templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline]
############## interferometry ################
templateDict['int_flag'] = '1' # 1 means do interferometry
templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase
templateDict['unw_flag'] = '1' # unwrap process
templateDict['geo_flag'] = '0' # geocode process
############## select network #############
templateDict['endDate'] = '21000101'
templateDict['startDate'] = '19000101'
templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars
templateDict['conNumb'] = '2' # connect number for sequential
templateDict['max_tb'] = '50000'
templateDict['max_sb'] = '50000'
############## time-series ################
templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided
templateDict['down_parallel'] = '1' # multi-processor number used for downloading
templateDict['raw2slc_all'] = '0' # i.e., download 2 slc
templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used
templateDict['extract_burst_all'] = '0' # for TOPS SLC only
templateDict['extract_all_parallel'] = '1' # multi-processor number used
templateDict['coreg_all'] = '1'
templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8]
templateDict['select_pairs'] = '1'
templateDict['diff_all'] = '1'
templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8]
templateDict['unwrap_all'] = '1'
templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10]
templateDict['geocode_all'] = '0'
templateDict['geocode_all_parallel'] = '1' # multi-processor number used
templateDict['load_data'] = '0' # loading data for mintpy processing
templateDict0 = read_template(template_file, delimiter='=')
for key, value in templateDict0.items():
templateDict[key] = str(value)
return templateDict
####################### time-related function ################
def is_number(s):
try:
int(s)
return True
except ValueError:
return False
def print_process_time(start, end):
hours, rem = divmod(end-start, 3600)
minutes, seconds = divmod(rem, 60)
print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds))
return
def date_add1d(date0):
format_str = '%Y%m%d' # The format
date_obj = datetime.datetime.strptime(date0,format_str)
date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d')
return date1
def date_minus1d(date0):
format_str = '%Y%m%d' # The format
date_obj = datetime.datetime.strptime(date0,format_str)
date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d')
return date1
def generate_random_name(sufix):
nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time
randomNum = random.randint(0,100)
randomNum1 = random.randint(0,100)
Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix
return Nm
def yyyymmdd2yyyyddd(date):
dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object
day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1
doy = day_of_year.days + 1
year = dt.year
doy = str(doy)
if len(doy) ==1:
doy = '00' + doy
elif len(doy) ==2:
doy ='0'+ doy
year = str(year)
return year, doy
def get_txt_lines(txt):
count=0
myfile=open(txt,"r")
for line in myfile:
count=count+1
return count
def read_txt2list(txt):
A = np.loadtxt(txt,dtype=np.str)
if np.array(A).size ==1:
A = [A]
elif isinstance(A[0],bytes):
A = A.astype(str)
A = list(A)
return A
def read_txt2array(txt):
A = np.loadtxt(txt,dtype=np.str)
if np.array(A).size ==1:
A = [A]
elif isinstance(A[0],bytes):
A = A.astype(str)
#A = list(A)
return A
def get_sardata_swath(start_swath,end_swath):
k_swath = '0'
if (start_swath == '1') and (end_swath == '1'):
k_swath = '1'
elif (start_swath == '2') and (end_swath == '2'):
k_swath = '2'
elif (start_swath == '3') and (end_swath == '3'):
k_swath = '3'
elif (start_swath == '1') and (end_swath == '2'):
k_swath = '4'
elif (start_swath == '2') and (end_swath == '3'):
k_swath = '5'
elif (start_swath == '2') and (end_swath == '3'):
k_swath = '-'
return k_swath
def get_filelist_filesize(url0):
ttt = generate_random_name('.txt')
call_str = 'curl -s ' + url0 + ' > ' + ttt
os.system(call_str)
A = read_txt2array(ttt)
A_size = A[:,4]
A_size = A_size.astype(int)
A_size = A_size/1024/1024 #bytes to Mb
A_name = A[:,8]
if os.path.isfile(ttt):
os.remove(ttt)
return A_name, A_size
def yyyymmdd(date0):
if len(date0) ==6:
if float(date0[0:2]) > 90:
date1 = '19' + date0
else:
date1 = '20' + date0
elif len(date0) ==8:
date1 = date0
else:
print('The input date is invalid!!')
date1 = ''
return date1
def yymmdd(date0):
if len(date0) ==6:
date1 = date0
elif len(date0) ==8:
date1 = date0[2:8]
else:
print('The input date is invalid!!')
date1 = ''
return date1
def parallel_process(array, function, n_jobs=16, use_kwargs=False):
"""
A parallel version of the map function with a progress bar.
Args:
array (array-like): An array to iterate over.
function (function): A python function to apply to the elements of array
n_jobs (int, default=16): The number of cores to use
use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of
keyword arguments to function
Returns:
[function(array[0]), function(array[1]), ...]
"""
#We run the first few iterations serially to catch bugs
#If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging.
if n_jobs==1:
return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])]
#Assemble the workers
with ProcessPoolExecutor(max_workers=n_jobs) as pool:
#Pass the elements of array into function
if use_kwargs:
futures = [pool.submit(function, **a) for a in array[:]]
else:
futures = [pool.submit(function, a) for a in array[:]]
kwargs = {
'total': len(futures),
'unit': 'it',
'unit_scale': True,
'leave': True
}
#Print out the progress as tasks complete
for f in tqdm(as_completed(futures), **kwargs):
pass
out = []
#Get the results from the futures.
for i, future in tqdm(enumerate(futures)):
try:
out.append(future.result())
except Exception as e:
out.append(e)
return out
def sort_unique_list(numb_list):
list_out = sorted(set(numb_list))
return list_out
############################# download data ###################################
def StrNum(S):
S = str(S)
if len(S)==1:
S='0' +S
return S
def download_s1_orbit(date,save_path,satellite='A'):
DATE = date
if len(DATE)==6:
DATE = '20' + DATE
ST = satellite
YEAR = int(DATE[0:4])
MON = int(DATE[4:6])
DAY = int(DATE[6:8])
MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31]
if YEAR%4==0:
MON_DAY[1]=29
if MON ==1 and DAY ==1:
DAY0 = 31
MON0 = 12
YEAR0 =YEAR -1
elif MON!=1 and DAY ==1:
DAY0 = MON_DAY[MON-2]
MON0 = MON-1
YEAR0 = YEAR
else:
DAY0 = DAY -1
MON0 = MON
YEAR0 = YEAR
MONDAY0 = MON_DAY[MON0-1]
TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0]
T0 = []
for k in range(len(TT)):
T0.append(TT[k])
TT[k] = TT[k]-DAY0
for k in range(len(TT)):
if k == len(TT)-1:
if TT[k]==0:
ff = k-1
else:
if TT[k]<=0 and TT[k+1]>0:
ff = k
DAY1 = T0[ff]
DAY2 = T0[ff+1]
S1 = StrNum(YEAR0) + '-' + StrNum(MON0)
S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1)
S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2)
S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0)
#SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4
SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST
tt ='tt_orb_' + date
tt0 ='tt0_orb_' + date
tt00 ='tt00_orb_' + date
tt000 ='tt000_orb_' + date
SS = "'" + SS + "'"
#print(SS)
call_str = 'curl -s -l ' + SS + ' > ' + tt
os.system(call_str)
call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0
os.system(call_str)
call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00
os.system(call_str)
call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000
os.system(call_str)
SS=linecache.getline(tt000, 1)
SS = SS.split('"')[-1]
filename = os.path.basename(SS)
call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename
os.system(call_str)
call_str = "downlaod the precise Orbit of" + date
print(call_str)
filename = os.path.basename(SS)
os.remove(tt)
os.remove(tt0)
os.remove(tt00)
os.remove(tt000)
return filename
############################# write & read #####################################
def copy_file(file0,file1):
call_str = 'cp ' + file0 + ' ' + file1
os.system(call_str)
return
def get_project_slcList(projectName):
scratchDir = os.getenv('SCRATCHDIR')
slcDir = scratchDir + '/' + projectName + "/SLC"
#slcDir = scratchDir + '/SLC'
fn = os.listdir(slcDir)
slc_list0 = [os.path.basename(fname) for fname in sorted(fn)]
#slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))]
slc_list = []
for k0 in slc_list0:
if is_number(k0):
slc_list.append(k0)
slc_list = sorted(slc_list)
return slc_list
def createBlankFile(strFile):
f = open(strFile,'w')
for i in range (10):
f.write('\n')
f.close()
def read_attr(fname):
# read hdf5
with h5py.File(fname, 'r') as f:
atr = dict(f.attrs)
return atr
def read_hdf5(fname, datasetName=None, box=None):
# read hdf5
with h5py.File(fname, 'r') as f:
data = f[datasetName][:]
atr = dict(f.attrs)
return data, atr
def get_dataNames(FILE):
with h5py.File(FILE, 'r') as f:
dataNames = []
for k0 in f.keys():
dataNames.append(k0)
return dataNames
def check_variable_name(path):
s=path.split("/")[0]
if len(s)>0 and s[0]=="$":
p0=os.getenv(s[1:])
path=path.replace(path.split("/")[0],p0)
return path
def read_template(File, delimiter='='):
'''Reads the template file into a python dictionary structure.
Input : string, full path to the template file
Output: dictionary, pysar template content
Example:
tmpl = read_template(KyushuT424F610_640AlosA.template)
tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':')
'''
template_dict = {}
for line in open(File):
line = line.strip()
c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter
if len(c) < 2 or line.startswith('%') or line.startswith('#'):
next #ignore commented lines or those without variables
else:
atrName = c[0]
atrValue = str.replace(c[1],'\n','').split("#")[0].strip()
atrValue = check_variable_name(atrValue)
template_dict[atrName] = atrValue
return template_dict
def read_gamma_par(inFile, task, keyword):
if task == "read":
f = open(inFile, "r")
while 1:
line = f.readline()
if not line: break
if line.count(keyword) == 1:
strtemp = line.split(":")
value = strtemp[1].strip()
return value
print("Keyword " + keyword + " doesn't exist in " + inFile)
f.close
def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None):
if os.path.isfile(out_file):
print('delete exsited file: {}'.format(out_file))
os.remove(out_file)
print('create HDF5 file: {} with w mode'.format(out_file))
dt = h5py.special_dtype(vlen=np.dtype('float64'))
with h5py.File(out_file, 'w') as f:
for dsName in datasetDict.keys():
data = datasetDict[dsName]
ds = f.create_dataset(dsName,
data=data,
compression=compression)
for key, value in metadata.items():
f.attrs[key] = str(value)
#print(key + ': ' + value)
print('finished writing to {}'.format(out_file))
return out_file
######################################################################
class progressBar:
"""Creates a text-based progress bar. Call the object with
the simple print command to see the progress bar, which looks
something like this:
[=======> 22% ]
You may specify the progress bar's min and max values on init.
note:
modified from mintPy (https://github.com/insarlab/MintPy/wiki)
Code originally from http://code.activestate.com/recipes/168639/
example:
from mintpy.utils import ptime
date12_list = ptime.list_ifgram2date12(ifgram_list)
prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:')
for i in range(1000):
prog_bar.update(i+1, suffix=date)
prog_bar.update(i+1, suffix=date12_list[i])
prog_bar.close()
"""
def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True):
self.prog_bar = "[]" # This holds the progress bar string
self.min = minValue
self.max = maxValue
self.span = maxValue - minValue
self.suffix = ''
self.prefix = prefix
self.print_msg = print_msg
## calculate total width based on console width
#rows, columns = os.popen('stty size', 'r').read().split()
#self.width = round(int(columns) * 0.7 / 10) * 10
self.width = totalWidth
self.reset()
def reset(self):
self.start_time = time.time()
self.amount = 0 # When amount == max, we are 100% done
self.update_amount(0) # Build progress bar string
def update_amount(self, newAmount=0, suffix=''):
""" Update the progress bar with the new amount (with min and max
values set at initialization; if it is over or under, it takes the
min or max value as a default. """
if newAmount < self.min:
newAmount = self.min
if newAmount > self.max:
newAmount = self.max
self.amount = newAmount
# Figure out the new percent done, round to an integer
diffFromMin = np.float(self.amount - self.min)
percentDone = (diffFromMin / np.float(self.span)) * 100.0
percentDone = np.int(np.round(percentDone))
# Figure out how many hash bars the percentage should be
allFull = self.width - 2 - 18
numHashes = (percentDone / 100.0) * allFull
numHashes = np.int(np.round(numHashes))
# Build a progress bar with an arrow of equal signs; special cases for
# empty and full
if numHashes == 0:
self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1))
elif numHashes == allFull:
self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull)
if suffix:
self.prog_bar += ' %s' % (suffix)
else:
self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes))
# figure out where to put the percentage, roughly centered
percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone)))
percentString = ' ' + str(percentDone) + '% '
# slice the percentage into the bar
self.prog_bar = ''.join([self.prog_bar[0:percentPlace],
percentString,
self.prog_bar[percentPlace+len(percentString):]])
# prefix and suffix
self.prog_bar = self.prefix + self.prog_bar
if suffix:
self.prog_bar += ' %s' % (suffix)
# time info - elapsed time and estimated remaining time
if percentDone > 0:
elapsed_time = time.time() - self.start_time
self.prog_bar += '%5ds / %5ds' % (int(elapsed_time),
int(elapsed_time * (100./percentDone-1)))
def update(self, value, every=1, suffix=''):
""" Updates the amount, and writes to stdout. Prints a
carriage return first, so it will overwrite the current
line in stdout."""
if value % every == 0 or value >= self.max:
self.update_amount(newAmount=value, suffix=suffix)
if self.print_msg:
sys.stdout.write('\r' + self.prog_bar)
sys.stdout.flush()
def close(self):
"""Prints a blank space at the end to ensure proper printing
of future statements."""
if self.print_msg:
print(' ')