Apply current workspace changes
This commit is contained in:
Vendored
+689
@@ -0,0 +1,689 @@
|
||||
############################################################
|
||||
# Program is part of PyINT V2.1 #
|
||||
# Copyright 2017-2019 Yunmeng Cao #
|
||||
# Contact: ymcmrs@gmail.com #
|
||||
############################################################
|
||||
#from datetime import datetime
|
||||
import datetime
|
||||
import urllib.request
|
||||
import os
|
||||
import numpy as np
|
||||
import random
|
||||
import h5py
|
||||
from pathlib import Path
|
||||
import linecache
|
||||
import time
|
||||
import glob
|
||||
|
||||
from tqdm import tqdm
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
|
||||
# 无条件清理 OMP_PLACES / OMP_PROC_BIND，避免 libgomp 报错
|
||||
for _k in ('OMP_PLACES', 'OMP_PROC_BIND'):
|
||||
os.environ.pop(_k, None)
|
||||
|
||||
####################### update template #############################
|
||||
def update_template(template_file):
|
||||
|
||||
templateDict = {}
|
||||
|
||||
######### download using SSARA ######
|
||||
#templateDict['sensor'] = 'sentinel-1'
|
||||
#templateDict['track'] = '14'
|
||||
#templateDict['frame'] = '75'
|
||||
templateDict['start_time'] = '1989-01-01'
|
||||
templateDict['end_time'] = '2189-01-01'
|
||||
|
||||
######### download using ASF API ######
|
||||
templateDict['asf_username'] = '' # ASF username (can be read from ~/.netrc)
|
||||
templateDict['asf_password'] = '' # ASF password (can be read from ~/.netrc)
|
||||
templateDict['bbox'] = '' # bounding box: min_lon,min_lat,max_lon,max_lat
|
||||
templateDict['shapefile'] = '' # shapefile path for spatial filtering
|
||||
templateDict['relative_orbit'] = '' # relative orbit number (optional)
|
||||
templateDict['frame'] = '' # frame number (optional)
|
||||
templateDict['flight_direction'] = 'a' # flight direction: a=ascending, d=descending, all
|
||||
templateDict['acquisition_mode'] = 'IW' # acquisition mode: IW, EW, SM, all
|
||||
|
||||
####### basic parameters for interferometry ######
|
||||
templateDict['start_swath'] = '1'
|
||||
templateDict['end_swath'] = '3'
|
||||
|
||||
templateDict['start_burst'] = '1'
|
||||
templateDict['end_burst'] = '20'
|
||||
|
||||
templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m
|
||||
templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table
|
||||
templateDict['dem_num_workers'] = '4' # number of parallel workers for DEM download
|
||||
templateDict['dem_source'] = 'copernicus' # DEM data source: copernicus or nasadem
|
||||
|
||||
templateDict['Igram_Spsflg'] = '1' # Range spectral filtering
|
||||
templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering
|
||||
|
||||
templateDict['rwin4cor'] = '256' # range window length for coregistration
|
||||
templateDict['azwin4cor'] = '256' # azimuth window length for coregistration
|
||||
|
||||
templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters
|
||||
templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters
|
||||
|
||||
templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR
|
||||
|
||||
templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both]
|
||||
templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant
|
||||
|
||||
templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave
|
||||
templateDict['Igram_Cor_awin'] = '5' # used for cc_wave
|
||||
|
||||
templateDict['Igram_Cor_Win'] = '5' # used for adf
|
||||
templateDict['adf_alpha'] = '0.4' # used for adf
|
||||
######## sim phase ##################
|
||||
templateDict['Igram_Flag_TDM'] = 'N'
|
||||
templateDict['Simphase_rpos'] = '-'
|
||||
templateDict['Simphase_azpos'] = '-'
|
||||
templateDict['Simphase_rwin'] = '256'
|
||||
templateDict['Simphase_azwin'] = '256'
|
||||
templateDict['Simphase_thresh'] = '-'
|
||||
|
||||
#### unwrap phase ###########
|
||||
templateDict['auto_unw'] = '1' # auto reference-point selection
|
||||
templateDict['make_mask'] = '1' # coherence-mask switch for unwrap
|
||||
templateDict['init_flag'] = '1' # set reference-point phase to 0.0
|
||||
templateDict['r_refer'] = '-' # range reference pixel for manual unwrap
|
||||
templateDict['a_refer'] = '-' # azimuth reference pixel for manual unwrap
|
||||
templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;]
|
||||
templateDict['unwrap_patr'] = '4'
|
||||
templateDict['unwrap_pataz'] = '4'
|
||||
templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap
|
||||
|
||||
#### geocode #########
|
||||
templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline]
|
||||
templateDict['geocode_products'] = 'hyp3,licsbas' # 产品类型: hyp3,licsbas,pot (逗号分隔多选)
|
||||
|
||||
############## interferometry ################
|
||||
templateDict['int_flag'] = '1' # 1 means do interferometry
|
||||
templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase
|
||||
templateDict['unw_flag'] = '1' # unwrap process
|
||||
templateDict['geo_flag'] = '0' # geocode process
|
||||
|
||||
############## select network #############
|
||||
templateDict['endDate'] = '21000101'
|
||||
templateDict['startDate'] = '19000101'
|
||||
templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars
|
||||
templateDict['conNumb'] = '2' # connect number for sequential
|
||||
templateDict['max_tb'] = '50000'
|
||||
templateDict['max_sb'] = '50000'
|
||||
|
||||
############## time-series ################
|
||||
templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided
|
||||
templateDict['down_parallel'] = '1' # multi-processor number used for downloading
|
||||
|
||||
templateDict['raw2slc_all'] = '0' # i.e., download 2 slc
|
||||
templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used
|
||||
|
||||
templateDict['extract_burst_all'] = '0' # for TOPS SLC only
|
||||
templateDict['extract_all_parallel'] = '1' # multi-processor number used
|
||||
|
||||
templateDict['coreg_all'] = '1'
|
||||
templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8]
|
||||
|
||||
templateDict['select_pairs'] = '1'
|
||||
|
||||
templateDict['diff_all'] = '1'
|
||||
templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8]
|
||||
|
||||
templateDict['unwrap_all'] = '1'
|
||||
templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10]
|
||||
|
||||
templateDict['atmcor_all'] = '0' # if 1, run atmospheric correction
|
||||
templateDict['atmcor_all_parallel'] = '1' # multi-processor number used
|
||||
|
||||
templateDict['geocode_all'] = '0'
|
||||
templateDict['geocode_all_parallel'] = '1' # multi-processor number used
|
||||
templateDict['gamma2licsbas_all'] = '0' # legacy conversion stage switch
|
||||
templateDict['gamma2licsbas_all_parallel'] = '1'
|
||||
templateDict['hyp3format_all'] = '0' # legacy conversion stage switch
|
||||
templateDict['hyp3format_all_parallel'] = '1'
|
||||
|
||||
############## GACOS atmospheric correction #############
|
||||
templateDict['gacos_correction'] = '0' # if 1, apply GACOS atmospheric correction
|
||||
templateDict['gacos_all_parallel'] = '1' # multi-processor number used for GACOS correction
|
||||
templateDict['gacos_dir'] = '' # directory for GACOS ZTD files (default: projectName/GACOS)
|
||||
templateDict['gacos_email'] = '' # email for GACOS auto-download
|
||||
templateDict['gacos_email_user'] = '' # IMAP username
|
||||
templateDict['gacos_email_pass'] = '' # IMAP authorization code
|
||||
templateDict['gacos_email_host'] = 'imap.163.com' # IMAP server host
|
||||
templateDict['gacos_email_port'] = '993' # IMAP server port
|
||||
templateDict['gacos_email_ssl'] = '1' # Use SSL
|
||||
templateDict['gacos_check_interval'] = '60' # Email check interval (seconds)
|
||||
|
||||
############## Pixel Offset Tracking (POT) #############
|
||||
templateDict['pot_all'] = '0' # if 1, run offset tracking
|
||||
templateDict['pot_all_parallel'] = '1' # multi-processor number used
|
||||
templateDict['pot_rstep'] = '50' # range step in SLC pixels
|
||||
templateDict['pot_azstep'] = '10' # azimuth step in SLC pixels
|
||||
templateDict['pot_rwin'] = '256' # range window size (Round 1)
|
||||
templateDict['pot_azwin'] = '128' # azimuth window size (Round 1)
|
||||
templateDict['pot_rwin2'] = '128' # range window size (Round 2)
|
||||
templateDict['pot_azwin2'] = '64' # azimuth window size (Round 2)
|
||||
templateDict['pot_ovr'] = '1' # SLC oversampling factor (1=no deramp needed)
|
||||
templateDict['pot_snr_thresh'] = '0.01' # SNR threshold for offset_pwr_tracking
|
||||
templateDict['pot_ccp_thresh'] = '0.05' # cross-correlation threshold for masking
|
||||
templateDict['pot_roff_min'] = '-50' # min range offset (pixels) for masking
|
||||
templateDict['pot_roff_max'] = '50' # max range offset (pixels) for masking
|
||||
templateDict['pot_azoff_min'] = '-20' # min azimuth offset (pixels) for masking
|
||||
templateDict['pot_azoff_max'] = '20' # max azimuth offset (pixels) for masking
|
||||
templateDict['pot_drange_thresh'] = '2.0' # deviation threshold range (pixels)
|
||||
templateDict['pot_dazimuth_thresh'] = '1.0' # deviation threshold azimuth (pixels)
|
||||
templateDict['pot_median_win'] = '7' # median filter window size
|
||||
templateDict['pot_median_nmin'] = '15' # min valid samples for median filter
|
||||
templateDict['pot_two_rounds'] = '1' # 1=two-round estimation, 0=single round
|
||||
templateDict['pot_geocode'] = '1' # 1=geocode displacement results
|
||||
templateDict['pot_disp_max'] = '100' # max displacement for BMP display (meters)
|
||||
|
||||
templateDict['load_data'] = '0' # loading data for mintpy processing
|
||||
|
||||
templateDict0 = read_template(template_file, delimiter='=')
|
||||
for key, value in templateDict0.items():
|
||||
templateDict[key] = str(value)
|
||||
|
||||
return templateDict
|
||||
|
||||
####################### time-related function ################
|
||||
|
||||
def is_number(s):
|
||||
try:
|
||||
int(s)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def print_process_time(start, end):
|
||||
hours, rem = divmod(end-start, 3600)
|
||||
minutes, seconds = divmod(rem, 60)
|
||||
print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds))
|
||||
return
|
||||
|
||||
def date_add1d(date0):
|
||||
format_str = '%Y%m%d' # The format
|
||||
date_obj = datetime.datetime.strptime(date0,format_str)
|
||||
date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d')
|
||||
return date1
|
||||
|
||||
def date_minus1d(date0):
|
||||
format_str = '%Y%m%d' # The format
|
||||
date_obj = datetime.datetime.strptime(date0,format_str)
|
||||
date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d')
|
||||
return date1
|
||||
|
||||
|
||||
def generate_random_name(sufix):
|
||||
|
||||
nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time
|
||||
randomNum = random.randint(0,100)
|
||||
randomNum1 = random.randint(0,100)
|
||||
Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix
|
||||
|
||||
return Nm
|
||||
|
||||
def yyyymmdd2yyyyddd(date):
|
||||
dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object
|
||||
day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1
|
||||
doy = day_of_year.days + 1
|
||||
year = dt.year
|
||||
|
||||
doy = str(doy)
|
||||
if len(doy) ==1:
|
||||
doy = '00' + doy
|
||||
elif len(doy) ==2:
|
||||
doy ='0'+ doy
|
||||
year = str(year)
|
||||
|
||||
return year, doy
|
||||
|
||||
def get_txt_lines(txt):
|
||||
count=0
|
||||
myfile=open(txt,"r")
|
||||
for line in myfile:
|
||||
count=count+1
|
||||
|
||||
return count
|
||||
|
||||
def read_txt2list(txt):
|
||||
A = np.loadtxt(txt,dtype=str)
|
||||
if np.array(A).size ==1:
|
||||
A = [A]
|
||||
elif isinstance(A[0],bytes):
|
||||
A = A.astype(str)
|
||||
A = list(A)
|
||||
return A
|
||||
|
||||
def read_txt2array(txt):
|
||||
A = np.loadtxt(txt,dtype=str)
|
||||
if np.array(A).size ==1:
|
||||
A = [A]
|
||||
elif isinstance(A[0],bytes):
|
||||
A = A.astype(str)
|
||||
#A = list(A)
|
||||
return A
|
||||
def get_sardata_swath(start_swath,end_swath):
|
||||
k_swath = '0'
|
||||
if (start_swath == '1') and (end_swath == '1'):
|
||||
k_swath = '1'
|
||||
elif (start_swath == '2') and (end_swath == '2'):
|
||||
k_swath = '2'
|
||||
elif (start_swath == '3') and (end_swath == '3'):
|
||||
k_swath = '3'
|
||||
elif (start_swath == '1') and (end_swath == '2'):
|
||||
k_swath = '4'
|
||||
elif (start_swath == '2') and (end_swath == '3'):
|
||||
k_swath = '5'
|
||||
elif (start_swath == '2') and (end_swath == '3'):
|
||||
k_swath = '-'
|
||||
|
||||
return k_swath
|
||||
|
||||
def get_filelist_filesize(url0):
|
||||
ttt = generate_random_name('.txt')
|
||||
call_str = 'curl -s ' + url0 + ' > ' + ttt
|
||||
os.system(call_str)
|
||||
A = read_txt2array(ttt)
|
||||
|
||||
A_size = A[:,4]
|
||||
A_size = A_size.astype(int)
|
||||
A_size = A_size/1024/1024 #bytes to Mb
|
||||
A_name = A[:,8]
|
||||
if os.path.isfile(ttt):
|
||||
os.remove(ttt)
|
||||
|
||||
return A_name, A_size
|
||||
|
||||
def yyyymmdd(date0):
|
||||
if len(date0) ==6:
|
||||
if float(date0[0:2]) > 90:
|
||||
date1 = '19' + date0
|
||||
else:
|
||||
date1 = '20' + date0
|
||||
elif len(date0) ==8:
|
||||
date1 = date0
|
||||
else:
|
||||
print('The input date is invalid!!')
|
||||
date1 = ''
|
||||
return date1
|
||||
|
||||
def yymmdd(date0):
|
||||
if len(date0) ==6:
|
||||
date1 = date0
|
||||
elif len(date0) ==8:
|
||||
date1 = date0[2:8]
|
||||
else:
|
||||
print('The input date is invalid!!')
|
||||
date1 = ''
|
||||
return date1
|
||||
|
||||
def parallel_process(array, function, n_jobs=16, use_kwargs=False):
|
||||
"""
|
||||
A parallel version of the map function with a progress bar.
|
||||
|
||||
Args:
|
||||
array (array-like): An array to iterate over.
|
||||
function (function): A python function to apply to the elements of array
|
||||
n_jobs (int, default=16): The number of cores to use
|
||||
use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of
|
||||
keyword arguments to function
|
||||
Returns:
|
||||
[function(array[0]), function(array[1]), ...]
|
||||
"""
|
||||
#We run the first few iterations serially to catch bugs
|
||||
#If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging.
|
||||
if n_jobs==1:
|
||||
return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])]
|
||||
#Assemble the workers
|
||||
with ProcessPoolExecutor(max_workers=n_jobs) as pool:
|
||||
#Pass the elements of array into function
|
||||
if use_kwargs:
|
||||
futures = [pool.submit(function, **a) for a in array[:]]
|
||||
else:
|
||||
futures = [pool.submit(function, a) for a in array[:]]
|
||||
kwargs = {
|
||||
'total': len(futures),
|
||||
'unit': 'it',
|
||||
'unit_scale': True,
|
||||
'leave': True
|
||||
}
|
||||
#Print out the progress as tasks complete
|
||||
for f in tqdm(as_completed(futures), **kwargs):
|
||||
pass
|
||||
out = []
|
||||
#Get the results from the futures.
|
||||
for i, future in tqdm(enumerate(futures)):
|
||||
try:
|
||||
out.append(future.result())
|
||||
except Exception as e:
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
def sort_unique_list(numb_list):
|
||||
list_out = sorted(set(numb_list))
|
||||
return list_out
|
||||
|
||||
|
||||
############################# download data ###################################
|
||||
def StrNum(S):
|
||||
S = str(S)
|
||||
if len(S)==1:
|
||||
S='0' +S
|
||||
return S
|
||||
|
||||
def download_s1_orbit(date,save_path,satellite='A'):
|
||||
|
||||
DATE = date
|
||||
if len(DATE)==6:
|
||||
DATE = '20' + DATE
|
||||
ST = satellite
|
||||
YEAR = int(DATE[0:4])
|
||||
MON = int(DATE[4:6])
|
||||
DAY = int(DATE[6:8])
|
||||
|
||||
MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31]
|
||||
|
||||
if YEAR%4==0:
|
||||
MON_DAY[1]=29
|
||||
|
||||
if MON ==1 and DAY ==1:
|
||||
DAY0 = 31
|
||||
MON0 = 12
|
||||
YEAR0 =YEAR -1
|
||||
elif MON!=1 and DAY ==1:
|
||||
DAY0 = MON_DAY[MON-2]
|
||||
MON0 = MON-1
|
||||
YEAR0 = YEAR
|
||||
else:
|
||||
DAY0 = DAY -1
|
||||
MON0 = MON
|
||||
YEAR0 = YEAR
|
||||
|
||||
MONDAY0 = MON_DAY[MON0-1]
|
||||
|
||||
TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0]
|
||||
|
||||
T0 = []
|
||||
for k in range(len(TT)):
|
||||
T0.append(TT[k])
|
||||
TT[k] = TT[k]-DAY0
|
||||
|
||||
for k in range(len(TT)):
|
||||
if k == len(TT)-1:
|
||||
if TT[k]==0:
|
||||
ff = k-1
|
||||
else:
|
||||
if TT[k]<=0 and TT[k+1]>0:
|
||||
ff = k
|
||||
|
||||
|
||||
DAY1 = T0[ff]
|
||||
DAY2 = T0[ff+1]
|
||||
|
||||
S1 = StrNum(YEAR0) + '-' + StrNum(MON0)
|
||||
S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1)
|
||||
S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2)
|
||||
S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0)
|
||||
|
||||
#SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4
|
||||
SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST
|
||||
|
||||
tt ='tt_orb_' + date
|
||||
tt0 ='tt0_orb_' + date
|
||||
tt00 ='tt00_orb_' + date
|
||||
tt000 ='tt000_orb_' + date
|
||||
SS = "'" + SS + "'"
|
||||
#print(SS)
|
||||
call_str = 'curl -s -l ' + SS + ' > ' + tt
|
||||
os.system(call_str)
|
||||
|
||||
call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0
|
||||
os.system(call_str)
|
||||
|
||||
call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00
|
||||
os.system(call_str)
|
||||
|
||||
call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000
|
||||
os.system(call_str)
|
||||
|
||||
SS=linecache.getline(tt000, 1)
|
||||
SS = SS.split('"')[1]
|
||||
filename = os.path.basename(SS)
|
||||
if not os.path.isfile(filename):
|
||||
call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename
|
||||
os.system(call_str)
|
||||
call_str = "downlaod the precise Orbit of" + date
|
||||
print(call_str)
|
||||
|
||||
filename = os.path.basename(SS)
|
||||
os.remove(tt)
|
||||
os.remove(tt0)
|
||||
os.remove(tt00)
|
||||
os.remove(tt000)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
############################# write & read #####################################
|
||||
def copy_file(file0,file1):
|
||||
call_str = 'cp ' + file0 + ' ' + file1
|
||||
os.system(call_str)
|
||||
|
||||
return
|
||||
|
||||
def get_project_slcList(projectName):
|
||||
scratchDir = os.getenv('SCRATCHDIR')
|
||||
slcDir = scratchDir + '/' + projectName + "/SLC"
|
||||
#slcDir = scratchDir + '/SLC'
|
||||
fn = os.listdir(slcDir)
|
||||
slc_list0 = [os.path.basename(fname) for fname in sorted(fn)]
|
||||
#slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))]
|
||||
|
||||
slc_list = []
|
||||
for k0 in slc_list0:
|
||||
if is_number(k0):
|
||||
slc_list.append(k0)
|
||||
slc_list = sorted(slc_list)
|
||||
|
||||
return slc_list
|
||||
|
||||
def createBlankFile(strFile):
|
||||
f = open(strFile,'w')
|
||||
for i in range (10):
|
||||
f.write('\n')
|
||||
f.close()
|
||||
|
||||
def read_attr(fname):
|
||||
# read hdf5
|
||||
with h5py.File(fname, 'r') as f:
|
||||
atr = dict(f.attrs)
|
||||
|
||||
return atr
|
||||
|
||||
def read_hdf5(fname, datasetName=None, box=None):
|
||||
# read hdf5
|
||||
with h5py.File(fname, 'r') as f:
|
||||
data = f[datasetName][:]
|
||||
atr = dict(f.attrs)
|
||||
|
||||
return data, atr
|
||||
|
||||
def get_dataNames(FILE):
|
||||
with h5py.File(FILE, 'r') as f:
|
||||
dataNames = []
|
||||
for k0 in f.keys():
|
||||
dataNames.append(k0)
|
||||
return dataNames
|
||||
|
||||
def check_variable_name(path):
|
||||
s=path.split("/")[0]
|
||||
if len(s)>0 and s[0]=="$":
|
||||
p0=os.getenv(s[1:])
|
||||
path=path.replace(path.split("/")[0],p0)
|
||||
return path
|
||||
|
||||
def read_template(File, delimiter='='):
|
||||
'''Reads the template file into a python dictionary structure.
|
||||
Input : string, full path to the template file
|
||||
Output: dictionary, pysar template content
|
||||
Example:
|
||||
tmpl = read_template(KyushuT424F610_640AlosA.template)
|
||||
tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':')
|
||||
'''
|
||||
template_dict = {}
|
||||
for line in open(File):
|
||||
line = line.strip()
|
||||
c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter
|
||||
if len(c) < 2 or line.startswith('%') or line.startswith('#'):
|
||||
next #ignore commented lines or those without variables
|
||||
else:
|
||||
atrName = c[0]
|
||||
atrValue = str.replace(c[1],'\n','').split("#")[0].strip()
|
||||
atrValue = check_variable_name(atrValue)
|
||||
template_dict[atrName] = atrValue
|
||||
return template_dict
|
||||
|
||||
def read_gamma_par(inFile, task, keyword):
|
||||
if task == "read":
|
||||
f = open(inFile, "r")
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line: break
|
||||
if line.count(keyword) == 1:
|
||||
strtemp = line.split(":")
|
||||
value = strtemp[1].strip()
|
||||
return value
|
||||
print("Keyword " + keyword + " doesn't exist in " + inFile)
|
||||
f.close
|
||||
|
||||
|
||||
def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None):
|
||||
|
||||
if os.path.isfile(out_file):
|
||||
print('delete exsited file: {}'.format(out_file))
|
||||
os.remove(out_file)
|
||||
|
||||
print('create HDF5 file: {} with w mode'.format(out_file))
|
||||
dt = h5py.special_dtype(vlen=np.dtype('float64'))
|
||||
|
||||
|
||||
with h5py.File(out_file, 'w') as f:
|
||||
for dsName in datasetDict.keys():
|
||||
data = datasetDict[dsName]
|
||||
ds = f.create_dataset(dsName,
|
||||
data=data,
|
||||
compression=compression)
|
||||
|
||||
for key, value in metadata.items():
|
||||
f.attrs[key] = str(value)
|
||||
#print(key + ': ' + value)
|
||||
print('finished writing to {}'.format(out_file))
|
||||
|
||||
return out_file
|
||||
|
||||
######################################################################
|
||||
class progressBar:
|
||||
"""Creates a text-based progress bar. Call the object with
|
||||
the simple print command to see the progress bar, which looks
|
||||
something like this:
|
||||
[=======> 22% ]
|
||||
You may specify the progress bar's min and max values on init.
|
||||
|
||||
note:
|
||||
modified from mintPy (https://github.com/insarlab/MintPy/wiki)
|
||||
Code originally from http://code.activestate.com/recipes/168639/
|
||||
|
||||
example:
|
||||
from mintpy.utils import ptime
|
||||
date12_list = ptime.list_ifgram2date12(ifgram_list)
|
||||
prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:')
|
||||
for i in range(1000):
|
||||
prog_bar.update(i+1, suffix=date)
|
||||
prog_bar.update(i+1, suffix=date12_list[i])
|
||||
prog_bar.close()
|
||||
"""
|
||||
|
||||
def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True):
|
||||
self.prog_bar = "[]" # This holds the progress bar string
|
||||
self.min = minValue
|
||||
self.max = maxValue
|
||||
self.span = maxValue - minValue
|
||||
self.suffix = ''
|
||||
self.prefix = prefix
|
||||
|
||||
self.print_msg = print_msg
|
||||
## calculate total width based on console width
|
||||
#rows, columns = os.popen('stty size', 'r').read().split()
|
||||
#self.width = round(int(columns) * 0.7 / 10) * 10
|
||||
self.width = totalWidth
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.start_time = time.time()
|
||||
self.amount = 0 # When amount == max, we are 100% done
|
||||
self.update_amount(0) # Build progress bar string
|
||||
|
||||
def update_amount(self, newAmount=0, suffix=''):
|
||||
""" Update the progress bar with the new amount (with min and max
|
||||
values set at initialization; if it is over or under, it takes the
|
||||
min or max value as a default. """
|
||||
if newAmount < self.min:
|
||||
newAmount = self.min
|
||||
if newAmount > self.max:
|
||||
newAmount = self.max
|
||||
self.amount = newAmount
|
||||
|
||||
# Figure out the new percent done, round to an integer
|
||||
diffFromMin = np.float(self.amount - self.min)
|
||||
percentDone = (diffFromMin / np.float(self.span)) * 100.0
|
||||
percentDone = np.int(np.round(percentDone))
|
||||
|
||||
# Figure out how many hash bars the percentage should be
|
||||
allFull = self.width - 2 - 18
|
||||
numHashes = (percentDone / 100.0) * allFull
|
||||
numHashes = np.int(np.round(numHashes))
|
||||
|
||||
# Build a progress bar with an arrow of equal signs; special cases for
|
||||
# empty and full
|
||||
if numHashes == 0:
|
||||
self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1))
|
||||
elif numHashes == allFull:
|
||||
self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull)
|
||||
if suffix:
|
||||
self.prog_bar += ' %s' % (suffix)
|
||||
else:
|
||||
self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes))
|
||||
# figure out where to put the percentage, roughly centered
|
||||
percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone)))
|
||||
percentString = ' ' + str(percentDone) + '% '
|
||||
# slice the percentage into the bar
|
||||
self.prog_bar = ''.join([self.prog_bar[0:percentPlace],
|
||||
percentString,
|
||||
self.prog_bar[percentPlace+len(percentString):]])
|
||||
# prefix and suffix
|
||||
self.prog_bar = self.prefix + self.prog_bar
|
||||
if suffix:
|
||||
self.prog_bar += ' %s' % (suffix)
|
||||
# time info - elapsed time and estimated remaining time
|
||||
if percentDone > 0:
|
||||
elapsed_time = time.time() - self.start_time
|
||||
self.prog_bar += '%5ds / %5ds' % (int(elapsed_time),
|
||||
int(elapsed_time * (100./percentDone-1)))
|
||||
|
||||
def update(self, value, every=1, suffix=''):
|
||||
""" Updates the amount, and writes to stdout. Prints a
|
||||
carriage return first, so it will overwrite the current
|
||||
line in stdout."""
|
||||
if value % every == 0 or value >= self.max:
|
||||
self.update_amount(newAmount=value, suffix=suffix)
|
||||
if self.print_msg:
|
||||
sys.stdout.write('\r' + self.prog_bar)
|
||||
sys.stdout.flush()
|
||||
|
||||
def close(self):
|
||||
"""Prints a blank space at the end to ensure proper printing
|
||||
of future statements."""
|
||||
if self.print_msg:
|
||||
print(' ')
|
||||
Reference in New Issue
Block a user