Apply current workspace changes
This commit is contained in:
+158
@@ -0,0 +1,158 @@
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from data_downloader import downloader
|
||||
from faninsar.query import BoundingBox
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from .parse_email import GACOSEmail
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(
|
||||
self,
|
||||
url_file: Union[Path, str],
|
||||
output_dir: Union[Path, str],
|
||||
tar_gz_dir: Optional[Union[Path, str]] = None,
|
||||
keep_original: bool = False,
|
||||
times: Optional[Union[float, list[float]]] = None,
|
||||
bounds: Optional[tuple[float, float, float, float]] = None,
|
||||
) -> None:
|
||||
"""Initialize Downloader class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url_file : Union[Path, str]
|
||||
Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls`
|
||||
output_dir : Union[Path, str]
|
||||
directory to output gacos files
|
||||
tar_gz_dir : Optional[Union[Path, str]], optional
|
||||
directory to store downloaded *.tar.gz files. If None, then
|
||||
`output_dir` is used. Default is None.
|
||||
keep_original : bool, optional
|
||||
Whether to keep original files (*.tar.gz). Default is False.
|
||||
times : Optional[float], optional
|
||||
times of acquisition, used to filter out files that are not needed.
|
||||
this can be a single time or a list of times. times differ by less
|
||||
than 10 minutes are considered the same. Default is None.
|
||||
bounds : Optional[tuple[float, float, float, float]], optional
|
||||
bounds of area of interest with order (W, S, E, N), used to filter
|
||||
out files that are not needed. Default is None.
|
||||
"""
|
||||
self.url_file = Path(url_file)
|
||||
self.output_dir = Path(output_dir)
|
||||
if tar_gz_dir is None:
|
||||
self.tar_gz_dir = self.output_dir
|
||||
self.keep_original = keep_original
|
||||
|
||||
if not self.url_file.exists():
|
||||
raise FileNotFoundError(f"{self.url_file} does not exist")
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
if not self.tar_gz_dir.exists():
|
||||
self.tar_gz_dir.mkdir(parents=True)
|
||||
|
||||
self.df_urls = pd.read_csv(self.url_file, header=0)
|
||||
|
||||
# only keep urls that intersect with bounds
|
||||
if bounds is not None:
|
||||
mask_bbox = self._bbox_mask(BoundingBox(*bounds))
|
||||
|
||||
# only keep urls that acquisition time is within 10 minutes of `time`
|
||||
if times is not None:
|
||||
if isinstance(times, float):
|
||||
times = [times]
|
||||
mask_time = self._time_mask(times)
|
||||
|
||||
if bounds is not None and times is not None:
|
||||
self.mask = mask_bbox & mask_time
|
||||
elif bounds is not None:
|
||||
self.mask = mask_bbox
|
||||
elif times is not None:
|
||||
self.mask = mask_time
|
||||
else:
|
||||
self.mask = np.ones(self.df_urls.shape[0], dtype=bool)
|
||||
|
||||
self.mask = self.mask & self.date_mask
|
||||
|
||||
def _bbox_mask(self, bounds) -> np.ndarray:
|
||||
intersection_bbox = np.array(
|
||||
[
|
||||
BoundingBox(*b).intersects(bounds)
|
||||
for b in zip(
|
||||
self.df_urls["south"].astype(float),
|
||||
self.df_urls["west"].astype(float),
|
||||
self.df_urls["north"].astype(float),
|
||||
self.df_urls["east"].astype(float),
|
||||
)
|
||||
]
|
||||
)
|
||||
return intersection_bbox
|
||||
|
||||
def _time_mask(self, times) -> np.ndarray:
|
||||
"""Only keep urls that acquisition time is within 10 minutes of `time`"""
|
||||
intersection_times = []
|
||||
for time in times:
|
||||
intersection_times.append(
|
||||
np.array(
|
||||
np.abs(self.df_urls["time"].astype(float) - time)
|
||||
<= 1 / 60 * 10 # 10 minutes
|
||||
)
|
||||
)
|
||||
intersection_time = np.any(intersection_times, axis=0)
|
||||
return intersection_time
|
||||
|
||||
@property
|
||||
def date_mask(self) -> np.ndarray:
|
||||
"""Remove urls that all acquisition dates have been downloaded"""
|
||||
dates_urls = self.df_urls["date"].map(lambda x: eval(x))
|
||||
intersection_dates = []
|
||||
for dt_url in dates_urls:
|
||||
intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded)))
|
||||
return np.array(intersection_dates)
|
||||
|
||||
@property
|
||||
def dates_downloaded(self) -> np.ndarray:
|
||||
"""Return dates that have been downloaded"""
|
||||
gacos_files = list(self.output_dir.rglob("*.ztd.tif"))
|
||||
dates = []
|
||||
for i in gacos_files:
|
||||
stem = i.stem.split(".")[0]
|
||||
if len(stem) == 8:
|
||||
dates.append(stem)
|
||||
return np.array(dates)
|
||||
|
||||
def download(self) -> None:
|
||||
"""Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`"""
|
||||
urls_used = self.df_urls[self.mask]["url"].values
|
||||
|
||||
for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"):
|
||||
gz_file = self.tar_gz_dir / Path(url).name
|
||||
downloader.download_data(url, file_name=gz_file)
|
||||
self._extract_tar_gz(gz_file)
|
||||
if not self.keep_original:
|
||||
self._delete_file(gz_file)
|
||||
|
||||
def _extract_tar_gz(self, gz_file) -> None:
|
||||
"""Unzip/extract downloaded GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
with tarfile.open(gz_file, "r:gz") as tar:
|
||||
tar.extractall(path=self.output_dir)
|
||||
|
||||
def _delete_file(self, gz_file) -> None:
|
||||
"""Delete original GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
gz_file.unlink()
|
||||
Reference in New Issue
Block a user