Apply current workspace changes
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
from .datasets import LiCSARDataset, SarDataset, HyP3Dataset
|
||||
from .parse_email import GACOSEmail
|
||||
from .submit import Submitter
|
||||
from .download import Downloader
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
try:
|
||||
from faninsar.datasets import HyP3, LiCSAR
|
||||
except ImportError:
|
||||
try:
|
||||
from faninsar.datasets import hyp3 as HyP3, licsar as LiCSAR
|
||||
except ImportError:
|
||||
HyP3 = None
|
||||
LiCSAR = None
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
class SarDataset:
|
||||
def __init__(
|
||||
self,
|
||||
bounds: tuple[float, float, float, float],
|
||||
date_times: pd.DatetimeIndex,
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize SarDataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
bounds : tuple[float, float, float, float]
|
||||
The bounding box of the dataset.
|
||||
date_times : pd.DatetimeIndex
|
||||
The datetime index of the dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.bounds = bounds
|
||||
self._date_times = date_times
|
||||
|
||||
self._dates = date_times.strftime("%Y%m%d")
|
||||
|
||||
if gacos_dir is not None:
|
||||
self._dates_remain = self._get_dates_remain(gacos_dir)
|
||||
else:
|
||||
self._dates_remain = self.dates
|
||||
|
||||
hour = date_times.hour
|
||||
minute = np.round((date_times.second / 60) + date_times.minute).astype(int)
|
||||
times = pd.Series([f"{h:02d}:{m:02d}" for h, m in zip(hour, minute)])
|
||||
self._times = times
|
||||
|
||||
self._times_remain = self._get_times_remain()
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"{self.__class__.__name__}(\n"
|
||||
f" bounds={self.bounds}, \n"
|
||||
f" times={len(self.times)}, \n"
|
||||
f" dates={len(self.dates)}\n"
|
||||
")"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
def _get_dates_remain(self, gacos_dir: Union[Path, str]):
|
||||
"""Get the dates that are not downloaded yet.
|
||||
Parameters
|
||||
----------
|
||||
gacos_dir : Union[Path, str]
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dates_remain : np.ndarray
|
||||
The dates that are not downloaded yet.
|
||||
"""
|
||||
gacos_files = list(Path(gacos_dir).rglob("*.ztd.tif"))
|
||||
gacos_dates = []
|
||||
for i in gacos_files:
|
||||
stem = i.stem.split(".")[0]
|
||||
if len(stem) == 8:
|
||||
gacos_dates.append(stem)
|
||||
dates_remain = np.setdiff1d(self.dates, gacos_dates)
|
||||
return dates_remain
|
||||
|
||||
def _get_times_remain(self):
|
||||
"""Get the times corresponding to the dates that are not downloaded yet."""
|
||||
idx = np.where(np.isin(self.dates, self.dates_remain))[0]
|
||||
times_remain = self._times[idx]
|
||||
return times_remain
|
||||
|
||||
@property
|
||||
def dates(self):
|
||||
"""The dates (YYYYMMDD) of the acquisitions parsed from dataset."""
|
||||
return self._dates
|
||||
|
||||
@property
|
||||
def times(self):
|
||||
"""The times (HH:MM) of the acquisitions parsed from dataset."""
|
||||
return self._times.unique()
|
||||
|
||||
@property
|
||||
def date_times(self):
|
||||
"""The datetime of the acquisitions parsed from dataset."""
|
||||
return self._date_times
|
||||
|
||||
@property
|
||||
def dates_remain(self):
|
||||
"""The dates that are not downloaded yet. If gacos_dir is None, then
|
||||
dates_remain is the same as dates."""
|
||||
return self._dates_remain
|
||||
|
||||
@property
|
||||
def times_remain(self):
|
||||
"""The times corresponding to the dates that are not downloaded yet."""
|
||||
return self._times_remain
|
||||
|
||||
def gen_datetime_patches(
|
||||
self,
|
||||
mode: Literal["all", "remain"] = "remain",
|
||||
) -> dict:
|
||||
"""Generate datetime patches.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mode : Literal["all", "remain"], optional
|
||||
The mode to generate datetime patches. If "all", then generate all
|
||||
the datetime patches. If "remain", then generate the datetime
|
||||
patches of the dates that are not downloaded yet. Default is
|
||||
"remain".
|
||||
|
||||
Returns
|
||||
-------
|
||||
datetime_patches : dict
|
||||
The datetime patches. The key is the time (HH:MM) and the value is
|
||||
the datetime patches.
|
||||
"""
|
||||
nums = 20
|
||||
datetime_patches = {}
|
||||
|
||||
if mode == "all":
|
||||
for _time in self.times:
|
||||
_dts = self.dates[self._times == _time]
|
||||
n_patch = np.ceil(len(_dts) / nums)
|
||||
dates_patch = np.array_split(_dts, n_patch)
|
||||
datetime_patches[_time] = dates_patch
|
||||
elif mode == "remain":
|
||||
for _time in self.times_remain:
|
||||
_dts = self.dates_remain[self._times_remain == _time]
|
||||
n_patch = np.ceil(len(_dts) / nums)
|
||||
dates_patch = np.array_split(_dts, n_patch)
|
||||
datetime_patches[_time] = dates_patch
|
||||
|
||||
return datetime_patches
|
||||
|
||||
def gen_post_data(
|
||||
self,
|
||||
dates: Union[list, np.ndarray],
|
||||
times: Union[tuple[int, int], tuple[str, str]],
|
||||
email: str,
|
||||
):
|
||||
"""Generate post data for gacos website.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dates : list or np.ndarray
|
||||
The list of dates.
|
||||
times : tuple[int, int]
|
||||
The time of the acquisition (hour, minute).
|
||||
email : str
|
||||
The email address to receive the gacos data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
post_data : dict
|
||||
The post data.
|
||||
"""
|
||||
if isinstance(dates, np.ndarray):
|
||||
dates = dates.tolist()
|
||||
times = [int(t) for t in times]
|
||||
|
||||
post_data = {
|
||||
"N": self.bounds[3],
|
||||
"W": self.bounds[0],
|
||||
"S": self.bounds[1],
|
||||
"E": self.bounds[2],
|
||||
"H": times[0],
|
||||
"M": times[1],
|
||||
"date": "\n".join(dates),
|
||||
"type": "2",
|
||||
"email": email,
|
||||
}
|
||||
return post_data
|
||||
|
||||
|
||||
class LiCSARDataset(SarDataset):
|
||||
def __init__(
|
||||
self,
|
||||
home_dir: Union[Path, str],
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize LiCSARDataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
home_dir : Union[Path, str]
|
||||
The home directory of LiCSAR dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.home_dir = Path(home_dir)
|
||||
self.dataset = LiCSAR(home_dir)
|
||||
bounds = self.dataset.bounds
|
||||
time = self._get_time()
|
||||
dates = self.dataset.pairs.dates
|
||||
date_times = pd.to_datetime([f"{d} {time[0]}:{time[1]}:00" for d in dates])
|
||||
super().__init__(bounds, date_times, gacos_dir)
|
||||
|
||||
def _get_time(self):
|
||||
"""Get the acquisition time of acquisitions.
|
||||
|
||||
Returns
|
||||
-------
|
||||
time: tuple[int, int]
|
||||
A tuple of hour and minute representing the acquisition time.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If no center_time found in metadata.txt.
|
||||
"""
|
||||
meta_file = sorted(self.home_dir.rglob("metadata.txt"))[0]
|
||||
|
||||
with open(meta_file) as f:
|
||||
lines = f.readlines()
|
||||
time = None
|
||||
for line in lines:
|
||||
line_split = line.split("=")
|
||||
key, value = (line_split[0].strip(), line_split[1])
|
||||
if "center_time" == key:
|
||||
center_time = value.strip()
|
||||
hour, minute, second = center_time.split(":")
|
||||
hour, minute, second = int(hour), int(minute), float(second)
|
||||
minute = minute + int(np.round(second / 60, 0))
|
||||
return hour, minute
|
||||
else:
|
||||
continue
|
||||
if time is None:
|
||||
raise ValueError(f"No center_time found in {meta_file}")
|
||||
|
||||
|
||||
class HyP3Dataset(SarDataset):
|
||||
def __init__(
|
||||
self,
|
||||
home_dir: Union[Path, str],
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize HyP3Dataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
home_dir : Union[Path, str]
|
||||
The home directory of HyP3 dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.dataset = HyP3(home_dir)
|
||||
bounds = self.dataset.bounds.to_crs("epsg:4326")
|
||||
date_times = self.dataset.datetime
|
||||
|
||||
super().__init__(bounds, date_times, gacos_dir)
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from data_downloader import downloader
|
||||
from faninsar.query import BoundingBox
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from .parse_email import GACOSEmail
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(
|
||||
self,
|
||||
url_file: Union[Path, str],
|
||||
output_dir: Union[Path, str],
|
||||
tar_gz_dir: Optional[Union[Path, str]] = None,
|
||||
keep_original: bool = False,
|
||||
times: Optional[Union[float, list[float]]] = None,
|
||||
bounds: Optional[tuple[float, float, float, float]] = None,
|
||||
) -> None:
|
||||
"""Initialize Downloader class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url_file : Union[Path, str]
|
||||
Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls`
|
||||
output_dir : Union[Path, str]
|
||||
directory to output gacos files
|
||||
tar_gz_dir : Optional[Union[Path, str]], optional
|
||||
directory to store downloaded *.tar.gz files. If None, then
|
||||
`output_dir` is used. Default is None.
|
||||
keep_original : bool, optional
|
||||
Whether to keep original files (*.tar.gz). Default is False.
|
||||
times : Optional[float], optional
|
||||
times of acquisition, used to filter out files that are not needed.
|
||||
this can be a single time or a list of times. times differ by less
|
||||
than 10 minutes are considered the same. Default is None.
|
||||
bounds : Optional[tuple[float, float, float, float]], optional
|
||||
bounds of area of interest with order (W, S, E, N), used to filter
|
||||
out files that are not needed. Default is None.
|
||||
"""
|
||||
self.url_file = Path(url_file)
|
||||
self.output_dir = Path(output_dir)
|
||||
if tar_gz_dir is None:
|
||||
self.tar_gz_dir = self.output_dir
|
||||
self.keep_original = keep_original
|
||||
|
||||
if not self.url_file.exists():
|
||||
raise FileNotFoundError(f"{self.url_file} does not exist")
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
if not self.tar_gz_dir.exists():
|
||||
self.tar_gz_dir.mkdir(parents=True)
|
||||
|
||||
self.df_urls = pd.read_csv(self.url_file, header=0)
|
||||
|
||||
# only keep urls that intersect with bounds
|
||||
if bounds is not None:
|
||||
mask_bbox = self._bbox_mask(BoundingBox(*bounds))
|
||||
|
||||
# only keep urls that acquisition time is within 10 minutes of `time`
|
||||
if times is not None:
|
||||
if isinstance(times, float):
|
||||
times = [times]
|
||||
mask_time = self._time_mask(times)
|
||||
|
||||
if bounds is not None and times is not None:
|
||||
self.mask = mask_bbox & mask_time
|
||||
elif bounds is not None:
|
||||
self.mask = mask_bbox
|
||||
elif times is not None:
|
||||
self.mask = mask_time
|
||||
else:
|
||||
self.mask = np.ones(self.df_urls.shape[0], dtype=bool)
|
||||
|
||||
self.mask = self.mask & self.date_mask
|
||||
|
||||
def _bbox_mask(self, bounds) -> np.ndarray:
|
||||
intersection_bbox = np.array(
|
||||
[
|
||||
BoundingBox(*b).intersects(bounds)
|
||||
for b in zip(
|
||||
self.df_urls["south"].astype(float),
|
||||
self.df_urls["west"].astype(float),
|
||||
self.df_urls["north"].astype(float),
|
||||
self.df_urls["east"].astype(float),
|
||||
)
|
||||
]
|
||||
)
|
||||
return intersection_bbox
|
||||
|
||||
def _time_mask(self, times) -> np.ndarray:
|
||||
"""Only keep urls that acquisition time is within 10 minutes of `time`"""
|
||||
intersection_times = []
|
||||
for time in times:
|
||||
intersection_times.append(
|
||||
np.array(
|
||||
np.abs(self.df_urls["time"].astype(float) - time)
|
||||
<= 1 / 60 * 10 # 10 minutes
|
||||
)
|
||||
)
|
||||
intersection_time = np.any(intersection_times, axis=0)
|
||||
return intersection_time
|
||||
|
||||
@property
|
||||
def date_mask(self) -> np.ndarray:
|
||||
"""Remove urls that all acquisition dates have been downloaded"""
|
||||
dates_urls = self.df_urls["date"].map(lambda x: eval(x))
|
||||
intersection_dates = []
|
||||
for dt_url in dates_urls:
|
||||
intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded)))
|
||||
return np.array(intersection_dates)
|
||||
|
||||
@property
|
||||
def dates_downloaded(self) -> np.ndarray:
|
||||
"""Return dates that have been downloaded"""
|
||||
gacos_files = list(self.output_dir.rglob("*.ztd.tif"))
|
||||
dates = []
|
||||
for i in gacos_files:
|
||||
stem = i.stem.split(".")[0]
|
||||
if len(stem) == 8:
|
||||
dates.append(stem)
|
||||
return np.array(dates)
|
||||
|
||||
def download(self) -> None:
|
||||
"""Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`"""
|
||||
urls_used = self.df_urls[self.mask]["url"].values
|
||||
|
||||
for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"):
|
||||
gz_file = self.tar_gz_dir / Path(url).name
|
||||
downloader.download_data(url, file_name=gz_file)
|
||||
self._extract_tar_gz(gz_file)
|
||||
if not self.keep_original:
|
||||
self._delete_file(gz_file)
|
||||
|
||||
def _extract_tar_gz(self, gz_file) -> None:
|
||||
"""Unzip/extract downloaded GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
with tarfile.open(gz_file, "r:gz") as tar:
|
||||
tar.extractall(path=self.output_dir)
|
||||
|
||||
def _delete_file(self, gz_file) -> None:
|
||||
"""Delete original GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
gz_file.unlink()
|
||||
@@ -0,0 +1,391 @@
|
||||
import email
|
||||
import email.message
|
||||
import getpass
|
||||
import imaplib
|
||||
import poplib
|
||||
import re
|
||||
from email.parser import Parser
|
||||
from email.utils import parseaddr
|
||||
from pathlib import Path
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
class GACOSEmail:
|
||||
"""a class to retrieve gacos urls from email.
|
||||
|
||||
.. note::
|
||||
The IMAP server is used to retrieve content from email. Some email
|
||||
service providers may need to enable the IMAP service in the settings.
|
||||
**You are recommended to use a new email account to receive gacos urls to
|
||||
avoid polluting your own email account**.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
host: str,
|
||||
prompt: bool = False,
|
||||
email_protocol: Literal["imap", "pop3"] = "imap",
|
||||
port: Optional[int] = None,
|
||||
gacos_email: str = "gacos2017@foxmail.com",
|
||||
gacos_suffix: str = "tar.gz",
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
date_args: Optional[dict] = None,
|
||||
ssl: bool = False,
|
||||
) -> None:
|
||||
"""Retrieve gacos urls from email.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
username : str
|
||||
The username of the email address.
|
||||
password : str
|
||||
The password.
|
||||
host : str
|
||||
The host of the email address. For example, the host of gmail for
|
||||
imap is "imap.gmail.com". You can find the host of your email
|
||||
settings or search it on the Internet.
|
||||
prompt: bool, optional
|
||||
Prompt for username and/or password interactively when they are not
|
||||
provided as keyword parameters. Default is False.
|
||||
email_protocol : str, one of ["imap", "pop3"], optional
|
||||
The protocol of the email. Default is "imap".
|
||||
port : int, optional
|
||||
The port of the host of your email. If None, the default port will be
|
||||
used. Default is None.
|
||||
gacos_email : str, optional
|
||||
The email address of gacos. Default is "gacos2017@foxmail.com".
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
start_date / end_date: str, optional
|
||||
The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime.
|
||||
date_args : dict, optional
|
||||
The arguments are passed to pandas.to_datetime. Default is None.
|
||||
ssl : bool, optional
|
||||
Whether to use SSL connection. Default is False.
|
||||
"""
|
||||
if prompt:
|
||||
self.username = None
|
||||
self.password = None
|
||||
else:
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.host = host
|
||||
self.email_protocol = email_protocol
|
||||
self.port = port
|
||||
self.gacos_email = gacos_email
|
||||
self.gacos_suffix = gacos_suffix
|
||||
self.ssl = ssl
|
||||
|
||||
# date part
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
if date_args is None:
|
||||
date_args = {}
|
||||
self.date_args = date_args
|
||||
|
||||
def _retrieve_gacos_urls_pop3(self):
|
||||
server = login_in_email_pop3(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
print(server.getwelcome())
|
||||
|
||||
nums = server.stat()[0]
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"):
|
||||
response, msgLines, octets = server.retr(i)
|
||||
msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore")
|
||||
messageObject = Parser().parsestr(msgLinesToStr)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.quit()
|
||||
|
||||
return gacos
|
||||
|
||||
def _retrieve_gacos_urls_imap(self):
|
||||
server = login_in_email_imap(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
if server is None:
|
||||
print("IMAP server connection failed, skipping email check.")
|
||||
return []
|
||||
server.select("inbox")
|
||||
status, data = server.search(None, "ALL")
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"):
|
||||
res, msg = server.fetch(i, "(RFC822)")
|
||||
for response_part in msg:
|
||||
if isinstance(response_part, tuple):
|
||||
msgLines = response_part[1].decode("utf8", "ignore")
|
||||
break
|
||||
|
||||
messageObject = Parser().parsestr(msgLines)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.close()
|
||||
|
||||
return gacos
|
||||
|
||||
def retrieve_gacos_urls(
|
||||
self,
|
||||
output_file: Union[str, Path],
|
||||
):
|
||||
"""Retrieve gacos urls from username.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
output_file : str or Path
|
||||
The output file used to save the gacos urls.
|
||||
"""
|
||||
if self.email_protocol == "pop3":
|
||||
gacos = self._retrieve_gacos_urls_pop3()
|
||||
elif self.email_protocol == "imap":
|
||||
gacos = self._retrieve_gacos_urls_imap()
|
||||
else:
|
||||
raise ValueError("email_protocol must be 'pop3' or 'imap'.")
|
||||
|
||||
cols = ["url", "south", "north", "west", "east", "time", "date"]
|
||||
df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url")
|
||||
|
||||
# save to file
|
||||
try:
|
||||
df_gacos.to_csv(output_file)
|
||||
print(f"Save gacos urls to {output_file}")
|
||||
except Exception as e:
|
||||
self.df_gacos = df_gacos
|
||||
print(e)
|
||||
print("Save gacos urls failed")
|
||||
print("You can access the gacos urls by `df_gacos` attribute.")
|
||||
|
||||
|
||||
def in_date_range(date, start_date, end_date, date_args={}):
|
||||
start_date = pd.to_datetime(start_date, **date_args)
|
||||
end_date = pd.to_datetime(end_date, **date_args)
|
||||
start_none = start_date is None or pd.isna(start_date)
|
||||
end_none = end_date is None or pd.isna(end_date)
|
||||
if start_none and end_none:
|
||||
return True
|
||||
elif start_none:
|
||||
return date <= end_date
|
||||
elif end_none:
|
||||
return date >= start_date
|
||||
else:
|
||||
return (date >= start_date) and (date <= end_date)
|
||||
|
||||
|
||||
def decodeBody(msgPart: email.message.Message):
|
||||
"""decode email body
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgPart : email.message.Message
|
||||
The email message object.
|
||||
"""
|
||||
contentType = msgPart.get_content_type()
|
||||
textContent = ""
|
||||
if contentType == "text/plain" or contentType == "text/html":
|
||||
content = msgPart.get_payload(decode=True)
|
||||
charset = msgPart.get_charset()
|
||||
if charset is None:
|
||||
contentType = msgPart.get("Content-Type", "").lower()
|
||||
position = contentType.find("charset=")
|
||||
if position >= 0:
|
||||
charset = contentType[position + 8 :].strip()
|
||||
if charset:
|
||||
textContent = content.decode(charset)
|
||||
return textContent
|
||||
|
||||
|
||||
def get_content(messageObject):
|
||||
msgBodyContents = []
|
||||
if messageObject.is_multipart(): # parse multipart email
|
||||
messageParts = messageObject.get_payload()
|
||||
for messagePart in messageParts:
|
||||
bodyContent = decodeBody(messagePart)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
else:
|
||||
bodyContent = decodeBody(messageObject)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
return msgBodyContents
|
||||
|
||||
|
||||
def login_in_email_pop3(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 995
|
||||
server = poplib.POP3_SSL(host, port)
|
||||
else:
|
||||
if port is None:
|
||||
port = 110
|
||||
server = poplib.POP3(host, port)
|
||||
|
||||
server.user(username)
|
||||
server.pass_(password)
|
||||
return server
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("login failed")
|
||||
|
||||
|
||||
def login_in_email_imap(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 993
|
||||
else:
|
||||
if port is None:
|
||||
port = 143
|
||||
|
||||
# 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达
|
||||
import socket as _socket
|
||||
_orig_getaddrinfo = _socket.getaddrinfo
|
||||
def _ipv4_only_getaddrinfo(*args, **kwargs):
|
||||
return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET,
|
||||
*args[3:], **kwargs)
|
||||
_socket.getaddrinfo = _ipv4_only_getaddrinfo
|
||||
try:
|
||||
if ssl:
|
||||
server = imaplib.IMAP4_SSL(host, port)
|
||||
else:
|
||||
server = imaplib.IMAP4(host, port)
|
||||
finally:
|
||||
_socket.getaddrinfo = _orig_getaddrinfo
|
||||
|
||||
server.login(username, password)
|
||||
|
||||
# 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT
|
||||
if '163.com' in host or '126.com' in host or 'yeah.net' in host:
|
||||
try:
|
||||
tag = server._new_tag()
|
||||
server.send(tag + b' ID ("name" "pyint" "version" "1.0" '
|
||||
b'"vendor" "pyint")\r\n')
|
||||
while True:
|
||||
resp = server.readline()
|
||||
if resp.startswith(tag):
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return server
|
||||
except Exception as e:
|
||||
print(f"IMAP login failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"):
|
||||
"""Parse gacos info from email body.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgBodyContents : list
|
||||
The email body contents.
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
"""
|
||||
|
||||
url, south, north, west, east, _time, date_list = (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
date_list = []
|
||||
for contents in msgBodyContents:
|
||||
lines = [i.strip() for i in contents.split("\n") if i]
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
loc = line.split("=")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "MinLat" == parameter:
|
||||
south = float(value)
|
||||
if "MaxLat" == parameter:
|
||||
north = float(value)
|
||||
if "MinLon" == parameter:
|
||||
west = float(value)
|
||||
if "MaxLon" == parameter:
|
||||
east = float(value)
|
||||
loc = line.split(":")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "Time" == parameter:
|
||||
_time = float(value)
|
||||
|
||||
if len(line) == 8 and line.isdigit():
|
||||
date_list.append(line)
|
||||
|
||||
for i in ["http", "ftp", "https"]:
|
||||
result = re.search(f"\({i}.*{gacos_suffix}\)", line)
|
||||
if result:
|
||||
url = result.group()[1:-1]
|
||||
break
|
||||
|
||||
if url == south == north == west == east == _time:
|
||||
return None
|
||||
else:
|
||||
return url, south, north, west, east, _time, date_list
|
||||
@@ -0,0 +1,83 @@
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from .datasets import SarDataset
|
||||
|
||||
|
||||
class Submitter:
|
||||
def __init__(
|
||||
self,
|
||||
dataset: SarDataset,
|
||||
email: str,
|
||||
sleep_time_range: tuple[int, int] = (60 / 2, 60 * 5),
|
||||
gacos_url="http://www.gacos.net/M/action_page.php",
|
||||
) -> None:
|
||||
"""Initialize Submitter class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dataset : SarDataset
|
||||
The SarDataset object.
|
||||
email : str
|
||||
The email address to submit to gacos.
|
||||
sleep_time_range : tuple[int, int], optional
|
||||
The range of sleep time in seconds. Default is (60, 60 * 5).
|
||||
gacos_url : str, optional
|
||||
The url of gacos website. Default is "http://www.gacos.net/M/action_page.php".
|
||||
"""
|
||||
self.dataset = dataset
|
||||
self.email = email
|
||||
self.sleep_time_range = sleep_time_range
|
||||
self.gacos_url = gacos_url
|
||||
|
||||
self._failed = []
|
||||
self._succeed = []
|
||||
|
||||
def _post_data(self, data):
|
||||
"""Post data to gacos website."""
|
||||
r = requests.post(self.gacos_url, data=data)
|
||||
return "Thanks for using GACOS!" in r.text
|
||||
|
||||
def post_requests(self):
|
||||
# post gacos info to website
|
||||
datetime_patches = self.dataset.gen_datetime_patches()
|
||||
for _key, _dates in tqdm(
|
||||
datetime_patches.items(),
|
||||
desc="submitting times",
|
||||
unit="times",
|
||||
):
|
||||
try:
|
||||
for _dt in tqdm(_dates, desc="submitting dates", unit="dates"):
|
||||
post_data = self.dataset.gen_post_data(
|
||||
_dt, _key.split(":"), self.email
|
||||
)
|
||||
status_ok = self._post_data(post_data)
|
||||
if status_ok:
|
||||
self._succeed.append(post_data)
|
||||
tqdm.write(f">>> succeed post: {post_data}")
|
||||
else:
|
||||
self._failed.append(post_data)
|
||||
tqdm.write(f">>> failed post: {post_data}")
|
||||
|
||||
# wait to avoid be rejected
|
||||
sleep_time = np.random.randint(*self.sleep_time_range)
|
||||
tqdm.write(f" sleeping for {sleep_time} seconds...")
|
||||
time.sleep(sleep_time)
|
||||
except:
|
||||
self._failed.append(post_data)
|
||||
tqdm.write(f">>> failed post: {post_data}")
|
||||
|
||||
@property
|
||||
def failed(self):
|
||||
"""A list of failed post data."""
|
||||
return self._failed
|
||||
|
||||
@property
|
||||
def succeed(self):
|
||||
"""A list of succeed post data."""
|
||||
return self._succeed
|
||||
Reference in New Issue
Block a user