Apply current workspace changes
This commit is contained in:
@@ -0,0 +1,167 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# mac
|
||||
.DS_Store
|
||||
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
cover/
|
||||
tests
|
||||
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
.pybuilder/
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
# For a library or package, you might want to ignore these files since the code is
|
||||
# intended to run in multiple environments; otherwise, check them in:
|
||||
# .python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# poetry
|
||||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
||||
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
||||
# commonly ignored for libraries.
|
||||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
||||
#poetry.lock
|
||||
|
||||
# pdm
|
||||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
||||
#pdm.lock
|
||||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
||||
# in version control.
|
||||
# https://pdm.fming.dev/#use-with-ide
|
||||
.pdm.toml
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# Cython debug symbols
|
||||
cython_debug/
|
||||
|
||||
# PyCharm
|
||||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
||||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
||||
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
||||
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
||||
#.idea/
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 Fanchengyan
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,2 @@
|
||||
# AutoGACOS
|
||||
A Python library for automatically submitting and downloading GACOS data
|
||||
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
SPHINXOPTS ?=
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
@@ -0,0 +1,6 @@
|
||||
recommonmark
|
||||
sphinx>=7
|
||||
myst-parser
|
||||
myst_nb
|
||||
sphinx_rtd_theme
|
||||
Jinja2
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
InSAR Datasets
|
||||
==============
|
||||
|
||||
To automatically submit requests to the GACOS service, the user needs to provide
|
||||
the coordinates of the area of interest and the acquisition dates and times of
|
||||
the SAR images. We provide a set of classes to automatically retrieve the
|
||||
those information from well known InSAR datasets.
|
||||
|
||||
Currently, the following InSAR Datasets are supported:
|
||||
* :class:`.HyP3Dataset`
|
||||
* :class:`.LiCSARDataset`
|
||||
|
||||
HyP3Dataset
|
||||
-----------
|
||||
|
||||
.. autoclass:: gacos.HyP3Dataset
|
||||
:members:
|
||||
:undoc-members:
|
||||
:member-order: bysource
|
||||
:show-inheritance:
|
||||
|
||||
LiCSARDataset
|
||||
-------------
|
||||
|
||||
.. autoclass:: gacos.LiCSARDataset
|
||||
:members:
|
||||
:undoc-members:
|
||||
:member-order: bysource
|
||||
:show-inheritance:
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
Python API Reference
|
||||
====================
|
||||
|
||||
|
||||
.. toctree::
|
||||
|
||||
datasets/datasets
|
||||
submit/submit
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
Requests Submitter
|
||||
==================
|
||||
|
||||
The GACOS limits the number of acquisition requests that can be submitted at a time.
|
||||
This is to prevent the system from being overloaded. But, if you have a large number
|
||||
of acquisition requests to submit, you can submit them in batches. We designed a
|
||||
:class:`gacos.Submitter` class to help you submit the requests in batches. The
|
||||
acquisitions will be split into batches which not exceed the 20 acquisitions per
|
||||
request.
|
||||
|
||||
.. autoclass:: gacos.Submitter
|
||||
:members:
|
||||
:undoc-members:
|
||||
:member-order: bysource
|
||||
:show-inheritance:
|
||||
@@ -0,0 +1,58 @@
|
||||
# Configuration file for the Sphinx documentation builder.
|
||||
#
|
||||
# For the full list of built-in configuration values, see the documentation:
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html
|
||||
|
||||
# -- Project information -----------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
|
||||
|
||||
project = "FanSAR"
|
||||
copyright = "2023, Fancy"
|
||||
author = "Fancy"
|
||||
release = "v1.0"
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
|
||||
|
||||
extensions = [
|
||||
"myst_nb",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.doctest",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx.ext.todo",
|
||||
]
|
||||
source_suffix = {
|
||||
".rst": "restructuredtext",
|
||||
".md": "markdown",
|
||||
".ipynb": "myst-nb",
|
||||
}
|
||||
|
||||
# templates_path = ['_templates']
|
||||
exclude_patterns = []
|
||||
|
||||
# to disable execution of notebooks
|
||||
nb_execution_mode = "off"
|
||||
nb_execution_excludepatterns = ["quickstart.ipynb"]
|
||||
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
|
||||
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
# html_static_path = ['_static']
|
||||
|
||||
# General information about the project.
|
||||
project = "faninsar"
|
||||
copyright = "2023, Fancy"
|
||||
author = "Chengyan Fan"
|
||||
|
||||
video_enforce_extra_source = True
|
||||
|
||||
autodoc_default_options = {
|
||||
"members": True,
|
||||
"undoc-members": True,
|
||||
"member-order": "bysource",
|
||||
"special-members": "__init__",
|
||||
":show-inheritance:": True,
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
=====================================
|
||||
Welcome to AutoGACOS's documentation!
|
||||
=====================================
|
||||
|
||||
|
||||
Introduction
|
||||
------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 4
|
||||
:caption: Contents:
|
||||
|
||||
intro
|
||||
user_guide/quickstart
|
||||
AutoGACOS API Reference <api/index>
|
||||
terminology
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
Introduction
|
||||
============
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
.. _terminology:
|
||||
|
||||
Terminology
|
||||
===========
|
||||
|
||||
.. glossary::
|
||||
|
||||
Acquisition
|
||||
A single SAR acquisition, and is expressed as ``datetime.datetime`` object.
|
||||
|
||||
Pair
|
||||
A pair is a combination of two SAR acquisitions.
|
||||
|
||||
Pairs
|
||||
A collection of pairs.
|
||||
|
||||
Loop
|
||||
A loop
|
||||
|
||||
Loops
|
||||
A collection of loops.
|
||||
|
||||
SBASNetwork
|
||||
A collection of loops and pairs.
|
||||
|
||||
CRS
|
||||
A coordinate reference system (CRS) is a coordinate-based local, regional or global system used to locate geographical entities. In FanInSAR, the CRS is handled by the ``rasterio`` package.
|
||||
+635
@@ -0,0 +1,635 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Quickstart for AutoGACOS\n",
|
||||
"\n",
|
||||
"**AutoGACOS** is a Python library for automatically submitting and downloading GACOS data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import gacos"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"path = \"/Volumes/Data/Hyp3/descending_roi\"\n",
|
||||
"ds = gacos.HyP3Dataset(path)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"check all acquisition dates in Hyp3 dateset"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"DatetimeIndex(['2015-11-12 23:26:31', '2015-12-06 23:26:31',\n",
|
||||
" '2015-12-30 23:26:30', '2016-01-23 23:26:29',\n",
|
||||
" '2016-02-16 23:26:28', '2016-03-11 23:26:28',\n",
|
||||
" '2016-04-04 23:26:29', '2016-04-28 23:26:30',\n",
|
||||
" '2016-05-22 23:26:34', '2016-06-15 23:26:36',\n",
|
||||
" ...\n",
|
||||
" '2023-01-10 23:27:15', '2023-01-22 23:27:15',\n",
|
||||
" '2023-02-03 23:27:15', '2023-02-15 23:27:14',\n",
|
||||
" '2023-02-27 23:27:15', '2023-03-11 23:27:14',\n",
|
||||
" '2023-03-23 23:27:14', '2023-04-04 23:27:15',\n",
|
||||
" '2023-08-14 23:27:22', '2023-09-07 23:27:23'],\n",
|
||||
" dtype='datetime64[ns]', length=204, freq=None)\n",
|
||||
"Index(['20151112', '20151206', '20151230', '20160123', '20160216', '20160311',\n",
|
||||
" '20160404', '20160428', '20160522', '20160615',\n",
|
||||
" ...\n",
|
||||
" '20230110', '20230122', '20230203', '20230215', '20230227', '20230311',\n",
|
||||
" '20230323', '20230404', '20230814', '20230907'],\n",
|
||||
" dtype='object', length=204)\n",
|
||||
"['23:25' '23:26' '23:27']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"print(ds.date_times)\n",
|
||||
"print(ds.dates)\n",
|
||||
"print(ds.times)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Submit GACOS requests automatically"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "bf9575b9deea497089cc8167b8c8a438",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/3 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a9dafb6856a5407da0dbf6b965189f73",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/1 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 25, 'date': '20161007\\n20161031\\n20161124\\n20161218\\n20170111\\n20170204', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
" sleeping for 546 seconds...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "8fd9eb9543774f3b87058eef331792df",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/6 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20151112\\n20151206\\n20151230\\n20160123\\n20160216\\n20160311\\n20160404\\n20160428\\n20160522\\n20160615\\n20160709\\n20160802\\n20160826\\n20160919\\n20170222\\n20170318\\n20170330', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20170411\\n20170423\\n20170505\\n20170517\\n20170529\\n20170610\\n20170622\\n20170704\\n20170716\\n20170728\\n20170809\\n20170821\\n20170902\\n20170914\\n20171008\\n20171020\\n20171101', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20171113\\n20171125\\n20171207\\n20171219\\n20171231\\n20180112\\n20180124\\n20180205\\n20180217\\n20180301\\n20180313\\n20180325\\n20180406\\n20180418\\n20180430\\n20180512\\n20180524', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20180605\\n20180617\\n20180629\\n20180711\\n20180723\\n20180804\\n20180816\\n20180921\\n20181003\\n20181015\\n20181027\\n20181108\\n20181120\\n20181202\\n20181214\\n20181226\\n20190107', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190119\\n20190131\\n20190212\\n20190224\\n20190308\\n20190320\\n20190401\\n20190413\\n20190425\\n20190507\\n20190519\\n20190531\\n20190612\\n20190624\\n20190706\\n20190718\\n20190730', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190811\\n20190823\\n20190904\\n20191221\\n20200102\\n20200114\\n20200126\\n20200207\\n20200219\\n20200302\\n20200314\\n20200326\\n20200407\\n20200419\\n20200501\\n20200513', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
" sleeping for 288 seconds...\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "4aa82286c5414723adcddcc8d864c133",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/5 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20190916\\n20190928\\n20191010\\n20191022\\n20191103\\n20191115\\n20191127\\n20191209\\n20200525\\n20200606\\n20200618\\n20200630\\n20200712\\n20200724\\n20200805\\n20200817\\n20200829\\n20200910\\n20200922\\n20201004', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20201016\\n20201028\\n20201109\\n20201121\\n20201203\\n20201215\\n20201227\\n20210108\\n20210120\\n20210201\\n20210213\\n20210225\\n20210309\\n20210321\\n20210402\\n20210414\\n20210426\\n20210508\\n20210520\\n20210601', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20210613\\n20210625\\n20210707\\n20210719\\n20210731\\n20210812\\n20210824\\n20210905\\n20210917\\n20210929\\n20211011\\n20211023\\n20211104\\n20211116\\n20211128\\n20211210\\n20211222\\n20220103\\n20220115', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220127\\n20220208\\n20220220\\n20220304\\n20220316\\n20220328\\n20220409\\n20220421\\n20220503\\n20220515\\n20220527\\n20220620\\n20220702\\n20220714\\n20220726\\n20220807\\n20220819\\n20220831\\n20220912', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220924\\n20221006\\n20221018\\n20221030\\n20221111\\n20221123\\n20221205\\n20221217\\n20221229\\n20230110\\n20230122\\n20230203\\n20230215\\n20230227\\n20230311\\n20230323\\n20230404\\n20230814\\n20230907', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n",
|
||||
" sleeping for 743 seconds...\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# initialize a submitter with the dataset and your email\n",
|
||||
"submitter = gacos.Submitter(ds, email=\"your_email@xxx.com\")\n",
|
||||
"\n",
|
||||
"# submit all gacos requests for acquisitions in the Hyp3 dataset\n",
|
||||
"submitter.post_requests()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# check the status of all requests\n",
|
||||
"print(f\"Successfully submitted requests: \\n{submitter.succeed}\")\n",
|
||||
"print(f\"Failed to submit requests: \\n{submitter.failed}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Retrieve GACOS results urls from your email automatically"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "667dccc73a434102a87226ebcb453389",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Retrieving GACOS urls: 0%| | 0/240 [00:00<?, ? emails/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"output_file = \"/Volumes/Data/Hyp3/descending_gacos/gacos_urls.csv\"\n",
|
||||
"\n",
|
||||
"email = gacos.GACOSEmail(\n",
|
||||
" username=\"your_email\",\n",
|
||||
" password=\"password_of_your_email\",\n",
|
||||
" host=\"imap.xxx.com\", # imap server of your email\n",
|
||||
" start_date=\"2023-11-01\", # only download gacos results for the email after this date\n",
|
||||
")\n",
|
||||
"email.retrieve_gacos_urls(output_file)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Download GACOS results using the urls "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b99066adfc1c4f24aa2164540cd03ce7",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/24 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ed443926d42e480ca0d5d835ee23b377",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T010337VzTsCTN1h.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a40ea2c7a8da49eebc8dcb46d6e528ae",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011307V7OXOyZBe.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "478bae1d3f9545e7936e0436e21f48f3",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011337VC4LikOAs.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "35e60690588147028abbc997e1b0fdf6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011408V6h3BniJT.tar.gz: 0%| | 0.00/41.1M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b5797b66e4044865acac24bb6bdbe4bc",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011237Vy7m1l2al.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ab235b358f7d477c9a8f367b1c969e04",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011738Vniy7uNPV.tar.gz: 0%| | 0.00/51.2M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "edd581a5a24049cfa884c7d47ff5fd6c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011808VcEaTHazf.tar.gz: 0%| | 0.00/48.8M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f4501c922a984f8e890e2196c8717742",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T011838VJaHXOaZJ.tar.gz: 0%| | 0.00/48.8M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "025da6a325394f6195555963ebc48f30",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T142253VdcxwhZjh.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1e10f5c2420047dd9a20476e990be884",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T142324VXIAESwIW.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7f048fd91a7246f1adaa81de3c33a6f8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T142354VQj8kS9I0.tar.gz: 0%| | 0.00/43.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b819434541c540898948615a24650eab",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T142424VMFCCq5W7.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "9b2abbf34daf4b12b44a44dc6b902ed2",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T144425VSlcfYAm2.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "89261f821810404281862ba336f56eb6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T144455VCncFDojU.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d55f37d845454a0d90254ab82686b5d0",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T144525VisGJO2bU.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "75bd725bd9bf43b0bfe03682db91c6a2",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T144555VcE1J5wo4.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "53773e7ba770412da3b45fb63d014a98",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T145756VCHtI731i.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b7848836c32244759118c616b458a3c4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T145827VhbjZbz5b.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f4f396e98e134ea08c8b6f5f9aa6138f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T145857VUeNu4WnY.tar.gz: 0%| | 0.00/43.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7ef6ff94335c4db38c7f4d20a07d9f29",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T145927V2I8j7eo6.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1979604e57d3473b9ea7af1cd2e112bb",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T151057VQMukfvJ5.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7fcd89254e0e4335a20440b2cff80bcc",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T151158Vx3KrT8Va.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "7550835c8d834f8895efea89ede2d308",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T151228VcpWSrjqg.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d42edcb7dfd546aca10f4cd4635e3cc1",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"20231116T151258VfhhwgAli.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"url_file = output_file\n",
|
||||
"output_dir = \"/Volumes/Data/Hyp3/descending_gacos\"\n",
|
||||
"\n",
|
||||
"gacos_dl = gacos.Downloader(url_file, output_dir, time=23.43)\n",
|
||||
"gacos_dl.download()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "geo",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
from .datasets import LiCSARDataset, SarDataset, HyP3Dataset
|
||||
from .parse_email import GACOSEmail
|
||||
from .submit import Submitter
|
||||
from .download import Downloader
|
||||
@@ -0,0 +1,276 @@
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
try:
|
||||
from faninsar.datasets import HyP3, LiCSAR
|
||||
except ImportError:
|
||||
try:
|
||||
from faninsar.datasets import hyp3 as HyP3, licsar as LiCSAR
|
||||
except ImportError:
|
||||
HyP3 = None
|
||||
LiCSAR = None
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
|
||||
|
||||
class SarDataset:
|
||||
def __init__(
|
||||
self,
|
||||
bounds: tuple[float, float, float, float],
|
||||
date_times: pd.DatetimeIndex,
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize SarDataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
bounds : tuple[float, float, float, float]
|
||||
The bounding box of the dataset.
|
||||
date_times : pd.DatetimeIndex
|
||||
The datetime index of the dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.bounds = bounds
|
||||
self._date_times = date_times
|
||||
|
||||
self._dates = date_times.strftime("%Y%m%d")
|
||||
|
||||
if gacos_dir is not None:
|
||||
self._dates_remain = self._get_dates_remain(gacos_dir)
|
||||
else:
|
||||
self._dates_remain = self.dates
|
||||
|
||||
hour = date_times.hour
|
||||
minute = np.round((date_times.second / 60) + date_times.minute).astype(int)
|
||||
times = pd.Series([f"{h:02d}:{m:02d}" for h, m in zip(hour, minute)])
|
||||
self._times = times
|
||||
|
||||
self._times_remain = self._get_times_remain()
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"{self.__class__.__name__}(\n"
|
||||
f" bounds={self.bounds}, \n"
|
||||
f" times={len(self.times)}, \n"
|
||||
f" dates={len(self.dates)}\n"
|
||||
")"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return self.__str__()
|
||||
|
||||
def _get_dates_remain(self, gacos_dir: Union[Path, str]):
|
||||
"""Get the dates that are not downloaded yet.
|
||||
Parameters
|
||||
----------
|
||||
gacos_dir : Union[Path, str]
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dates_remain : np.ndarray
|
||||
The dates that are not downloaded yet.
|
||||
"""
|
||||
gacos_files = list(Path(gacos_dir).rglob("*.ztd.tif"))
|
||||
gacos_dates = []
|
||||
for i in gacos_files:
|
||||
stem = i.stem.split(".")[0]
|
||||
if len(stem) == 8:
|
||||
gacos_dates.append(stem)
|
||||
dates_remain = np.setdiff1d(self.dates, gacos_dates)
|
||||
return dates_remain
|
||||
|
||||
def _get_times_remain(self):
|
||||
"""Get the times corresponding to the dates that are not downloaded yet."""
|
||||
idx = np.where(np.isin(self.dates, self.dates_remain))[0]
|
||||
times_remain = self._times[idx]
|
||||
return times_remain
|
||||
|
||||
@property
|
||||
def dates(self):
|
||||
"""The dates (YYYYMMDD) of the acquisitions parsed from dataset."""
|
||||
return self._dates
|
||||
|
||||
@property
|
||||
def times(self):
|
||||
"""The times (HH:MM) of the acquisitions parsed from dataset."""
|
||||
return self._times.unique()
|
||||
|
||||
@property
|
||||
def date_times(self):
|
||||
"""The datetime of the acquisitions parsed from dataset."""
|
||||
return self._date_times
|
||||
|
||||
@property
|
||||
def dates_remain(self):
|
||||
"""The dates that are not downloaded yet. If gacos_dir is None, then
|
||||
dates_remain is the same as dates."""
|
||||
return self._dates_remain
|
||||
|
||||
@property
|
||||
def times_remain(self):
|
||||
"""The times corresponding to the dates that are not downloaded yet."""
|
||||
return self._times_remain
|
||||
|
||||
def gen_datetime_patches(
|
||||
self,
|
||||
mode: Literal["all", "remain"] = "remain",
|
||||
) -> dict:
|
||||
"""Generate datetime patches.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mode : Literal["all", "remain"], optional
|
||||
The mode to generate datetime patches. If "all", then generate all
|
||||
the datetime patches. If "remain", then generate the datetime
|
||||
patches of the dates that are not downloaded yet. Default is
|
||||
"remain".
|
||||
|
||||
Returns
|
||||
-------
|
||||
datetime_patches : dict
|
||||
The datetime patches. The key is the time (HH:MM) and the value is
|
||||
the datetime patches.
|
||||
"""
|
||||
nums = 20
|
||||
datetime_patches = {}
|
||||
|
||||
if mode == "all":
|
||||
for _time in self.times:
|
||||
_dts = self.dates[self._times == _time]
|
||||
n_patch = np.ceil(len(_dts) / nums)
|
||||
dates_patch = np.array_split(_dts, n_patch)
|
||||
datetime_patches[_time] = dates_patch
|
||||
elif mode == "remain":
|
||||
for _time in self.times_remain:
|
||||
_dts = self.dates_remain[self._times_remain == _time]
|
||||
n_patch = np.ceil(len(_dts) / nums)
|
||||
dates_patch = np.array_split(_dts, n_patch)
|
||||
datetime_patches[_time] = dates_patch
|
||||
|
||||
return datetime_patches
|
||||
|
||||
def gen_post_data(
|
||||
self,
|
||||
dates: Union[list, np.ndarray],
|
||||
times: Union[tuple[int, int], tuple[str, str]],
|
||||
email: str,
|
||||
):
|
||||
"""Generate post data for gacos website.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dates : list or np.ndarray
|
||||
The list of dates.
|
||||
times : tuple[int, int]
|
||||
The time of the acquisition (hour, minute).
|
||||
email : str
|
||||
The email address to receive the gacos data.
|
||||
|
||||
Returns
|
||||
-------
|
||||
post_data : dict
|
||||
The post data.
|
||||
"""
|
||||
if isinstance(dates, np.ndarray):
|
||||
dates = dates.tolist()
|
||||
times = [int(t) for t in times]
|
||||
|
||||
post_data = {
|
||||
"N": self.bounds[3],
|
||||
"W": self.bounds[0],
|
||||
"S": self.bounds[1],
|
||||
"E": self.bounds[2],
|
||||
"H": times[0],
|
||||
"M": times[1],
|
||||
"date": "\n".join(dates),
|
||||
"type": "2",
|
||||
"email": email,
|
||||
}
|
||||
return post_data
|
||||
|
||||
|
||||
class LiCSARDataset(SarDataset):
|
||||
def __init__(
|
||||
self,
|
||||
home_dir: Union[Path, str],
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize LiCSARDataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
home_dir : Union[Path, str]
|
||||
The home directory of LiCSAR dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.home_dir = Path(home_dir)
|
||||
self.dataset = LiCSAR(home_dir)
|
||||
bounds = self.dataset.bounds
|
||||
time = self._get_time()
|
||||
dates = self.dataset.pairs.dates
|
||||
date_times = pd.to_datetime([f"{d} {time[0]}:{time[1]}:00" for d in dates])
|
||||
super().__init__(bounds, date_times, gacos_dir)
|
||||
|
||||
def _get_time(self):
|
||||
"""Get the acquisition time of acquisitions.
|
||||
|
||||
Returns
|
||||
-------
|
||||
time: tuple[int, int]
|
||||
A tuple of hour and minute representing the acquisition time.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If no center_time found in metadata.txt.
|
||||
"""
|
||||
meta_file = sorted(self.home_dir.rglob("metadata.txt"))[0]
|
||||
|
||||
with open(meta_file) as f:
|
||||
lines = f.readlines()
|
||||
time = None
|
||||
for line in lines:
|
||||
line_split = line.split("=")
|
||||
key, value = (line_split[0].strip(), line_split[1])
|
||||
if "center_time" == key:
|
||||
center_time = value.strip()
|
||||
hour, minute, second = center_time.split(":")
|
||||
hour, minute, second = int(hour), int(minute), float(second)
|
||||
minute = minute + int(np.round(second / 60, 0))
|
||||
return hour, minute
|
||||
else:
|
||||
continue
|
||||
if time is None:
|
||||
raise ValueError(f"No center_time found in {meta_file}")
|
||||
|
||||
|
||||
class HyP3Dataset(SarDataset):
|
||||
def __init__(
|
||||
self,
|
||||
home_dir: Union[Path, str],
|
||||
gacos_dir: Optional[Union[Path, str]] = None,
|
||||
) -> None:
|
||||
"""Initialize HyP3Dataset class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
home_dir : Union[Path, str]
|
||||
The home directory of HyP3 dataset.
|
||||
gacos_dir : Optional[Union[Path, str]], optional
|
||||
The directory used to save gacos data. Used to check if the data is
|
||||
already downloaded and avoid resubmitting. Default is None.
|
||||
"""
|
||||
self.dataset = HyP3(home_dir)
|
||||
bounds = self.dataset.bounds.to_crs("epsg:4326")
|
||||
date_times = self.dataset.datetime
|
||||
|
||||
super().__init__(bounds, date_times, gacos_dir)
|
||||
@@ -0,0 +1,158 @@
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from data_downloader import downloader
|
||||
from faninsar.query import BoundingBox
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from .parse_email import GACOSEmail
|
||||
|
||||
|
||||
class Downloader:
|
||||
def __init__(
|
||||
self,
|
||||
url_file: Union[Path, str],
|
||||
output_dir: Union[Path, str],
|
||||
tar_gz_dir: Optional[Union[Path, str]] = None,
|
||||
keep_original: bool = False,
|
||||
times: Optional[Union[float, list[float]]] = None,
|
||||
bounds: Optional[tuple[float, float, float, float]] = None,
|
||||
) -> None:
|
||||
"""Initialize Downloader class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
url_file : Union[Path, str]
|
||||
Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls`
|
||||
output_dir : Union[Path, str]
|
||||
directory to output gacos files
|
||||
tar_gz_dir : Optional[Union[Path, str]], optional
|
||||
directory to store downloaded *.tar.gz files. If None, then
|
||||
`output_dir` is used. Default is None.
|
||||
keep_original : bool, optional
|
||||
Whether to keep original files (*.tar.gz). Default is False.
|
||||
times : Optional[float], optional
|
||||
times of acquisition, used to filter out files that are not needed.
|
||||
this can be a single time or a list of times. times differ by less
|
||||
than 10 minutes are considered the same. Default is None.
|
||||
bounds : Optional[tuple[float, float, float, float]], optional
|
||||
bounds of area of interest with order (W, S, E, N), used to filter
|
||||
out files that are not needed. Default is None.
|
||||
"""
|
||||
self.url_file = Path(url_file)
|
||||
self.output_dir = Path(output_dir)
|
||||
if tar_gz_dir is None:
|
||||
self.tar_gz_dir = self.output_dir
|
||||
self.keep_original = keep_original
|
||||
|
||||
if not self.url_file.exists():
|
||||
raise FileNotFoundError(f"{self.url_file} does not exist")
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
if not self.tar_gz_dir.exists():
|
||||
self.tar_gz_dir.mkdir(parents=True)
|
||||
|
||||
self.df_urls = pd.read_csv(self.url_file, header=0)
|
||||
|
||||
# only keep urls that intersect with bounds
|
||||
if bounds is not None:
|
||||
mask_bbox = self._bbox_mask(BoundingBox(*bounds))
|
||||
|
||||
# only keep urls that acquisition time is within 10 minutes of `time`
|
||||
if times is not None:
|
||||
if isinstance(times, float):
|
||||
times = [times]
|
||||
mask_time = self._time_mask(times)
|
||||
|
||||
if bounds is not None and times is not None:
|
||||
self.mask = mask_bbox & mask_time
|
||||
elif bounds is not None:
|
||||
self.mask = mask_bbox
|
||||
elif times is not None:
|
||||
self.mask = mask_time
|
||||
else:
|
||||
self.mask = np.ones(self.df_urls.shape[0], dtype=bool)
|
||||
|
||||
self.mask = self.mask & self.date_mask
|
||||
|
||||
def _bbox_mask(self, bounds) -> np.ndarray:
|
||||
intersection_bbox = np.array(
|
||||
[
|
||||
BoundingBox(*b).intersects(bounds)
|
||||
for b in zip(
|
||||
self.df_urls["south"].astype(float),
|
||||
self.df_urls["west"].astype(float),
|
||||
self.df_urls["north"].astype(float),
|
||||
self.df_urls["east"].astype(float),
|
||||
)
|
||||
]
|
||||
)
|
||||
return intersection_bbox
|
||||
|
||||
def _time_mask(self, times) -> np.ndarray:
|
||||
"""Only keep urls that acquisition time is within 10 minutes of `time`"""
|
||||
intersection_times = []
|
||||
for time in times:
|
||||
intersection_times.append(
|
||||
np.array(
|
||||
np.abs(self.df_urls["time"].astype(float) - time)
|
||||
<= 1 / 60 * 10 # 10 minutes
|
||||
)
|
||||
)
|
||||
intersection_time = np.any(intersection_times, axis=0)
|
||||
return intersection_time
|
||||
|
||||
@property
|
||||
def date_mask(self) -> np.ndarray:
|
||||
"""Remove urls that all acquisition dates have been downloaded"""
|
||||
dates_urls = self.df_urls["date"].map(lambda x: eval(x))
|
||||
intersection_dates = []
|
||||
for dt_url in dates_urls:
|
||||
intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded)))
|
||||
return np.array(intersection_dates)
|
||||
|
||||
@property
|
||||
def dates_downloaded(self) -> np.ndarray:
|
||||
"""Return dates that have been downloaded"""
|
||||
gacos_files = list(self.output_dir.rglob("*.ztd.tif"))
|
||||
dates = []
|
||||
for i in gacos_files:
|
||||
stem = i.stem.split(".")[0]
|
||||
if len(stem) == 8:
|
||||
dates.append(stem)
|
||||
return np.array(dates)
|
||||
|
||||
def download(self) -> None:
|
||||
"""Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`"""
|
||||
urls_used = self.df_urls[self.mask]["url"].values
|
||||
|
||||
for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"):
|
||||
gz_file = self.tar_gz_dir / Path(url).name
|
||||
downloader.download_data(url, file_name=gz_file)
|
||||
self._extract_tar_gz(gz_file)
|
||||
if not self.keep_original:
|
||||
self._delete_file(gz_file)
|
||||
|
||||
def _extract_tar_gz(self, gz_file) -> None:
|
||||
"""Unzip/extract downloaded GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
with tarfile.open(gz_file, "r:gz") as tar:
|
||||
tar.extractall(path=self.output_dir)
|
||||
|
||||
def _delete_file(self, gz_file) -> None:
|
||||
"""Delete original GACOS files
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gz_file : Path
|
||||
path to downloaded GACOS file (*.tar.gz)
|
||||
"""
|
||||
gz_file.unlink()
|
||||
@@ -0,0 +1,391 @@
|
||||
import email
|
||||
import email.message
|
||||
import getpass
|
||||
import imaplib
|
||||
import poplib
|
||||
import re
|
||||
from email.parser import Parser
|
||||
from email.utils import parseaddr
|
||||
from pathlib import Path
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
class GACOSEmail:
|
||||
"""a class to retrieve gacos urls from email.
|
||||
|
||||
.. note::
|
||||
The IMAP server is used to retrieve content from email. Some email
|
||||
service providers may need to enable the IMAP service in the settings.
|
||||
**You are recommended to use a new email account to receive gacos urls to
|
||||
avoid polluting your own email account**.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
host: str,
|
||||
prompt: bool = False,
|
||||
email_protocol: Literal["imap", "pop3"] = "imap",
|
||||
port: Optional[int] = None,
|
||||
gacos_email: str = "gacos2017@foxmail.com",
|
||||
gacos_suffix: str = "tar.gz",
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
date_args: Optional[dict] = None,
|
||||
ssl: bool = False,
|
||||
) -> None:
|
||||
"""Retrieve gacos urls from email.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
username : str
|
||||
The username of the email address.
|
||||
password : str
|
||||
The password.
|
||||
host : str
|
||||
The host of the email address. For example, the host of gmail for
|
||||
imap is "imap.gmail.com". You can find the host of your email
|
||||
settings or search it on the Internet.
|
||||
prompt: bool, optional
|
||||
Prompt for username and/or password interactively when they are not
|
||||
provided as keyword parameters. Default is False.
|
||||
email_protocol : str, one of ["imap", "pop3"], optional
|
||||
The protocol of the email. Default is "imap".
|
||||
port : int, optional
|
||||
The port of the host of your email. If None, the default port will be
|
||||
used. Default is None.
|
||||
gacos_email : str, optional
|
||||
The email address of gacos. Default is "gacos2017@foxmail.com".
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
start_date / end_date: str, optional
|
||||
The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime.
|
||||
date_args : dict, optional
|
||||
The arguments are passed to pandas.to_datetime. Default is None.
|
||||
ssl : bool, optional
|
||||
Whether to use SSL connection. Default is False.
|
||||
"""
|
||||
if prompt:
|
||||
self.username = None
|
||||
self.password = None
|
||||
else:
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.host = host
|
||||
self.email_protocol = email_protocol
|
||||
self.port = port
|
||||
self.gacos_email = gacos_email
|
||||
self.gacos_suffix = gacos_suffix
|
||||
self.ssl = ssl
|
||||
|
||||
# date part
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
if date_args is None:
|
||||
date_args = {}
|
||||
self.date_args = date_args
|
||||
|
||||
def _retrieve_gacos_urls_pop3(self):
|
||||
server = login_in_email_pop3(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
print(server.getwelcome())
|
||||
|
||||
nums = server.stat()[0]
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"):
|
||||
response, msgLines, octets = server.retr(i)
|
||||
msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore")
|
||||
messageObject = Parser().parsestr(msgLinesToStr)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.quit()
|
||||
|
||||
return gacos
|
||||
|
||||
def _retrieve_gacos_urls_imap(self):
|
||||
server = login_in_email_imap(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
if server is None:
|
||||
print("IMAP server connection failed, skipping email check.")
|
||||
return []
|
||||
server.select("inbox")
|
||||
status, data = server.search(None, "ALL")
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"):
|
||||
res, msg = server.fetch(i, "(RFC822)")
|
||||
for response_part in msg:
|
||||
if isinstance(response_part, tuple):
|
||||
msgLines = response_part[1].decode("utf8", "ignore")
|
||||
break
|
||||
|
||||
messageObject = Parser().parsestr(msgLines)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.close()
|
||||
|
||||
return gacos
|
||||
|
||||
def retrieve_gacos_urls(
|
||||
self,
|
||||
output_file: Union[str, Path],
|
||||
):
|
||||
"""Retrieve gacos urls from username.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
output_file : str or Path
|
||||
The output file used to save the gacos urls.
|
||||
"""
|
||||
if self.email_protocol == "pop3":
|
||||
gacos = self._retrieve_gacos_urls_pop3()
|
||||
elif self.email_protocol == "imap":
|
||||
gacos = self._retrieve_gacos_urls_imap()
|
||||
else:
|
||||
raise ValueError("email_protocol must be 'pop3' or 'imap'.")
|
||||
|
||||
cols = ["url", "south", "north", "west", "east", "time", "date"]
|
||||
df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url")
|
||||
|
||||
# save to file
|
||||
try:
|
||||
df_gacos.to_csv(output_file)
|
||||
print(f"Save gacos urls to {output_file}")
|
||||
except Exception as e:
|
||||
self.df_gacos = df_gacos
|
||||
print(e)
|
||||
print("Save gacos urls failed")
|
||||
print("You can access the gacos urls by `df_gacos` attribute.")
|
||||
|
||||
|
||||
def in_date_range(date, start_date, end_date, date_args={}):
|
||||
start_date = pd.to_datetime(start_date, **date_args)
|
||||
end_date = pd.to_datetime(end_date, **date_args)
|
||||
start_none = start_date is None or pd.isna(start_date)
|
||||
end_none = end_date is None or pd.isna(end_date)
|
||||
if start_none and end_none:
|
||||
return True
|
||||
elif start_none:
|
||||
return date <= end_date
|
||||
elif end_none:
|
||||
return date >= start_date
|
||||
else:
|
||||
return (date >= start_date) and (date <= end_date)
|
||||
|
||||
|
||||
def decodeBody(msgPart: email.message.Message):
|
||||
"""decode email body
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgPart : email.message.Message
|
||||
The email message object.
|
||||
"""
|
||||
contentType = msgPart.get_content_type()
|
||||
textContent = ""
|
||||
if contentType == "text/plain" or contentType == "text/html":
|
||||
content = msgPart.get_payload(decode=True)
|
||||
charset = msgPart.get_charset()
|
||||
if charset is None:
|
||||
contentType = msgPart.get("Content-Type", "").lower()
|
||||
position = contentType.find("charset=")
|
||||
if position >= 0:
|
||||
charset = contentType[position + 8 :].strip()
|
||||
if charset:
|
||||
textContent = content.decode(charset)
|
||||
return textContent
|
||||
|
||||
|
||||
def get_content(messageObject):
|
||||
msgBodyContents = []
|
||||
if messageObject.is_multipart(): # parse multipart email
|
||||
messageParts = messageObject.get_payload()
|
||||
for messagePart in messageParts:
|
||||
bodyContent = decodeBody(messagePart)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
else:
|
||||
bodyContent = decodeBody(messageObject)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
return msgBodyContents
|
||||
|
||||
|
||||
def login_in_email_pop3(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 995
|
||||
server = poplib.POP3_SSL(host, port)
|
||||
else:
|
||||
if port is None:
|
||||
port = 110
|
||||
server = poplib.POP3(host, port)
|
||||
|
||||
server.user(username)
|
||||
server.pass_(password)
|
||||
return server
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("login failed")
|
||||
|
||||
|
||||
def login_in_email_imap(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 993
|
||||
else:
|
||||
if port is None:
|
||||
port = 143
|
||||
|
||||
# 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达
|
||||
import socket as _socket
|
||||
_orig_getaddrinfo = _socket.getaddrinfo
|
||||
def _ipv4_only_getaddrinfo(*args, **kwargs):
|
||||
return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET,
|
||||
*args[3:], **kwargs)
|
||||
_socket.getaddrinfo = _ipv4_only_getaddrinfo
|
||||
try:
|
||||
if ssl:
|
||||
server = imaplib.IMAP4_SSL(host, port)
|
||||
else:
|
||||
server = imaplib.IMAP4(host, port)
|
||||
finally:
|
||||
_socket.getaddrinfo = _orig_getaddrinfo
|
||||
|
||||
server.login(username, password)
|
||||
|
||||
# 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT
|
||||
if '163.com' in host or '126.com' in host or 'yeah.net' in host:
|
||||
try:
|
||||
tag = server._new_tag()
|
||||
server.send(tag + b' ID ("name" "pyint" "version" "1.0" '
|
||||
b'"vendor" "pyint")\r\n')
|
||||
while True:
|
||||
resp = server.readline()
|
||||
if resp.startswith(tag):
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return server
|
||||
except Exception as e:
|
||||
print(f"IMAP login failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"):
|
||||
"""Parse gacos info from email body.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgBodyContents : list
|
||||
The email body contents.
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
"""
|
||||
|
||||
url, south, north, west, east, _time, date_list = (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
date_list = []
|
||||
for contents in msgBodyContents:
|
||||
lines = [i.strip() for i in contents.split("\n") if i]
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
loc = line.split("=")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "MinLat" == parameter:
|
||||
south = float(value)
|
||||
if "MaxLat" == parameter:
|
||||
north = float(value)
|
||||
if "MinLon" == parameter:
|
||||
west = float(value)
|
||||
if "MaxLon" == parameter:
|
||||
east = float(value)
|
||||
loc = line.split(":")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "Time" == parameter:
|
||||
_time = float(value)
|
||||
|
||||
if len(line) == 8 and line.isdigit():
|
||||
date_list.append(line)
|
||||
|
||||
for i in ["http", "ftp", "https"]:
|
||||
result = re.search(f"\({i}.*{gacos_suffix}\)", line)
|
||||
if result:
|
||||
url = result.group()[1:-1]
|
||||
break
|
||||
|
||||
if url == south == north == west == east == _time:
|
||||
return None
|
||||
else:
|
||||
return url, south, north, west, east, _time, date_list
|
||||
@@ -0,0 +1,83 @@
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from .datasets import SarDataset
|
||||
|
||||
|
||||
class Submitter:
|
||||
def __init__(
|
||||
self,
|
||||
dataset: SarDataset,
|
||||
email: str,
|
||||
sleep_time_range: tuple[int, int] = (60 / 2, 60 * 5),
|
||||
gacos_url="http://www.gacos.net/M/action_page.php",
|
||||
) -> None:
|
||||
"""Initialize Submitter class
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dataset : SarDataset
|
||||
The SarDataset object.
|
||||
email : str
|
||||
The email address to submit to gacos.
|
||||
sleep_time_range : tuple[int, int], optional
|
||||
The range of sleep time in seconds. Default is (60, 60 * 5).
|
||||
gacos_url : str, optional
|
||||
The url of gacos website. Default is "http://www.gacos.net/M/action_page.php".
|
||||
"""
|
||||
self.dataset = dataset
|
||||
self.email = email
|
||||
self.sleep_time_range = sleep_time_range
|
||||
self.gacos_url = gacos_url
|
||||
|
||||
self._failed = []
|
||||
self._succeed = []
|
||||
|
||||
def _post_data(self, data):
|
||||
"""Post data to gacos website."""
|
||||
r = requests.post(self.gacos_url, data=data)
|
||||
return "Thanks for using GACOS!" in r.text
|
||||
|
||||
def post_requests(self):
|
||||
# post gacos info to website
|
||||
datetime_patches = self.dataset.gen_datetime_patches()
|
||||
for _key, _dates in tqdm(
|
||||
datetime_patches.items(),
|
||||
desc="submitting times",
|
||||
unit="times",
|
||||
):
|
||||
try:
|
||||
for _dt in tqdm(_dates, desc="submitting dates", unit="dates"):
|
||||
post_data = self.dataset.gen_post_data(
|
||||
_dt, _key.split(":"), self.email
|
||||
)
|
||||
status_ok = self._post_data(post_data)
|
||||
if status_ok:
|
||||
self._succeed.append(post_data)
|
||||
tqdm.write(f">>> succeed post: {post_data}")
|
||||
else:
|
||||
self._failed.append(post_data)
|
||||
tqdm.write(f">>> failed post: {post_data}")
|
||||
|
||||
# wait to avoid be rejected
|
||||
sleep_time = np.random.randint(*self.sleep_time_range)
|
||||
tqdm.write(f" sleeping for {sleep_time} seconds...")
|
||||
time.sleep(sleep_time)
|
||||
except:
|
||||
self._failed.append(post_data)
|
||||
tqdm.write(f">>> failed post: {post_data}")
|
||||
|
||||
@property
|
||||
def failed(self):
|
||||
"""A list of failed post data."""
|
||||
return self._failed
|
||||
|
||||
@property
|
||||
def succeed(self):
|
||||
"""A list of succeed post data."""
|
||||
return self._succeed
|
||||
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "AutoGACOS"
|
||||
authors = [{name = "Fancy", email = "fanchengyan@outlook.com"}]
|
||||
description = "A Python library for automatically submitting and downloading GACOS data"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.6"
|
||||
dependencies = [
|
||||
"data_downloader",
|
||||
"faninsar",
|
||||
]
|
||||
readme = "README.md"
|
||||
license = {file = "LICENSE"}
|
||||
keywords = ["GACOS", "data", "downloader"]
|
||||
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest"
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/Fanchengyan/AutoGACOS"
|
||||
Repository = "https://github.com/Fanchengyan/AutoGACOS"
|
||||
Reference in New Issue
Block a user