Apply current workspace changes
This commit is contained in:
@@ -0,0 +1,391 @@
|
||||
import email
|
||||
import email.message
|
||||
import getpass
|
||||
import imaplib
|
||||
import poplib
|
||||
import re
|
||||
from email.parser import Parser
|
||||
from email.utils import parseaddr
|
||||
from pathlib import Path
|
||||
from typing import Literal, Optional, Union
|
||||
|
||||
import pandas as pd
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
|
||||
class GACOSEmail:
|
||||
"""a class to retrieve gacos urls from email.
|
||||
|
||||
.. note::
|
||||
The IMAP server is used to retrieve content from email. Some email
|
||||
service providers may need to enable the IMAP service in the settings.
|
||||
**You are recommended to use a new email account to receive gacos urls to
|
||||
avoid polluting your own email account**.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
host: str,
|
||||
prompt: bool = False,
|
||||
email_protocol: Literal["imap", "pop3"] = "imap",
|
||||
port: Optional[int] = None,
|
||||
gacos_email: str = "gacos2017@foxmail.com",
|
||||
gacos_suffix: str = "tar.gz",
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
date_args: Optional[dict] = None,
|
||||
ssl: bool = False,
|
||||
) -> None:
|
||||
"""Retrieve gacos urls from email.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
username : str
|
||||
The username of the email address.
|
||||
password : str
|
||||
The password.
|
||||
host : str
|
||||
The host of the email address. For example, the host of gmail for
|
||||
imap is "imap.gmail.com". You can find the host of your email
|
||||
settings or search it on the Internet.
|
||||
prompt: bool, optional
|
||||
Prompt for username and/or password interactively when they are not
|
||||
provided as keyword parameters. Default is False.
|
||||
email_protocol : str, one of ["imap", "pop3"], optional
|
||||
The protocol of the email. Default is "imap".
|
||||
port : int, optional
|
||||
The port of the host of your email. If None, the default port will be
|
||||
used. Default is None.
|
||||
gacos_email : str, optional
|
||||
The email address of gacos. Default is "gacos2017@foxmail.com".
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
start_date / end_date: str, optional
|
||||
The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime.
|
||||
date_args : dict, optional
|
||||
The arguments are passed to pandas.to_datetime. Default is None.
|
||||
ssl : bool, optional
|
||||
Whether to use SSL connection. Default is False.
|
||||
"""
|
||||
if prompt:
|
||||
self.username = None
|
||||
self.password = None
|
||||
else:
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.host = host
|
||||
self.email_protocol = email_protocol
|
||||
self.port = port
|
||||
self.gacos_email = gacos_email
|
||||
self.gacos_suffix = gacos_suffix
|
||||
self.ssl = ssl
|
||||
|
||||
# date part
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
if date_args is None:
|
||||
date_args = {}
|
||||
self.date_args = date_args
|
||||
|
||||
def _retrieve_gacos_urls_pop3(self):
|
||||
server = login_in_email_pop3(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
print(server.getwelcome())
|
||||
|
||||
nums = server.stat()[0]
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"):
|
||||
response, msgLines, octets = server.retr(i)
|
||||
msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore")
|
||||
messageObject = Parser().parsestr(msgLinesToStr)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.quit()
|
||||
|
||||
return gacos
|
||||
|
||||
def _retrieve_gacos_urls_imap(self):
|
||||
server = login_in_email_imap(
|
||||
self.username, self.password, self.host, self.port, ssl=self.ssl
|
||||
)
|
||||
if server is None:
|
||||
print("IMAP server connection failed, skipping email check.")
|
||||
return []
|
||||
server.select("inbox")
|
||||
status, data = server.search(None, "ALL")
|
||||
|
||||
gacos = []
|
||||
for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"):
|
||||
res, msg = server.fetch(i, "(RFC822)")
|
||||
for response_part in msg:
|
||||
if isinstance(response_part, tuple):
|
||||
msgLines = response_part[1].decode("utf8", "ignore")
|
||||
break
|
||||
|
||||
messageObject = Parser().parsestr(msgLines)
|
||||
|
||||
senderContent = messageObject["From"]
|
||||
senderRealName, senderAdr = parseaddr(senderContent)
|
||||
if senderAdr == self.gacos_email:
|
||||
if not in_date_range(
|
||||
pd.to_datetime(messageObject["Date"]).tz_localize(None),
|
||||
self.start_date,
|
||||
self.end_date,
|
||||
self.date_args,
|
||||
):
|
||||
continue
|
||||
|
||||
msgBodyContents = get_content(messageObject)
|
||||
info = parse_gacos_info(
|
||||
msgBodyContents,
|
||||
gacos_suffix=self.gacos_suffix,
|
||||
)
|
||||
if info is not None:
|
||||
gacos.append(info)
|
||||
|
||||
server.close()
|
||||
|
||||
return gacos
|
||||
|
||||
def retrieve_gacos_urls(
|
||||
self,
|
||||
output_file: Union[str, Path],
|
||||
):
|
||||
"""Retrieve gacos urls from username.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
output_file : str or Path
|
||||
The output file used to save the gacos urls.
|
||||
"""
|
||||
if self.email_protocol == "pop3":
|
||||
gacos = self._retrieve_gacos_urls_pop3()
|
||||
elif self.email_protocol == "imap":
|
||||
gacos = self._retrieve_gacos_urls_imap()
|
||||
else:
|
||||
raise ValueError("email_protocol must be 'pop3' or 'imap'.")
|
||||
|
||||
cols = ["url", "south", "north", "west", "east", "time", "date"]
|
||||
df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url")
|
||||
|
||||
# save to file
|
||||
try:
|
||||
df_gacos.to_csv(output_file)
|
||||
print(f"Save gacos urls to {output_file}")
|
||||
except Exception as e:
|
||||
self.df_gacos = df_gacos
|
||||
print(e)
|
||||
print("Save gacos urls failed")
|
||||
print("You can access the gacos urls by `df_gacos` attribute.")
|
||||
|
||||
|
||||
def in_date_range(date, start_date, end_date, date_args={}):
|
||||
start_date = pd.to_datetime(start_date, **date_args)
|
||||
end_date = pd.to_datetime(end_date, **date_args)
|
||||
start_none = start_date is None or pd.isna(start_date)
|
||||
end_none = end_date is None or pd.isna(end_date)
|
||||
if start_none and end_none:
|
||||
return True
|
||||
elif start_none:
|
||||
return date <= end_date
|
||||
elif end_none:
|
||||
return date >= start_date
|
||||
else:
|
||||
return (date >= start_date) and (date <= end_date)
|
||||
|
||||
|
||||
def decodeBody(msgPart: email.message.Message):
|
||||
"""decode email body
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgPart : email.message.Message
|
||||
The email message object.
|
||||
"""
|
||||
contentType = msgPart.get_content_type()
|
||||
textContent = ""
|
||||
if contentType == "text/plain" or contentType == "text/html":
|
||||
content = msgPart.get_payload(decode=True)
|
||||
charset = msgPart.get_charset()
|
||||
if charset is None:
|
||||
contentType = msgPart.get("Content-Type", "").lower()
|
||||
position = contentType.find("charset=")
|
||||
if position >= 0:
|
||||
charset = contentType[position + 8 :].strip()
|
||||
if charset:
|
||||
textContent = content.decode(charset)
|
||||
return textContent
|
||||
|
||||
|
||||
def get_content(messageObject):
|
||||
msgBodyContents = []
|
||||
if messageObject.is_multipart(): # parse multipart email
|
||||
messageParts = messageObject.get_payload()
|
||||
for messagePart in messageParts:
|
||||
bodyContent = decodeBody(messagePart)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
else:
|
||||
bodyContent = decodeBody(messageObject)
|
||||
if bodyContent:
|
||||
msgBodyContents.append(bodyContent)
|
||||
return msgBodyContents
|
||||
|
||||
|
||||
def login_in_email_pop3(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 995
|
||||
server = poplib.POP3_SSL(host, port)
|
||||
else:
|
||||
if port is None:
|
||||
port = 110
|
||||
server = poplib.POP3(host, port)
|
||||
|
||||
server.user(username)
|
||||
server.pass_(password)
|
||||
return server
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print("login failed")
|
||||
|
||||
|
||||
def login_in_email_imap(username, password, host, port, ssl=False):
|
||||
try:
|
||||
if username is None:
|
||||
username = input("username: ")
|
||||
if password is None:
|
||||
password = getpass.getpass("password: ")
|
||||
|
||||
if ssl:
|
||||
if port is None:
|
||||
port = 993
|
||||
else:
|
||||
if port is None:
|
||||
port = 143
|
||||
|
||||
# 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达
|
||||
import socket as _socket
|
||||
_orig_getaddrinfo = _socket.getaddrinfo
|
||||
def _ipv4_only_getaddrinfo(*args, **kwargs):
|
||||
return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET,
|
||||
*args[3:], **kwargs)
|
||||
_socket.getaddrinfo = _ipv4_only_getaddrinfo
|
||||
try:
|
||||
if ssl:
|
||||
server = imaplib.IMAP4_SSL(host, port)
|
||||
else:
|
||||
server = imaplib.IMAP4(host, port)
|
||||
finally:
|
||||
_socket.getaddrinfo = _orig_getaddrinfo
|
||||
|
||||
server.login(username, password)
|
||||
|
||||
# 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT
|
||||
if '163.com' in host or '126.com' in host or 'yeah.net' in host:
|
||||
try:
|
||||
tag = server._new_tag()
|
||||
server.send(tag + b' ID ("name" "pyint" "version" "1.0" '
|
||||
b'"vendor" "pyint")\r\n')
|
||||
while True:
|
||||
resp = server.readline()
|
||||
if resp.startswith(tag):
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return server
|
||||
except Exception as e:
|
||||
print(f"IMAP login failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"):
|
||||
"""Parse gacos info from email body.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
msgBodyContents : list
|
||||
The email body contents.
|
||||
gacos_suffix : str, optional
|
||||
The suffix of the gacos file url. Default is "tar.gz". The suffix is used
|
||||
to filter urls in the email. This parameter is used to avoid the
|
||||
situation that the email contains other urls.
|
||||
"""
|
||||
|
||||
url, south, north, west, east, _time, date_list = (
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
date_list = []
|
||||
for contents in msgBodyContents:
|
||||
lines = [i.strip() for i in contents.split("\n") if i]
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
loc = line.split("=")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "MinLat" == parameter:
|
||||
south = float(value)
|
||||
if "MaxLat" == parameter:
|
||||
north = float(value)
|
||||
if "MinLon" == parameter:
|
||||
west = float(value)
|
||||
if "MaxLon" == parameter:
|
||||
east = float(value)
|
||||
loc = line.split(":")
|
||||
if len(loc) == 2:
|
||||
parameter, value = loc
|
||||
parameter, value = (parameter.strip(), value.strip())
|
||||
if "Time" == parameter:
|
||||
_time = float(value)
|
||||
|
||||
if len(line) == 8 and line.isdigit():
|
||||
date_list.append(line)
|
||||
|
||||
for i in ["http", "ftp", "https"]:
|
||||
result = re.search(f"\({i}.*{gacos_suffix}\)", line)
|
||||
if result:
|
||||
url = result.group()[1:-1]
|
||||
break
|
||||
|
||||
if url == south == north == west == east == _time:
|
||||
return None
|
||||
else:
|
||||
return url, south, north, west, east, _time, date_list
|
||||
Reference in New Issue
Block a user