manage imap/smtp directly. remove srmail dependency

pull/6/head
Yax 5 years ago
parent 3281dd1aae
commit 7f5ea9c4d3

@ -1,3 +1,4 @@
;
; Default configuration ; Default configuration
[main] [main]
lang = fr lang = fr
@ -5,18 +6,24 @@ db_url = sqlite:///db.sqlite
newcomment_polling = 60 newcomment_polling = 60
[http] [http]
root_url = http://localhost:8100 host = 127.0.0.1
host = 0.0.0.0
port = 8100 port = 8100
[security]
salt = BRRJRqXgGpXWrgTidBPcixIThHpDuKc0
secret = Uqca5Kc8xuU6THz9
[rss] [rss]
proto = http proto = https
file = comments.xml file = comments.xml
[mail] [imap]
fetch_polling = 30 polling = 120
mailer_url = http://localhost:8000 host = mail.gandi.net
ssl = false
port = 993
login = blog@mydomain.com
password = MYPASSWORD
[smtp]
host = mail.gandi.net
starttls = true
port = 587
login = blog@mydomain.com
password = MYPASSWORD

18
poetry.lock generated

@ -363,6 +363,17 @@ urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"] security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"] socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
[[package]]
category = "dev"
description = "a python refactoring library..."
name = "rope"
optional = false
python-versions = "*"
version = "0.16.0"
[package.extras]
dev = ["pytest"]
[[package]] [[package]]
category = "main" category = "main"
description = "Python 2 and 3 compatibility utilities" description = "Python 2 and 3 compatibility utilities"
@ -449,7 +460,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"]
testing = ["pathlib2", "contextlib2", "unittest2"] testing = ["pathlib2", "contextlib2", "unittest2"]
[metadata] [metadata]
content-hash = "6270dbd1455ca926e89cdd874748cf8cee18c2ef5a10a05b6dc7f7100e2482d5" content-hash = "d698fc06cf58f4d228449cb76f48e8d739c323abd535427746d70dbbcaa4924c"
python-versions = "^3.7" python-versions = "^3.7"
[metadata.files] [metadata.files]
@ -618,6 +629,11 @@ requests = [
{file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"}, {file = "requests-2.22.0-py2.py3-none-any.whl", hash = "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"},
{file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"}, {file = "requests-2.22.0.tar.gz", hash = "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4"},
] ]
rope = [
{file = "rope-0.16.0-py2-none-any.whl", hash = "sha256:ae1fa2fd56f64f4cc9be46493ce54bed0dd12dee03980c61a4393d89d84029ad"},
{file = "rope-0.16.0-py3-none-any.whl", hash = "sha256:52423a7eebb5306a6d63bdc91a7c657db51ac9babfb8341c9a1440831ecf3203"},
{file = "rope-0.16.0.tar.gz", hash = "sha256:d2830142c2e046f5fc26a022fe680675b6f48f81c7fc1f03a950706e746e9dfe"},
]
six = [ six = [
{file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"},
{file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"},

@ -19,6 +19,7 @@ requests = "^2.22.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
pytest = "^5.2" pytest = "^5.2"
black = {version = "^19.10b0", allow-prereleases = true} black = {version = "^19.10b0", allow-prereleases = true}
rope = "^0.16.0"
[build-system] [build-system]
requires = ["poetry>=0.12"] requires = ["poetry>=0.12"]

@ -4,23 +4,30 @@
import profig import profig
# constants # constants
FLASK_APP = "flask.app" FLASK_APP = 'flask.app'
DB_URL = "main.db_url" DB_URL = 'main.db_url'
LANG = "main.lang" LANG = 'main.lang'
COMMENT_POLLING = 'main.newcomment_polling'
HTTP_HOST = "http.host" HTTP_HOST = 'http.host'
HTTP_PORT = "http.port" HTTP_PORT = 'http.port'
SECURITY_SALT = "security.salt" RSS_PROTO = 'rss.proto'
SECURITY_SECRET = "security.secret" RSS_FILE = 'rss.file'
RSS_PROTO = "rss.proto" IMAP_POLLING = 'imap.polling'
RSS_FILE = "rss.file" IMAP_SSL = 'imap.ssl'
IMAP_HOST = 'imap.host'
IMAP_PORT = 'imap.port'
IMAP_LOGIN = 'imap.login'
IMAP_PASSWORD = 'imap.password'
MAIL_POLLING = "mail.fetch_polling" SMTP_STARTTLS = 'smtp.starttls'
COMMENT_POLLING = "main.newcomment_polling" SMTP_HOST = 'smtp.host'
MAILER_URL = "mail.mailer_url" SMTP_PORT = 'smtp.port'
SMTP_LOGIN = 'smtp.login'
SMTP_PASSWORD = 'smtp.password'
# variable # variable
@ -38,16 +45,12 @@ def get(key):
return params[key] return params[key]
def getInt(key): def get_int(key):
return int(params[key]) return int(params[key])
def _str2bool(v): def get_bool(key):
return v.lower() in ("yes", "true", "t", "1") return params[key].lower() in ('yes', 'true', '1')
def getBool(key):
return _str2bool(params[key])
def flaskapp(): def flaskapp():

@ -2,21 +2,21 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging import logging
from datetime import datetime
import time
import re import re
from core import mailer import time
from datetime import datetime
from core import mailer, rss
from core.templater import get_template from core.templater import get_template
from core import rss from model.comment import Comment, Site
from model.comment import Comment from model.email import Email
from model.comment import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def cron(func): def cron(func):
def wrapper(): def wrapper():
logger.debug("execute CRON " + func.__name__) logger.debug('execute CRON ' + func.__name__)
func() func()
return wrapper return wrapper
@ -26,10 +26,10 @@ def cron(func):
def fetch_mail_answers(): def fetch_mail_answers():
for msg in mailer.fetch(): for msg in mailer.fetch():
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg["subject"], re.DOTALL): if re.search(r'.*STACOSYS.*\[(\d+)\:(\w+)\]', msg.subject, re.DOTALL):
full_msg = mailer.get(msg["id"]) if full_msg and _reply_comment_email(msg):
if full_msg and reply_comment_email(full_msg['email']): mailer.delete(msg.id)
mailer.delete(msg["id"])
@cron @cron
def submit_new_comment(): def submit_new_comment():
@ -37,42 +37,36 @@ def submit_new_comment():
for comment in Comment.select().where(Comment.notified.is_null()): for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = ( comment_list = (
"author: %s" % comment.author_name, 'author: %s' % comment.author_name,
"site: %s" % comment.author_site, 'site: %s' % comment.author_site,
"date: %s" % comment.created, 'date: %s' % comment.created,
"url: %s" % comment.url, 'url: %s' % comment.url,
"", '',
"%s" % comment.content, '%s' % comment.content,
"", '',
) )
comment_text = "\n".join(comment_list) comment_text = '\n'.join(comment_list)
email_body = get_template("new_comment").render( email_body = get_template('new_comment').render(
url=comment.url, comment=comment_text url=comment.url, comment=comment_text
) )
# send email # send email
site = Site.get(Site.id == comment.site) site = Site.get(Site.id == comment.site)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token) subject = 'STACOSYS %s: [%d:%s]' % (site.name, comment.id, site.token)
mailer.send(site.admin_email, subject, email_body) if mailer.send(site.admin_email, subject, email_body):
logger.debug("new comment processed ") logger.debug('new comment processed ')
# notify site admin and save notification datetime
comment.notify_site_admin()
# notify site admin and save notification datetime
comment.notify_site_admin()
else:
logger.warn('rescheduled. send mail failure ' + subject)
def reply_comment_email(data):
from_email = data["from"] def _reply_comment_email(email):
subject = data["subject"]
message = ""
for part in data["parts"]:
if part["content-type"] == "text/plain":
message = part["content"]
break
m = re.search(r"\[(\d+)\:(\w+)\]", subject) m = re.search(r'\[(\d+)\:(\w+)\]', email.subject)
if not m: if not m:
logger.warn("ignore corrupted email. No token %s" % subject) logger.warn('ignore corrupted email. No token %s' % email.subject)
return return
comment_id = int(m.group(1)) comment_id = int(m.group(1))
token = m.group(2) token = m.group(2)
@ -81,37 +75,39 @@ def reply_comment_email(data):
try: try:
comment = Comment.select().where(Comment.id == comment_id).get() comment = Comment.select().where(Comment.id == comment_id).get()
except: except:
logger.warn("unknown comment %d" % comment_id) logger.warn('unknown comment %d' % comment_id)
return True return True
if comment.published: if comment.published:
logger.warn("ignore already published email. token %d" % comment_id) logger.warn('ignore already published email. token %d' % comment_id)
return return
if comment.site.token != token: if comment.site.token != token:
logger.warn("ignore corrupted email. Unknown token %d" % comment_id) logger.warn('ignore corrupted email. Unknown token %d' % comment_id)
return return
if not message: if not email.content:
logger.warn("ignore empty email") logger.warn('ignore empty email')
return return
# safe logic: no answer or unknown answer is a go for publishing # safe logic: no answer or unknown answer is a go for publishing
if message[:2].upper() in ("NO"): if email.content[:2].upper() in ('NO'):
logger.info("discard comment: %d" % comment_id) logger.info('discard comment: %d' % comment_id)
comment.delete_instance() comment.delete_instance()
email_body = get_template("drop_comment").render(original=message) new_email_body = get_template('drop_comment').render(original=email.content)
mailer.send(from_email, "Re: " + subject, email_body) if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send rejection mail ' + email.subject)
else: else:
# save publishing datetime # save publishing datetime
comment.publish() comment.publish()
logger.info("commit comment: %d" % comment_id) logger.info('commit comment: %d' % comment_id)
# rebuild RSS # rebuild RSS
rss.generate_site(token) rss.generate_site(token)
# send approval confirmation email to admin # send approval confirmation email to admin
email_body = get_template("approve_comment").render(original=message) new_email_body = get_template('approve_comment').render(original=email.content)
mailer.send(from_email, "Re: " + subject, email_body) if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body):
logger.warn('minor failure. cannot send approval email ' + email.subject)
return True return True

@ -1,9 +1,10 @@
#!/usr/bin/python #!/usr/bin/python
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
from conf import config
from playhouse.db_url import connect from playhouse.db_url import connect
from conf import config
def get_db(): def get_db():
return connect(config.get(config.DB_URL)) return connect(config.get(config.DB_URL))

@ -0,0 +1,152 @@
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import base64
import datetime
import email
import imaplib
import logging
import re
filename_re = re.compile('filename="(.+)"|filename=([^;\n\r"\']+)', re.I | re.S)
class Mailbox(object):
def __init__(self, host, port, ssl, login, password):
self.logger = logging.getLogger(__name__)
self.host = host
self.port = port
self.ssl = ssl
self.login = login
self.password = password
def __enter__(self):
if self.ssl:
self.imap = imaplib.IMAP4_SSL(self.host, self.port)
else:
self.imap = imaplib.IMAP4(self.host, self.port)
self.imap.login(self.login, self.password)
return self
def __exit__(self, type, value, traceback):
self.imap.close()
self.imap.logout()
def get_count(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
return sum(1 for num in data[0].split())
def fetch_raw_message(self, num):
self.imap.select('Inbox')
_, data = self.imap.fetch(str(num), '(RFC822)')
email_msg = email.message_from_bytes(data[0][1])
return email_msg
def fetch_message(self, num):
raw_msg = self.fetch_raw_message(num)
msg = {}
msg['encoding'] = 'UTF-8'
msg['index'] = num
dt = parse_date(raw_msg['Date']).strftime('%Y-%m-%d %H:%M:%S')
msg['datetime'] = dt
msg['from'] = raw_msg['From']
msg['to'] = raw_msg['To']
subject = email_nonascii_to_uft8(raw_msg['Subject'])
msg['subject'] = subject
parts = []
attachments = []
for part in raw_msg.walk():
if part.is_multipart():
continue
content_disposition = part.get('Content-Disposition', None)
if content_disposition:
# we have attachment
r = filename_re.findall(content_disposition)
if r:
filename = sorted(r[0])[1]
else:
filename = 'undefined'
content = base64.b64encode(part.get_payload(decode=True))
content = content.decode()
a = {
'filename': email_nonascii_to_uft8(filename),
'content': content,
'content-type': part.get_content_type(),
}
attachments.append(a)
else:
part_item = {}
content = part.get_payload(decode=True)
content_type = part.get_content_type()
try:
charset = part.get_param('charset', None)
if charset:
content = to_utf8(content, charset)
elif type(content) == bytes:
content = content.decode('utf8')
except:
self.logger.exception()
# RFC 3676: remove automatic word-wrapping
content = content.replace(' \r\n', ' ')
part_item['content'] = content
part_item['content-type'] = content_type
parts.append(part_item)
if parts:
msg['parts'] = parts
if attachments:
msg['attachments'] = attachments
return msg
def delete_message(self, num):
self.imap.select('Inbox')
self.imap.store(str(num), '+FLAGS', r'\Deleted')
self.imap.expunge()
def delete_all(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
for num in data[0].split():
self.imap.store(num, '+FLAGS', r'\Deleted')
self.imap.expunge()
def print_msgs(self):
self.imap.select('Inbox')
_, data = self.imap.search(None, 'ALL')
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
self.logger.debug('Message %s\n%s\n' % (num, data[0][1]))
def parse_date(v):
if v is None:
return datetime.datetime.now()
tt = email.utils.parsedate_tz(v)
if tt is None:
return datetime.datetime.now()
timestamp = email.utils.mktime_tz(tt)
date = datetime.datetime.fromtimestamp(timestamp)
return date
def to_utf8(string, charset):
return string.decode(charset).encode('UTF-8').decode('UTF-8')
def email_nonascii_to_uft8(string):
# RFC 1342 is a recommendation that provides a way to represent non ASCII
# characters inside e-mail in a way that wont confuse e-mail servers
subject = ''
for v, charset in email.header.decode_header(string):
if charset is None:
if type(v) is bytes:
v = v.decode()
subject = subject + v
else:
subject = subject + to_utf8(v, charset)
return subject

@ -1,43 +1,85 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging
import json import json
import logging
import smtplib
from email.mime.text import MIMEText
import requests import requests
from conf import config from conf import config
from core import imap
from model.email import Email
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def fetch(): def _open_mailbox():
mails = [] return imap.Mailbox(
r = requests.get(config.get(config.MAILER_URL) + "/mbox") config.get(config.IMAP_HOST),
if r.status_code == 200: config.get_int(config.IMAP_PORT),
payload = r.json() config.get_bool(config.IMAP_SSL),
if payload["count"] > 0: config.get(config.IMAP_LOGIN),
mails = payload["emails"] config.get(config.IMAP_PASSWORD),
return mails )
def get(id): def _to_dto(msg):
payload = None content = 'no plain-text part found in email'
r = requests.get(config.get(config.MAILER_URL) + "/mbox/" + str(id)) for part in msg['parts']:
if r.status_code == 200: if part['content-type'] == 'text/plain':
payload = r.json() content = part['content']
return payload break
return Email(
id=msg['index'],
encoding=msg['encoding'],
date=msg['datetime'],
from_addr=msg['from'],
to_addr=msg['to'],
subject=msg['subject'],
content=content,
)
def fetch():
msgs = []
try:
with _open_mailbox() as mbox:
count = mbox.get_count()
for num in range(count):
msg = _to_dto(mbox.fetch_message(num + 1))
msgs.append(msg)
except:
logger.exception('fetch mail exception')
return msgs
def send(to_email, subject, message): def send(to_email, subject, message):
headers = {"Content-Type": "application/json; charset=utf-8"}
msg = {"to": to_email, "subject": subject, "content": message} # Create the container (outer) email message.
r = requests.post( msg = MIMEText(message)
config.get(config.MAILER_URL) + "/mbox", data=json.dumps(msg), headers=headers msg['Subject'] = subject
) msg['To'] = to_email
if r.status_code in (200, 201): msg['From'] = config.get(config.SMTP_LOGIN)
logger.debug("Email for %s posted" % to_email)
else: success = True
logger.warn("Cannot post email for %s" % to_email) try:
s = smtplib.SMTP(config.get(config.SMTP_HOST), config.getInt(config.SMTP_PORT))
if config.get_bool(config.SMTP_STARTTLS):
s.starttls()
s.login(config.get(config.SMTP_LOGIN), config.get(config.SMTP_PASSWORD))
s.send_message(msg)
s.quit()
except:
logger.exception('send mail exception')
success = False
return success
def delete(id): def delete(id):
requests.delete(config.get(config.MAILER_URL) + "/mbox/" + str(id)) try:
with _open_mailbox() as mbox:
mbox.delete_message(id)
except:
logger.exception('delete mail exception')

@ -2,15 +2,18 @@
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
from datetime import datetime from datetime import datetime
import markdown import markdown
import PyRSS2Gen import PyRSS2Gen
from model.site import Site
from model.comment import Comment
from core.templater import get_template
from conf import config from conf import config
from core.templater import get_template
from model.comment import Comment
from model.site import Site
def generate_all(): def generate_all():
for site in Site.select(): for site in Site.select():
generate_site(site.token) generate_site(site.token)
@ -18,7 +21,7 @@ def generate_all():
def generate_site(token): def generate_site(token):
site = Site.select().where(Site.token == token).get() site = Site.select().where(Site.token == token).get()
rss_title = get_template("rss_title_message").render(site=site.name) rss_title = get_template('rss_title_message').render(site=site.name)
md = markdown.Markdown() md = markdown.Markdown()
items = [] items = []
@ -29,24 +32,23 @@ def generate_site(token):
.order_by(-Comment.published) .order_by(-Comment.published)
.limit(10) .limit(10)
): ):
item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url) item_link = '%s://%s%s' % (config.get(config.RSS_PROTO), site.url, row.url)
items.append( items.append(
PyRSS2Gen.RSSItem( PyRSS2Gen.RSSItem(
title="%s - %s://%s%s" title='%s - %s://%s%s'
% (config.get(config.RSS_PROTO), row.author_name, site.url, row.url), % (config.get(config.RSS_PROTO), row.author_name, site.url, row.url),
link=item_link, link=item_link,
description=md.convert(row.content), description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)), guid=PyRSS2Gen.Guid('%s/%d' % (item_link, row.id)),
pubDate=row.published, pubDate=row.published,
) )
) )
rss = PyRSS2Gen.RSS2( rss = PyRSS2Gen.RSS2(
title=rss_title, title=rss_title,
link="%s://%s" % (config.get(config.RSS_PROTO), site.url), link='%s://%s' % (config.get(config.RSS_PROTO), site.url),
description="Commentaires du site '%s'" % site.name, description='Commentaires du site "%s"' % site.name,
lastBuildDate=datetime.now(), lastBuildDate=datetime.now(),
items=items, items=items,
) )
rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8") rss.write_xml(open(config.get(config.RSS_FILE), 'w'), encoding='utf-8')

@ -2,14 +2,15 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os import os
from jinja2 import Environment
from jinja2 import FileSystemLoader from jinja2 import Environment, FileSystemLoader
from conf import config from conf import config
current_path = os.path.dirname(__file__) current_path = os.path.dirname(__file__)
template_path = os.path.abspath(os.path.join(current_path, "../templates")) template_path = os.path.abspath(os.path.join(current_path, '../templates'))
env = Environment(loader=FileSystemLoader(template_path)) env = Environment(loader=FileSystemLoader(template_path))
def get_template(name): def get_template(name):
return env.get_template(config.get(config.LANG) + "/" + name + ".tpl") return env.get_template(config.get(config.LANG) + '/' + name + '.tpl')

@ -1,16 +0,0 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import hashlib
from conf import config
def salt(value):
string = "%s%s" % (value, config.get(config.SECURITY_SALT))
dk = hashlib.sha256(string.encode())
return dk.hexdigest()
def md5(value):
dk = hashlib.md5(value.encode())
return dk.hexdigest()

@ -2,29 +2,31 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging import logging
from flask import request, jsonify, abort
from model.site import Site from flask import abort, jsonify, request
from model.comment import Comment
from conf import config from conf import config
from model.comment import Comment
from model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = config.flaskapp() app = config.flaskapp()
@app.route("/ping", methods=["GET"]) @app.route('/ping', methods=['GET'])
def ping(): def ping():
return "OK" return 'OK'
@app.route("/comments", methods=["GET"]) @app.route('/comments', methods=['GET'])
def query_comments(): def query_comments():
comments = [] comments = []
try: try:
token = request.args.get("token", "") token = request.args.get('token', '')
url = request.args.get("url", "") url = request.args.get('url', '')
logger.info("retrieve comments for token %s, url %s" % (token, url)) logger.info('retrieve comments for url %s' % (url))
for comment in ( for comment in (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .join(Site)
@ -35,29 +37,30 @@ def query_comments():
) )
.order_by(+Comment.published) .order_by(+Comment.published)
): ):
d = {} d = {
d["author"] = comment.author_name 'author': comment.author_name,
d["content"] = comment.content 'content': comment.content,
'avatar': comment.author_gravatar,
'date': comment.published.strftime('%Y-%m-%d %H:%M:%S')
}
if comment.author_site: if comment.author_site:
d["site"] = comment.author_site d['site'] = comment.author_site
d["avatar"] = comment.author_gravatar
d["date"] = comment.published.strftime("%Y-%m-%d %H:%M:%S")
logger.debug(d) logger.debug(d)
comments.append(d) comments.append(d)
r = jsonify({"data": comments}) r = jsonify({'data': comments})
r.status_code = 200 r.status_code = 200
except: except:
logger.warn("bad request") logger.warn('bad request')
r = jsonify({"data": []}) r = jsonify({'data': []})
r.status_code = 400 r.status_code = 400
return r return r
@app.route("/comments/count", methods=["GET"]) @app.route('/comments/count', methods=['GET'])
def get_comments_count(): def get_comments_count():
try: try:
token = request.args.get("token", "") token = request.args.get('token', '')
url = request.args.get("url", "") url = request.args.get('url', '')
count = ( count = (
Comment.select(Comment) Comment.select(Comment)
.join(Site) .join(Site)
@ -68,9 +71,9 @@ def get_comments_count():
) )
.count() .count()
) )
r = jsonify({"count": count}) r = jsonify({'count': count})
r.status_code = 200 r.status_code = 200
except: except:
r = jsonify({"count": 0}) r = jsonify({'count': 0})
r.status_code = 200 r.status_code = 200
return r return r

@ -3,52 +3,53 @@
import logging import logging
from datetime import datetime from datetime import datetime
from flask import request, abort, redirect
from model.site import Site from flask import abort, redirect, request
from model.comment import Comment
from conf import config from conf import config
from helper.hashing import md5 from model.comment import Comment
from model.site import Site
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = config.flaskapp() app = config.flaskapp()
@app.route("/newcomment", methods=["POST"]) @app.route('/newcomment', methods=['POST'])
def new_form_comment(): def new_form_comment():
try: try:
data = request.form data = request.form
logger.info("form data " + str(data)) logger.info('form data ' + str(data))
# validate token: retrieve site entity # validate token: retrieve site entity
token = data.get("token", "") token = data.get('token', '')
site = Site.select().where(Site.token == token).get() site = Site.select().where(Site.token == token).get()
if site is None: if site is None:
logger.warn("Unknown site %s" % token) logger.warn('Unknown site %s' % token)
abort(400) abort(400)
# honeypot for spammers # honeypot for spammers
captcha = data.get("remarque", "") captcha = data.get('remarque', '')
if captcha: if captcha:
logger.warn("discard spam: data %s" % data) logger.warn('discard spam: data %s' % data)
abort(400) abort(400)
url = data.get("url", "") url = data.get('url', '')
author_name = data.get("author", "").strip() author_name = data.get('author', '').strip()
author_gravatar = data.get("email", "").strip() author_gravatar = data.get('email', '').strip()
author_site = data.get("site", "").lower().strip() author_site = data.get('site', '').lower().strip()
if author_site and author_site[:4] != "http": if author_site and author_site[:4] != 'http':
author_site = "http://" + author_site author_site = 'http://' + author_site
message = data.get("message", "") message = data.get('message', '')
# anti-spam again # anti-spam again
if not url or not author_name or not message: if not url or not author_name or not message:
logger.warn("empty field: data %s" % data) logger.warn('empty field: data %s' % data)
abort(400) abort(400)
check_form_data(data) check_form_data(data)
# add a row to Comment table # add a row to Comment table
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") created = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
comment = Comment( comment = Comment(
site=site, site=site,
url=url, url=url,
@ -63,18 +64,18 @@ def new_form_comment():
comment.save() comment.save()
except: except:
logger.exception("new comment failure") logger.exception('new comment failure')
abort(400) abort(400)
return redirect("/redirect/", code=302) return redirect('/redirect/', code=302)
def check_form_data(data): def check_form_data(data):
fields = ["url", "message", "site", "remarque", "author", "token", "email"] fields = ['url', 'message', 'site', 'remarque', 'author', 'token', 'email']
d = data.to_dict() d = data.to_dict()
for field in fields: for field in fields:
if field in d: if field in d:
del d[field] del d[field]
if d: if d:
logger.warn("additional field: data %s" % data) logger.warn('additional field: data %s' % data)
abort(400) abort(400)

@ -17,18 +17,18 @@ class Comment(Model):
notified = DateTimeField(null=True, default=None) notified = DateTimeField(null=True, default=None)
published = DateTimeField(null=True, default=None) published = DateTimeField(null=True, default=None)
author_name = CharField() author_name = CharField()
author_site = CharField(default="") author_site = CharField(default='')
author_gravatar = CharField(default="") author_gravatar = CharField(default='')
content = TextField() content = TextField()
site = ForeignKeyField(Site, related_name="site") site = ForeignKeyField(Site, related_name='site')
class Meta: class Meta:
database = get_db() database = get_db()
def notify_site_admin(self): def notify_site_admin(self):
self.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.notified = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.save() self.save()
def publish(self): def publish(self):
self.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.published = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.save() self.save()

@ -0,0 +1,14 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from typing import NamedTuple
from datetime import datetime
class Email(NamedTuple):
id: int
encoding: str
date: datetime
from_addr: str
to_addr: str
subject: str
content: str

@ -2,12 +2,15 @@
# -*- coding: UTF-8 -*- # -*- coding: UTF-8 -*-
import argparse import argparse
import os
import logging import logging
import os
from flask import Flask from flask import Flask
from flask_apscheduler import APScheduler from flask_apscheduler import APScheduler
from conf import config from conf import config
# configure logging # configure logging
def configure_logging(level): def configure_logging(level):
root_logger = logging.getLogger() root_logger = logging.getLogger()
@ -15,7 +18,7 @@ def configure_logging(level):
ch = logging.StreamHandler() ch = logging.StreamHandler()
ch.setLevel(level) ch.setLevel(level)
# create formatter # create formatter
formatter = logging.Formatter("[%(asctime)s] %(name)s %(levelname)s %(message)s") formatter = logging.Formatter('[%(asctime)s] %(name)s %(levelname)s %(message)s')
# add formatter to ch # add formatter to ch
ch.setFormatter(formatter) ch.setFormatter(formatter)
# add ch to logger # add ch to logger
@ -26,21 +29,21 @@ class JobConfig(object):
JOBS = [] JOBS = []
SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 20}} SCHEDULER_EXECUTORS = {'default': {'type': 'threadpool', 'max_workers': 4}}
def __init__(self, mail_polling_seconds, new_comment_polling_seconds): def __init__(self, imap_polling_seconds, new_comment_polling_seconds):
self.JOBS = [ self.JOBS = [
{ {
"id": "fetch_mail", 'id': 'fetch_mail',
"func": "core.cron:fetch_mail_answers", 'func': 'core.cron:fetch_mail_answers',
"trigger": "interval", 'trigger': 'interval',
"seconds": mail_polling_seconds, 'seconds': imap_polling_seconds,
}, },
{ {
"id": "submit_new_comment", 'id': 'submit_new_comment',
"func": "core.cron:submit_new_comment", 'func': 'core.cron:submit_new_comment',
"trigger": "interval", 'trigger': 'interval',
"seconds": new_comment_polling_seconds, 'seconds': new_comment_polling_seconds,
}, },
] ]
@ -53,8 +56,8 @@ def stacosys_server(config_pathname):
# configure logging # configure logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
configure_logging(logging.INFO) configure_logging(logging.INFO)
logging.getLogger("werkzeug").level = logging.WARNING logging.getLogger('werkzeug').level = logging.WARNING
logging.getLogger("apscheduler.executors").level = logging.WARNING logging.getLogger('apscheduler.executors').level = logging.WARNING
# initialize database # initialize database
from core import database from core import database
@ -64,14 +67,14 @@ def stacosys_server(config_pathname):
# cron email fetcher # cron email fetcher
app.config.from_object( app.config.from_object(
JobConfig( JobConfig(
config.getInt(config.MAIL_POLLING), config.getInt(config.COMMENT_POLLING) config.get_int(config.IMAP_POLLING), config.get_int(config.COMMENT_POLLING)
) )
) )
scheduler = APScheduler() scheduler = APScheduler()
scheduler.init_app(app) scheduler.init_app(app)
scheduler.start() scheduler.start()
logger.info("Start Stacosys application") logger.info('Start Stacosys application')
# generate RSS for all sites # generate RSS for all sites
from core import rss from core import rss
@ -80,10 +83,12 @@ def stacosys_server(config_pathname):
# start Flask # start Flask
from interface import api from interface import api
logger.info('Load interface %s' % api)
from interface import form from interface import form
logger.debug("Load interface %s" % api) logger.info('Load interface %s' % form)
logger.debug("Load interface %s" % form)
app.run( app.run(
host=config.get(config.HTTP_HOST), host=config.get(config.HTTP_HOST),
@ -93,8 +98,8 @@ def stacosys_server(config_pathname):
) )
if __name__ == "__main__": if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("config", help="config path name") parser.add_argument('config', help='config path name')
args = parser.parse_args() args = parser.parse_args()
stacosys_server(args.config) stacosys_server(args.config)

Loading…
Cancel
Save