From adc6451116ca4d31b720e9146061b417694a11f6 Mon Sep 17 00:00:00 2001 From: Yax <1949284+kianby@users.noreply.github.com> Date: Sun, 3 Jan 2021 18:24:16 +0100 Subject: [PATCH] improve encapsulation --- run.py | 83 ++++++++++------------ stacosys/core/cron.py | 117 +++++++++++++++----------------- stacosys/core/mailer.py | 110 ++++++++++++++++++------------ stacosys/interface/__init__.py | 5 ++ stacosys/interface/api.py | 42 ++++++------ stacosys/interface/form.py | 42 ++++++------ stacosys/interface/scheduler.py | 37 ++++++++++ 7 files changed, 240 insertions(+), 196 deletions(-) create mode 100644 stacosys/interface/scheduler.py diff --git a/run.py b/run.py index 5958b8b..efa984d 100644 --- a/run.py +++ b/run.py @@ -1,19 +1,21 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- +import sys +import os import argparse import logging -import os -import sys - from flask import Flask -from flask_apscheduler import APScheduler import stacosys.conf.config as config from stacosys.core import database -from stacosys.core import rss -#from stacosys.interface import api -#from stacosys.interface import form +from stacosys.core.rss import Rss +from stacosys.core.mailer import Mailer +from stacosys.interface import app +from stacosys.interface import api +from stacosys.interface import form +from stacosys.interface import scheduler + # configure logging def configure_logging(level): @@ -29,33 +31,8 @@ def configure_logging(level): root_logger.addHandler(ch) -class JobConfig(object): - - JOBS = [] - - SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}} - - def __init__(self, imap_polling_seconds, new_comment_polling_seconds): - self.JOBS = [ - { - "id": "fetch_mail", - "func": "stacosys.core.cron:fetch_mail_answers", - "trigger": "interval", - "seconds": imap_polling_seconds, - }, - { - "id": "submit_new_comment", - "func": "stacosys.core.cron:submit_new_comment", - "trigger": "interval", - "seconds": new_comment_polling_seconds, - }, - ] - - def stacosys_server(config_pathname): - app = Flask(__name__) - conf = config.Config.load(config_pathname) # configure logging @@ -68,26 +45,38 @@ def stacosys_server(config_pathname): db = database.Database() db.setup(conf.get(config.DB_URL)) - # cron email fetcher - app.config.from_object( - JobConfig( - conf.get_int(config.IMAP_POLLING), conf.get_int(config.COMMENT_POLLING) - ) - ) - scheduler = APScheduler() - scheduler.init_app(app) - scheduler.start() - logger.info("Start Stacosys application") # generate RSS for all sites - rss_manager = rss.Rss(conf.get(config.LANG), conf.get(config.RSS_FILE), conf.get(config.RSS_PROTO)) - rss_manager.generate_all() + rss = Rss( + conf.get(config.LANG), conf.get(config.RSS_FILE), conf.get(config.RSS_PROTO) + ) + rss.generate_all() + + # configure mailer + mailer = Mailer( + conf.get(config.IMAP_HOST), + conf.get_int(config.IMAP_PORT), + conf.get_bool(config.IMAP_SSL), + conf.get(config.IMAP_LOGIN), + conf.get(config.IMAP_PASSWORD), + conf.get(config.SMTP_HOST), + conf.get_int(config.SMTP_PORT), + conf.get_bool(config.SMTP_STARTTLS), + conf.get(config.SMTP_LOGIN), + conf.get(config.SMTP_PASSWORD), + ) - # start Flask - #logger.info("Load interface %s" % api) - #logger.info("Load interface %s" % form) + # configure scheduler + scheduler.configure( + conf.get_int(config.IMAP_POLLING), + conf.get_int(config.COMMENT_POLLING), + conf.get(config.LANG), + mailer, + rss, + ) + # start Flask app.run( host=conf.get(config.HTTP_HOST), port=conf.get(config.HTTP_PORT), diff --git a/stacosys/core/cron.py b/stacosys/core/cron.py index 974ab17..95951ee 100644 --- a/stacosys/core/cron.py +++ b/stacosys/core/cron.py @@ -6,7 +6,7 @@ import re import time from datetime import datetime -from stacosys.core import mailer, rss +from stacosys.core import rss from stacosys.core.templater import get_template from stacosys.model.comment import Comment, Site from stacosys.model.email import Email @@ -14,59 +14,18 @@ from stacosys.model.email import Email logger = logging.getLogger(__name__) -def cron(func): - def wrapper(): - logger.debug('execute CRON ' + func.__name__) - func() - - return wrapper - - -@cron -def fetch_mail_answers(): - +def fetch_mail_answers(lang, mailer, rss): for msg in mailer.fetch(): - if re.search(r'.*STACOSYS.*\[(\d+)\:(\w+)\]', msg.subject, re.DOTALL): - if _reply_comment_email(msg): + if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL): + if _reply_comment_email(lang, mailer, rss, msg): mailer.delete(msg.id) -@cron -def submit_new_comment(): - - for comment in Comment.select().where(Comment.notified.is_null()): - - comment_list = ( - 'author: %s' % comment.author_name, - 'site: %s' % comment.author_site, - 'date: %s' % comment.created, - 'url: %s' % comment.url, - '', - '%s' % comment.content, - '', - ) - comment_text = '\n'.join(comment_list) - email_body = get_template('new_comment').render( - url=comment.url, comment=comment_text - ) - - # send email - site = Site.get(Site.id == comment.site) - subject = 'STACOSYS %s: [%d:%s]' % (site.name, comment.id, site.token) - if mailer.send(site.admin_email, subject, email_body): - logger.debug('new comment processed ') - - # notify site admin and save notification datetime - comment.notify_site_admin() - else: - logger.warn('rescheduled. send mail failure ' + subject) - - -def _reply_comment_email(email : Email): +def _reply_comment_email(lang, mailer, rss, email: Email): - m = re.search(r'\[(\d+)\:(\w+)\]', email.subject) + m = re.search(r"\[(\d+)\:(\w+)\]", email.subject) if not m: - logger.warn('ignore corrupted email. No token %s' % email.subject) + logger.warn("ignore corrupted email. No token %s" % email.subject) return comment_id = int(m.group(1)) token = m.group(2) @@ -75,39 +34,75 @@ def _reply_comment_email(email : Email): try: comment = Comment.select().where(Comment.id == comment_id).get() except: - logger.warn('unknown comment %d' % comment_id) + logger.warn("unknown comment %d" % comment_id) return True if comment.published: - logger.warn('ignore already published email. token %d' % comment_id) + logger.warn("ignore already published email. token %d" % comment_id) return if comment.site.token != token: - logger.warn('ignore corrupted email. Unknown token %d' % comment_id) + logger.warn("ignore corrupted email. Unknown token %d" % comment_id) return if not email.plain_text_content: - logger.warn('ignore empty email') + logger.warn("ignore empty email") return # safe logic: no answer or unknown answer is a go for publishing - if email.plain_text_content[:2].upper() in ('NO'): - logger.info('discard comment: %d' % comment_id) + if email.plain_text_content[:2].upper() in ("NO"): + logger.info("discard comment: %d" % comment_id) comment.delete_instance() - new_email_body = get_template('drop_comment').render(original=email.plain_text_content) - if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body): - logger.warn('minor failure. cannot send rejection mail ' + email.subject) + new_email_body = get_template(lang, "drop_comment").render( + original=email.plain_text_content + ) + if not mailer.send(email.from_addr, "Re: " + email.subject, new_email_body): + logger.warn("minor failure. cannot send rejection mail " + email.subject) else: # save publishing datetime comment.publish() - logger.info('commit comment: %d' % comment_id) + logger.info("commit comment: %d" % comment_id) # rebuild RSS rss.generate_site(token) # send approval confirmation email to admin - new_email_body = get_template('approve_comment').render(original=email.plain_text_content) - if not mailer.send(email.from_addr, 'Re: ' + email.subject, new_email_body): - logger.warn('minor failure. cannot send approval email ' + email.subject) + new_email_body = get_template(lang, "approve_comment").render( + original=email.plain_text_content + ) + if not mailer.send(email.from_addr, "Re: " + email.subject, new_email_body): + logger.warn("minor failure. cannot send approval email " + email.subject) return True + + +def submit_new_comment(lang, mailer): + + for comment in Comment.select().where(Comment.notified.is_null()): + + comment_list = ( + "author: %s" % comment.author_name, + "site: %s" % comment.author_site, + "date: %s" % comment.created, + "url: %s" % comment.url, + "", + "%s" % comment.content, + "", + ) + comment_text = "\n".join(comment_list) + # TODO use constants for template names + email_body = get_template(lang, "new_comment").render( + url=comment.url, comment=comment_text + ) + + # send email + site = Site.get(Site.id == comment.site) + subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token) + if mailer.send(site.admin_email, subject, email_body): + logger.debug("new comment processed ") + + # notify site admin and save notification datetime + comment.notify_site_admin() + else: + logger.warn("rescheduled. send mail failure " + subject) + diff --git a/stacosys/core/mailer.py b/stacosys/core/mailer.py index bf5fcc3..d4b48c1 100644 --- a/stacosys/core/mailer.py +++ b/stacosys/core/mailer.py @@ -15,53 +15,75 @@ from stacosys.model.email import Email logger = logging.getLogger(__name__) -def _open_mailbox(): - return imap.Mailbox( - config.get(config.IMAP_HOST), - config.get_int(config.IMAP_PORT), - config.get_bool(config.IMAP_SSL), - config.get(config.IMAP_LOGIN), - config.get(config.IMAP_PASSWORD), - ) +class Mailer: + def __init__( + self, + imap_host, + imap_port, + imap_ssl, + imap_login, + imap_password, + smtp_host, + smtp_port, + smtp_starttls, + smtp_login, + smtp_password, + ): + self._imap_host = imap_host + self._imap_port = imap_port + self._imap_ssl = imap_ssl + self._imap_login = imap_login + self._imap_password = imap_password + self._smtp_host = smtp_host + self._smtp_port = smtp_port + self._smtp_starttls = smtp_starttls + self._smtp_login = smtp_login + self._smtp_password = smtp_password + def _open_mailbox(self): + return imap.Mailbox( + self._imap_host, + self._imap_port, + self._imap_ssl, + self._imap_login, + self._imap_password, + ) -def fetch(): - msgs = [] - try: - with _open_mailbox() as mbox: - count = mbox.get_count() - for num in range(count): - msgs.append(mbox.fetch_message(num + 1)) - except: - logger.exception("fetch mail exception") - return msgs + def fetch(self): + msgs = [] + try: + with self._open_mailbox() as mbox: + count = mbox.get_count() + for num in range(count): + msgs.append(mbox.fetch_message(num + 1)) + except: + logger.exception("fetch mail exception") + return msgs + def send(self, to_email, subject, message): -def send(to_email, subject, message): + # Create the container (outer) email message. + msg = MIMEText(message) + msg["Subject"] = subject + msg["To"] = to_email + msg["From"] = self._smtp_login - # Create the container (outer) email message. - msg = MIMEText(message) - msg["Subject"] = subject - msg["To"] = to_email - msg["From"] = config.get(config.SMTP_LOGIN) + success = True + try: + s = smtplib.SMTP(self._smtp_host, self._smtp_port) + if self._smtp_starttls: + s.starttls() + s.login(self._smtp_login, self._smtp_password) + s.send_message(msg) + s.quit() + except: + logger.exception("send mail exception") + success = False + return success - success = True - try: - s = smtplib.SMTP(config.get(config.SMTP_HOST), config.get_int(config.SMTP_PORT)) - if config.get_bool(config.SMTP_STARTTLS): - s.starttls() - s.login(config.get(config.SMTP_LOGIN), config.get(config.SMTP_PASSWORD)) - s.send_message(msg) - s.quit() - except: - logger.exception("send mail exception") - success = False - return success - - -def delete(id): - try: - with _open_mailbox() as mbox: - mbox.delete_message(id) - except: - logger.exception("delete mail exception") + def delete(self, id): + try: + with self._open_mailbox() as mbox: + mbox.delete_message(id) + except: + logger.exception("delete mail exception") diff --git a/stacosys/interface/__init__.py b/stacosys/interface/__init__.py index e69de29..b7714c3 100644 --- a/stacosys/interface/__init__.py +++ b/stacosys/interface/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from flask import Flask +app = Flask(__name__) \ No newline at end of file diff --git a/stacosys/interface/api.py b/stacosys/interface/api.py index dcc63c1..90d192a 100644 --- a/stacosys/interface/api.py +++ b/stacosys/interface/api.py @@ -2,31 +2,29 @@ # -*- coding: utf-8 -*- import logging - from flask import abort, jsonify, request -from stacosys.conf import config +from stacosys.interface import app from stacosys.model.comment import Comment from stacosys.model.site import Site logger = logging.getLogger(__name__) -app = config.flaskapp() -@app.route('/ping', methods=['GET']) +@app.route("/ping", methods=["GET"]) def ping(): - return 'OK' + return "OK" -@app.route('/comments', methods=['GET']) +@app.route("/comments", methods=["GET"]) def query_comments(): comments = [] try: - token = request.args.get('token', '') - url = request.args.get('url', '') + token = request.args.get("token", "") + url = request.args.get("url", "") - logger.info('retrieve comments for url %s' % (url)) + logger.info("retrieve comments for url %s" % (url)) for comment in ( Comment.select(Comment) .join(Site) @@ -38,29 +36,29 @@ def query_comments(): .order_by(+Comment.published) ): d = { - 'author': comment.author_name, - 'content': comment.content, - 'avatar': comment.author_gravatar, - 'date': comment.published.strftime('%Y-%m-%d %H:%M:%S') + "author": comment.author_name, + "content": comment.content, + "avatar": comment.author_gravatar, + "date": comment.published.strftime("%Y-%m-%d %H:%M:%S"), } if comment.author_site: - d['site'] = comment.author_site + d["site"] = comment.author_site logger.debug(d) comments.append(d) - r = jsonify({'data': comments}) + r = jsonify({"data": comments}) r.status_code = 200 except: - logger.warn('bad request') - r = jsonify({'data': []}) + logger.warn("bad request") + r = jsonify({"data": []}) r.status_code = 400 return r -@app.route('/comments/count', methods=['GET']) +@app.route("/comments/count", methods=["GET"]) def get_comments_count(): try: - token = request.args.get('token', '') - url = request.args.get('url', '') + token = request.args.get("token", "") + url = request.args.get("url", "") count = ( Comment.select(Comment) .join(Site) @@ -71,9 +69,9 @@ def get_comments_count(): ) .count() ) - r = jsonify({'count': count}) + r = jsonify({"count": count}) r.status_code = 200 except: - r = jsonify({'count': 0}) + r = jsonify({"count": 0}) r.status_code = 200 return r diff --git a/stacosys/interface/form.py b/stacosys/interface/form.py index 738a958..df0684c 100644 --- a/stacosys/interface/form.py +++ b/stacosys/interface/form.py @@ -3,53 +3,51 @@ import logging from datetime import datetime - from flask import abort, redirect, request -from stacosys.conf import config +from stacosys.interface import app from stacosys.model.comment import Comment from stacosys.model.site import Site logger = logging.getLogger(__name__) -app = config.flaskapp() -@app.route('/newcomment', methods=['POST']) +@app.route("/newcomment", methods=["POST"]) def new_form_comment(): try: data = request.form - logger.info('form data ' + str(data)) + logger.info("form data " + str(data)) # validate token: retrieve site entity - token = data.get('token', '') + token = data.get("token", "") site = Site.select().where(Site.token == token).get() if site is None: - logger.warn('Unknown site %s' % token) + logger.warn("Unknown site %s" % token) abort(400) # honeypot for spammers - captcha = data.get('remarque', '') + captcha = data.get("remarque", "") if captcha: - logger.warn('discard spam: data %s' % data) + logger.warn("discard spam: data %s" % data) abort(400) - url = data.get('url', '') - author_name = data.get('author', '').strip() - author_gravatar = data.get('email', '').strip() - author_site = data.get('site', '').lower().strip() - if author_site and author_site[:4] != 'http': - author_site = 'http://' + author_site - message = data.get('message', '') + url = data.get("url", "") + author_name = data.get("author", "").strip() + author_gravatar = data.get("email", "").strip() + author_site = data.get("site", "").lower().strip() + if author_site and author_site[:4] != "http": + author_site = "http://" + author_site + message = data.get("message", "") # anti-spam again if not url or not author_name or not message: - logger.warn('empty field: data %s' % data) + logger.warn("empty field: data %s" % data) abort(400) check_form_data(data) # add a row to Comment table - created = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") comment = Comment( site=site, url=url, @@ -64,18 +62,18 @@ def new_form_comment(): comment.save() except: - logger.exception('new comment failure') + logger.exception("new comment failure") abort(400) - return redirect('/redirect/', code=302) + return redirect("/redirect/", code=302) def check_form_data(data): - fields = ['url', 'message', 'site', 'remarque', 'author', 'token', 'email'] + fields = ["url", "message", "site", "remarque", "author", "token", "email"] d = data.to_dict() for field in fields: if field in d: del d[field] if d: - logger.warn('additional field: data %s' % data) + logger.warn("additional field: data %s" % data) abort(400) diff --git a/stacosys/interface/scheduler.py b/stacosys/interface/scheduler.py new file mode 100644 index 0000000..0127282 --- /dev/null +++ b/stacosys/interface/scheduler.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from flask_apscheduler import APScheduler +from stacosys.interface import app + + +class JobConfig(object): + + JOBS = [] + + SCHEDULER_EXECUTORS = {"default": {"type": "threadpool", "max_workers": 4}} + + def __init__(self, imap_polling_seconds, new_comment_polling_seconds, lang, mailer, rss): + self.JOBS = [ + { + "id": "fetch_mail", + "func": "stacosys.core.cron:fetch_mail_answers", + "args": [lang, mailer, rss], + "trigger": "interval", + "seconds": imap_polling_seconds, + }, + { + "id": "submit_new_comment", + "func": "stacosys.core.cron:submit_new_comment", + "args": [lang, mailer], + "trigger": "interval", + "seconds": new_comment_polling_seconds, + }, + ] + + +def configure(imap_polling, comment_polling, lang, mailer, rss): + app.config.from_object(JobConfig(imap_polling, comment_polling, lang, mailer, rss)) + scheduler = APScheduler() + scheduler.init_app(app) + scheduler.start()