diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app/conf/config.py b/app/conf/config.py index 9994bc2..a9cd9b3 100644 --- a/app/conf/config.py +++ b/app/conf/config.py @@ -7,6 +7,7 @@ import profig FLASK_APP = "flask.app" DB_URL = "main.db_url" +LANG = "main.lang" HTTP_HOST = "http.host" HTTP_PORT = "http.port" @@ -14,8 +15,13 @@ HTTP_PORT = "http.port" SECURITY_SALT = "security.salt" SECURITY_SECRET = "security.secret" -MAIL_POLLING = "polling.newmail" -COMMENT_POLLING = "polling.newcomment" +RSS_PROTO = "rss.proto" +RSS_FILE = "rss.file" + +MAIL_POLLING = "mail.fetch_polling" +COMMENT_POLLING = "main.newcomment_polling" +MAILER_URL = "mail.mailer_url" + # variable params = dict() diff --git a/app/core/cron.py b/app/core/cron.py index 9788efc..932bf83 100644 --- a/app/core/cron.py +++ b/app/core/cron.py @@ -2,30 +2,48 @@ # -*- coding: utf-8 -*- import logging +from datetime import datetime import time +import re from core import mailer -from core import templater +from core.templater import get_template +from core import rss from model.comment import Comment from model.comment import Site logger = logging.getLogger(__name__) +client_ips = {} + +def cron(func): + def wrapper(): + logger.debug("execute fun " + func) + func() + + return wrapper + + +@cron def fetch_mail_answers(): - logger.info("DEBUT POP MAIL") - time.sleep(80) - logger.info("FIN POP MAIL") + msg = {} + + if msg["request"] == "new_mail": + reply_comment_email(msg["data"]) + mailer.delete(msg["data"]) + # data = request.get_json() # logger.debug(data) # processor.enqueue({'request': 'new_mail', 'data': data}) +@cron def submit_new_comment(): for comment in Comment.select().where(Comment.notified.is_null()): - + comment_list = ( "author: %s" % comment.author_name, "site: %s" % comment.author_site, @@ -36,10 +54,83 @@ def submit_new_comment(): "", ) comment_text = "\n".join(comment_list) - email_body = templater.get_template("new_comment").render(url=comment.url, comment=comment_text) + email_body = get_template("new_comment").render( + url=comment.url, comment=comment_text + ) site = Site.select().where(Site.id == Comment.site).get() # send email subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token) - mailer.send_mail(site.admin_email, subject, email_body) + mailer.send(site.admin_email, subject, email_body) logger.debug("new comment processed ") + + +def reply_comment_email(data): + + from_email = data["from"] + subject = data["subject"] + message = "" + for part in data["parts"]: + if part["content-type"] == "text/plain": + message = part["content"] + break + + m = re.search(r"\[(\d+)\:(\w+)\]", subject) + if not m: + logger.warn("ignore corrupted email. No token %s" % subject) + return + comment_id = int(m.group(1)) + token = m.group(2) + + # retrieve site and comment rows + try: + comment = Comment.select().where(Comment.id == comment_id).get() + except: + logger.warn("unknown comment %d" % comment_id) + return + + if comment.published: + logger.warn("ignore already published email. token %d" % comment_id) + return + + if comment.site.token != token: + logger.warn("ignore corrupted email. Unknown token %d" % comment_id) + return + + if not message: + logger.warn("ignore empty email") + return + + # safe logic: no answer or unknown answer is a go for publishing + if message[:2].upper() in ("NO", "SP"): + + # put a log to help fail2ban + if message[:2].upper() == "SP": # SPAM + if comment_id in client_ips: + logger.info( + "SPAM comment from %s: %d" % (client_ips[comment_id], comment_id) + ) + else: + logger.info("cannot identify SPAM source: %d" % comment_id) + + # forget client IP + if comment_id in client_ips: + del client_ips[comment_id] + + logger.info("discard comment: %d" % comment_id) + comment.delete_instance() + email_body = get_template("drop_comment").render(original=message) + mailer.send(from_email, "Re: " + subject, email_body) + else: + # update Comment row + comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + comment.save() + logger.info("commit comment: %d" % comment_id) + + # rebuild RSS + rss.generate_site(token) + + # send approval confirmation email to admin + email_body = get_template("approve_comment").render(original=message) + mailer.send(from_email, "Re: " + subject, email_body) + diff --git a/app/core/mailer.py b/app/core/mailer.py index 3a1213c..b58e6e1 100644 --- a/app/core/mailer.py +++ b/app/core/mailer.py @@ -1,3 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + +def fetch(): + pass + + +def send(email, subject, body): + pass + + +def delete(content): + # TODO delete mail + pass diff --git a/app/core/processor.py b/app/core/processor.py index aa63c27..148924d 100644 --- a/app/core/processor.py +++ b/app/core/processor.py @@ -12,7 +12,7 @@ from threading import Thread from queue import Queue from model.site import Site from model.comment import Comment -from helpers.hashing import md5 +from helper.hashing import md5 from conf import config from core import mailer @@ -26,95 +26,7 @@ env = None client_ips = {} -class Processor(Thread): - def stop(self): - logger.info("stop requested") - self.is_running = False - def run(self): - - logger.info("processor thread started") - self.is_running = True - while self.is_running: - try: - msg = queue.get() - if msg["request"] == "new_mail": - reply_comment_email(msg["data"]) - send_delete_command(msg["data"]) - else: - logger.info("throw unknown request " + str(msg)) - except: - logger.exception("processing failure") - - -def reply_comment_email(data): - - from_email = data["from"] - subject = data["subject"] - message = "" - for part in data["parts"]: - if part["content-type"] == "text/plain": - message = part["content"] - break - - m = re.search("\[(\d+)\:(\w+)\]", subject) - if not m: - logger.warn("ignore corrupted email. No token %s" % subject) - return - comment_id = int(m.group(1)) - token = m.group(2) - - # retrieve site and comment rows - try: - comment = Comment.select().where(Comment.id == comment_id).get() - except: - logger.warn("unknown comment %d" % comment_id) - return - - if comment.published: - logger.warn("ignore already published email. token %d" % comment_id) - return - - if comment.site.token != token: - logger.warn("ignore corrupted email. Unknown token %d" % comment_id) - return - - if not message: - logger.warn("ignore empty email") - return - - # safe logic: no answer or unknown answer is a go for publishing - if message[:2].upper() in ("NO", "SP"): - - # put a log to help fail2ban - if message[:2].upper() == "SP": # SPAM - if comment_id in client_ips: - logger.info( - "SPAM comment from %s: %d" % (client_ips[comment_id], comment_id) - ) - else: - logger.info("cannot identify SPAM source: %d" % comment_id) - - # forget client IP - if comment_id in client_ips: - del client_ips[comment_id] - - logger.info("discard comment: %d" % comment_id) - comment.delete_instance() - email_body = get_template("drop_comment").render(original=message) - mail(from_email, "Re: " + subject, email_body) - else: - # update Comment row - comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - comment.save() - logger.info("commit comment: %d" % comment_id) - - # rebuild RSS - rss(token) - - # send approval confirmation email to admin - email_body = get_template("approve_comment").render(original=message) - mail(from_email, "Re: " + subject, email_body) def get_email_metadata(message): @@ -126,73 +38,3 @@ def get_email_metadata(message): return email -def rss(token, onstart=False): - - if onstart and os.path.isfile(config.rss["file"]): - return - - site = Site.select().where(Site.token == token).get() - rss_title = get_template("rss_title_message").render(site=site.name) - md = markdown.Markdown() - - items = [] - for row in ( - Comment.select() - .join(Site) - .where(Site.token == token, Comment.published) - .order_by(-Comment.published) - .limit(10) - ): - item_link = "%s://%s%s" % (config.rss["proto"], site.url, row.url) - items.append( - PyRSS2Gen.RSSItem( - title="%s - %s://%s%s" - % (config.rss["proto"], row.author_name, site.url, row.url), - link=item_link, - description=md.convert(row.content), - guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)), - pubDate=row.published, - ) - ) - - rss = PyRSS2Gen.RSS2( - title=rss_title, - link="%s://%s" % (config.rss["proto"], site.url), - description="Commentaires du site '%s'" % site.name, - lastBuildDate=datetime.now(), - items=items, - ) - rss.write_xml(open(config.rss["file"], "w"), encoding="utf-8") - - -def send_delete_command(content): - # TODO delete mail - pass - - -def get_template(name): - return env.get_template(config.general["lang"] + "/" + name + ".tpl") - - -def enqueue(something): - queue.put(something) - - -def get_processor(): - return proc - - -def start(template_dir): - global proc, env - - # initialize Jinja 2 templating - logger.info("load templates from directory %s" % template_dir) - env = Environment(loader=FileSystemLoader(template_dir)) - - # generate RSS for all sites - for site in Site.select(): - rss(site.token, True) - - # start processor thread - proc = Processor() - proc.start() diff --git a/app/core/rss.py b/app/core/rss.py new file mode 100644 index 0000000..97f4637 --- /dev/null +++ b/app/core/rss.py @@ -0,0 +1,52 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from datetime import datetime +import markdown +import PyRSS2Gen +from model.site import Site +from model.comment import Comment +from core.templater import get_template +from conf import config + + +def generate_all(): + for site in Site.select(): + generate_site(site.token) + + +def generate_site(token): + + site = Site.select().where(Site.token == token).get() + rss_title = get_template("rss_title_message").render(site=site.name) + md = markdown.Markdown() + + items = [] + for row in ( + Comment.select() + .join(Site) + .where(Site.token == token, Comment.published) + .order_by(-Comment.published) + .limit(10) + ): + item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url) + items.append( + PyRSS2Gen.RSSItem( + title="%s - %s://%s%s" + % (config.get(config.RSS_PROTO), row.author_name, site.url, row.url), + link=item_link, + description=md.convert(row.content), + guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)), + pubDate=row.published, + ) + ) + + rss = PyRSS2Gen.RSS2( + title=rss_title, + link="%s://%s" % (config.get(config.RSS_PROTO), site.url), + description="Commentaires du site '%s'" % site.name, + lastBuildDate=datetime.now(), + items=items, + ) + rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8") + diff --git a/app/core/templater.py b/app/core/templater.py index 55d59fe..56b5278 100644 --- a/app/core/templater.py +++ b/app/core/templater.py @@ -1,4 +1,3 @@ - #!/usr/bin/env python # -*- coding: utf-8 -*- @@ -13,4 +12,4 @@ env = Environment(loader=FileSystemLoader(template_path)) def get_template(name): - return env.get_template(config.general["lang"] + "/" + name + ".tpl") + return env.get_template(config.get(config.LANG) + "/" + name + ".tpl") diff --git a/app/helpers/hashing.py b/app/helper/hashing.py similarity index 100% rename from app/helpers/hashing.py rename to app/helper/hashing.py diff --git a/app/interface/form.py b/app/interface/form.py index 36620e3..00cc149 100644 --- a/app/interface/form.py +++ b/app/interface/form.py @@ -7,7 +7,7 @@ from flask import request, abort, redirect from model.site import Site from model.comment import Comment from conf import config -from helpers.hashing import md5 +from helper.hashing import md5 logger = logging.getLogger(__name__) app = config.flaskapp() diff --git a/app/run.py b/app/run.py index 4dfb781..c57aa3a 100644 --- a/app/run.py +++ b/app/run.py @@ -59,9 +59,6 @@ def stacosys_server(config_pathname): database.setup() - # start processor - from core import processor - # cron email fetcher app.config.from_object( JobConfig( @@ -74,10 +71,18 @@ def stacosys_server(config_pathname): logger.info("Start Stacosys application") + # generate RSS for all sites + from core import rss + + rss.generate_all() + # start Flask from interface import api from interface import form + logger.debug("Load interface %s" % api) + logger.debug("Load interface %s" % form) + app.run( host=config.get(config.HTTP_HOST), port=config.get(config.HTTP_PORT), diff --git a/config.ini b/config.ini index c91f89d..bd26c6a 100755 --- a/config.ini +++ b/config.ini @@ -2,6 +2,7 @@ [main] lang = fr db_url = sqlite:///db.sqlite +newcomment_polling = 60 [http] root_url = http://localhost:8100 @@ -16,6 +17,6 @@ secret = Uqca5Kc8xuU6THz9 proto = http file = comments.xml -[polling] -newmail = 15 -newcomment = 60 +[mail] +fetch_polling = 15 +mailer_url = http://localhost:8000