From daed9e9cb559a0eb2aa5b88975d93b03099ac39e Mon Sep 17 00:00:00 2001 From: Yax <1949284+kianby@users.noreply.github.com> Date: Fri, 15 May 2015 19:51:02 +0200 Subject: [PATCH] Merge pecosys work around comment processing with stacosys --- app/controllers/api.py | 22 +-- app/models/comment.py | 2 +- app/run.py | 13 +- app/services/processor.py | 231 +++++++++++++++++++++++++ app/templates/en/approve_comment.tpl | 9 + app/templates/en/drop_comment.tpl | 9 + app/templates/en/new_comment.tpl | 16 ++ app/templates/en/notify_message.tpl | 1 + app/templates/en/notify_reader.tpl | 9 + app/templates/en/notify_subscriber.tpl | 13 ++ app/templates/en/unsubscribe_page.tpl | 2 + app/templates/fr/approve_comment.tpl | 9 + app/templates/fr/drop_comment.tpl | 9 + app/templates/fr/new_comment.tpl | 16 ++ app/templates/fr/notify_message.tpl | 1 + app/templates/fr/notify_reader.tpl | 9 + app/templates/fr/notify_subscriber.tpl | 13 ++ app/templates/fr/unsubscribe_page.tpl | 2 + config.py | 2 + demo/public/index.html | 3 +- demo/public/js/page.js | 5 +- demo/public/js/stacosys.js | 6 +- demo/public/redirect.html | 67 +++++++ requirements.txt | 1 + tools/pecosys2stacosys.py | 17 +- 25 files changed, 456 insertions(+), 31 deletions(-) create mode 100644 app/services/processor.py create mode 100644 app/templates/en/approve_comment.tpl create mode 100644 app/templates/en/drop_comment.tpl create mode 100644 app/templates/en/new_comment.tpl create mode 100644 app/templates/en/notify_message.tpl create mode 100644 app/templates/en/notify_reader.tpl create mode 100644 app/templates/en/notify_subscriber.tpl create mode 100644 app/templates/en/unsubscribe_page.tpl create mode 100644 app/templates/fr/approve_comment.tpl create mode 100644 app/templates/fr/drop_comment.tpl create mode 100644 app/templates/fr/new_comment.tpl create mode 100644 app/templates/fr/notify_message.tpl create mode 100644 app/templates/fr/notify_reader.tpl create mode 100644 app/templates/fr/notify_subscriber.tpl create mode 100644 app/templates/fr/unsubscribe_page.tpl create mode 100644 demo/public/redirect.html diff --git a/app/controllers/api.py b/app/controllers/api.py index c9362ab..0c9a765 100644 --- a/app/controllers/api.py +++ b/app/controllers/api.py @@ -7,6 +7,7 @@ from app import app from app.models.site import Site from app.models.comment import Comment from app.helpers.hashing import md5 +from app.services import processor logger = logging.getLogger(__name__) @@ -22,6 +23,7 @@ def query_comments(): logger.info('retrieve comments for token %s, url %s' % (token, url)) for comment in Comment.select(Comment).join(Site).where( (Comment.url == url) & + (Comment.published.is_null(False)) & (Site.token == token)).order_by(+Comment.published): d = {} d['author'] = comment.author_name @@ -49,6 +51,7 @@ def get_comments_count(): url = request.args.get('url', '') count = Comment.select(Comment).join(Site).where( (Comment.url == url) & + (Comment.published.is_null(False)) & (Site.token == token)).count() r = jsonify({'count': count}) r.status_code = 200 @@ -72,24 +75,13 @@ def new_comment(): logger.warn('Unknown site %s' % token) abort(400) - # get values - url = data.get('url', '') - author_name = data.get('author', '') - author_email = data.get('email', '') - author_site = data.get('site', '') - message = data.get('message', '') - subscribe = data.get('subscribe', '') - # honeypot for spammers captcha = data.get('captcha', '') if captcha: - logger.warn('discard spam: captcha %s author %s email %s site %s url %s message %s' - % (captcha, author_name, author_email, author_site, url, message)) - else: - # TODO push new comment to backend service - logger.info('process: captcha %s author %s email %s site %s url %s message %s subscribe %s' - % (captcha, author_name, author_email, author_site, - url, message, subscribe)) + logger.warn('discard spam: data %s' % data) + abort(400) + + processor.enqueue({'request': 'new_comment', 'data': data}) except: logger.exception("new comment failure") diff --git a/app/models/comment.py b/app/models/comment.py index ab80933..1b79a58 100644 --- a/app/models/comment.py +++ b/app/models/comment.py @@ -13,7 +13,7 @@ from app.services.database import get_db class Comment(Model): url = CharField() created = DateTimeField() - published = DateTimeField() + published = DateTimeField(null=True,default=None) author_name = CharField() author_email = CharField(default='') author_site = CharField(default='') diff --git a/app/run.py b/app/run.py index 0a5e674..c513b6d 100644 --- a/app/run.py +++ b/app/run.py @@ -7,10 +7,10 @@ import logging from werkzeug.contrib.fixers import ProxyFix from flask.ext.cors import CORS -# add current and parent path to syspath -currentPath = os.path.dirname(__file__) -parentPath = os.path.abspath(os.path.join(currentPath, os.path.pardir)) -paths = [currentPath, parentPath] +# add current path and parent path to syspath +current_path = os.path.dirname(__file__) +parent_path = os.path.abspath(os.path.join(current_path, os.path.pardir)) +paths = [current_path, parent_path] for path in paths: if path not in sys.path: sys.path.insert(0, path) @@ -18,6 +18,7 @@ for path in paths: # more imports import config from app.services import database +from app.services import processor from app.controllers import api from app import app @@ -42,6 +43,10 @@ logger = logging.getLogger(__name__) # initialize database database.setup() +# start processor +template_path = os.path.abspath(os.path.join(current_path, 'templates')) +processor.start(template_path) + app.wsgi_app = ProxyFix(app.wsgi_app) logger.info("Start Stacosys application") diff --git a/app/services/processor.py b/app/services/processor.py new file mode 100644 index 0000000..5d0fbf9 --- /dev/null +++ b/app/services/processor.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import logging +import re +from datetime import datetime +from threading import Thread +from queue import Queue +import chardet +from jinja2 import Environment, FileSystemLoader +from app.models.site import Site +from app.models.comment import Comment + + +logger = logging.getLogger(__name__) +queue = Queue() +proc = None +env = None + + +class Processor(Thread): + + def stop(self): + logger.info("stop requested") + self.is_running = False + + def run(self): + + self.is_running = True + while self.is_running: + msg = queue.get() + if msg['request'] == 'new_comment': + new_comment(msg['data']) + #elif msg['type'] == 'reply_comment_email': + # reply_comment_email(req['From'], req['Subject'], req['Body']) + #elif req['type'] == 'unsubscribe': + # unsubscribe_reader(req['email'], req['article']) + else: + logger.info("Dequeue unknown request " + msg) + + +def new_comment(data): + + try: + token = data.get('token', '') + url = data.get('url', '') + author_name = data.get('author', '') + author_email = data.get('email', '') + author_site = data.get('site', '') + message = data.get('message', '') + subscribe = data.get('subscribe', '') + + # create a new comment row + site = Site.select().where(Site.token == token).get() + + logger.info('new comment received: %s' % data) + + if author_site and author_site[:4] != 'http': + author_site = 'http://' + author_site + + created = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + comment = Comment(site=site, url=url, author_name=author_name, + author_site=author_site, author_email=author_email, + content=message, created=created, published=None) + comment.save() + + 1 / 0 + # Render email body template + email_body = get_template('new_comment').render(url=url, comment=comment) + + # Send email + mail(pecosys.get_config('post', 'from_email'), + pecosys.get_config('post', 'to_email'), + '[' + branch_name + '-' + article + ']', email_body) + + # Reader subscribes to further comments + if subscribe and email: + subscribe_reader(email, article, url) + + logger.debug("new comment processed ") + except: + logger.exception("new_comment failure") + + +def reply_comment_email(from_email, subject, message): + try: + m = re.search('\[(\d+)\-(\w+)\]', subject) + branch_name = m.group(1) + article = m.group(2) + + message = decode_best_effort(message) + + # safe logic: no answer or unknown answer is a go for publishing + if message[:2].upper() == 'NO': + logger.info('discard comment: %s' % branch_name) + email_body = get_template('drop_comment').render(original=message) + mail(pecosys.get_config('post', 'from_email'), + pecosys.get_config('post', 'to_email'), + 'Re: ' + subject, email_body) + else: + if pecosys.get_config("git", "disabled"): + logger.debug("GIT usage disabled (debug mode)") + else: + git.merge(branch_name) + if pecosys.get_config("git", "remote"): + git.push() + logger.info('commit comment: %s' % branch_name) + + # send approval confirmation email to admin + email_body = get_template('approve_comment').render(original=message) + mail(pecosys.get_config('post', 'from_email'), + pecosys.get_config('post', 'to_email'), + 'Re: ' + subject, email_body) + + # notify reader once comment is published + reader_email, article_url = get_email_metadata(message) + if reader_email: + notify_reader(reader_email, article_url) + + # notify subscribers every time a new comment is published + notify_subscribers(article) + + if pecosys.get_config("git", "disabled"): + logger.debug("GIT usage disabled (debug mode)") + else: + git.branch("-D", branch_name) + except: + logger.exception("new email failure") + + +def get_email_metadata(message): + # retrieve metadata reader email and URL from email body sent by admin + email = "" + url = "" + m = re.search('email:\s(.+@.+\..+)', message) + if m: + email = m.group(1) + + m = re.search('url:\s(.+)', message) + if m: + url = m.group(1) + return (email, url) + + +def subscribe_reader(email, article, url): + logger.info("subscribe reader %s to %s (%s)" % (email, article, url)) + db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json') + db.insert({'email': email, 'article': article, 'url': url}) + + +def unsubscribe_reader(email, article): + logger.info("unsubscribe reader %s from %s" % (email, article)) + db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json') + db.remove((where('email') == email) & (where('article') == article)) + + +def notify_subscribers(article): + logger.info('notify subscribers for article %s' % article) + db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json') + for item in db.search(where('article') == article): + logger.info(item) + to_email = item['email'] + logger.info("notify reader %s for article %s" % (to_email, article)) + unsubscribe_url = pecosys.get_config('subscription', 'url') + '?email=' + to_email + '&article=' + article + email_body = get_template('notify_subscriber').render(article_url=item['url'], + unsubscribe_url=unsubscribe_url) + subject = get_template('notify_message').render() + mail(pecosys.get_config('subscription', 'from_email'), to_email, subject, email_body) + + +def notify_reader(email, url): + logger.info('notify reader: email %s about URL %s' % (email, url)) + email_body = get_template('notify_reader').render(article_url=url) + subject = get_template('notify_message').render() + mail(pecosys.get_config('subscription', 'from_email'), email, subject, email_body) + + +def decode_best_effort(string): + info = chardet.detect(string) + if info['confidence'] < 0.5: + return string.decode('utf8', errors='replace') + else: + return string.decode(info['encoding'], errors='replace') + + +def mail(from_email, to_email, subject, *messages): + + # Create the container (outer) email message. + msg = MIMEMultipart() + msg['Subject'] = subject + msg['From'] = from_email + msg['To'] = to_email + msg.preamble = subject + + for message in messages: + part = MIMEText(message, 'plain') + msg.attach(part) + + s = smtplib.SMTP(pecosys.get_config('smtp', 'host'), + pecosys.get_config('smtp', 'port')) + if(pecosys.get_config('smtp', 'starttls')): + s.starttls() + s.login(pecosys.get_config('smtp', 'login'), + pecosys.get_config('smtp', 'password')) + s.sendmail(from_email, to_email, msg.as_string()) + s.quit() + + +def get_template(name): + return env.get_template(pecosys.get_config('global', 'lang') + '/' + name + '.tpl') + + +def enqueue(something): + queue.put(something) + + +def get_processor(): + return proc + + +def start(template_dir): + global proc, env + + # initialize Jinja 2 templating + logger.info("load templates from directory %s" % template_dir) + env = Environment(loader=FileSystemLoader(template_dir)) + + # start processor thread + proc = Processor() + proc.start() diff --git a/app/templates/en/approve_comment.tpl b/app/templates/en/approve_comment.tpl new file mode 100644 index 0000000..2daa5b2 --- /dev/null +++ b/app/templates/en/approve_comment.tpl @@ -0,0 +1,9 @@ +Hi, + +The comment should be published soon. It has been approved. + +-- +Pecosys + + +{{ original }} diff --git a/app/templates/en/drop_comment.tpl b/app/templates/en/drop_comment.tpl new file mode 100644 index 0000000..c3b0362 --- /dev/null +++ b/app/templates/en/drop_comment.tpl @@ -0,0 +1,9 @@ +Hi, + +The comment will not be published. It has been dropped. + +-- +Pecosys + + +{{ original }} diff --git a/app/templates/en/new_comment.tpl b/app/templates/en/new_comment.tpl new file mode 100644 index 0000000..a07f7a3 --- /dev/null +++ b/app/templates/en/new_comment.tpl @@ -0,0 +1,16 @@ +Hi, + +A new comment has been submitted for post {{ url }}. + +You have two choices: +- reject the comment by replying NO (or no), +- accept the comment by sending back the email as it is. + +If you choose the latter option, Pecosys is going to publish the commennt. + +Please find comment details below: + +{{ comment }} + +-- +Pecosys diff --git a/app/templates/en/notify_message.tpl b/app/templates/en/notify_message.tpl new file mode 100644 index 0000000..94a261f --- /dev/null +++ b/app/templates/en/notify_message.tpl @@ -0,0 +1 @@ +New comment diff --git a/app/templates/en/notify_reader.tpl b/app/templates/en/notify_reader.tpl new file mode 100644 index 0000000..c2d2a0e --- /dev/null +++ b/app/templates/en/notify_reader.tpl @@ -0,0 +1,9 @@ +Hi, + +Your comment has been approved. It should be published in few minutes. + + {{ article_url }} + +-- +Pecosys + diff --git a/app/templates/en/notify_subscriber.tpl b/app/templates/en/notify_subscriber.tpl new file mode 100644 index 0000000..d1d9e6c --- /dev/null +++ b/app/templates/en/notify_subscriber.tpl @@ -0,0 +1,13 @@ +Hi, + +A new comment has been published for an article you have subscribed to. + + {{ article_url }} + +You can unsubscribe at any time using this link: + + {{ unsubscribe_url }} + +-- +Pecosys + diff --git a/app/templates/en/unsubscribe_page.tpl b/app/templates/en/unsubscribe_page.tpl new file mode 100644 index 0000000..a52afd7 --- /dev/null +++ b/app/templates/en/unsubscribe_page.tpl @@ -0,0 +1,2 @@ +Your request has been sent. In case of issue please contact site +administrator. diff --git a/app/templates/fr/approve_comment.tpl b/app/templates/fr/approve_comment.tpl new file mode 100644 index 0000000..a381c21 --- /dev/null +++ b/app/templates/fr/approve_comment.tpl @@ -0,0 +1,9 @@ +Bonjour, + +Le commentaire sera bientôt publié. Il a été approuvé. + +-- +Pecosys + + +{{ original }} diff --git a/app/templates/fr/drop_comment.tpl b/app/templates/fr/drop_comment.tpl new file mode 100644 index 0000000..714decf --- /dev/null +++ b/app/templates/fr/drop_comment.tpl @@ -0,0 +1,9 @@ +Bonjour, + +Le commentaire ne sera pas publié. Il a été rejeté. + +-- +Pecosys + + +{{ original }} diff --git a/app/templates/fr/new_comment.tpl b/app/templates/fr/new_comment.tpl new file mode 100644 index 0000000..b3346fe --- /dev/null +++ b/app/templates/fr/new_comment.tpl @@ -0,0 +1,16 @@ +Bonjour, + +Un nouveau commentaire a été posté pour l'article {{ url }}. + +Vous avez deux réponses possibles : +- rejeter le commentaire en répondant NO (ou no), +- accepter le commentaire en renvoyant cet email tel quel. + +Si cette dernière option est choisie, Pecosys publiera le commentaire très bientôt. + +Voici les détails concernant le commentaire : + +{{ comment }} + +-- +Pecosys diff --git a/app/templates/fr/notify_message.tpl b/app/templates/fr/notify_message.tpl new file mode 100644 index 0000000..5455f77 --- /dev/null +++ b/app/templates/fr/notify_message.tpl @@ -0,0 +1 @@ +Nouveau commentaire diff --git a/app/templates/fr/notify_reader.tpl b/app/templates/fr/notify_reader.tpl new file mode 100644 index 0000000..c8d2956 --- /dev/null +++ b/app/templates/fr/notify_reader.tpl @@ -0,0 +1,9 @@ +Bonjour, + +Votre commentaire a été approuvé. Il sera publié dans quelques minutes. + + {{ article_url }} + +-- +Pecosys + diff --git a/app/templates/fr/notify_subscriber.tpl b/app/templates/fr/notify_subscriber.tpl new file mode 100644 index 0000000..29804ee --- /dev/null +++ b/app/templates/fr/notify_subscriber.tpl @@ -0,0 +1,13 @@ +Bonjour, + +Un nouveau commentaire a été publié pour un article auquel vous êtes abonné. + + {{ article_url }} + +Vous pouvez vous désinscrire à tout moment en suivant ce lien : + + {{ unsubscribe_url }} + +-- +Pecosys + diff --git a/app/templates/fr/unsubscribe_page.tpl b/app/templates/fr/unsubscribe_page.tpl new file mode 100644 index 0000000..3cd63e8 --- /dev/null +++ b/app/templates/fr/unsubscribe_page.tpl @@ -0,0 +1,2 @@ +Votre requête a été envoyée. En cas de problème, contactez l'administrateur du +site. diff --git a/config.py b/config.py index 9777272..8fef52b 100644 --- a/config.py +++ b/config.py @@ -2,6 +2,8 @@ DEBUG = True +LANG = "en" + #DB_URL = "mysql://stacosys_user:stacosys_password@localhost:3306/stacosys" DB_URL = "sqlite:///db.sqlite" diff --git a/demo/public/index.html b/demo/public/index.html index ad7c883..7e016ef 100644 --- a/demo/public/index.html +++ b/demo/public/index.html @@ -203,7 +203,8 @@ instance d'ici peu.

var STACOSYS_URL = 'http://127.0.0.1:8000'; var STACOSYS_TOKEN = '9fb3fc042c572cb831005fd16186126765140fa2bd9bb2d4a28e47a9457dc26c'; //STACOSYS_PAGE = 'blogduyax.madyanne.fr/mes-applications-pour-blackberry.html' -var STACOSYS_PAGE = 'blogduyax.madyanne.fr/migration-du-blog-sous-pelican.html' +//var STACOSYS_PAGE = 'blogduyax.madyanne.fr/migration-du-blog-sous-pelican.html' +var STACOSYS_PAGE = 'migration-du-blog-sous-pelican.html' window.onload = initialize_comments(); diff --git a/demo/public/js/page.js b/demo/public/js/page.js index 77c4813..06d6a97 100644 --- a/demo/public/js/page.js +++ b/demo/public/js/page.js @@ -65,9 +65,10 @@ function new_comment() { var email = document.getElementById('email').value; var site = document.getElementById('site').value; var captcha = document.getElementById('captcha').value; - //var subscribe = document.getElementById('subscribe').value; + var subscribe = document.getElementById('subscribe').value; + var message = document.getElementById('message').value; - stacosys_new_comment(author, email, site, captcha, submit_success, submit_failure); + stacosys_new_comment(author, email, site, captcha, subscribe, message, submit_success, submit_failure); } function submit_success(data) { diff --git a/demo/public/js/stacosys.js b/demo/public/js/stacosys.js index 8efba1a..0277443 100644 --- a/demo/public/js/stacosys.js +++ b/demo/public/js/stacosys.js @@ -57,7 +57,7 @@ function stacosys_load_comments(callback, errback) { xdr(url, 'GET', null, {}, callback, errback); } -function stacosys_new_comment(author, email, site, captcha, callback, errback) { +function stacosys_new_comment(author, email, site, captcha, subscribe, message, callback, errback) { var url = STACOSYS_URL + '/comments'; var data = { 'token': STACOSYS_TOKEN, @@ -65,7 +65,9 @@ function stacosys_new_comment(author, email, site, captcha, callback, errback) { 'author': author, 'email': email, 'site': site, - 'captcha': captcha + 'captcha': captcha, + 'subscribe': subscribe, + 'message': message }; var header = { 'Content-type': 'application/json' diff --git a/demo/public/redirect.html b/demo/public/redirect.html new file mode 100644 index 0000000..c7dc481 --- /dev/null +++ b/demo/public/redirect.html @@ -0,0 +1,67 @@ + + + + + + + + Le blog du Yax + + + + + + + + + + + + + + + + + +
+ +
+

+ +
+
+
+

Le commentaire a été envoyé à l'administrateur du site.

+ Cliquez sur ce lien pour retourner à l'article. +
+
+
+ + + + + diff --git a/requirements.txt b/requirements.txt index ce353a8..cd3fc9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +chardet==2.3.0 clize==2.4 Flask==0.10.1 Flask-Cors==2.0.1 diff --git a/tools/pecosys2stacosys.py b/tools/pecosys2stacosys.py index f68873b..408db43 100644 --- a/tools/pecosys2stacosys.py +++ b/tools/pecosys2stacosys.py @@ -35,7 +35,12 @@ logger.addHandler(ch) # regex regex = re.compile(r"(\w+):\s*(.*)") -def convert_comment(db, site, filename): +def remove_from_string(line, start): + if line[:len(start)] == start: + line = line[len(start):].strip() + return line + +def convert_comment(db, site, root_url, filename): logger.info('convert %s' % filename) d = {} content = '' @@ -62,10 +67,10 @@ def convert_comment(db, site, filename): if 'site' in d: comment.author_site = d['site'].strip() if 'url' in d: - if d['url'][:7] == 'http://': - comment.url = d['url'][7:].strip() - elif d['url'][:8] == 'https://': - comment.url = d['url'][8:].strip() + url = remove_from_string(d['url'], 'https://') + url = remove_from_string(url, 'http://') + url = remove_from_string(url, root_url) + comment.url = remove_from_string(url, '/') # else: # comment.url = d['article'] if 'date' in d: @@ -94,7 +99,7 @@ def convert(db, site_name, url, comment_dir): for filename in files: if filename.endswith(('.md',)): comment_file = '/'.join([dirpath, filename]) - convert_comment(db, site, comment_file) + convert_comment(db, site, url, comment_file) else: logger.warn('ignore file %s' % filename)