Merge pecosys work around comment processing with stacosys

app/controllers/api.py

@@ -7,6 +7,7 @@ from app import app
 from app.models.site import Site
 from app.models.comment import Comment
 from app.helpers.hashing import md5
+from app.services import processor
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +23,7 @@ def query_comments():
         logger.info('retrieve comments for token %s, url %s' % (token, url))
         for comment in Comment.select(Comment).join(Site).where(
                (Comment.url == url) &
+               (Comment.published.is_null(False)) &
                (Site.token == token)).order_by(+Comment.published):
             d = {}
             d['author'] = comment.author_name
@@ -49,6 +51,7 @@ def get_comments_count():
         url = request.args.get('url', '')
         count = Comment.select(Comment).join(Site).where(
                   (Comment.url == url) & 
+                  (Comment.published.is_null(False)) & 
                   (Site.token == token)).count()
         r = jsonify({'count': count})
         r.status_code = 200
@@ -72,24 +75,13 @@ def new_comment():
             logger.warn('Unknown site %s' % token)
             abort(400)
 
-        # get values
-        url = data.get('url', '')
-        author_name = data.get('author', '')
-        author_email = data.get('email', '')
-        author_site = data.get('site', '')
-        message = data.get('message', '')
-        subscribe = data.get('subscribe', '')
-
         # honeypot for spammers
         captcha = data.get('captcha', '')
         if captcha:
-            logger.warn('discard spam: captcha %s author %s email %s site %s url  %s message %s'
+            logger.warn('discard spam: data %s' % data)
-                          % (captcha, author_name, author_email, author_site, url, message))
+            abort(400)
-        else:
+
-            # TODO push new comment to backend service
+        processor.enqueue({'request': 'new_comment', 'data': data})
-            logger.info('process: captcha %s author %s email %s site %s url  %s message %s subscribe %s'
-                          % (captcha, author_name, author_email, author_site,
-                              url, message, subscribe))
 
     except:
         logger.exception("new comment failure")

app/models/comment.py

@@ -13,7 +13,7 @@ from app.services.database import get_db
 class Comment(Model):
     url = CharField()
     created = DateTimeField()
-    published = DateTimeField()
+    published = DateTimeField(null=True,default=None)
     author_name = CharField()
     author_email = CharField(default='')
     author_site = CharField(default='')

app/run.py

@@ -7,10 +7,10 @@ import logging
 from werkzeug.contrib.fixers import ProxyFix
 from flask.ext.cors import CORS
 
-# add current and parent path to syspath
+# add current path and parent path to syspath
-currentPath = os.path.dirname(__file__)
+current_path = os.path.dirname(__file__)
-parentPath = os.path.abspath(os.path.join(currentPath, os.path.pardir))
+parent_path = os.path.abspath(os.path.join(current_path, os.path.pardir))
-paths = [currentPath, parentPath]
+paths = [current_path, parent_path]
 for path in paths:
     if path not in sys.path:
         sys.path.insert(0, path)
@@ -18,6 +18,7 @@ for path in paths:
 # more imports
 import config
 from app.services import database
+from app.services import processor
 from app.controllers import api
 from app import app
 
@@ -42,6 +43,10 @@ logger = logging.getLogger(__name__)
 # initialize database
 database.setup()
 
+# start processor 
+template_path = os.path.abspath(os.path.join(current_path, 'templates'))
+processor.start(template_path)
+
 app.wsgi_app = ProxyFix(app.wsgi_app)
 
 logger.info("Start Stacosys application")

app/services/processor.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+import re
+from datetime import datetime
+from threading import Thread
+from queue import Queue
+import chardet
+from jinja2 import Environment, FileSystemLoader
+from app.models.site import Site
+from app.models.comment import Comment
+
+
+logger = logging.getLogger(__name__)
+queue = Queue()
+proc = None
+env = None
+
+
+class Processor(Thread):
+
+    def stop(self):
+        logger.info("stop requested")
+        self.is_running = False
+
+    def run(self):
+
+        self.is_running = True
+        while self.is_running:
+            msg = queue.get()
+            if msg['request'] == 'new_comment':
+                new_comment(msg['data'])
+            #elif msg['type'] == 'reply_comment_email':
+            #    reply_comment_email(req['From'], req['Subject'], req['Body'])
+            #elif req['type'] == 'unsubscribe':
+            #    unsubscribe_reader(req['email'], req['article'])
+            else:
+                logger.info("Dequeue unknown request " + msg)
+
+
+def new_comment(data):
+
+    try:
+        token = data.get('token', '')
+        url = data.get('url', '')
+        author_name = data.get('author', '')
+        author_email = data.get('email', '')
+        author_site = data.get('site', '')
+        message = data.get('message', '')
+        subscribe = data.get('subscribe', '')
+
+        # create a new comment row
+        site = Site.select().where(Site.token == token).get()
+        
+        logger.info('new comment received: %s' % data)
+
+        if author_site and author_site[:4] != 'http':
+            author_site = 'http://' + author_site
+
+        created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        comment = Comment(site=site, url=url, author_name=author_name,
+                author_site=author_site, author_email=author_email,
+                content=message, created=created, published=None)
+        comment.save()
+
+        1 / 0 
+        # Render email body template
+        email_body = get_template('new_comment').render(url=url, comment=comment)
+
+        # Send email
+        mail(pecosys.get_config('post', 'from_email'),
+             pecosys.get_config('post', 'to_email'),
+             '[' + branch_name + '-' + article + ']',  email_body)
+
+        # Reader subscribes to further comments
+        if subscribe and email:
+            subscribe_reader(email, article, url)
+
+        logger.debug("new comment processed ")
+    except:
+        logger.exception("new_comment failure")
+
+
+def reply_comment_email(from_email, subject, message):
+    try:
+        m = re.search('\[(\d+)\-(\w+)\]', subject)
+        branch_name = m.group(1)
+        article = m.group(2)
+
+        message = decode_best_effort(message)
+
+        # safe logic: no answer or unknown answer is a go for publishing
+        if message[:2].upper() == 'NO':
+            logger.info('discard comment: %s' % branch_name)
+            email_body = get_template('drop_comment').render(original=message)
+            mail(pecosys.get_config('post', 'from_email'),
+                 pecosys.get_config('post', 'to_email'),
+                 'Re: ' + subject, email_body)
+        else:
+            if pecosys.get_config("git", "disabled"):
+                logger.debug("GIT usage disabled (debug mode)")
+            else:
+                git.merge(branch_name)
+                if pecosys.get_config("git", "remote"):
+                    git.push()
+                logger.info('commit comment: %s' % branch_name)
+
+            # send approval confirmation email to admin
+            email_body = get_template('approve_comment').render(original=message)
+            mail(pecosys.get_config('post', 'from_email'),
+                 pecosys.get_config('post', 'to_email'),
+                 'Re: ' + subject, email_body)
+
+            # notify reader once comment is published
+            reader_email, article_url = get_email_metadata(message)
+            if reader_email:
+                notify_reader(reader_email, article_url)
+
+            # notify subscribers every time a new comment is published
+            notify_subscribers(article)
+
+        if pecosys.get_config("git", "disabled"):
+            logger.debug("GIT usage disabled (debug mode)")
+        else:
+            git.branch("-D", branch_name)
+    except:
+        logger.exception("new email failure")
+
+
+def get_email_metadata(message):
+    # retrieve metadata reader email and URL from email body sent by admin
+    email = ""
+    url = ""
+    m = re.search('email:\s(.+@.+\..+)', message)
+    if m:
+        email = m.group(1)
+
+    m = re.search('url:\s(.+)', message)
+    if m:
+        url = m.group(1)
+    return (email, url)
+
+
+def subscribe_reader(email, article, url):
+    logger.info("subscribe reader %s to %s (%s)" % (email, article, url))
+    db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json')
+    db.insert({'email': email, 'article': article, 'url': url})
+
+
+def unsubscribe_reader(email, article):
+    logger.info("unsubscribe reader %s from %s" % (email, article))
+    db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json')
+    db.remove((where('email') == email) & (where('article') == article))
+
+
+def notify_subscribers(article):
+    logger.info('notify subscribers for article %s' % article)
+    db = TinyDB(pecosys.get_config('global', 'cwd') + '/db.json')
+    for item in db.search(where('article') == article):
+        logger.info(item)
+        to_email = item['email']
+        logger.info("notify reader %s for article %s" % (to_email, article))
+        unsubscribe_url = pecosys.get_config('subscription', 'url') + '?email=' + to_email + '&article=' + article
+        email_body = get_template('notify_subscriber').render(article_url=item['url'],
+                                                              unsubscribe_url=unsubscribe_url)
+        subject = get_template('notify_message').render()
+        mail(pecosys.get_config('subscription', 'from_email'), to_email, subject, email_body)
+
+
+def notify_reader(email, url):
+    logger.info('notify reader: email %s about URL %s' % (email, url))
+    email_body = get_template('notify_reader').render(article_url=url)
+    subject = get_template('notify_message').render()
+    mail(pecosys.get_config('subscription', 'from_email'), email, subject, email_body)
+
+
+def decode_best_effort(string):
+    info = chardet.detect(string)
+    if info['confidence'] < 0.5:
+        return string.decode('utf8', errors='replace')
+    else:
+        return string.decode(info['encoding'], errors='replace')
+
+
+def mail(from_email, to_email, subject, *messages):
+
+    # Create the container (outer) email message.
+    msg = MIMEMultipart()
+    msg['Subject'] = subject
+    msg['From'] = from_email
+    msg['To'] = to_email
+    msg.preamble = subject
+
+    for message in messages:
+        part = MIMEText(message, 'plain')
+        msg.attach(part)
+
+    s = smtplib.SMTP(pecosys.get_config('smtp', 'host'),
+                     pecosys.get_config('smtp', 'port'))
+    if(pecosys.get_config('smtp', 'starttls')):
+        s.starttls()
+    s.login(pecosys.get_config('smtp', 'login'),
+            pecosys.get_config('smtp', 'password'))
+    s.sendmail(from_email, to_email, msg.as_string())
+    s.quit()
+
+
+def get_template(name):
+    return env.get_template(pecosys.get_config('global', 'lang') + '/' + name + '.tpl')
+
+
+def enqueue(something):
+    queue.put(something)
+
+
+def get_processor():
+    return proc
+
+
+def start(template_dir):
+    global proc, env
+
+    # initialize Jinja 2 templating
+    logger.info("load templates from directory %s" % template_dir)
+    env = Environment(loader=FileSystemLoader(template_dir))
+
+    # start processor thread
+    proc = Processor()
+    proc.start()

app/templates/en/approve_comment.tpl

@@ -0,0 +1,9 @@
+Hi,
+
+The comment should be published soon. It has been approved.
+
+--
+Pecosys
+
+
+{{ original }}

app/templates/en/drop_comment.tpl

@@ -0,0 +1,9 @@
+Hi,
+
+The comment will not be published. It has been dropped.
+
+--
+Pecosys
+
+
+{{ original }}

app/templates/en/new_comment.tpl

@@ -0,0 +1,16 @@
+Hi,
+
+A new comment has been submitted for post {{ url }}.
+
+You have two choices:
+- reject the comment by replying NO (or no),
+- accept the comment by sending back the email as it is.
+
+If you choose the latter option, Pecosys is going to publish the commennt.
+
+Please find comment details below:
+
+{{ comment }}
+
+--
+Pecosys

app/templates/en/notify_message.tpl

@@ -0,0 +1 @@
+New comment

app/templates/en/notify_reader.tpl

@@ -0,0 +1,9 @@
+Hi,
+
+Your comment has been approved. It should be published in few minutes.
+
+    {{ article_url }}
+
+--
+Pecosys
+

app/templates/en/notify_subscriber.tpl

@@ -0,0 +1,13 @@
+Hi,
+
+A new comment has been published for an article you have subscribed to.
+
+    {{ article_url }}
+
+You can unsubscribe at any time using this link:
+
+   {{ unsubscribe_url }}
+
+--
+Pecosys
+

app/templates/en/unsubscribe_page.tpl

@@ -0,0 +1,2 @@
+Your request has been sent. In case of issue please contact site
+administrator.

app/templates/fr/approve_comment.tpl

@@ -0,0 +1,9 @@
+Bonjour,
+
+Le commentaire sera bientôt publié. Il a été approuvé.
+
+--
+Pecosys
+
+
+{{ original }}

app/templates/fr/drop_comment.tpl

@@ -0,0 +1,9 @@
+Bonjour,
+
+Le commentaire ne sera pas publié. Il a été rejeté.
+
+--
+Pecosys
+
+
+{{ original }}

app/templates/fr/new_comment.tpl

@@ -0,0 +1,16 @@
+Bonjour,
+
+Un nouveau commentaire a été posté pour l'article {{ url }}.
+
+Vous avez deux réponses possibles :
+- rejeter le commentaire en répondant NO (ou no),
+- accepter le commentaire en renvoyant cet email tel quel.
+
+Si cette dernière option est choisie, Pecosys publiera le commentaire très bientôt.
+
+Voici les détails concernant le commentaire :
+
+{{ comment }}
+
+--
+Pecosys

app/templates/fr/notify_message.tpl

@@ -0,0 +1 @@
+Nouveau commentaire

app/templates/fr/notify_reader.tpl

@@ -0,0 +1,9 @@
+Bonjour,
+
+Votre commentaire a été approuvé. Il sera publié dans quelques minutes.
+
+    {{ article_url }}
+
+--
+Pecosys
+

app/templates/fr/notify_subscriber.tpl

@@ -0,0 +1,13 @@
+Bonjour,
+
+Un nouveau commentaire a été publié pour un article auquel vous êtes abonné.
+
+    {{ article_url }}
+
+Vous pouvez vous désinscrire à tout moment en suivant ce lien :
+
+   {{ unsubscribe_url }}
+
+--
+Pecosys
+

app/templates/fr/unsubscribe_page.tpl

@@ -0,0 +1,2 @@
+Votre requête a été envoyée. En cas de problème, contactez l'administrateur du
+site.

config.py

@@ -2,6 +2,8 @@
 
 DEBUG = True
 
+LANG = "en" 
+
 #DB_URL = "mysql://stacosys_user:stacosys_password@localhost:3306/stacosys"
 DB_URL = "sqlite:///db.sqlite"
 

demo/public/index.html

@@ -203,7 +203,8 @@ instance d'ici peu.</p>
 var STACOSYS_URL = 'http://127.0.0.1:8000';
 var STACOSYS_TOKEN = '9fb3fc042c572cb831005fd16186126765140fa2bd9bb2d4a28e47a9457dc26c';
 //STACOSYS_PAGE = 'blogduyax.madyanne.fr/mes-applications-pour-blackberry.html'
-var STACOSYS_PAGE = 'blogduyax.madyanne.fr/migration-du-blog-sous-pelican.html'
+//var STACOSYS_PAGE = 'blogduyax.madyanne.fr/migration-du-blog-sous-pelican.html'
+var STACOSYS_PAGE = 'migration-du-blog-sous-pelican.html'
 
 window.onload = initialize_comments();
 

demo/public/js/page.js

@@ -65,9 +65,10 @@ function new_comment() {
   var email = document.getElementById('email').value;
   var site = document.getElementById('site').value;
   var captcha = document.getElementById('captcha').value;
-  //var subscribe = document.getElementById('subscribe').value;
+  var subscribe = document.getElementById('subscribe').value;
+  var message = document.getElementById('message').value;
   
-  stacosys_new_comment(author, email, site, captcha, submit_success, submit_failure);
+  stacosys_new_comment(author, email, site, captcha, subscribe, message, submit_success, submit_failure);
 }
 
 function submit_success(data) { 

demo/public/js/stacosys.js

@@ -57,7 +57,7 @@ function stacosys_load_comments(callback, errback) {
   xdr(url, 'GET', null, {}, callback, errback);
 }
 
-function stacosys_new_comment(author, email, site, captcha, callback, errback) {
+function stacosys_new_comment(author, email, site, captcha, subscribe, message, callback, errback) {
   var url = STACOSYS_URL + '/comments';
   var data = {
     'token': STACOSYS_TOKEN,
@@ -65,7 +65,9 @@ function stacosys_new_comment(author, email, site, captcha, callback, errback) {
     'author': author,
     'email': email,
     'site': site,
-    'captcha': captcha
+    'captcha': captcha,
+    'subscribe': subscribe,
+    'message': message
   };
   var header = {
     'Content-type': 'application/json'

demo/public/redirect.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="Yax">
+    <title>Le blog du Yax</title>
+
+    <!-- Le styles -->
+    <link rel="stylesheet" href="theme/css/pure/pure-0.5.0-min.css" type="text/css" />
+    <!--[if lte IE 8]>
+        <link rel="stylesheet" href="theme/css/pure/grids-responsive-old-ie-min.css">
+    <![endif]-->
+    <!--[if gt IE 8]><!-->
+        <link rel="stylesheet" href="theme/css/pure/grids-responsive-min.css">
+    <!--<![endif]-->    
+    <link rel="stylesheet" href="theme/css/style.css" type="text/css" />
+    <link rel="stylesheet" href="theme/css/font-awesome.min.css" type="text/css" />
+
+    <!-- Le fav and touch icons -->
+    <link rel="shortcut icon" href="theme/img/favicon.png">
+
+</head>
+
+<body onload="display()">
+
+    <div class="pure-g">
+        <div class="pure-u-1 l-box" id="banner">
+            <a href="http://blogduyax.madyanne.fr">Le blog du Yax&nbsp;<i class="fa fa-coffee"></i></a>
+            <p>GNU/Linux et autres libertés</p>
+        </div>
+    </div>
+    <p></p>
+
+    <div class="pure-g">
+        <div class="pure-u-lg-2-3 pure-u-1" id="content">
+            <div class="l-box">
+                <p><strong>Le commentaire a été envoyé à l'administrateur du site.</strong></p> 
+                <a id="linkurl" href="#">Cliquez sur ce lien pour retourner à l'article.</a>
+            </div>
+        </div>
+    </div>
+
+<script type="text/javascript"><!--
+
+function gup( name )
+{
+  name = name.replace(/[\[]/,"\\\[").replace(/[\]]/,"\\\]");
+  var regexS = "[\\?&]"+name+"=([^&#]*)";
+  var regex = new RegExp( regexS );
+  var results = regex.exec( window.location.href );
+  if( results == null )
+    return null;
+  else
+    return results[1];
+}
+
+function display() {
+  var referer_param = gup( 'referer' );
+  document.getElementById('linkurl').href = referer_param;
+}
+
+--></script>
+
+</body>
+</html>

requirements.txt

@@ -1,3 +1,4 @@
+chardet==2.3.0
 clize==2.4
 Flask==0.10.1
 Flask-Cors==2.0.1

tools/pecosys2stacosys.py

@@ -35,7 +35,12 @@ logger.addHandler(ch)
 # regex
 regex = re.compile(r"(\w+):\s*(.*)")
 
-def convert_comment(db, site, filename):
+def remove_from_string(line, start):
+    if line[:len(start)] == start:
+        line = line[len(start):].strip()
+    return line
+
+def convert_comment(db, site, root_url, filename):
     logger.info('convert %s' % filename)
     d = {}
     content = ''
@@ -62,10 +67,10 @@ def convert_comment(db, site, filename):
     if 'site' in d:
         comment.author_site = d['site'].strip()
     if 'url' in d:
-        if d['url'][:7] == 'http://':
+        url = remove_from_string(d['url'], 'https://')
-            comment.url = d['url'][7:].strip()
+        url = remove_from_string(url, 'http://')
-        elif d['url'][:8] == 'https://':
+        url = remove_from_string(url, root_url)
-            comment.url = d['url'][8:].strip()
+        comment.url = remove_from_string(url, '/')
     # else:
     #    comment.url = d['article']
     if 'date' in d:
@@ -94,7 +99,7 @@ def convert(db, site_name, url, comment_dir):
         for filename in files:
             if filename.endswith(('.md',)):
                 comment_file = '/'.join([dirpath, filename])
-                convert_comment(db, site, comment_file)
+                convert_comment(db, site, url, comment_file)
             else:
                 logger.warn('ignore file %s' % filename)