From 2c5b63fcf5b58607a5f7c2f459d22bb5c87db40a Mon Sep 17 00:00:00 2001 From: Yax <1949284+kianby@users.noreply.github.com> Date: Tue, 16 Jan 2018 18:49:19 +0100 Subject: [PATCH] fetch emails through Zero MQ --- app/conf/config.py | 6 ++++ app/{controllers => interface}/api.py | 0 app/{controllers => interface}/form.py | 0 app/{controllers => interface}/mail.py | 0 app/{controllers => interface}/reader.py | 0 app/{controllers => interface}/report.py | 0 app/interface/zclient.py | 43 ++++++++++++++++++++++++ app/run.py | 25 +++++--------- app/services/processor.py | 4 +-- requirements.txt | 11 ++---- 10 files changed, 62 insertions(+), 27 deletions(-) create mode 100644 app/conf/config.py rename app/{controllers => interface}/api.py (100%) rename app/{controllers => interface}/form.py (100%) rename app/{controllers => interface}/mail.py (100%) rename app/{controllers => interface}/reader.py (100%) rename app/{controllers => interface}/report.py (100%) create mode 100644 app/interface/zclient.py diff --git a/app/conf/config.py b/app/conf/config.py new file mode 100644 index 0000000..7ca283f --- /dev/null +++ b/app/conf/config.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# TODO move to JSON config + +zmq = {'pub_port': 7701, 'sub_port':7702} \ No newline at end of file diff --git a/app/controllers/api.py b/app/interface/api.py similarity index 100% rename from app/controllers/api.py rename to app/interface/api.py diff --git a/app/controllers/form.py b/app/interface/form.py similarity index 100% rename from app/controllers/form.py rename to app/interface/form.py diff --git a/app/controllers/mail.py b/app/interface/mail.py similarity index 100% rename from app/controllers/mail.py rename to app/interface/mail.py diff --git a/app/controllers/reader.py b/app/interface/reader.py similarity index 100% rename from app/controllers/reader.py rename to app/interface/reader.py diff --git a/app/controllers/report.py b/app/interface/report.py similarity index 100% rename from app/controllers/report.py rename to app/interface/report.py diff --git a/app/interface/zclient.py b/app/interface/zclient.py new file mode 100644 index 0000000..69bfb8b --- /dev/null +++ b/app/interface/zclient.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import zmq +from conf import config +from threading import Thread +import logging +import json +from app.services import processor + +logger = logging.getLogger(__name__) + +context = zmq.Context() + + +def process(message): + data = json.loads(message) + if data['topic'] == 'email:newmail': + logger.info('newmail => {}'.format(data)) + processor.enqueue({'request': 'new_mail', 'data': data}) + + +class Consumer(Thread): + + def run(self): + zsub = context.socket(zmq.SUB) + zsub.connect('tcp://127.0.0.1:{}'.format(config.zmq['pub_port'])) + zsub.setsockopt_string(zmq.SUBSCRIBE, '') + self.loop = True + while self.loop: + message = zsub.recv() + try: + process(message) + except: + logger.exception('cannot process broker message') + + def stop(self): + self.loop = False + + +def start(): + c = Consumer() + c.start() diff --git a/app/run.py b/app/run.py index 1494fc9..746596f 100644 --- a/app/run.py +++ b/app/run.py @@ -18,10 +18,11 @@ for path in paths: import config from app.services import database from app.services import processor -from app.controllers import api -from app.controllers import form -from app.controllers import report -from app.controllers import mail +from app.interface import api +from app.interface import form +from app.interface import report +#from app.controllers import mail +from app.interface import zclient from app import app @@ -45,20 +46,10 @@ configure_logging(logging_level) logger = logging.getLogger(__name__) # initialize database -database.setup() +database.setup() -#from app.helpers.hashing import md5 -#from app.models.comment import Comment -#for comment in Comment.select(): -# email = comment.author_email.strip().lower() -# if email: -# comment.author_gravatar = md5(email) -# comment.author_email = '' -# comment.save() - -# routes -logger.debug('imported: %s ' % api.__name__) -logger.debug('imported: %s ' % mail.__name__) +# start broker client +zclient.start() # start processor template_path = os.path.abspath(os.path.join(current_path, 'templates')) diff --git a/app/services/processor.py b/app/services/processor.py index ea496f1..7fbecb3 100644 --- a/app/services/processor.py +++ b/app/services/processor.py @@ -14,7 +14,6 @@ from app.models.reader import Reader from app.models.report import Report from app.models.comment import Comment from app.helpers.hashing import md5 -import requests import json import config import PyRSS2Gen @@ -392,7 +391,8 @@ def mail(to_email, subject, message): 'subject': subject, 'content': message } - r = requests.post(config.MAIL_URL, data=json.dumps(msg), headers=headers) + # do something smart here + # r = requests.post(config.MAIL_URL, data=json.dumps(msg), headers=headers) if r.status_code in (200, 201): logger.debug('Email for %s posted' % to_email) else: diff --git a/requirements.txt b/requirements.txt index 311f04f..33c72d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,11 @@ -certifi==2017.11.5 chardet==3.0.4 click==6.7 Flask==0.12.2 -Flask-Cors==3.0.3 -idna==2.6 itsdangerous==0.24 Jinja2==2.10 -Markdown==2.6.9 +Markdown==2.6.11 MarkupSafe==1.0 peewee==2.10.2 PyRSS2Gen==1.1 -requests==2.18.4 -six==1.11.0 -urllib3==1.22 -Werkzeug==0.12.2 +pyzmq==16.0.3 +Werkzeug==0.14.1