diff --git a/app/services/__init__.py b/app/conf/__init__.py similarity index 100% rename from app/services/__init__.py rename to app/conf/__init__.py diff --git a/app/conf/schema.py b/app/conf/schema.py new file mode 100644 index 0000000..f76d18a --- /dev/null +++ b/app/conf/schema.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Created with https://app.quicktype.io +# name: stacosys + +json_schema = """ +{ + "$ref": "#/definitions/Stacosys", + "definitions": { + "Stacosys": { + "type": "object", + "additionalProperties": false, + "properties": { + "general": { + "$ref": "#/definitions/General" + }, + "http": { + "$ref": "#/definitions/HTTP" + }, + "security": { + "$ref": "#/definitions/Security" + }, + "rss": { + "$ref": "#/definitions/RSS" + }, + "zmq": { + "$ref": "#/definitions/Zmq" + } + }, + "required": [ + "general", + "http", + "rss", + "security", + "zmq" + ], + "title": "stacosys" + }, + "General": { + "type": "object", + "additionalProperties": false, + "properties": { + "debug": { + "type": "boolean" + }, + "lang": { + "type": "string" + }, + "db_url": { + "type": "string" + } + }, + "required": [ + "db_url", + "debug", + "lang" + ], + "title": "general" + }, + "HTTP": { + "type": "object", + "additionalProperties": false, + "properties": { + "root_url": { + "type": "string" + }, + "host": { + "type": "string" + }, + "port": { + "type": "integer" + } + }, + "required": [ + "host", + "port", + "root_url" + ], + "title": "http" + }, + "RSS": { + "type": "object", + "additionalProperties": false, + "properties": { + "proto": { + "type": "string" + }, + "file": { + "type": "string" + } + }, + "required": [ + "file", + "proto" + ], + "title": "rss" + }, + "Security": { + "type": "object", + "additionalProperties": false, + "properties": { + "salt": { + "type": "string" + }, + "secret": { + "type": "string" + }, + "private": { + "type": "boolean" + } + }, + "required": [ + "private", + "salt", + "secret" + ], + "title": "security" + }, + "Zmq": { + "type": "object", + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "host": { + "type": "string" + }, + "pub_port": { + "type": "integer" + }, + "sub_port": { + "type": "integer" + } + }, + "required": [ + "active", + "host", + "pub_port", + "sub_port" + ], + "title": "zmq" + } + } +} +""" \ No newline at end of file diff --git a/app/run.py b/app/core/__init__.py similarity index 71% rename from app/run.py rename to app/core/__init__.py index 746596f..a85d304 100644 --- a/app/run.py +++ b/app/core/__init__.py @@ -4,7 +4,12 @@ import os import sys import logging +from flask import Flask from flask.ext.cors import CORS +from conf import config +from jsonschema import validate + +app = Flask(__name__) # add current path and parent path to syspath current_path = os.path.dirname(__file__) @@ -15,16 +20,12 @@ for path in paths: sys.path.insert(0, path) # more imports -import config -from app.services import database -from app.services import processor -from app.interface import api -from app.interface import form -from app.interface import report -#from app.controllers import mail -from app.interface import zclient -from app import app - +import database +import processor +from interface import api +from interface import form +from interface import report +from interface import zclient # configure logging def configure_logging(level): @@ -40,7 +41,7 @@ def configure_logging(level): # add ch to logger root_logger.addHandler(ch) -logging_level = (20, 10)[config.DEBUG] +logging_level = (20, 10)[config.general['debug']] configure_logging(logging_level) logger = logging.getLogger(__name__) @@ -52,25 +53,23 @@ database.setup() zclient.start() # start processor -template_path = os.path.abspath(os.path.join(current_path, 'templates')) +template_path = os.path.abspath(os.path.join(current_path, '../templates')) processor.start(template_path) # less feature in private mode -if not config.PRIVATE: +if not config.security['private']: # enable CORS cors = CORS(app, resources={r"/comments/*": {"origins": "*"}}) from app.controllers import reader logger.debug('imported: %s ' % reader.__name__) # tune logging level -if not config.DEBUG: +if not config.general['debug']: logging.getLogger('app.cors').level = logging.WARNING logging.getLogger('werkzeug').level = logging.WARNING logger.info("Start Stacosys application") -if __name__ == '__main__': - - app.run(host=config.HTTP_ADDRESS, - port=config.HTTP_PORT, - debug=config.DEBUG, use_reloader=False) +app.run(host=config.http['host'], + port=config.http['port'], + debug=config.general['debug'], use_reloader=False) \ No newline at end of file diff --git a/app/services/database.py b/app/core/database.py similarity index 63% rename from app/services/database.py rename to app/core/database.py index 0075e13..7f96f5d 100644 --- a/app/services/database.py +++ b/app/core/database.py @@ -1,13 +1,13 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- -import config +from conf import config import functools from playhouse.db_url import connect def get_db(): - return connect(config.DB_URL) + return connect(config.general['db_url']) def provide_db(func): @@ -21,9 +21,9 @@ def provide_db(func): @provide_db def setup(db): - from app.models.site import Site - from app.models.comment import Comment - from app.models.reader import Reader - from app.models.report import Report + from models.site import Site + from models.comment import Comment + from models.reader import Reader + from models.report import Report db.create_tables([Site, Comment, Reader, Report], safe=True) diff --git a/app/services/processor.py b/app/core/processor.py similarity index 91% rename from app/services/processor.py rename to app/core/processor.py index 7fbecb3..f75b986 100644 --- a/app/services/processor.py +++ b/app/core/processor.py @@ -9,21 +9,27 @@ from threading import Thread from queue import Queue from jinja2 import Environment from jinja2 import FileSystemLoader -from app.models.site import Site -from app.models.reader import Reader -from app.models.report import Report -from app.models.comment import Comment -from app.helpers.hashing import md5 +from models.site import Site +from models.reader import Reader +from models.report import Report +from models.comment import Comment +from helpers.hashing import md5 import json -import config +from conf import config import PyRSS2Gen import markdown +import zmq logger = logging.getLogger(__name__) queue = Queue() proc = None env = None +if config.zmq['active']: + context = zmq.Context() + zpub = context.socket(zmq.PUB) + zpub.connect('tcp://127.0.0.1:{}'.format(config.zmq['sub_port'])) + class Processor(Thread): @@ -69,7 +75,7 @@ def new_comment(data): subscribe = data.get('subscribe', '') # private mode: email contains gravar md5 hash - if config.PRIVATE: + if config.security['private']: author_gravatar = author_email author_email = '' else: @@ -112,7 +118,7 @@ def new_comment(data): mail(site.admin_email, subject, email_body) # Reader subscribes to further comments - if not config.PRIVATE and subscribe and author_email: + if not config.security['private'] and subscribe and author_email: subscribe_reader(author_email, token, url) logger.debug("new comment processed ") @@ -171,7 +177,7 @@ def reply_comment_email(data): mail(from_email, 'Re: ' + subject, email_body) # notify reader once comment is published - if not config.PRIVATE: + if not config.security['private']: reader_email = get_email_metadata(message) if reader_email: notify_reader(from_email, reader_email, comment.site.token, @@ -258,7 +264,7 @@ def notify_subscribed_readers(token, site_url, url): to_email = reader.email logger.info('notify reader %s' % to_email) unsubscribe_url = '%s/unsubscribe?email=%s&token=%s&url=%s' % ( - config.ROOT_URL, to_email, token, reader.url) + config.http['root_url'], to_email, token, reader.url) email_body = get_template( 'notify_subscriber').render(article_url=article_url, unsubscribe_url=unsubscribe_url) @@ -337,8 +343,8 @@ def report(token): unsubscribed.append({'url': "http://" + site.url + row.url, 'name': row.name, 'email': row.email}) - email_body = get_template('report').render(secret=config.SECRET, - root_url=config.ROOT_URL, + email_body = get_template('report').render(secret=config.security['secret'], + root_url=config.http['root_url'], standbys=standbys, published=published, rejected=rejected, @@ -354,7 +360,7 @@ def report(token): def rss(token, onstart=False): - if onstart and os.path.isfile(config.RSS_FILE): + if onstart and os.path.isfile(config.rss['file']): return site = Site.select().where(Site.token == token).get() @@ -365,9 +371,9 @@ def rss(token, onstart=False): for row in Comment.select().join(Site).where( Site.token == token, Comment.published).order_by( -Comment.published).limit(10): - item_link = "%s://%s%s" % (config.RSS_URL_PROTO, site.url, row.url) + item_link = "%s://%s%s" % (config.rss['proto'], site.url, row.url) items.append(PyRSS2Gen.RSSItem( - title='%s - %s://%s%s' % (config.RSS_URL_PROTO, row.author_name, site.url, row.url), + title='%s - %s://%s%s' % (config.rss['proto'], row.author_name, site.url, row.url), link=item_link, description=md.convert(row.content), guid=PyRSS2Gen.Guid('%s/%d' % (item_link, row.id)), @@ -376,11 +382,11 @@ def rss(token, onstart=False): rss = PyRSS2Gen.RSS2( title=rss_title, - link='%s://%s' % (config.RSS_URL_PROTO, site.url), + link='%s://%s' % (config.rss['proto'], site.url), description="Commentaires du site '%s'" % site.name, lastBuildDate=datetime.now(), items=items) - rss.write_xml(open(config.RSS_FILE, "w"), encoding="utf-8") + rss.write_xml(open(config.rss['file'], 'w'), encoding='utf-8') def mail(to_email, subject, message): @@ -400,7 +406,7 @@ def mail(to_email, subject, message): def get_template(name): - return env.get_template(config.LANG + '/' + name + '.tpl') + return env.get_template(config.general['lang'] + '/' + name + '.tpl') def enqueue(something): diff --git a/app/helpers/hashing.py b/app/helpers/hashing.py index 2d91635..207da43 100644 --- a/app/helpers/hashing.py +++ b/app/helpers/hashing.py @@ -2,11 +2,11 @@ # -*- coding: UTF-8 -*- import hashlib -import config +from conf import config def salt(value): - string = '%s%s' % (value, config.SALT) + string = '%s%s' % (value, config.security['salt']) dk = hashlib.sha256(string.encode()) return dk.hexdigest() diff --git a/app/interface/api.py b/app/interface/api.py index e03591d..db35e22 100644 --- a/app/interface/api.py +++ b/app/interface/api.py @@ -2,12 +2,11 @@ # -*- coding: utf-8 -*- import logging -import config from flask import request, jsonify, abort -from app import app -from app.models.site import Site -from app.models.comment import Comment -from app.services import processor +from core import app +from models.site import Site +from models.comment import Comment +from core import processor logger = logging.getLogger(__name__) diff --git a/app/interface/form.py b/app/interface/form.py index dbcf714..9998b2c 100644 --- a/app/interface/form.py +++ b/app/interface/form.py @@ -2,13 +2,12 @@ # -*- coding: utf-8 -*- import logging -import config from flask import request, jsonify, abort, redirect -from app import app -from app.models.site import Site -from app.models.comment import Comment -from app.helpers.hashing import md5 -from app.services import processor +from core import app +from models.site import Site +from models.comment import Comment +from helpers.hashing import md5 +from core import processor logger = logging.getLogger(__name__) diff --git a/app/interface/mail.py b/app/interface/mail.py index 7ade3fc..ccebbb9 100644 --- a/app/interface/mail.py +++ b/app/interface/mail.py @@ -3,8 +3,8 @@ import logging from flask import request, abort -from app import app -from app.services import processor +from core import app +from core import processor logger = logging.getLogger(__name__) diff --git a/app/interface/reader.py b/app/interface/reader.py index 2673105..fb6149b 100644 --- a/app/interface/reader.py +++ b/app/interface/reader.py @@ -3,8 +3,8 @@ import logging from flask import request, abort -from app import app -from app.services import processor +from core import app +from core import processor logger = logging.getLogger(__name__) diff --git a/app/interface/report.py b/app/interface/report.py index c3467fd..297b76c 100644 --- a/app/interface/report.py +++ b/app/interface/report.py @@ -2,13 +2,13 @@ # -*- coding: utf-8 -*- import logging -import config +from conf import config from flask import request, jsonify, abort -from app import app -from app.models.site import Site -from app.models.comment import Comment -from app.helpers.hashing import md5 -from app.services import processor +from core import app +from models.site import Site +from models.comment import Comment +from helpers.hashing import md5 +from core import processor logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def report(): token = request.args.get('token', '') secret = request.args.get('secret', '') - if secret != config.SECRET: + if secret != config.security['secret']: logger.warn('Unauthorized request') abort(401) @@ -45,7 +45,7 @@ def accept_comment(): id = request.args.get('comment', '') secret = request.args.get('secret', '') - if secret != config.SECRET: + if secret != config.security['secret']: logger.warn('Unauthorized request') abort(401) @@ -65,7 +65,7 @@ def reject_comment(): id = request.args.get('comment', '') secret = request.args.get('secret', '') - if secret != config.SECRET: + if secret != config.security['secret']: logger.warn('Unauthorized request') abort(401) diff --git a/app/interface/zclient.py b/app/interface/zclient.py index 69bfb8b..a467428 100644 --- a/app/interface/zclient.py +++ b/app/interface/zclient.py @@ -6,7 +6,7 @@ from conf import config from threading import Thread import logging import json -from app.services import processor +from core import processor logger = logging.getLogger(__name__) diff --git a/app/models/comment.py b/app/models/comment.py index 9ebdfb7..502f0bf 100644 --- a/app/models/comment.py +++ b/app/models/comment.py @@ -6,8 +6,8 @@ from peewee import CharField from peewee import TextField from peewee import DateTimeField from peewee import ForeignKeyField -from app.models.site import Site -from app.services.database import get_db +from models.site import Site +from core.database import get_db class Comment(Model): diff --git a/app/models/reader.py b/app/models/reader.py index a1eae15..212c74b 100644 --- a/app/models/reader.py +++ b/app/models/reader.py @@ -4,8 +4,8 @@ from peewee import Model from peewee import CharField from peewee import ForeignKeyField -from app.services.database import get_db -from app.models.site import Site +from core.database import get_db +from models.site import Site class Reader(Model): diff --git a/app/models/report.py b/app/models/report.py index 44e8ac0..bda6e68 100644 --- a/app/models/report.py +++ b/app/models/report.py @@ -5,8 +5,8 @@ from peewee import Model from peewee import CharField from peewee import BooleanField from peewee import ForeignKeyField -from app.services.database import get_db -from app.models.site import Site +from core.database import get_db +from models.site import Site class Report(Model): name = CharField() diff --git a/app/models/site.py b/app/models/site.py index 9c34cb2..156ebb3 100644 --- a/app/models/site.py +++ b/app/models/site.py @@ -3,7 +3,7 @@ from peewee import Model from peewee import CharField -from app.services.database import get_db +from core.database import get_db class Site(Model): diff --git a/app/stacosys.py b/app/stacosys.py new file mode 100644 index 0000000..8d26708 --- /dev/null +++ b/app/stacosys.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +import logging +import json +from clize import clize, run +from jsonschema import validate +from conf import config, schema + +def load_json(filename): + jsondoc = None + with open(filename, 'rt') as json_file: + jsondoc = json.loads(json_file.read()) + return jsondoc + + +@clize +def stacosys_server(config_pathname): + + # load and validate startup config + conf = load_json(config_pathname) + json_schema = json.loads(schema.json_schema) + v = validate(conf, json_schema) + print('validation: {}'.format(v)) + + # set configuration + config.general = conf['general'] + config.http = conf['http'] + config.security = conf['security'] + config.rss = conf['rss'] + config.zmq = conf['zmq'] + + # start application + from core import app + +if __name__ == '__main__': + run(stacosys_server) \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..7b6be9e --- /dev/null +++ b/config.json @@ -0,0 +1,27 @@ +{ + "general" : { + "debug": true, + "lang": "fr", + "db_url": "sqlite:///db.sqlite" + }, + "http": { + "root_url": "http://localhost:8100", + "host": "127.0.0.1", + "port": 8100 + }, + "security": { + "salt": "BRRJRqXgGpXWrgTidBPcixIThHpDuKc0", + "secret": "Uqca5Kc8xuU6THz9", + "private": true + }, + "rss": { + "proto": "http", + "file": "comments.xml" + }, + "zmq": { + "active": true, + "host": "127.0.0.1", + "pub_port": 7701, + "sub_port": 7702 + } +} diff --git a/config.py b/config.py deleted file mode 100644 index 219e3e7..0000000 --- a/config.py +++ /dev/null @@ -1,25 +0,0 @@ -# Configuration file - -DEBUG = True - -LANG = "fr" - -# DB_URL = "mysql://stacosys_user:stacosys_password@localhost:3306/stacosys" -DB_URL = "sqlite:///db.sqlite" - -MAIL_URL = "http://localhost:8025/mbox" - -HTTP_ADDRESS = "127.0.0.1" -HTTP_PORT = 8100 -HTTP_WORKERS = 1 - -SALT = "BRRJRqXgGpXWrgTidBPcixIThHpDuKc0" - -SECRET = "Uqca5Kc8xuU6THz9" - -ROOT_URL = 'http://localhost:8100' - -RSS_URL_PROTO = 'http' -RSS_FILE = 'comments.xml' - -PRIVATE = True \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 33c72d6..c5fd0c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,19 @@ +attrs==17.4.0 chardet==3.0.4 click==6.7 +clize==4.0.3 +docutils==0.14 Flask==0.12.2 +Flask-Cors==3.0.3 itsdangerous==0.24 Jinja2==2.10 +jsonschema==2.6.0 Markdown==2.6.11 MarkupSafe==1.0 +od==1.0 peewee==2.10.2 PyRSS2Gen==1.1 pyzmq==16.0.3 +sigtools==2.0.1 +six==1.11.0 Werkzeug==0.14.1 diff --git a/run.sh b/run.sh index a3b0364..3300b92 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/sh -python app/run.py "$@" +python app/stacosys.py "$@"