From 7b5c0822ff97843dc5d59e62250c192b71c45734 Mon Sep 17 00:00:00 2001 From: Yax <1949284+kianby@users.noreply.github.com> Date: Fri, 1 May 2015 18:57:17 +0200 Subject: [PATCH] Conversion tool is near from complete. Use Peewee as ORM --- app/models/comment.py | 24 +++++++++++++ app/models/site.py | 15 +++++++++ app/services/database.py | 39 +++++++++++++++++++++ config.py | 3 ++ requirements.txt | 4 ++- tools/config.json | 3 -- tools/pecosys2stacosys.py | 71 ++++++++++++++++++++++++++++++--------- 7 files changed, 140 insertions(+), 19 deletions(-) create mode 100644 app/models/comment.py create mode 100644 app/models/site.py create mode 100644 app/services/database.py create mode 100644 config.py delete mode 100644 tools/config.json diff --git a/app/models/comment.py b/app/models/comment.py new file mode 100644 index 0000000..f86d4e1 --- /dev/null +++ b/app/models/comment.py @@ -0,0 +1,24 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from peewee import Model +from peewee import CharField +from peewee import DateTimeField +from peewee import IntegerField +from peewee import ForeignKeyField +from app.models.site import Site +from app.services.database import get_db + + +class Comment(Model): + url = CharField() + date = DateTimeField() + rel_index = IntegerField() + author_name = CharField() + author_email = CharField(default='') + author_site = CharField() + content = CharField() + site = ForeignKeyField(Site, related_name='site') + + class Meta: + database = get_db() diff --git a/app/models/site.py b/app/models/site.py new file mode 100644 index 0000000..413dd82 --- /dev/null +++ b/app/models/site.py @@ -0,0 +1,15 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from peewee import Model +from peewee import CharField +from app.services.database import get_db + + +class Site(Model): + name = CharField(unique=True) + url = CharField() + token = CharField() + + class Meta: + database = get_db() diff --git a/app/services/database.py b/app/services/database.py new file mode 100644 index 0000000..1d6028e --- /dev/null +++ b/app/services/database.py @@ -0,0 +1,39 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +import hashlib +import config +import functools +from config import DB_URL +from playhouse.db_url import connect + + +def get_db(): + return connect(DB_URL) + + +def provide_db(func): + + @functools.wraps(func) + def new_function(*args, **kwargs): + return func(get_db(), *args, **kwargs) + + return new_function + + +def hash(value): + string = '%s%s' % (value, config.SALT) + dk = hashlib.sha256(string.encode()) + return dk.hexdigest() + + +@provide_db +def setup(db): + from app.models.user import User + db.create_tables([User], safe=True) + + # create admin user if user table is empty + if User.select().count() == 0: + admin_user = User(username='admin', password=hash('admin'), + displayname='Admin') + admin_user.save() diff --git a/config.py b/config.py new file mode 100644 index 0000000..c0fe729 --- /dev/null +++ b/config.py @@ -0,0 +1,3 @@ +# Configuration file + +DB_URL = "mysql://stacosys_user:stacosys_password@localhost:3306/stacosys" diff --git a/requirements.txt b/requirements.txt index b11b0cb..6f03f6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ +clize==2.4 Flask==0.10.1 itsdangerous==0.24 Jinja2==2.7.3 MarkupSafe==0.23 -SQLAlchemy==1.0.2 +peewee==2.6.0 +PyMySQL==0.6.6 Werkzeug==0.10.4 diff --git a/tools/config.json b/tools/config.json deleted file mode 100644 index ffdef07..0000000 --- a/tools/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "comments": "/home/yannic/work/coding/blog/blogduyax/comments" -} diff --git a/tools/pecosys2stacosys.py b/tools/pecosys2stacosys.py index c8da9c7..dcca732 100644 --- a/tools/pecosys2stacosys.py +++ b/tools/pecosys2stacosys.py @@ -1,12 +1,24 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- +import sys import os import re -import json import logging from clize import clize, run +# add necessary directories to PATH +current_path = os.path.realpath('.') +parent_path = os.path.abspath(os.path.join(current_path, '..')) +paths = [current_path, parent_path] +for path in paths: + if path not in sys.path: + sys.path.insert(0, path) + +# import database models +from app.services.database import provide_db +from app.models.site import Site +from app.models.comment import Comment # configure logging level = logging.DEBUG @@ -22,9 +34,10 @@ logger.addHandler(ch) regex = re.compile(r"(\w+):\s*(.*)") -def convert_comment(config, filename): +def convert_comment(db, site, filename): logger.info('convert %s' % filename) d = {} + content = '' with open(filename) as f: for line in f: match = regex.match(line) @@ -32,32 +45,60 @@ def convert_comment(config, filename): d[match.group(1)] = match.group(2) else: break + is_header = True + for line in f: + if is_header: + if line.strip(): + is_header = False + else: + continue + content = content + line logger.debug(d) + logger.debug(content) + + # create DB record + comment = Comment(site=site, author_name=d['author'], content=content) + if 'email' in d: + comment.author_email = d['email'] + if 'site' in d: + comment.author_site = d['site'] + if 'url' in d: + comment.url = d['url'] + # else: + # comment.url = d['article'] + if 'date' in d: + comment.date = d['date'] + comment.save() + + +@provide_db +def convert(db, site_name, url, comment_dir): + # create DB tables if needed + db.create_tables([Site, Comment], safe=True) + + # delete site record + try: + site = Site.select().where(Site.name == site_name).get() + site.delete_instance(recursive=True) + except Site.DoesNotExist: + pass + + site = Site.create(name=site_name, url=url, token='') -def convert(config): - comment_dir = config['comments'] logger.info('Comment directory %s' % comment_dir) for dirpath, dirs, files in os.walk(comment_dir): for filename in files: if filename.endswith(('.md',)): comment_file = '/'.join([dirpath, filename]) - convert_comment(config, comment_file) + convert_comment(db, site, comment_file) else: logger.debug('ignore file %s' % filename) -def load_config(config_pathname): - logger.info("Load config from %s" % config_pathname) - with open(config_pathname, 'rt') as config_file: - config = json.loads(config_file.read()) - return config - - @clize -def pecosys2stacosys(config_pathname): - config = load_config(config_pathname) - convert(config) +def pecosys2stacosys(site, url, comment_dir): + convert(site, url, comment_dir) if __name__ == '__main__':