yax
/
stacosys
mirror of https://github.com/kianby/stacosys
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move DB operations to DAO

Browse Source
pull/6/head
Yax 3 years ago
parent 3eb1b86246
commit c175b4a120
8 changed files with 92 additions and 236 deletions
Show Stats Download Patch File Download Diff File

135
poetry.lock generated
Unescape Escape View File

@ -32,28 +32,6 @@ tornado = ["tornado (>=4.3)"]
twisted = ["twisted"]
zookeeper = ["kazoo"]
[[package]]
name = "atomicwrites"
version = "1.4.0"
description = "Atomic file writes."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "attrs"
version = "21.2.0"
description = "Classes Without Boilerplate"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.extras]
dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"]
docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"]
tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"]
[[package]]
name = "black"
version = "20.8b1"
@ -179,14 +157,6 @@ category = "main"
optional = false
python-versions = ">=3.5"
[[package]]
name = "iniconfig"
version = "1.1.1"
description = "iniconfig: brain-dead simple config-ini parsing"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "itsdangerous"
version = "2.0.1"
@ -260,17 +230,6 @@ category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "packaging"
version = "21.0"
description = "Core utilities for Python packages"
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
pyparsing = ">=2.0.2"
[[package]]
name = "pathspec"
version = "0.8.1"
@ -287,17 +246,6 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "pluggy"
version = "0.13.1"
description = "plugin and hook calling mechanisms for python"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[package.extras]
dev = ["pre-commit", "tox"]
[[package]]
name = "profig"
version = "0.5.1"
@ -306,14 +254,6 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "py"
version = "1.10.0"
description = "library with cross-python path, ini-parsing, io, code, log facilities"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pycodestyle"
version = "2.7.0"
@ -330,14 +270,6 @@ category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pyparsing"
version = "2.4.7"
description = "Python parsing module"
category = "dev"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "pyrss2gen"
version = "1.1"
@ -346,27 +278,6 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "pytest"
version = "6.2.4"
description = "pytest: simple powerful testing with Python"
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
attrs = ">=19.2.0"
colorama = {version = "*", markers = "sys_platform == \"win32\""}
iniconfig = "*"
packaging = "*"
pluggy = ">=0.12,<1.0.0a1"
py = ">=1.8.2"
toml = "*"
[package.extras]
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
name = "python-dateutil"
version = "2.8.2"
@ -431,14 +342,6 @@ category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "tinydb"
version = "4.5.0"
description = "TinyDB is a tiny, document oriented database optimized for your happiness :)"
category = "main"
optional = false
python-versions = ">=3.5,<4.0"
[[package]]
name = "toml"
version = "0.10.2"
@ -501,7 +404,7 @@ watchdog = ["watchdog"]
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "c2c76edb4fcae80581572bcb9699f0b8b0193cbcc5e08a4b6f431c15f09236e7"
content-hash = "8190054ee0a6bf5fccefd841ac71fa6851a4e7d057c5af6fb83ea2364711cb78"
[metadata.files]
appdirs = [
@ -512,14 +415,6 @@ apscheduler = [
{file = "APScheduler-3.7.0-py2.py3-none-any.whl", hash = "sha256:c06cc796d5bb9eb3c4f77727f6223476eb67749e7eea074d1587550702a7fbe3"},
{file = "APScheduler-3.7.0.tar.gz", hash = "sha256:1cab7f2521e107d07127b042155b632b7a1cd5e02c34be5a28ff62f77c900c6a"},
]
atomicwrites = [
{file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
{file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
]
attrs = [
{file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"},
{file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"},
]
black = [
{file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
]
@ -558,10 +453,6 @@ idna = [
{file = "idna-3.2-py3-none-any.whl", hash = "sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a"},
{file = "idna-3.2.tar.gz", hash = "sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"},
]
iniconfig = [
{file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
{file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
]
itsdangerous = [
{file = "itsdangerous-2.0.1-py3-none-any.whl", hash = "sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c"},
{file = "itsdangerous-2.0.1.tar.gz", hash = "sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0"},
@ -634,10 +525,6 @@ mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
packaging = [
{file = "packaging-21.0-py3-none-any.whl", hash = "sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14"},
{file = "packaging-21.0.tar.gz", hash = "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7"},
]
pathspec = [
{file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"},
{file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"},
@ -645,17 +532,9 @@ pathspec = [
peewee = [
{file = "peewee-3.14.4.tar.gz", hash = "sha256:9e356b327c2eaec6dd42ecea6f4ddded025793dba906a3d065a0452e726c51a2"},
]
pluggy = [
{file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
{file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
]
profig = [
{file = "profig-0.5.1.tar.gz", hash = "sha256:cb9c094325a93505fc6325d13f3e679b281093223f143a96a6df8ad9c2bfc9a6"},
]
py = [
{file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"},
{file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"},
]
pycodestyle = [
{file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"},
{file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"},
@ -664,17 +543,9 @@ pyflakes = [
{file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"},
{file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"},
]
pyparsing = [
{file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
{file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
]
pyrss2gen = [
{file = "PyRSS2Gen-1.1.tar.gz", hash = "sha256:7960aed7e998d2482bf58716c316509786f596426f879b05f8d84e98b82c6ee7"},
]
pytest = [
{file = "pytest-6.2.4-py3-none-any.whl", hash = "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"},
{file = "pytest-6.2.4.tar.gz", hash = "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b"},
]
python-dateutil = [
{file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
{file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
@ -739,10 +610,6 @@ six = [
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
tinydb = [
{file = "tinydb-4.5.0-py3-none-any.whl", hash = "sha256:ab2669b88ba1e1b3e1bd6da1a1e3ee284fde6fbf327fb16a206ac3954915f37f"},
{file = "tinydb-4.5.0.tar.gz", hash = "sha256:d287cd092f19a2b8553d0a6018f172c351268fb8619898eb87633d9e2c036344"},
]
toml = [
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},

2
pyproject.toml
Unescape Escape View File

@ -13,7 +13,6 @@ pyrss2gen = "^1.1"
profig = "^0.5.1"
markdown = "^3.1.1"
flask_apscheduler = "^1.11.0"
tinydb = "^4.3.0"
Flask = "^2.0.1"
peewee = "^3.14.0"
requests = "^2.25.1"
@ -23,7 +22,6 @@ rope = "^0.16.0"
mypy = "^0.790"
flake8-black = "^0.2.1"
black = "^20.8b1"
pytest = "^6.2.1"
[build-system]
requires = ["poetry>=0.12"]

64
stacosys/core/cron.py
Unescape Escape View File

@ -5,12 +5,13 @@ import logging
import os
import re
from stacosys.core.templater import Templater, Template
from stacosys.model.comment import Comment
from stacosys.model.email import Email
from stacosys.core.rss import Rss
from stacosys.core.mailer import Mailer
from stacosys.core.rss import Rss
from stacosys.core.templater import Templater, Template
from stacosys.db import dao
from stacosys.model.email import Email
REGEX_EMAIL_SUBJECT = r".*STACOSYS.*\[(\d+)\:(\w+)\]"
logger = logging.getLogger(__name__)
@ -21,41 +22,42 @@ templater = Templater(template_path)
def fetch_mail_answers(lang, mailer: Mailer, rss: Rss, site_token):
for msg in mailer.fetch():
if re.search(r".*STACOSYS.*\[(\d+)\:(\w+)\]", msg.subject, re.DOTALL):
if _reply_comment_email(lang, mailer, rss, msg, site_token):
mailer.delete(msg.id)
def _reply_comment_email(lang, mailer: Mailer, rss: Rss, email: Email, site_token):
m = re.search(r"\[(\d+)\:(\w+)\]", email.subject)
# filter stacosys e-mails
m = re.search(REGEX_EMAIL_SUBJECT, msg.subject, re.DOTALL)
if not m:
logger.warning("ignore corrupted email. No token %s" % email.subject)
return
continue
comment_id = int(m.group(1))
token = m.group(2)
if token != site_token:
submitted_token = m.group(2)
# validate token
if submitted_token != site_token:
logger.warning("ignore corrupted email. Unknown token %d" % comment_id)
return
continue
if not msg.plain_text_content:
logger.warning("ignore empty email")
continue
_reply_comment_email(lang, mailer, rss, msg, comment_id)
mailer.delete(msg.id)
# retrieve site and comment rows
comment = Comment
def _reply_comment_email(lang, mailer: Mailer, rss: Rss, email: Email, comment_id):
# retrieve comment
comment = dao.find_comment_by_id(comment_id)
if not comment:
logger.warning("unknown comment %d" % comment_id)
return True
return
if comment.published:
logger.warning("ignore already published email. token %d" % comment_id)
return
if not email.plain_text_content:
logger.warning("ignore empty email")
return
# safe logic: no answer or unknown answer is a go for publishing
if email.plain_text_content[:2].upper() == "NO":
logger.info("discard comment: %d" % comment_id)
comment.delete_instance()
dao.delete_comment(comment)
new_email_body = templater.get_template(lang, Template.DROP_COMMENT).render(
original=email.plain_text_content
)
@ -63,7 +65,7 @@ def _reply_comment_email(lang, mailer: Mailer, rss: Rss, email: Email, site_toke
logger.warning("minor failure. cannot send rejection mail " + email.subject)
else:
# save publishing datetime
dao.publish(comment)
dao.publish_comment(comment)
logger.info("commit comment: %d" % comment_id)
# rebuild RSS
@ -76,11 +78,9 @@ def _reply_comment_email(lang, mailer: Mailer, rss: Rss, email: Email, site_toke
if not mailer.send(email.from_addr, "Re: " + email.subject, new_email_body):
logger.warning("minor failure. cannot send approval email " + email.subject)
return True
def submit_new_comment(lang, site_name, site_token, site_admin_email, mailer):
for comment in Comment.select().where(Comment.notified.is_null()):
for comment in dao.find_not_notified_comments():
comment_list = (
"author: %s" % comment.author_name,
"site: %s" % comment.author_site,
@ -95,12 +95,12 @@ def submit_new_comment(lang, site_name, site_token, site_admin_email, mailer):
url=comment.url, comment=comment_text
)
# send email
# send email to notify admin
subject = "STACOSYS %s: [%d:%s]" % (site_name, comment.id, site_token)
if mailer.send(site_admin_email, subject, email_body):
logger.debug("new comment processed ")
# notify site admin and save notification datetime
dao.notify_site_admin(comment)
# save notification datetime
dao.notify_comment(comment)
else:
logger.warning("rescheduled. send mail failure " + subject)

39
stacosys/db/dao.py
Unescape Escape View File

@ -5,14 +5,49 @@ from datetime import datetime
from stacosys.model.comment import Comment
def notify_site_admin(comment: Comment):
def find_comment_by_id(id):
return Comment.get_by_id(id)
def notify_comment(comment: Comment):
comment.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.save()
def publish(comment: Comment):
def publish_comment(comment: Comment):
comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.save()
def delete_comment(comment: Comment):
comment.delete_instance()
def find_not_notified_comments():
return Comment.select().where(Comment.notified.is_null())
def find_published_comments_by_url(url):
return Comment.select(Comment).where((Comment.url == url) & (Comment.published.is_null(False))).order_by(
+Comment.published)
def count_published_comments(url):
return Comment.select(Comment).where(
(Comment.url == url) & (Comment.published.is_null(False))).count() if url else Comment.select(Comment).where(
Comment.publishd.is_null(False)).count()
def create_comment(url, author_name, author_site, author_gravatar, message):
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment = Comment(
url=url,
author_name=author_name,
author_site=author_site,
author_gravatar=author_gravatar,
content=message,
created=created,
notified=None,
published=None,
)
comment.save()

24
stacosys/db/database.py
Unescape Escape View File

@ -1,11 +1,8 @@
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
from peewee import Model
from playhouse.db_url import SqliteDatabase
from playhouse.shortcuts import model_to_dict
from tinydb import TinyDB
db = SqliteDatabase(None)
@ -20,29 +17,8 @@ class Database:
return db
def setup(self, db_url):
db.init(db_url)
db.connect()
from stacosys.model.comment import Comment
db.create_tables([Comment], safe=True)
# if config.exists(config.DB_BACKUP_JSON_FILE):
# _backup_db(config.DB_BACKUP_JSON_FILE, Comment)
def _tojson_model(comment):
dcomment = model_to_dict(comment)
# del dcomment["site"]
tcomment = json.dumps(dcomment, indent=4, sort_keys=True, default=str)
return json.loads(tcomment)
def _backup_db(db_file, Comment):
db = TinyDB(db_file, sort_keys=True, indent=4, separators=(",", ": "))
db.drop_tables()
table = db.table("comments")
for comment in Comment.select():
cc = _tojson_model(comment)
table.insert(cc)

23
stacosys/interface/api.py
Unescape Escape View File

@ -2,14 +2,15 @@
# -*- coding: utf-8 -*-
import logging
from flask import abort, jsonify, request
from stacosys.db import dao
from stacosys.interface import app
from stacosys.model.comment import Comment
logger = logging.getLogger(__name__)
@app.route("/ping", methods=["GET"])
def ping():
return "OK"
@ -23,12 +24,8 @@ def query_comments():
abort(401)
url = request.args.get("url", "")
logger.info("retrieve comments for url %s" % (url))
for comment in (
Comment.select(Comment)
.where((Comment.url == url) & (Comment.published.is_null(False)))
.order_by(+Comment.published)
):
logger.info("retrieve comments for url %s" % url)
for comment in dao.find_published_comments_by_url(url):
d = {
"author": comment.author_name,
"content": comment.content,
@ -48,12 +45,4 @@ def get_comments_count():
if token != app.config.get("SITE_TOKEN"):
abort(401)
url = request.args.get("url", "")
if url:
count = (
Comment.select(Comment)
.where((Comment.url == url) & (Comment.published.is_null(False)))
.count()
)
else:
count = Comment.select(Comment).where(Comment.publishd.is_null(False)).count()
return jsonify({"count": count})
return jsonify({"count": dao.count_published_comments(url)})

37
stacosys/interface/form.py
Unescape Escape View File

@ -2,18 +2,17 @@
# -*- coding: utf-8 -*-
import logging
from datetime import datetime
from flask import abort, redirect, request
from stacosys.db import dao
from stacosys.interface import app
from stacosys.model.comment import Comment
logger = logging.getLogger(__name__)
@app.route("/newcomment", methods=["POST"])
def new_form_comment():
try:
data = request.form
logger.info("form data " + str(data))
@ -26,7 +25,7 @@ def new_form_comment():
# honeypot for spammers
captcha = data.get("remarque", "")
if captcha:
logger.warn("discard spam: data %s" % data)
logger.warning("discard spam: data %s" % data)
abort(400)
url = data.get("url", "")
@ -39,23 +38,14 @@ def new_form_comment():
# anti-spam again
if not url or not author_name or not message:
logger.warn("empty field: data %s" % data)
logger.warning("empty field: data %s" % data)
abort(400)
if not check_form_data(data.to_dict()):
logger.warning("additional field: data %s" % data)
abort(400)
check_form_data(data)
# add a row to Comment table
created = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment = Comment(
url=url,
author_name=author_name,
author_site=author_site,
author_gravatar=author_gravatar,
content=message,
created=created,
notified=None,
published=None,
)
comment.save()
dao.create_comment(url, author_name, author_site, author_gravatar, message)
except Exception:
logger.exception("new comment failure")
@ -64,12 +54,13 @@ def new_form_comment():
return redirect("/redirect/", code=302)
def check_form_data(data):
def check_form_data(d):
fields = ["url", "message", "site", "remarque", "author", "token", "email"]
d = data.to_dict()
for field in fields:
if field in d:
del d[field]
if d:
logger.warn("additional field: data %s" % data)
abort(400)
# filtered = dict(filter(lambda x: x[0] not in fields, data.to_dict().items()))
return not d

4
stacosys/model/comment.py
Unescape Escape View File

@ -2,9 +2,9 @@
# -*- coding: UTF-8 -*-
from peewee import CharField
from peewee import TextField
from peewee import DateTimeField
from datetime import datetime
from peewee import TextField
from stacosys.db.database import BaseModel

Write Preview
Loading…
Cancel
Save