added admin functions (remove spam content)

use efficient sql SELECT for frontpage poll list
added link counter for spam detection
improve efficiency of post deletion by using sqlobject's "cascade" attribute
This commit is contained in:
lars 2013-02-22 01:19:32 +00:00
parent 4b089f4494
commit fa37629d2c

View file

@ -43,7 +43,6 @@ loader = genshi.template.TemplateLoader(os.path.join(BASE_DIR, 'templates'), aut
BLOG_DIR = os.path.join(BASE_DIR, "blog") BLOG_DIR = os.path.join(BASE_DIR, "blog")
BASE_DICT = { BASE_DICT = {
"base_url": "/", # the trailing slash is necessary "base_url": "/", # the trailing slash is necessary
"show_navbar": True, "show_navbar": True,
@ -51,6 +50,7 @@ BASE_DICT = {
"authlevel": "public", #authentiction level of template: one of "public" (e.g. frontpage), "poll_public" (e.g. poll_details),"poll_admin" (poll_admin_details","admin" (admin interface) "authlevel": "public", #authentiction level of template: one of "public" (e.g. frontpage), "poll_public" (e.g. poll_details),"poll_admin" (poll_admin_details","admin" (admin interface)
} }
# used as the default setting for expose/close dates # used as the default setting for expose/close dates
DEFAULT_DAYS_AHEAD = 7 DEFAULT_DAYS_AHEAD = 7
DATE_FORMAT = "%d.%m.%Y" DATE_FORMAT = "%d.%m.%Y"
@ -85,7 +85,7 @@ POLL_SETTING_TEMPLATES = {
class ContentSubmission(sqlobject.SQLObject): class ContentSubmission(sqlobject.SQLObject):
submitter = sqlobject.UnicodeCol() submitter = sqlobject.UnicodeCol()
content = sqlobject.UnicodeCol() content = sqlobject.UnicodeCol()
poll_id = sqlobject.ForeignKey("Poll") poll_id = sqlobject.ForeignKey("Poll", cascade=True)
timestamp_creation = sqlobject.DateTimeCol() timestamp_creation = sqlobject.DateTimeCol()
def get_creation_time_string(self): def get_creation_time_string(self):
@ -133,20 +133,20 @@ class Profile(sqlobject.SQLObject):
class ProfilePolls(sqlobject.SQLObject): class ProfilePolls(sqlobject.SQLObject):
user = sqlobject.ForeignKey("Profile") user = sqlobject.ForeignKey("Profile", cascade=True)
poll = sqlobject.ForeignKey("Poll") poll = sqlobject.ForeignKey("Poll", cascade=True)
is_admin = sqlobject.BoolCol() is_admin = sqlobject.BoolCol()
class PollSetting(sqlobject.SQLObject): class PollSetting(sqlobject.SQLObject):
poll_id = sqlobject.ForeignKey("Poll") poll_id = sqlobject.ForeignKey("Poll", cascade=True)
key = sqlobject.UnicodeCol() key = sqlobject.UnicodeCol()
value = sqlobject.UnicodeCol() value = sqlobject.UnicodeCol()
class PollRelation(sqlobject.SQLObject): class PollRelation(sqlobject.SQLObject):
first = sqlobject.ForeignKey("Poll") first = sqlobject.ForeignKey("Poll", cascade=True)
second = sqlobject.ForeignKey("Poll") second = sqlobject.ForeignKey("Poll", cascade=True)
class Poll(sqlobject.SQLObject): class Poll(sqlobject.SQLObject):
@ -252,15 +252,6 @@ class Poll(sqlobject.SQLObject):
def get_submissions(self): def get_submissions(self):
return ContentSubmission.selectBy(poll_id=self.id).orderBy("timestamp_creation") return ContentSubmission.selectBy(poll_id=self.id).orderBy("timestamp_creation")
def delete_poll(self):
submissions = ContentSubmission.selectBy(poll_id=self.id)
settings = PollSetting.selectBy(poll_id=self.id)
for submission in submissions:
submission.destroySelf()
for setting in settings:
setting.destroySelf()
self.destroySelf()
def get_url(self, absolute=False): def get_url(self, absolute=False):
return get_url_string("%s%s" % (BASE_DICT["base_url"], self.hash_key), absolute) return get_url_string("%s%s" % (BASE_DICT["base_url"], self.hash_key), absolute)
@ -589,12 +580,15 @@ def is_spam_submitter_name(name, errors_dict):
else: else:
return False return False
def is_spam_content(text, errors_dict): def count_urls(text, errors_dict):
hits = re.findall(r"(\swww\.|http://|https://)", text)
return len(hits)
def check_spam_content(text):
if re.search(r"(<a\s|\shref=|</a>)", text.lower()): if re.search(r"(<a\s|\shref=|</a>)", text.lower()):
errors_dict["content"] = \
"Spam-Verdacht: Inhalt darf keine HTML-Tags enthalten"
return True return True
else: if count_urls(text) > 2:
return True
return False return False
@bobo.query('/profile/logout') @bobo.query('/profile/logout')
@ -866,7 +860,11 @@ def submit_content(bobo_request, hash_key=None, submitter=None, content=None):
value_dict["errors"] = {} value_dict["errors"] = {}
value_dict["authlevel"] = "poll_public" value_dict["authlevel"] = "poll_public"
data = {} data = {}
if content and (not is_spam_content(content, value_dict["errors"])): if content and check_spam_content(content):
value_dict["errors"]["content"] = \
"Spam-Verdacht: Inhalt darf keine HTML-Tags und nicht " + \
"zuviele Links enthalten"
else:
data["content"] = content data["content"] = content
if submitter and (not is_spam_submitter_name(submitter, value_dict["errors"])): if submitter and (not is_spam_submitter_name(submitter, value_dict["errors"])):
data["submitter"] = submitter data["submitter"] = submitter
@ -899,7 +897,7 @@ def delete_poll(bobo_request, admin_hash_key=None):
admin_poll_id = get_poll_admin_id(admin_hash_key) admin_poll_id = get_poll_admin_id(admin_hash_key)
if not admin_poll_id is None: if not admin_poll_id is None:
poll = Poll.get(admin_poll_id) poll = Poll.get(admin_poll_id)
poll.delete_poll() poll.destroySelf()
return bobo.redirect(BASE_DICT["base_url"]) return bobo.redirect(BASE_DICT["base_url"])
@bobo.query('/:admin_hash_key/vote_enable') @bobo.query('/:admin_hash_key/vote_enable')
@ -1111,15 +1109,21 @@ def serve_blog(bobo_request, blog_id=None):
value_dict["blog_list"] = blog_list value_dict["blog_list"] = blog_list
return render("blog_list.html", **value_dict) return render("blog_list.html", **value_dict)
def show_poll_list(bobo_request, render_file, page_size, page=None, filter_private=True): def show_poll_list(bobo_request, render_file, page_size, page=None,
value_dict = get_default_values(bobo_request) filter_private=True, value_dict=None):
polls = Poll.select().orderBy("-timestamp_creation") if value_dict is None:
# TODO: speed the filtering up by using SQL statements (see sqlobject "filter") value_dict = {}
value_dict.update(get_default_values(bobo_request))
true_string = get_poll_setting_string("public", True)
if filter_private: if filter_private:
polls = [poll for poll in polls if poll.get_settings()["public"]] polls = Poll.select(sqlobject.sqlbuilder.AND(
PollSetting.q.key == "public",
PollSetting.q.value == true_string,
PollSetting.q.poll_id == Poll.q.id))
else: else:
# convert the sql query into a list (probably an expensive operation) polls = Poll.select()
polls = [poll for poll in polls] polls = polls.orderBy("-timestamp_creation")
poll_count = polls.count()
if page is None: if page is None:
page = 1 page = 1
else: else:
@ -1130,15 +1134,16 @@ def show_poll_list(bobo_request, render_file, page_size, page=None, filter_priva
# "page" should at least be 1 - zero shows an empty list # "page" should at least be 1 - zero shows an empty list
page = max(1, page) page = max(1, page)
start = (page - 1) * page_size start = (page - 1) * page_size
if start >= len(polls): if start >= poll_count:
start = len(polls) - page_size start = poll_count - page_size
page = (len(polls) + page_size - 1) / page_size page = (poll_count + page_size - 1) / page_size
end = start + page_size - 1 end = start + page_size - 1
value_dict["polls"] = polls[start : end + 1] value_dict["polls"] = polls[start : end + 1]
# show a link for the next page, if more polls are available # show a link for the next page, if more polls are available
value_dict["show_next_link"] = (end + 1 < len(polls)) value_dict["show_next_link"] = (end + 1 < poll_count)
value_dict["show_previous_link"] = (start > 0) value_dict["show_previous_link"] = (start > 0)
value_dict["page"] = page value_dict["page"] = page
value_dict["page_size"] = page_size
return render(render_file, **value_dict) return render(render_file, **value_dict)
def render_poll_admin(bobo_request, poll, add_related, del_related, count, page, show_delete): def render_poll_admin(bobo_request, poll, add_related, del_related, count, page, show_delete):
@ -1176,22 +1181,47 @@ def render_poll_admin(bobo_request, poll, add_related, del_related, count, page,
value_dict["authlevel"] = "poll_admin" value_dict["authlevel"] = "poll_admin"
return render("poll_admin_details.html", **value_dict) return render("poll_admin_details.html", **value_dict)
@bobo.query('/admin/maintenance')
def admin_maintenance(age_days=60, keyword="viagra", keyword_submission="viagra", method=None):
if method == "Remove old":
age_days = int(age_days)
now = datetime.datetime.now()
oldest = now - datetime.timedelta(days=age_days)
for poll in Poll.select(Poll.q.timestamp_creation < oldest):
if poll.get_num_of_submissions() == 0:
poll.destroySelf()
elif method == "Remove poll by keyword" and keyword:
keyword = keyword.lower()
for poll in Poll.select():
if (keyword in poll.title.lower()) or (keyword in poll.description.lower()):
poll.destroySelf()
elif method == "Remove submission by keyword" and keyword:
keyword = keyword_submission.lower()
for submission in ContentSubmission.select():
if keyword in submission.content.lower():
submission.destroySelf()
return bobo.redirect("../admin")
@bobo.query('/admin') @bobo.query('/admin')
@bobo.query('/admin/') @bobo.query('/admin/')
@bobo.query('/admin/page/:page') @bobo.query('/admin/page/:page')
def show_admin_page(bobo_request, page=None, page_size=20): def show_admin_page(bobo_request, page=None, page_size=50):
try: try:
page_size = int(page_size) page_size = int(page_size)
except ValueError: except ValueError:
page_size = 30 page_size = 50
return show_poll_list(bobo_request, "admin.html", page_size, page, filter_private=False) value_dict = {}
value_dict["poll_table"] = Poll
value_dict["submission_table"] = ContentSubmission
return show_poll_list(bobo_request, "admin.html", page_size, page=page,
filter_private=False, value_dict=value_dict)
@bobo.query('/') @bobo.query('/')
@bobo.query('/public') @bobo.query('/public')
@bobo.query('/public/') @bobo.query('/public/')
@bobo.query('/public/page/:page') @bobo.query('/public/page/:page')
def show_frontpage(bobo_request, page=None): def show_frontpage(bobo_request, page=None):
return show_poll_list(bobo_request, "frontpage.html", 20, page) return show_poll_list(bobo_request, "frontpage.html", 20, page=page)
@bobo.query('') @bobo.query('')
def base(): def base():