From 95e911981726a4087ce52053e854468cc8e3b24d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 26 Feb 2022 08:44:19 -0800 Subject: [PATCH 1/5] Adds django celery beat --- bookwyrm/settings.py | 1 + bw-dev | 1 + docker-compose.yml | 13 +++++++++++++ requirements.txt | 1 + 4 files changed, 16 insertions(+) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 0fbe3b73..5d4eb781 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -90,6 +90,7 @@ INSTALLED_APPS = [ "sass_processor", "bookwyrm", "celery", + 'django_celery_beat', "imagekit", "storages", ] diff --git a/bw-dev b/bw-dev index 89cc8d8c..d4b5263d 100755 --- a/bw-dev +++ b/bw-dev @@ -215,6 +215,7 @@ case "$CMD" in ;; setup) migrate + migrate django_celery_beat initdb runweb python manage.py collectstatic --no-input admin_code diff --git a/docker-compose.yml b/docker-compose.yml index 0994aa00..e45cae0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,6 +70,19 @@ services: - db - redis_broker restart: on-failure + celery_beat: + env_file: .env + build: . + networks: + - main + command: celery -A celerywyrm beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler + volumes: + - .:/app + - static_volume:/app/static + - media_volume:/app/images + depends_on: + - celery_worker + restart: on-failure flower: build: . command: celery -A celerywyrm flower --basic_auth=${FLOWER_USER}:${FLOWER_PASSWORD} diff --git a/requirements.txt b/requirements.txt index 26582e00..8eb44d95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ celery==5.2.2 colorthief==0.2.1 Django==3.2.12 +django-celery-beat==2.2.1 django-compressor==2.4.1 django-imagekit==4.1.0 django-model-utils==4.0.0 From 2a436800c4f801715631993aa95818fe66a18d64 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 26 Feb 2022 10:13:44 -0800 Subject: [PATCH 2/5] Schedules automod task --- bookwyrm/forms.py | 7 +++ bookwyrm/models/antispam.py | 1 + bookwyrm/settings.py | 2 +- .../templates/settings/automod/rules.html | 41 +++++++++++++-- bookwyrm/urls.py | 11 +++- bookwyrm/views/__init__.py | 2 +- bookwyrm/views/admin/automod.py | 50 ++++++++++++++++--- celerywyrm/settings.py | 5 ++ 8 files changed, 105 insertions(+), 14 deletions(-) diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 7ae4e446..00e6d5d8 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -8,6 +8,7 @@ from django.forms import ModelForm, PasswordInput, widgets, ChoiceField from django.forms.widgets import Textarea from django.utils import timezone from django.utils.translation import gettext_lazy as _ +from django_celery_beat.models import IntervalSchedule from bookwyrm import models from bookwyrm.models.fields import ClearableFileInputWithWarning @@ -556,3 +557,9 @@ class AutoModRuleForm(CustomForm): class Meta: model = models.AutoMod fields = ["string_match", "flag_users", "flag_statuses", "created_by"] + + +class IntervalScheduleForm(CustomForm): + class Meta: + model = IntervalSchedule + fields = ["every", "period"] diff --git a/bookwyrm/models/antispam.py b/bookwyrm/models/antispam.py index f506b6f1..bce02780 100644 --- a/bookwyrm/models/antispam.py +++ b/bookwyrm/models/antispam.py @@ -54,6 +54,7 @@ class AutoMod(models.Model): @app.task(queue="low_priority") def automod_task(): """Create reports""" + print("TASK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") if not AutoMod.objects.exists(): return reporter = AutoMod.objects.first().created_by diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 5d4eb781..16e5ccb3 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -90,7 +90,7 @@ INSTALLED_APPS = [ "sass_processor", "bookwyrm", "celery", - 'django_celery_beat', + "django_celery_beat", "imagekit", "storages", ] diff --git a/bookwyrm/templates/settings/automod/rules.html b/bookwyrm/templates/settings/automod/rules.html index 8205b3d7..2a28cb64 100644 --- a/bookwyrm/templates/settings/automod/rules.html +++ b/bookwyrm/templates/settings/automod/rules.html @@ -1,5 +1,6 @@ {% extends 'settings/layout.html' %} {% load i18n %} +{% load humanize %} {% load utilities %} {% block title %} @@ -16,12 +17,46 @@

{% trans "Auto-moderation rules will create reports for any local user or status with fields matching the provided string." %} {% trans "Users or statuses that have already been reported (regardless of whether the report was resolved) will not be flagged." %} - {% trans "At this time, reports are not being generated automatically, and you must manually trigger a scan." %}

-
+ {% if task %} +
+
+ {% trans "Schedule:" %} +
+
+ {{ task.schedule }} +
+ +
+ {% trans "Last run:" %} +
+
+ {{ task.last_run_at|naturaltime }} +
+ +
+ {% trans "Total run count:" %} +
+
+ {{ task.total_run_count }} +
+ +
+ {% trans "Enabled:" %} +
+
+ + {{ task.enabled|yesno }} + +
+
+ {% else %} + {% csrf_token %} - + {{ task_form.as_p }} +
+ {% endif %} {% if success %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index d2caa76e..5abe7ac2 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -217,11 +217,18 @@ urlpatterns = [ # auto-moderation rules re_path(r"^settings/automod/?$", views.AutoMod.as_view(), name="settings-automod"), re_path( - r"^settings/automod/(?P\d+)/delete?$", + r"^settings/automod/(?P\d+)/delete/?$", views.automod_delete, name="settings-automod-delete", ), - re_path(r"^settings/automod/run?$", views.run_automod, name="settings-automod-run"), + re_path( + r"^settings/automod/schedule/?$", + views.schedule_automod_task, + name="settings-automod-schedule", + ), + re_path( + r"^settings/automod/run/?$", views.run_automod, name="settings-automod-run" + ), # moderation re_path( r"^settings/reports/?$", views.ReportsAdmin.as_view(), name="settings-reports" diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 76e9ff02..aa4d7299 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -2,7 +2,7 @@ # site admin from .admin.announcements import Announcements, Announcement from .admin.announcements import EditAnnouncement, delete_announcement -from .admin.automod import AutoMod, automod_delete, run_automod +from .admin.automod import AutoMod, automod_delete, run_automod, schedule_automod_task from .admin.dashboard import Dashboard from .admin.federation import Federation, FederatedServer from .admin.federation import AddFederatedServer, ImportServerBlocklist diff --git a/bookwyrm/views/admin/automod.py b/bookwyrm/views/admin/automod.py index d9901d01..fbe6408c 100644 --- a/bookwyrm/views/admin/automod.py +++ b/bookwyrm/views/admin/automod.py @@ -1,10 +1,12 @@ """ moderation via flagged posts and users """ from django.contrib.auth.decorators import login_required, permission_required +from django.db import transaction from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.http import require_POST +from django_celery_beat.models import PeriodicTask from bookwyrm import forms, models @@ -24,8 +26,9 @@ class AutoMod(View): def get(self, request): """view rules""" - data = {"rules": models.AutoMod.objects.all(), "form": forms.AutoModRuleForm()} - return TemplateResponse(request, "settings/automod/rules.html", data) + return TemplateResponse( + request, "settings/automod/rules.html", automod_view_data() + ) def post(self, request): """add rule""" @@ -35,14 +38,32 @@ class AutoMod(View): form.save() form = forms.AutoModRuleForm() - data = { - "rules": models.AutoMod.objects.all(), - "form": form, - "success": success, - } + data = automod_view_data() + data["form"] = form return TemplateResponse(request, "settings/automod/rules.html", data) +@require_POST +@permission_required("bookwyrm.moderate_user", raise_exception=True) +@permission_required("bookwyrm.moderate_post", raise_exception=True) +def schedule_automod_task(request): + """scheduler""" + form = forms.IntervalScheduleForm(request.POST) + if not form.is_valid(): + data = automod_view_data() + data["task_form"] = form + return TemplateResponse(request, "settings/automod/rules.html", data) + + with transaction.atomic(): + schedule = form.save() + PeriodicTask.objects.get_or_create( + interval=schedule, + name="automod-task", + task="bookwyrm.models.antispam.automod_task", + ) + return redirect("settings-automod") + + @require_POST @permission_required("bookwyrm.moderate_user", raise_exception=True) @permission_required("bookwyrm.moderate_post", raise_exception=True) @@ -62,3 +83,18 @@ def run_automod(request): """run scan""" models.automod_task.delay() return redirect("settings-automod") + + +def automod_view_data(): + """helper to get data used in the template""" + try: + task = PeriodicTask.objects.get(name="automod-task") + except PeriodicTask.DoesNotExist: + task = None + + return { + "task": task, + "task_form": forms.IntervalScheduleForm(), + "rules": models.AutoMod.objects.all(), + "form": forms.AutoModRuleForm(), + } diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index bd7805e5..35eb3933 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -3,6 +3,7 @@ # pylint: disable=unused-wildcard-import from bookwyrm.settings import * +# pylint: disable=line-too-long REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None)) REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker") REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379) @@ -16,6 +17,10 @@ CELERY_DEFAULT_QUEUE = "low_priority" CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" CELERY_RESULT_SERIALIZER = "json" + +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" +CELERY_TIMEZONE = env("TIME_ZONE", "UTC") + FLOWER_PORT = env("FLOWER_PORT") INSTALLED_APPS = INSTALLED_APPS + [ From 0870eccad98c28026096f06eb520fc13ebcc4dc9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 26 Feb 2022 10:24:23 -0800 Subject: [PATCH 3/5] Adds unscheduler --- bookwyrm/templates/settings/automod/rules.html | 13 +++++++++++++ bookwyrm/urls.py | 5 +++++ bookwyrm/views/__init__.py | 3 ++- bookwyrm/views/admin/automod.py | 13 +++++++++++-- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/bookwyrm/templates/settings/automod/rules.html b/bookwyrm/templates/settings/automod/rules.html index 2a28cb64..0a1b1c3f 100644 --- a/bookwyrm/templates/settings/automod/rules.html +++ b/bookwyrm/templates/settings/automod/rules.html @@ -50,6 +50,19 @@ + +
+
+ {% csrf_token %} + +
+
+ {% csrf_token %} + +

{% trans "Last run date will not be updated" %}

+
+
+ {% else %}
{% csrf_token %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 5abe7ac2..b458084e 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -226,6 +226,11 @@ urlpatterns = [ views.schedule_automod_task, name="settings-automod-schedule", ), + re_path( + r"^settings/automod/unschedule/(?P\d+)/?$", + views.unschedule_automod_task, + name="settings-automod-unschedule", + ), re_path( r"^settings/automod/run/?$", views.run_automod, name="settings-automod-run" ), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index aa4d7299..f9392815 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -2,7 +2,8 @@ # site admin from .admin.announcements import Announcements, Announcement from .admin.announcements import EditAnnouncement, delete_announcement -from .admin.automod import AutoMod, automod_delete, run_automod, schedule_automod_task +from .admin.automod import AutoMod, automod_delete, run_automod +from .admin.automod import schedule_automod_task, unschedule_automod_task from .admin.dashboard import Dashboard from .admin.federation import Federation, FederatedServer from .admin.federation import AddFederatedServer, ImportServerBlocklist diff --git a/bookwyrm/views/admin/automod.py b/bookwyrm/views/admin/automod.py index fbe6408c..f8c3e8e6 100644 --- a/bookwyrm/views/admin/automod.py +++ b/bookwyrm/views/admin/automod.py @@ -64,14 +64,23 @@ def schedule_automod_task(request): return redirect("settings-automod") +@require_POST +@permission_required("bookwyrm.moderate_user", raise_exception=True) +@permission_required("bookwyrm.moderate_post", raise_exception=True) +# pylint: disable=unused-argument +def unschedule_automod_task(request, task_id): + """unscheduler""" + get_object_or_404(PeriodicTask, id=task_id).delete() + return redirect("settings-automod") + + @require_POST @permission_required("bookwyrm.moderate_user", raise_exception=True) @permission_required("bookwyrm.moderate_post", raise_exception=True) # pylint: disable=unused-argument def automod_delete(request, rule_id): """Remove a rule""" - rule = get_object_or_404(models.AutoMod, id=rule_id) - rule.delete() + get_object_or_404(models.AutoMod, id=rule_id).delete() return redirect("settings-automod") From 6b5bebdf781e067e97683507639343d35e6ade45 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 26 Feb 2022 10:44:50 -0800 Subject: [PATCH 4/5] Cleans up scheduler form --- bookwyrm/forms.py | 5 ++++ .../templates/settings/automod/rules.html | 28 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 00e6d5d8..561a86d0 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -563,3 +563,8 @@ class IntervalScheduleForm(CustomForm): class Meta: model = IntervalSchedule fields = ["every", "period"] + + widgets = { + "every": forms.NumberInput(attrs={"aria-describedby": "desc_every"}), + "period": forms.Select(attrs={"aria-describedby": "desc_period"}), + } diff --git a/bookwyrm/templates/settings/automod/rules.html b/bookwyrm/templates/settings/automod/rules.html index 0a1b1c3f..ef0a49be 100644 --- a/bookwyrm/templates/settings/automod/rules.html +++ b/bookwyrm/templates/settings/automod/rules.html @@ -18,8 +18,10 @@ {% trans "Auto-moderation rules will create reports for any local user or status with fields matching the provided string." %} {% trans "Users or statuses that have already been reported (regardless of whether the report was resolved) will not be flagged." %}

+ +
{% if task %} -
+
{% trans "Schedule:" %}
@@ -51,7 +53,7 @@
-
+
{% csrf_token %} @@ -64,9 +66,29 @@
{% else %} +

{% trans "Schedule scan" %}

{% csrf_token %} - {{ task_form.as_p }} +
+ + {{ task_form.every }} +

+ {{ task_form.every.help_text }} +

+
+
+ +
+ {{ task_form.period }} +
+

+ {{ task_form.period.help_text }} +

+
{% endif %} From c7c90f9e9b43ad322b0e382da2fceb2d4790f92f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 1 Mar 2022 10:09:53 -0800 Subject: [PATCH 5/5] Removes test print statement --- bookwyrm/models/antispam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bookwyrm/models/antispam.py b/bookwyrm/models/antispam.py index bce02780..f506b6f1 100644 --- a/bookwyrm/models/antispam.py +++ b/bookwyrm/models/antispam.py @@ -54,7 +54,6 @@ class AutoMod(models.Model): @app.task(queue="low_priority") def automod_task(): """Create reports""" - print("TASK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") if not AutoMod.objects.exists(): return reporter = AutoMod.objects.first().created_by