diff --git a/.env.dev.example b/.env.dev.example index 22e12de1..b65c1b02 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -26,15 +26,15 @@ POSTGRES_HOST=db MAX_STREAM_LENGTH=200 REDIS_ACTIVITY_HOST=redis_activity REDIS_ACTIVITY_PORT=6379 -#REDIS_ACTIVITY_PASSWORD=redispassword345 +REDIS_ACTIVITY_PASSWORD=redispassword345 # Redis as celery broker REDIS_BROKER_PORT=6379 -#REDIS_BROKER_PASSWORD=redispassword123 +REDIS_BROKER_PASSWORD=redispassword123 FLOWER_PORT=8888 -#FLOWER_USER=mouse -#FLOWER_PASSWORD=changeme +FLOWER_USER=mouse +FLOWER_PASSWORD=changeme EMAIL_HOST=smtp.mailgun.org EMAIL_PORT=587 diff --git a/.github/workflows/django-tests.yml b/.github/workflows/django-tests.yml index 03875193..4968b40c 100644 --- a/.github/workflows/django-tests.yml +++ b/.github/workflows/django-tests.yml @@ -46,6 +46,8 @@ jobs: POSTGRES_HOST: 127.0.0.1 CELERY_BROKER: "" REDIS_BROKER_PORT: 6379 + REDIS_BROKER_PASSWORD: beep + USE_LOCAL_CACHE: true FLOWER_PORT: 8888 EMAIL_HOST: "smtp.mailgun.org" EMAIL_PORT: 587 diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 5cc11afd..5edac57d 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,6 +1,8 @@ """ database schema for info about authors """ import re from django.contrib.postgres.indexes import GinIndex +from django.core.cache import cache +from django.core.cache.utils import make_template_fragment_key from django.db import models from bookwyrm import activitypub @@ -34,6 +36,17 @@ class Author(BookDataModel): ) bio = fields.HtmlField(null=True, blank=True) + def save(self, *args, **kwargs): + """clear related template caches""" + # clear template caches + if self.id: + cache_keys = [ + make_template_fragment_key("titleby", [book]) + for book in self.book_set.values_list("id", flat=True) + ] + cache.delete_many(cache_keys) + return super().save(*args, **kwargs) + @property def isni_link(self): """generate the url from the isni id""" diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 0a551bf2..a9dd9508 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -3,6 +3,8 @@ import re from django.contrib.postgres.search import SearchVectorField from django.contrib.postgres.indexes import GinIndex +from django.core.cache import cache +from django.core.cache.utils import make_template_fragment_key from django.db import models, transaction from django.db.models import Prefetch from django.dispatch import receiver @@ -185,6 +187,11 @@ class Book(BookDataModel): """can't be abstract for query reasons, but you shouldn't USE it""" if not isinstance(self, Edition) and not isinstance(self, Work): raise ValueError("Books should be added as Editions or Works") + + # clear template caches + cache_key = make_template_fragment_key("titleby", [self.id]) + cache.delete(cache_key) + return super().save(*args, **kwargs) def get_remote_id(self): diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index fc7a9df8..03417454 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -1,5 +1,7 @@ """ defines relationships between users """ from django.apps import apps +from django.core.cache import cache +from django.core.cache.utils import make_template_fragment_key from django.db import models, transaction, IntegrityError from django.db.models import Q @@ -36,6 +38,20 @@ class UserRelationship(BookWyrmModel): """the remote user needs to recieve direct broadcasts""" return [u for u in [self.user_subject, self.user_object] if not u.local] + def save(self, *args, **kwargs): + """clear the template cache""" + # invalidate the template cache + cache_keys = [ + make_template_fragment_key( + "follow_button", [self.user_subject.id, self.user_object.id] + ), + make_template_fragment_key( + "follow_button", [self.user_object.id, self.user_subject.id] + ), + ] + cache.delete_many(cache_keys) + super().save(*args, **kwargs) + class Meta: """relationships should be unique""" diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index c7c0a425..ee138d97 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -82,6 +82,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): if not self.reply_parent: self.thread_id = self.id + super().save(broadcast=False, update_fields=["thread_id"]) def delete(self, *args, **kwargs): # pylint: disable=unused-argument diff --git a/bookwyrm/redis_store.py b/bookwyrm/redis_store.py index 78f373a2..595868ff 100644 --- a/bookwyrm/redis_store.py +++ b/bookwyrm/redis_store.py @@ -5,7 +5,10 @@ import redis from bookwyrm import settings r = redis.Redis( - host=settings.REDIS_ACTIVITY_HOST, port=settings.REDIS_ACTIVITY_PORT, db=0 + host=settings.REDIS_ACTIVITY_HOST, + port=settings.REDIS_ACTIVITY_PORT, + password=settings.REDIS_ACTIVITY_PASSWORD, + db=0, ) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index f2068a16..57717fb9 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -119,6 +119,22 @@ STREAMS = [ {"key": "books", "name": _("Books Timeline"), "shortname": _("Books")}, ] +# Redis cache backend +if not env("USE_LOCAL_CACHE", False): + # pylint: disable=line-too-long + CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/0", + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + }, + } + } + + SESSION_ENGINE = "django.contrib.sessions.backends.cache" + SESSION_CACHE_ALIAS = "default" + # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index 5769ca8b..0c1d59b5 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -29,6 +29,7 @@ class SuggestedUsers(RedisStore): def get_counts_from_rank(self, rank): # pylint: disable=no-self-use """calculate mutuals count and shared books count from rank""" + # pylint: disable=c-extension-no-member return { "mutuals": math.floor(rank), # "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1, @@ -112,16 +113,17 @@ def get_annotated_users(viewer, *args, **kwargs): ), distinct=True, ), - # shared_books=Count( - # "shelfbook", - # filter=Q( - # ~Q(id=viewer.id), - # shelfbook__book__parent_work__in=[ - # s.book.parent_work for s in viewer.shelfbook_set.all() - # ], - # ), - # distinct=True, - # ), + # pylint: disable=line-too-long + # shared_books=Count( + # "shelfbook", + # filter=Q( + # ~Q(id=viewer.id), + # shelfbook__book__parent_work__in=[ + # s.book.parent_work for s in viewer.shelfbook_set.all() + # ], + # ), + # distinct=True, + # ), ) ) diff --git a/bookwyrm/templates/feed/layout.html b/bookwyrm/templates/feed/layout.html index 6e7ec849..5697f266 100644 --- a/bookwyrm/templates/feed/layout.html +++ b/bookwyrm/templates/feed/layout.html @@ -8,82 +8,7 @@
{% if user.is_authenticated %}
-
-

{% trans "Your Books" %}

- {% if not suggested_books %} -

{% trans "There are no books here right now! Try searching for a book to get started" %}

- {% else %} - {% with active_book=request.GET.book %} -
-
-
    - {% for shelf in suggested_books %} - {% if shelf.books %} - {% with shelf_counter=forloop.counter %} -
  • -

    - {% if shelf.identifier == 'to-read' %}{% trans "To Read" %} - {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %} - {% elif shelf.identifier == 'read' %}{% trans "Read" %} - {% else %}{{ shelf.name }}{% endif %} -

    -
    - -
    -
    • - {% endwith %} - {% endif %} - {% endfor %} -
-
- {% for shelf in suggested_books %} - {% with shelf_counter=forloop.counter %} - {% for book in shelf.books %} -
- -
-
-
-

{% include 'snippets/book_titleby.html' with book=book %}

- {% include 'snippets/shelve_button/shelve_button.html' with book=book %} -
-
-
- {% trans "Close" as button_text %} - {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %} -
-
-
- {% include 'snippets/create_status.html' with book=book %} -
-
- {% endfor %} - {% endwith %} - {% endfor %} -
- {% endwith %} - {% endif %} -
- + {% include "feed/suggested_books.html" %} {% if goal %}
diff --git a/bookwyrm/templates/feed/suggested_books.html b/bookwyrm/templates/feed/suggested_books.html new file mode 100644 index 00000000..b2f1b5d2 --- /dev/null +++ b/bookwyrm/templates/feed/suggested_books.html @@ -0,0 +1,83 @@ +{% load i18n %} +{% load cache %} +{% load bookwyrm_tags %} + +{# 6 month cache #} +{% cache 15552000 suggested_books request.user.id %} +{% suggested_books as suggested_books %} +
+

{% trans "Your Books" %}

+ {% if not suggested_books %} +

{% trans "There are no books here right now! Try searching for a book to get started" %}

+ {% else %} + {% with active_book=request.GET.book %} +
+
+
    + {% for shelf in suggested_books %} + {% if shelf.books %} + {% with shelf_counter=forloop.counter %} +
  • +

    + {% if shelf.identifier == 'to-read' %}{% trans "To Read" %} + {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %} + {% elif shelf.identifier == 'read' %}{% trans "Read" %} + {% else %}{{ shelf.name }}{% endif %} +

    +
    + +
    +
    • + {% endwith %} + {% endif %} + {% endfor %} +
+
+ {% for shelf in suggested_books %} + {% with shelf_counter=forloop.counter %} + {% for book in shelf.books %} +
+ +
+
+
+

{% include 'snippets/book_titleby.html' with book=book %}

+ {% include 'snippets/shelve_button/shelve_button.html' with book=book %} +
+
+
+ {% trans "Close" as button_text %} + {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %} +
+
+
+ {% include 'snippets/create_status.html' with book=book %} +
+
+ {% endfor %} + {% endwith %} + {% endfor %} +
+ {% endwith %} + {% endif %} +
+{% endcache %} diff --git a/bookwyrm/templates/snippets/book_titleby.html b/bookwyrm/templates/snippets/book_titleby.html index 6dbaeb26..5e35e36a 100644 --- a/bookwyrm/templates/snippets/book_titleby.html +++ b/bookwyrm/templates/snippets/book_titleby.html @@ -1,7 +1,11 @@ {% load i18n %} {% load utilities %} +{% load cache %} {% spaceless %} +{# 6 month cache #} +{% cache 15552000 titleby book.id %} + {% if book.authors.exists %} {% blocktrans trimmed with path=book.local_path title=book|book_title %} {{ title }} by @@ -10,4 +14,6 @@ {% else %} {{ book|book_title }} {% endif %} + +{% endcache %} {% endspaceless %} diff --git a/bookwyrm/templates/snippets/follow_button.html b/bookwyrm/templates/snippets/follow_button.html index 530322b9..a5c40b24 100644 --- a/bookwyrm/templates/snippets/follow_button.html +++ b/bookwyrm/templates/snippets/follow_button.html @@ -1,4 +1,8 @@ {% load i18n %} +{% load cache %} + +{# 6 month cache #} +{% cache 15552000 follow_button request.user.id user.id %} {% if request.user == user or not request.user.is_authenticated %} {% elif user in request.user.blocks.all %} {% include 'snippets/block_button.html' with blocks=True %} @@ -42,3 +46,4 @@ {% endif %}
{% endif %} +{% endcache %} diff --git a/bookwyrm/templates/snippets/shelve_button/shelve_button.html b/bookwyrm/templates/snippets/shelve_button/shelve_button.html index 38f6be38..0ffc708d 100644 --- a/bookwyrm/templates/snippets/shelve_button/shelve_button.html +++ b/bookwyrm/templates/snippets/shelve_button/shelve_button.html @@ -1,7 +1,10 @@ {% load bookwyrm_tags %} {% load utilities %} +{% load cache %} {% if request.user.is_authenticated %} +{# 6 month cache #} +{% cache 15552000 shelve_button request.user.id book.id %} {% with book.id|uuid as uuid %} {% active_shelf book as active_shelf %} @@ -32,4 +35,5 @@ {% include 'snippets/reading_modals/progress_update_modal.html' with book=active_shelf.book id=modal_id readthrough=readthrough class="" %} {% endwith %} +{% endcache %} {% endif %} diff --git a/bookwyrm/templates/snippets/status/headers/generatednote.html b/bookwyrm/templates/snippets/status/headers/generatednote.html index cc684a5f..7fc635ab 100644 --- a/bookwyrm/templates/snippets/status/headers/generatednote.html +++ b/bookwyrm/templates/snippets/status/headers/generatednote.html @@ -1,3 +1,7 @@ +{% load cache %} + +{# Three day cache #} +{% cache 259200 generated_note_header status.id %} {% if status.content == 'wants to read' %} {% include 'snippets/status/headers/to_read.html' with book=status.mention_books.first %} {% elif status.content == 'finished reading' %} @@ -7,3 +11,4 @@ {% else %} {{ status.content }} {% endif %} +{% endcache %} diff --git a/bookwyrm/templates/snippets/status/layout.html b/bookwyrm/templates/snippets/status/layout.html index 93620a08..174c379f 100644 --- a/bookwyrm/templates/snippets/status/layout.html +++ b/bookwyrm/templates/snippets/status/layout.html @@ -1,6 +1,7 @@ {% extends 'components/card.html' %} {% load i18n %} {% load utilities %} +{% load cache %} {% block card-header %}
- {% trans "Reply" as button_text %} - {% include 'snippets/toggle/toggle_button.html' with controls_text="show_comment" controls_uid=status.id text=button_text icon_with_text="comment" class="is-small is-light is-transparent toggle-button" focus="id_content_reply" %} -
-
- {% include 'snippets/boost_button.html' with status=status %} -
-
- {% include 'snippets/fav_button.html' with status=status %} -
-{% if not moderation_mode %} -
- {% include 'snippets/status/status_options.html' with class="is-small is-light is-transparent" right=True %} -
-{% endif %} + {% cache 259200 interact request.user.id status.id %} +
+ {% trans "Reply" as button_text %} + {% include 'snippets/toggle/toggle_button.html' with controls_text="show_comment" controls_uid=status.id text=button_text icon_with_text="comment" class="is-small is-light is-transparent toggle-button" focus="id_content_reply" %} +
+
+ {% include 'snippets/boost_button.html' with status=status %} +
+
+ {% include 'snippets/fav_button.html' with status=status %} +
+ {% if not moderation_mode %} +
+ {% include 'snippets/status/status_options.html' with class="is-small is-light is-transparent" right=True %} +
+ {% endif %} + {% endcache %} {% else %} -
- - - {% trans "Reply" %} - - - {% trans "Boost status" %} - +
+ + + {% trans "Reply" %} + - - {% trans "Like status" %} - - -
+ + {% trans "Boost status" %} + + + + {% trans "Like status" %} + + +
{% endif %} {% endblock %} diff --git a/bookwyrm/templatetags/bookwyrm_tags.py b/bookwyrm/templatetags/bookwyrm_tags.py index 9d84d1ff..c15df090 100644 --- a/bookwyrm/templatetags/bookwyrm_tags.py +++ b/bookwyrm/templatetags/bookwyrm_tags.py @@ -3,6 +3,7 @@ from django import template from django.db.models import Avg from bookwyrm import models +from bookwyrm.views.feed import get_suggested_books register = template.Library() @@ -115,3 +116,11 @@ def mutuals_count(context, user): if not viewer.is_authenticated: return None return user.followers.filter(followers=viewer).count() + + +@register.simple_tag(takes_context=True) +def suggested_books(context): + """get books for suggested books panel""" + # this happens here instead of in the view so that the template snippet can + # be cached in the template + return get_suggested_books(context["request"].user) diff --git a/bookwyrm/views/feed.py b/bookwyrm/views/feed.py index f6f9fb37..e060ae61 100644 --- a/bookwyrm/views/feed.py +++ b/bookwyrm/views/feed.py @@ -223,7 +223,6 @@ def feed_page_data(user): goal = models.AnnualGoal.objects.filter(user=user, year=timezone.now().year).first() return { - "suggested_books": get_suggested_books(user), "goal": goal, "goal_form": forms.GoalForm(), } diff --git a/bookwyrm/views/interaction.py b/bookwyrm/views/interaction.py index 910360d7..9e897beb 100644 --- a/bookwyrm/views/interaction.py +++ b/bookwyrm/views/interaction.py @@ -1,6 +1,8 @@ """ boosts and favs """ -from django.db import IntegrityError from django.contrib.auth.decorators import login_required +from django.core.cache import cache +from django.core.cache.utils import make_template_fragment_key +from django.db import IntegrityError from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import redirect from django.utils.decorators import method_decorator @@ -17,6 +19,7 @@ class Favorite(View): def post(self, request, status_id): """create a like""" + clear_cache(request.user.id, status_id) status = models.Status.objects.get(id=status_id) try: models.Favorite.objects.create(status=status, user=request.user) @@ -43,6 +46,7 @@ class Unfavorite(View): return HttpResponseNotFound() favorite.delete() + clear_cache(request.user.id, status_id) if is_api_request(request): return HttpResponse() return redirect(request.headers.get("Referer", "/")) @@ -70,6 +74,7 @@ class Boost(View): privacy=status.privacy, user=request.user, ) + clear_cache(request.user.id, status_id) if is_api_request(request): return HttpResponse() return redirect(request.headers.get("Referer", "/")) @@ -87,6 +92,13 @@ class Unboost(View): ).first() boost.delete() + clear_cache(request.user.id, status_id) if is_api_request(request): return HttpResponse() return redirect(request.headers.get("Referer", "/")) + + +def clear_cache(user_id, status_id): + """clear template cache""" + cache_key = make_template_fragment_key("interact", [user_id, status_id]) + cache.delete(cache_key) diff --git a/bookwyrm/views/landing/landing.py b/bookwyrm/views/landing/landing.py index c8bba066..74b5ee51 100644 --- a/bookwyrm/views/landing/landing.py +++ b/bookwyrm/views/landing/landing.py @@ -1,6 +1,8 @@ """ non-interactive pages """ from django.template.response import TemplateResponse from django.views import View +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page from bookwyrm import forms from bookwyrm.views import helpers @@ -31,6 +33,7 @@ class Home(View): class Landing(View): """preview of recently reviewed books""" + @method_decorator(cache_page(60 * 60), name="dispatch") def get(self, request): """tiled book activity page""" data = { diff --git a/bookwyrm/views/reading.py b/bookwyrm/views/reading.py index 35847558..c7eda10e 100644 --- a/bookwyrm/views/reading.py +++ b/bookwyrm/views/reading.py @@ -1,5 +1,7 @@ """ the good stuff! the books! """ from django.contrib.auth.decorators import login_required +from django.core.cache import cache +from django.core.cache.utils import make_template_fragment_key from django.db import transaction from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import get_object_or_404, redirect @@ -44,6 +46,13 @@ class ReadingStatus(View): if not identifier: return HttpResponseBadRequest() + # invalidate the template cache + cache_keys = [ + make_template_fragment_key("shelve_button", [request.user.id, book_id]), + make_template_fragment_key("suggested_books", [request.user.id]), + ] + cache.delete_many(cache_keys) + desired_shelf = get_object_or_404( models.Shelf, identifier=identifier, user=request.user ) diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 05ffdcab..24729345 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -3,11 +3,15 @@ # pylint: disable=unused-wildcard-import from bookwyrm.settings import * -CELERY_BROKER_URL = "redis://:{}@redis_broker:{}/0".format( - requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") +REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None)) +REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker") +REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379) + +CELERY_BROKER_URL = ( + f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0" ) -CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( - requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") +CELERY_RESULT_BACKEND = ( + f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0" ) CELERY_DEFAULT_QUEUE = "low_priority" diff --git a/docker-compose.yml b/docker-compose.yml index afa40b05..25a397ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,16 +38,17 @@ services: - 8000:8000 redis_activity: image: redis - command: ["redis-server", "--appendonly", "yes"] + command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT} env_file: .env networks: - main restart: on-failure volumes: + - ./redis.conf:/etc/redis/redis.conf - redis_activity_data:/data redis_broker: image: redis - command: ["redis-server", "--appendonly", "yes"] + command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT} env_file: .env ports: - 6379:6379 @@ -55,6 +56,7 @@ services: - main restart: on-failure volumes: + - ./redis.conf:/etc/redis/redis.conf - redis_broker_data:/data celery_worker: env_file: .env diff --git a/redis.conf b/redis.conf new file mode 100644 index 00000000..2a417579 --- /dev/null +++ b/redis.conf @@ -0,0 +1,9 @@ +bind 127.0.0.1 ::1 +protected-mode yes +port 6379 + +rename-command FLUSHDB "" +rename-command FLUSHALL "" +rename-command DEBUG "" +rename-command CONFIG "" +rename-command SHUTDOWN "" diff --git a/requirements.txt b/requirements.txt index 9c22a6ed..ec97e389 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ django-rename-app==0.1.2 pytz>=2021.1 boto3==1.17.88 django-storages==1.11.1 +django-redis==5.2.0 # Dev black==21.4b0