From 1d525d49cb5a4cb3b258e7fea7c1e97e47fff46f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 2 Apr 2021 10:44:30 -0700 Subject: [PATCH 1/6] Use sorted set for activitystreams --- bookwyrm/activitystreams.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index d811aa49..a7fa1637 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -23,15 +23,23 @@ class ActivityStream(ABC): """ the redis key for this user's unread count for this stream """ return "{}-unread".format(self.stream_id(user)) + def get_value(self, status): # pylint: disable=no-self-use + """ the status id and the rank (ie, published date) """ + return {status.id: status.published_date.timestamp()} + def add_status(self, status): """ add a status to users' feeds """ + value = self.get_value(status) # we want to do this as a bulk operation, hence "pipeline" pipeline = r.pipeline() for user in self.stream_users(status): # add the status to the feed - pipeline.lpush(self.stream_id(user), status.id) - pipeline.ltrim(self.stream_id(user), 0, settings.MAX_STREAM_LENGTH) - + pipeline.zadd(self.stream_id(user), value) + pipeline.zremrangebyrank( + self.stream_id(user), + settings.MAX_STREAM_LENGTH, + -1 + ) # add to the unread status count pipeline.incr(self.unread_id(user)) # and go! @@ -48,7 +56,12 @@ class ActivityStream(ABC): """ add a user's statuses to another user's feed """ pipeline = r.pipeline() for status in user.status_set.all()[: settings.MAX_STREAM_LENGTH]: - pipeline.lpush(self.stream_id(viewer), status.id) + pipeline.zadd(self.stream_id(viewer), self.get_value(status)) + pipeline.zremrangebyrank( + self.stream_id(user), + settings.MAX_STREAM_LENGTH, + -1 + ) pipeline.execute() def remove_user_statuses(self, viewer, user): @@ -63,7 +76,7 @@ class ActivityStream(ABC): # clear unreads for this feed r.set(self.unread_id(user), 0) - statuses = r.lrange(self.stream_id(user), 0, -1) + statuses = r.zrevrange(self.stream_id(user), 0, -1) return ( models.Status.objects.select_subclasses() .filter(id__in=statuses) @@ -81,7 +94,13 @@ class ActivityStream(ABC): stream_id = self.stream_id(user) for status in statuses.all()[: settings.MAX_STREAM_LENGTH]: - pipeline.lpush(stream_id, status.id) + pipeline.zadd(stream_id, self.get_value(status)) + + pipeline.zremrangebyrank( + stream_id, + settings.MAX_STREAM_LENGTH, + -1 + ) pipeline.execute() def stream_users(self, status): # pylint: disable=no-self-use From c0f7cbc5f0f07c26ec7367b190089e8114fc5fed Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 2 Apr 2021 11:20:56 -0700 Subject: [PATCH 2/6] Consistent wording in bw-dev command --- bw-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bw-dev b/bw-dev index b6406f2c..b9c4b2a1 100755 --- a/bw-dev +++ b/bw-dev @@ -109,7 +109,7 @@ case "$CMD" in black) makeitblack ;; - populate_feeds) + populate_streams) execweb python manage.py populate_streams ;; *) From 880323e838988ee7c760303d1958938d297a6549 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 2 Apr 2021 17:18:04 -0700 Subject: [PATCH 3/6] Python format --- bookwyrm/activitystreams.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index a7fa1637..0be90fc6 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -36,9 +36,7 @@ class ActivityStream(ABC): # add the status to the feed pipeline.zadd(self.stream_id(user), value) pipeline.zremrangebyrank( - self.stream_id(user), - settings.MAX_STREAM_LENGTH, - -1 + self.stream_id(user), settings.MAX_STREAM_LENGTH, -1 ) # add to the unread status count pipeline.incr(self.unread_id(user)) @@ -57,11 +55,7 @@ class ActivityStream(ABC): pipeline = r.pipeline() for status in user.status_set.all()[: settings.MAX_STREAM_LENGTH]: pipeline.zadd(self.stream_id(viewer), self.get_value(status)) - pipeline.zremrangebyrank( - self.stream_id(user), - settings.MAX_STREAM_LENGTH, - -1 - ) + pipeline.zremrangebyrank(self.stream_id(user), settings.MAX_STREAM_LENGTH, -1) pipeline.execute() def remove_user_statuses(self, viewer, user): @@ -96,11 +90,7 @@ class ActivityStream(ABC): for status in statuses.all()[: settings.MAX_STREAM_LENGTH]: pipeline.zadd(stream_id, self.get_value(status)) - pipeline.zremrangebyrank( - stream_id, - settings.MAX_STREAM_LENGTH, - -1 - ) + pipeline.zremrangebyrank(stream_id, settings.MAX_STREAM_LENGTH, -1) pipeline.execute() def stream_users(self, status): # pylint: disable=no-self-use From a91352499102df487cc9ed66425d35bf7534032d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Apr 2021 10:56:53 -0700 Subject: [PATCH 4/6] Fixes redis call so tests don't fail --- bookwyrm/activitystreams.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 0be90fc6..419e20c0 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -90,7 +90,9 @@ class ActivityStream(ABC): for status in statuses.all()[: settings.MAX_STREAM_LENGTH]: pipeline.zadd(stream_id, self.get_value(status)) - pipeline.zremrangebyrank(stream_id, settings.MAX_STREAM_LENGTH, -1) + # only trim the stream if statuses were added + if statuses.exists(): + pipeline.zremrangebyrank(stream_id, settings.MAX_STREAM_LENGTH, -1) pipeline.execute() def stream_users(self, status): # pylint: disable=no-self-use @@ -280,7 +282,7 @@ def add_statuses_on_unblock(sender, instance, *args, **kwargs): @receiver(signals.post_save, sender=models.User) # pylint: disable=unused-argument -def populate_feed_on_account_create(sender, instance, created, *args, **kwargs): +def populate_streams_on_account_create(sender, instance, created, *args, **kwargs): """ build a user's feeds when they join """ if not created or not instance.local: return From 44e87c01ab6c8daebfd7d8e929d8250c47165bc9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Apr 2021 11:07:08 -0700 Subject: [PATCH 5/6] Another place where redis may be called unnecessarily --- bookwyrm/activitystreams.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 419e20c0..689d396b 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -53,9 +53,11 @@ class ActivityStream(ABC): def add_user_statuses(self, viewer, user): """ add a user's statuses to another user's feed """ pipeline = r.pipeline() - for status in user.status_set.all()[: settings.MAX_STREAM_LENGTH]: + statuses = user.status_set.all()[: settings.MAX_STREAM_LENGTH] + for status in statuses: pipeline.zadd(self.stream_id(viewer), self.get_value(status)) - pipeline.zremrangebyrank(self.stream_id(user), settings.MAX_STREAM_LENGTH, -1) + if statuses: + pipeline.zremrangebyrank(self.stream_id(user), settings.MAX_STREAM_LENGTH, -1) pipeline.execute() def remove_user_statuses(self, viewer, user): From b9514ec81fa20c35b20417a4b5e0f20d391f511f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 3 Apr 2021 11:11:29 -0700 Subject: [PATCH 6/6] Python formatting --- bookwyrm/activitystreams.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 689d396b..d9236ec9 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -57,7 +57,9 @@ class ActivityStream(ABC): for status in statuses: pipeline.zadd(self.stream_id(viewer), self.get_value(status)) if statuses: - pipeline.zremrangebyrank(self.stream_id(user), settings.MAX_STREAM_LENGTH, -1) + pipeline.zremrangebyrank( + self.stream_id(user), settings.MAX_STREAM_LENGTH, -1 + ) pipeline.execute() def remove_user_statuses(self, viewer, user):