From 0736c7e160fe544db8f4be07840de2f197af4b68 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 10 Nov 2021 11:10:09 -0800 Subject: [PATCH 01/73] Uses general names for fields in parsed csvs --- bookwyrm/models/import_job.py | 44 ++++++++++++++--------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 22253fef..949e3edb 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -10,14 +10,6 @@ from bookwyrm.models import ReadThrough, User, Book from .fields import PrivacyLevels -# Mapping goodreads -> bookwyrm shelf titles. -GOODREADS_SHELVES = { - "read": "read", - "currently-reading": "reading", - "to-read": "to-read", -} - - def unquote_string(text): """resolve csv quote weirdness""" match = re.match(r'="([^"]*)"', text) @@ -106,56 +98,56 @@ class ImportItem(models.Model): @property def title(self): """get the book title""" - return self.data["Title"] + return self.data["title"] @property def author(self): - """get the book title""" - return self.data["Author"] + """get the book's authors""" + return self.data["authors"] @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.data["ISBN13"]) + return unquote_string(self.data["isbn_13"]) @property def shelf(self): """the goodreads shelf field""" - if self.data["Exclusive Shelf"]: - return GOODREADS_SHELVES.get(self.data["Exclusive Shelf"]) - return None + return self.data.get("shelf") @property def review(self): """a user-written review, to be imported with the book data""" - return self.data["My Review"] + return self.data["review_body"] @property def rating(self): """x/5 star rating for a book""" - if self.data.get("My Rating", None): - return int(self.data["My Rating"]) + if self.data.get("rating"): + return float(self.data["rating"]) return None @property def date_added(self): """when the book was added to this dataset""" - if self.data["Date Added"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Added"])) + if self.data.get("date_added"): + return timezone.make_aware(dateutil.parser.parse(self.data["date_added"])) return None @property def date_started(self): """when the book was started""" - if "Date Started" in self.data and self.data["Date Started"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Started"])) + if self.data.get("date_started"): + return timezone.make_aware(dateutil.parser.parse(self.data["date_started"])) return None @property def date_read(self): """the date a book was completed""" - if self.data["Date Read"]: - return timezone.make_aware(dateutil.parser.parse(self.data["Date Read"])) + if self.data.get("date_finished"): + return timezone.make_aware( + dateutil.parser.parse(self.data["date_finished"]) + ) return None @property @@ -185,8 +177,8 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r}Item {!r}>".format(self.data["import_source"], self.data["Title"]) + return "<{!r}Item {!r}>".format(self.data["import_source"], self.data["title"]) def __str__(self): # pylint: disable=consider-using-f-string - return "{} by {}".format(self.data["Title"], self.data["Author"]) + return "{} by {}".format(self.data["title"], self.data["authors"]) From 4ccd9fc633321970d2ecaf64631bf19282852e69 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 10 Nov 2021 16:49:54 -0800 Subject: [PATCH 02/73] Use generalized mappings to handle import --- bookwyrm/importers/goodreads_import.py | 7 -- bookwyrm/importers/importer.py | 87 ++++++++++++--- bookwyrm/importers/librarything_import.py | 34 +----- bookwyrm/importers/storygraph_import.py | 27 +---- bookwyrm/models/import_job.py | 40 ++++--- bookwyrm/tests/data/generic.csv | 8 +- bookwyrm/tests/importers/test_importer.py | 127 +++++++++------------- 7 files changed, 152 insertions(+), 178 deletions(-) diff --git a/bookwyrm/importers/goodreads_import.py b/bookwyrm/importers/goodreads_import.py index c62e6582..c0dc0ea2 100644 --- a/bookwyrm/importers/goodreads_import.py +++ b/bookwyrm/importers/goodreads_import.py @@ -7,10 +7,3 @@ class GoodreadsImporter(Importer): For a more complete example of overriding see librarything_import.py""" service = "Goodreads" - - def parse_fields(self, entry): - """handle the specific fields in goodreads csvs""" - entry.update({"import_source": self.service}) - # add missing 'Date Started' field - entry.update({"Date Started": None}) - return entry diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index a5243cd3..b0458bab 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -1,5 +1,6 @@ """ handle reading a csv from an external service, defaults are from Goodreads """ import csv +from dataclasses import dataclass import logging from django.utils import timezone @@ -18,30 +19,59 @@ class Importer: service = "Unknown" delimiter = "," encoding = "UTF-8" - mandatory_fields = ["Title", "Author"] + + # these are from Goodreads + row_mappings_guesses = { + "id": ["id", "book id"], + "title": ["title"], + "authors": ["author", "authors", "primary author"], + "isbn_13": ["isbn13", "isbn"], + "isbn_10": ["isbn10", "isbn"], + "shelf": ["shelf", "exclusive shelf", "read status"], + "review_name": [], + "review_body": ["my review"], + "rating": ["my rating", "rating", "star rating"], + "date_added": ["date added", "entry date", "added"], + "date_started": ["date started", "started"], + "date_finished": ["date finished", "last date read", "date read", "finished"], + } def create_job(self, user, csv_file, include_reviews, privacy): """check over a csv and creates a database entry for the job""" + csv_reader = csv.DictReader(csv_file, delimiter=self.delimiter) + rows = enumerate(list(csv_reader)) job = ImportJob.objects.create( - user=user, include_reviews=include_reviews, privacy=privacy + user=user, + include_reviews=include_reviews, + privacy=privacy, + mappings=self.create_row_mappings(csv_reader.fieldnames), ) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.delimiter)) - ): - if not all(x in entry for x in self.mandatory_fields): - raise ValueError("Author and title must be in data.") - entry = self.parse_fields(entry) - self.save_item(job, index, entry) + + for index, entry in rows: + print(index, entry) + self.create_item(job, index, entry) return job - def save_item(self, job, index, data): # pylint: disable=no-self-use - """creates and saves an import item""" - ImportItem(job=job, index=index, data=data).save() + def create_row_mappings(self, headers): + """guess what the headers mean""" + mappings = {} + for (key, guesses) in self.row_mappings_guesses.items(): + value = [h for h in headers if h.lower() in guesses] + value = value[0] if len(value) else None + if value: + headers.remove(value) + mappings[key] = value + return mappings - def parse_fields(self, entry): - """updates csv data with additional info""" - entry.update({"import_source": self.service}) - return entry + def create_item(self, job, index, data): + """creates and saves an import item""" + print(data) + normalized = self.normalize_row(data, job.mappings) + ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() + + def normalize_row(self, entry, mappings): # pylint: disable=no-self-use + """use the dataclass to create the formatted row of data""" + return {k: entry.get(v) for k, v in mappings.items()} def create_retry_job(self, user, original_job, items): """retry items that didn't import""" @@ -49,10 +79,13 @@ class Importer: user=user, include_reviews=original_job.include_reviews, privacy=original_job.privacy, + # TODO: allow users to adjust mappings + mappings=original_job.mappings, retry=True, ) for item in items: - self.save_item(job, item.index, item.data) + # this will re-normalize the raw data + self.create_item(job, item.index, item.data) return job def start_import(self, job): @@ -156,3 +189,23 @@ def handle_imported_book(source, user, item, include_reviews, privacy): ) # only broadcast this review to other bookwyrm instances review.save(software="bookwyrm") + + +@dataclass +class ImportEntry: + """data extracted from a line in a csv""" + + title: str + authors: str = None + isbn_13: str = None + isbn_10: str = None + shelf: str = None + review_name: str = None + review_rating: float = None + review_body: str = None + review_cw: str = None + rating: float = None + date_added: str = None + date_started: str = None + date_finished: str = None + import_source: str = "Unknown" diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index b3175a82..3d42e539 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -1,7 +1,4 @@ -""" handle reading a csv from librarything """ -import re -import math - +""" handle reading a tsv from librarything """ from . import Importer @@ -11,32 +8,3 @@ class LibrarythingImporter(Importer): service = "LibraryThing" delimiter = "\t" encoding = "ISO-8859-1" - # mandatory_fields : fields matching the book title and author - mandatory_fields = ["Title", "Primary Author"] - - def parse_fields(self, entry): - """custom parsing for librarything""" - data = {} - data["import_source"] = self.service - data["Book Id"] = entry["Book Id"] - data["Title"] = entry["Title"] - data["Author"] = entry["Primary Author"] - data["ISBN13"] = entry["ISBN"] - data["My Review"] = entry["Review"] - if entry["Rating"]: - data["My Rating"] = math.ceil(float(entry["Rating"])) - else: - data["My Rating"] = "" - data["Date Added"] = re.sub(r"\[|\]", "", entry["Entry Date"]) - data["Date Started"] = re.sub(r"\[|\]", "", entry["Date Started"]) - data["Date Read"] = re.sub(r"\[|\]", "", entry["Date Read"]) - - data["Exclusive Shelf"] = None - if data["Date Read"]: - data["Exclusive Shelf"] = "read" - elif data["Date Started"]: - data["Exclusive Shelf"] = "reading" - else: - data["Exclusive Shelf"] = "to-read" - - return data diff --git a/bookwyrm/importers/storygraph_import.py b/bookwyrm/importers/storygraph_import.py index 1333b8b9..9368115d 100644 --- a/bookwyrm/importers/storygraph_import.py +++ b/bookwyrm/importers/storygraph_import.py @@ -1,6 +1,4 @@ -""" handle reading a csv from librarything """ -import re - +""" handle reading a csv from storygraph""" from . import Importer @@ -8,26 +6,3 @@ class StorygraphImporter(Importer): """csv downloads from librarything""" service = "Storygraph" - # mandatory_fields : fields matching the book title and author - mandatory_fields = ["Title"] - - def parse_fields(self, entry): - """custom parsing for storygraph""" - data = {} - data["import_source"] = self.service - data["Title"] = entry["Title"] - data["Author"] = entry["Authors"] if "Authors" in entry else entry["Author"] - data["ISBN13"] = entry["ISBN"] - data["My Review"] = entry["Review"] - if entry["Star Rating"]: - data["My Rating"] = float(entry["Star Rating"]) - else: - data["My Rating"] = "" - - data["Date Added"] = re.sub(r"[/]", "-", entry["Date Added"]) - data["Date Read"] = re.sub(r"[/]", "-", entry["Last Date Read"]) - - data["Exclusive Shelf"] = ( - {"read": "read", "currently-reading": "reading", "to-read": "to-read"} - ).get(entry["Read Status"], None) - return data diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 949e3edb..6bca57f8 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -35,6 +35,7 @@ class ImportJob(models.Model): created_date = models.DateTimeField(default=timezone.now) task_id = models.CharField(max_length=100, null=True) include_reviews = models.BooleanField(default=True) + mappings = models.JSONField() complete = models.BooleanField(default=False) privacy = models.CharField( max_length=255, default="public", choices=PrivacyLevels.choices @@ -48,6 +49,7 @@ class ImportItem(models.Model): job = models.ForeignKey(ImportJob, on_delete=models.CASCADE, related_name="items") index = models.IntegerField() data = models.JSONField() + normalized_data = models.JSONField() book = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) book_guess = models.ForeignKey( Book, @@ -98,55 +100,59 @@ class ImportItem(models.Model): @property def title(self): """get the book title""" - return self.data["title"] + return self.normalized_data["title"] @property def author(self): """get the book's authors""" - return self.data["authors"] + return self.normalized_data["authors"] @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.data["isbn_13"]) + return unquote_string(self.normalized_data["isbn_13"]) @property def shelf(self): """the goodreads shelf field""" - return self.data.get("shelf") + return self.normalized_data.get("shelf") @property def review(self): """a user-written review, to be imported with the book data""" - return self.data["review_body"] + return self.normalized_data["review_body"] @property def rating(self): """x/5 star rating for a book""" - if self.data.get("rating"): - return float(self.data["rating"]) + if self.normalized_data.get("rating"): + return float(self.normalized_data["rating"]) return None @property def date_added(self): """when the book was added to this dataset""" - if self.data.get("date_added"): - return timezone.make_aware(dateutil.parser.parse(self.data["date_added"])) + if self.normalized_data.get("date_added"): + return timezone.make_aware( + dateutil.parser.parse(self.normalized_data["date_added"]) + ) return None @property def date_started(self): """when the book was started""" - if self.data.get("date_started"): - return timezone.make_aware(dateutil.parser.parse(self.data["date_started"])) + if self.normalized_data.get("date_started"): + return timezone.make_aware( + dateutil.parser.parse(self.normalized_data["date_started"]) + ) return None @property def date_read(self): """the date a book was completed""" - if self.data.get("date_finished"): + if self.normalized_data.get("date_finished"): return timezone.make_aware( - dateutil.parser.parse(self.data["date_finished"]) + dateutil.parser.parse(self.normalized_data["date_finished"]) ) return None @@ -177,8 +183,12 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r}Item {!r}>".format(self.data["import_source"], self.data["title"]) + return "<{!r}Item {!r}>".format( + self.normalized_data["import_source"], self.normalized_data["title"] + ) def __str__(self): # pylint: disable=consider-using-f-string - return "{} by {}".format(self.data["title"], self.data["authors"]) + return "{} by {}".format( + self.normalized_data["title"], self.normalized_data["authors"] + ) diff --git a/bookwyrm/tests/data/generic.csv b/bookwyrm/tests/data/generic.csv index a081a642..9c5b6f02 100644 --- a/bookwyrm/tests/data/generic.csv +++ b/bookwyrm/tests/data/generic.csv @@ -1,5 +1,5 @@ -id,title,author,ISBN,rating,shelf,review,added -38,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"9781250313195",,read,,2021-11-10 -48,Harrow the Ninth (The Locked Tomb #2),Tamsyn Muir,,3,read,,2021-11-10 +id,title,author,ISBN,rating,shelf,review,added,finished +38,Gideon the Ninth,Tamsyn Muir,"9781250313195",,read,,2021-11-10,2021-11-11 +48,Harrow the Ninth,Tamsyn Muir,,3,read,,2021-11-10 23,Subcutanean,Aaron A. Reed,,,read,,2021-11-10 -10,Patisserie at Home,Mélanie Dupuis,"9780062445315",2,read,"mixed feelings",2021-11-10 +10,Patisserie at Home,Mélanie Dupuis,"9780062445315",2,read,"mixed feelings",2021-11-10,2021-11-11 diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index f5d9af30..b2f0284d 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -1,6 +1,5 @@ """ testing import """ from collections import namedtuple -import csv import pathlib from unittest.mock import patch import datetime @@ -29,26 +28,7 @@ class GenericImporter(TestCase): def setUp(self): """use a test csv""" - class TestImporter(Importer): - """basic importer""" - - mandatory_fields = ["title", "author"] - - def parse_fields(self, entry): - return { - "id": entry["id"], - "Title": entry["title"], - "Author": entry["author"], - "ISBN13": entry["ISBN"], - "Star Rating": entry["rating"], - "My Rating": entry["rating"], - "My Review": entry["review"], - "Exclusive Shelf": entry["shelf"], - "Date Added": entry["added"], - "Date Read": None, - } - - self.importer = TestImporter() + self.importer = Importer() datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") self.csv = open(datafile, "r", encoding=self.importer.encoding) with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( @@ -77,13 +57,24 @@ class GenericImporter(TestCase): import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 4) self.assertEqual(import_items[0].index, 0) - self.assertEqual(import_items[0].data["id"], "38") + self.assertEqual(import_items[0].normalized_data["id"], "38") + self.assertEqual(import_items[0].normalized_data["title"], "Gideon the Ninth") + self.assertEqual(import_items[0].normalized_data["authors"], "Tamsyn Muir") + self.assertEqual(import_items[0].normalized_data["isbn_13"], "9781250313195") + self.assertIsNone(import_items[0].normalized_data["isbn_10"]) + self.assertEqual(import_items[0].normalized_data["shelf"], "read") + self.assertEqual(import_items[1].index, 1) - self.assertEqual(import_items[1].data["id"], "48") + self.assertEqual(import_items[1].normalized_data["id"], "48") + self.assertEqual(import_items[1].normalized_data["title"], "Harrow the Ninth") + self.assertEqual(import_items[2].index, 2) - self.assertEqual(import_items[2].data["id"], "23") + self.assertEqual(import_items[2].normalized_data["id"], "23") + self.assertEqual(import_items[2].normalized_data["title"], "Subcutanean") + self.assertEqual(import_items[3].index, 3) - self.assertEqual(import_items[3].data["id"], "10") + self.assertEqual(import_items[3].normalized_data["id"], "10") + self.assertEqual(import_items[3].normalized_data["title"], "Patisserie at Home") def test_create_retry_job(self, *_): """trying again with items that didn't import""" @@ -103,9 +94,9 @@ class GenericImporter(TestCase): retry_items = models.ImportItem.objects.filter(job=retry).all() self.assertEqual(len(retry_items), 2) self.assertEqual(retry_items[0].index, 0) - self.assertEqual(retry_items[0].data["id"], "38") + self.assertEqual(retry_items[0].normalized_data["id"], "38") self.assertEqual(retry_items[1].index, 1) - self.assertEqual(retry_items[1].data["id"], "48") + self.assertEqual(retry_items[1].normalized_data["id"], "48") def test_start_import(self, *_): """check that a task was created""" @@ -143,15 +134,12 @@ class GenericImporter(TestCase): shelf = self.local_user.shelf_set.filter(identifier="read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( @@ -172,15 +160,12 @@ class GenericImporter(TestCase): shelved_date=make_date(2020, 2, 2), ) - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( @@ -199,15 +184,12 @@ class GenericImporter(TestCase): def test_handle_import_twice(self, *_): """re-importing books""" shelf = self.local_user.shelf_set.filter(identifier="read").first() - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( @@ -219,18 +201,15 @@ class GenericImporter(TestCase): shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) + self.assertEqual(models.ReadThrough.objects.count(), 1) @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """review import""" - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[3] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book - ) + import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_item = import_job.items.filter(index=3).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): with patch("bookwyrm.models.Status.broadcast") as broadcast_mock: @@ -251,14 +230,12 @@ class GenericImporter(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """rating import""" - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[1] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" ) + import_item = import_job.items.filter(index=1).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( @@ -271,14 +248,12 @@ class GenericImporter(TestCase): def test_handle_imported_book_reviews_disabled(self, *_): """review import""" - import_job = models.ImportJob.objects.create(user=self.local_user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/generic.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[2] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" ) + import_item = import_job.items.filter(index=3).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( From 20baf9385d2253a328a69cf599a4b49e0dbf8b72 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 09:21:28 -0800 Subject: [PATCH 03/73] Updates goodreads tests --- bookwyrm/importers/importer.py | 6 +- bookwyrm/tests/data/goodreads-rating.csv | 5 - bookwyrm/tests/data/goodreads.csv | 2 +- .../tests/importers/test_goodreads_import.py | 76 +++++++-------- .../importers/test_librarything_import.py | 92 +++++++++---------- .../tests/importers/test_storygraph_import.py | 51 +++++----- 6 files changed, 109 insertions(+), 123 deletions(-) delete mode 100644 bookwyrm/tests/data/goodreads-rating.csv diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index b0458bab..f6abef0e 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -28,8 +28,8 @@ class Importer: "isbn_13": ["isbn13", "isbn"], "isbn_10": ["isbn10", "isbn"], "shelf": ["shelf", "exclusive shelf", "read status"], - "review_name": [], - "review_body": ["my review"], + "review_name": ["review name"], + "review_body": ["my review", "review"], "rating": ["my rating", "rating", "star rating"], "date_added": ["date added", "entry date", "added"], "date_started": ["date started", "started"], @@ -48,7 +48,6 @@ class Importer: ) for index, entry in rows: - print(index, entry) self.create_item(job, index, entry) return job @@ -65,7 +64,6 @@ class Importer: def create_item(self, job, index, data): """creates and saves an import item""" - print(data) normalized = self.normalize_row(data, job.mappings) ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() diff --git a/bookwyrm/tests/data/goodreads-rating.csv b/bookwyrm/tests/data/goodreads-rating.csv deleted file mode 100644 index fec0c77d..00000000 --- a/bookwyrm/tests/data/goodreads-rating.csv +++ /dev/null @@ -1,5 +0,0 @@ -Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID -42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",0,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, -52691223,Subcutanean,Aaron A. Reed,"Reed, Aaron A.",,"=""""","=""""",0,4.45,,Paperback,232,2020,,2020/03/06,2020/03/05,,,read,,,,1,,,0,,,,, -28694510,Patisserie at Home,Mélanie Dupuis,"Dupuis, Mélanie",Anne Cazor,"=""0062445316""","=""9780062445315""",2,4.60,Harper Design,Hardcover,288,2016,,,2019/07/08,,,read,,,,2,,,0,,,,, - diff --git a/bookwyrm/tests/data/goodreads.csv b/bookwyrm/tests/data/goodreads.csv index 5f124edc..a0a8232e 100644 --- a/bookwyrm/tests/data/goodreads.csv +++ b/bookwyrm/tests/data/goodreads.csv @@ -1,4 +1,4 @@ Book Id,Title,Author,Author l-f,Additional Authors,ISBN,ISBN13,My Rating,Average Rating,Publisher,Binding,Number of Pages,Year Published,Original Publication Year,Date Read,Date Added,Bookshelves,Bookshelves with positions,Exclusive Shelf,My Review,Spoiler,Private Notes,Read Count,Recommended For,Recommended By,Owned Copies,Original Purchase Date,Original Purchase Location,Condition,Condition Description,BCID -42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",0,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, +42036538,Gideon the Ninth (The Locked Tomb #1),Tamsyn Muir,"Muir, Tamsyn",,"=""1250313198""","=""9781250313195""",3,4.20,Tor,Hardcover,448,2019,2019,2020/10/25,2020/10/21,,,read,,,,1,,,0,,,,, 52691223,Subcutanean,Aaron A. Reed,"Reed, Aaron A.",,"=""""","=""""",0,4.45,,Paperback,232,2020,,2020/03/06,2020/03/05,,,read,,,,1,,,0,,,,, 28694510,Patisserie at Home,Mélanie Dupuis,"Dupuis, Mélanie",Anne Cazor,"=""0062445316""","=""9780062445315""",2,4.60,Harper Design,Hardcover,288,2016,,,2019/07/08,,,read,"mixed feelings",,,2,,,0,,,,, diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index a2451292..87624a0e 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -1,5 +1,4 @@ """ testing import """ -import csv import pathlib from unittest.mock import patch import datetime @@ -32,7 +31,7 @@ class GoodreadsImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mouse", "mouse@mouse.mouse", "password", local=True ) @@ -45,7 +44,9 @@ class GoodreadsImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 3) @@ -58,12 +59,16 @@ class GoodreadsImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -76,22 +81,19 @@ class GoodreadsImport(TestCase): def test_handle_imported_book(self, *_): """goodreads import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier="read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() @@ -100,7 +102,7 @@ class GoodreadsImport(TestCase): shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21) ) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2020, 10, 21)) self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25)) @@ -108,20 +110,16 @@ class GoodreadsImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """goodreads review import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[2] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book - ) + import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_item = import_job.items.get(index=2) + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.Review.objects.get(book=self.book, user=self.user) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "mixed feelings") self.assertEqual(review.rating, 2) self.assertEqual(review.published_date, make_date(2019, 7, 8)) @@ -130,23 +128,19 @@ class GoodreadsImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """goodreads rating import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath( - "../data/goodreads-rating.csv" - ) - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[2] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" ) + import_item = import_job.items.filter(index=0).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.ReviewRating.objects.get(book=self.book, user=self.user) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) - self.assertEqual(review.rating, 2) - self.assertEqual(review.published_date, make_date(2019, 7, 8)) + self.assertEqual(review.rating, 3) + self.assertEqual(review.published_date, make_date(2020, 10, 25)) self.assertEqual(review.privacy, "unlisted") diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index f76666a7..00bb0ffa 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -35,7 +35,7 @@ class LibrarythingImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mmai", "mmai@mmai.mmai", "password", local=True ) work = models.Work.objects.create(title="Test Work") @@ -47,8 +47,10 @@ class LibrarythingImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") - self.assertEqual(import_job.user, self.user) + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + self.assertEqual(import_job.user, self.local_user) self.assertEqual(import_job.include_reviews, False) self.assertEqual(import_job.privacy, "public") @@ -63,12 +65,16 @@ class LibrarythingImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -82,7 +88,9 @@ class LibrarythingImport(TestCase): @responses.activate def test_import_data(self, *_): """resolve entry""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) book = models.Edition.objects.create(title="Test Book") with patch( @@ -97,30 +105,25 @@ class LibrarythingImport(TestCase): def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="read").first() + shelf = self.local_user.shelf_set.filter(identifier="read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.importer.delimiter)) - ): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2007, 4, 16)) self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8)) @@ -128,31 +131,30 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): - shelf = self.user.shelf_set.filter(identifier="to-read").first() - models.ShelfBook.objects.create(shelf=shelf, user=self.user, book=self.book) - - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - for index, entry in enumerate( - list(csv.DictReader(csv_file, delimiter=self.importer.delimiter)) - ): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book + shelf = self.local_user.shelf_set.filter(identifier="to-read").first() + models.ShelfBook.objects.create( + shelf=shelf, user=self.local_user, book=self.book ) - break + + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) - self.assertIsNone(self.user.shelf_set.get(identifier="read").books.first()) + self.assertIsNone( + self.local_user.shelf_set.get(identifier="read").books.first() + ) - readthrough = models.ReadThrough.objects.get(user=self.user) + readthrough = models.ReadThrough.objects.get(user=self.local_user) self.assertEqual(readthrough.book, self.book) self.assertEqual(readthrough.start_date, make_date(2007, 4, 16)) self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8)) @@ -160,20 +162,16 @@ class LibrarythingImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """librarything review import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/librarything.tsv") - csv_file = open(datafile, "r", encoding=self.importer.encoding) - entry = list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))[0] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book - ) + import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_item = import_job.items.filter(index=0).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.Review.objects.get(book=self.book, user=self.user) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "chef d'oeuvre") self.assertEqual(review.rating, 5) self.assertEqual(review.published_date, make_date(2007, 5, 8)) diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index addf362c..91a5dfa6 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -32,7 +32,7 @@ class StorygraphImport(TestCase): with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - self.user = models.User.objects.create_user( + self.local_user = models.User.objects.create_user( "mouse", "mouse@mouse.mouse", "password", local=True ) @@ -45,7 +45,9 @@ class StorygraphImport(TestCase): def test_create_job(self, *_): """creates the import job entry and checks csv""" - import_job = self.importer.create_job(self.user, self.csv, False, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 2) @@ -57,12 +59,16 @@ class StorygraphImport(TestCase): def test_create_retry_job(self, *_): """trying again with items that didn't import""" - import_job = self.importer.create_job(self.user, self.csv, False, "unlisted") + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - retry = self.importer.create_retry_job(self.user, import_job, import_items) + retry = self.importer.create_retry_job( + self.local_user, import_job, import_items + ) self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.user) + self.assertEqual(retry.user, self.local_user) self.assertEqual(retry.include_reviews, False) self.assertEqual(retry.privacy, "unlisted") @@ -75,22 +81,19 @@ class StorygraphImport(TestCase): def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" - shelf = self.user.shelf_set.filter(identifier="to-read").first() + shelf = self.local_user.shelf_set.filter(identifier="to-read").first() self.assertIsNone(shelf.books.first()) - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/storygraph.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - for index, entry in enumerate(list(csv.DictReader(csv_file))): - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=index, data=entry, book=self.book - ) - break + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" + ) + import_item = import_job.items.first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, False, "public" + self.importer.service, self.local_user, import_item, False, "public" ) shelf.refresh_from_db() @@ -102,20 +105,18 @@ class StorygraphImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """storygraph rating import""" - import_job = models.ImportJob.objects.create(user=self.user) - datafile = pathlib.Path(__file__).parent.joinpath("../data/storygraph.csv") - csv_file = open(datafile, "r") # pylint: disable=unspecified-encoding - entry = list(csv.DictReader(csv_file))[1] - entry = self.importer.parse_fields(entry) - import_item = models.ImportItem.objects.create( - job_id=import_job.id, index=0, data=entry, book=self.book + import_job = self.importer.create_job( + self.local_user, self.csv, False, "public" ) + import_item = import_job.items.filter(index=1).first() + import_item.book = self.book + import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): handle_imported_book( - self.importer.service, self.user, import_item, True, "unlisted" + self.importer.service, self.local_user, import_item, True, "unlisted" ) - review = models.ReviewRating.objects.get(book=self.book, user=self.user) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) self.assertEqual(review.rating, 5.0) self.assertEqual(review.published_date, make_date(2021, 5, 10)) From 4d574a3536c88d1a51307b4e288ef2185ed71b13 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 09:54:36 -0800 Subject: [PATCH 04/73] Process dates in librarything import --- bookwyrm/importers/importer.py | 1 + bookwyrm/importers/librarything_import.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index f6abef0e..8acfb147 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -35,6 +35,7 @@ class Importer: "date_started": ["date started", "started"], "date_finished": ["date finished", "last date read", "date read", "finished"], } + date_fields = ["date_added", "date_started", "date_finished"] def create_job(self, user, csv_file, include_reviews, privacy): """check over a csv and creates a database entry for the job""" diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 3d42e539..12d841e9 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -1,4 +1,5 @@ """ handle reading a tsv from librarything """ +import re from . import Importer @@ -8,3 +9,11 @@ class LibrarythingImporter(Importer): service = "LibraryThing" delimiter = "\t" encoding = "ISO-8859-1" + + def normalize_row(self, entry, mappings): # pylint: disable=no-self-use + """use the dataclass to create the formatted row of data""" + normalized = {k: entry.get(v) for k, v in mappings.items()} + for date_field in self.date_fields: + date = normalized[date_field] + normalized[date_field] = re.sub(r"\[|\]", "", date) + return normalized From 147dd95e8d00a209ac1af1f87f36175a8da1afca Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 11:59:15 -0800 Subject: [PATCH 05/73] Removes unused import --- bookwyrm/tests/importers/test_librarything_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 00bb0ffa..aa52e2f1 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -1,5 +1,4 @@ """ testing import """ -import csv import pathlib from unittest.mock import patch import datetime From f3bcced0a019b119e02d2d191af41f49f9a65af8 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 12:29:38 -0800 Subject: [PATCH 06/73] Adds shelf mappings --- bookwyrm/importers/importer.py | 12 ++++++++++++ bookwyrm/importers/librarything_import.py | 7 +++++++ bookwyrm/tests/importers/test_librarything_import.py | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 8acfb147..0233ee3d 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -36,6 +36,11 @@ class Importer: "date_finished": ["date finished", "last date read", "date read", "finished"], } date_fields = ["date_added", "date_started", "date_finished"] + shelf_mapping_guesses = { + "to-read": ["to-read"], + "read": ["read"], + "reading": ["currently-reading", "reading"], + } def create_job(self, user, csv_file, include_reviews, privacy): """check over a csv and creates a database entry for the job""" @@ -66,8 +71,15 @@ class Importer: def create_item(self, job, index, data): """creates and saves an import item""" normalized = self.normalize_row(data, job.mappings) + normalized["shelf"] = self.get_shelf(normalized) ImportItem(job=job, index=index, data=data, normalized_data=normalized).save() + def get_shelf(self, normalized_row): + """determine which shelf to use""" + shelf_name = normalized_row["shelf"] + shelf = [s for (s, gs) in self.shelf_mapping_guesses if shelf_name in gs] + return shelf[0] if shelf else None + def normalize_row(self, entry, mappings): # pylint: disable=no-self-use """use the dataclass to create the formatted row of data""" return {k: entry.get(v) for k, v in mappings.items()} diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index 12d841e9..d6426de6 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -17,3 +17,10 @@ class LibrarythingImporter(Importer): date = normalized[date_field] normalized[date_field] = re.sub(r"\[|\]", "", date) return normalized + + def get_shelf(self, normalized_row): + if normalized_row["date_finished"]: + return "read" + if normalized_row["date_started"]: + return "reading" + return "to-read" diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index aa52e2f1..e9352896 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -172,6 +172,6 @@ class LibrarythingImport(TestCase): ) review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "chef d'oeuvre") - self.assertEqual(review.rating, 5) + self.assertEqual(review.rating, 4.5) self.assertEqual(review.published_date, make_date(2007, 5, 8)) self.assertEqual(review.privacy, "unlisted") From 59678348053f43f83912fa4e4ea11f046844f534 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 12:29:51 -0800 Subject: [PATCH 07/73] Adds migration --- .../migrations/0113_auto_20211110_2104.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 bookwyrm/migrations/0113_auto_20211110_2104.py diff --git a/bookwyrm/migrations/0113_auto_20211110_2104.py b/bookwyrm/migrations/0113_auto_20211110_2104.py new file mode 100644 index 00000000..572ba280 --- /dev/null +++ b/bookwyrm/migrations/0113_auto_20211110_2104.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.5 on 2021-11-10 21:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0112_auto_20211022_0844"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="normalized_data", + field=models.JSONField(default={}), + preserve_default=False, + ), + migrations.AddField( + model_name="importjob", + name="mappings", + field=models.JSONField(default={}), + preserve_default=False, + ), + ] From efcf7824dd47de77c3cc2e0bcc972bbd131b05af Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 12:39:12 -0800 Subject: [PATCH 08/73] iUpdates storygraph tests --- bookwyrm/importers/importer.py | 4 +++- bookwyrm/tests/importers/test_storygraph_import.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 0233ee3d..cf7841e6 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -77,7 +77,9 @@ class Importer: def get_shelf(self, normalized_row): """determine which shelf to use""" shelf_name = normalized_row["shelf"] - shelf = [s for (s, gs) in self.shelf_mapping_guesses if shelf_name in gs] + shelf = [ + s for (s, gs) in self.shelf_mapping_guesses.items() if shelf_name in gs + ] return shelf[0] if shelf else None def normalize_row(self, entry, mappings): # pylint: disable=no-self-use diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 91a5dfa6..2f0fd7ef 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -52,10 +52,12 @@ class StorygraphImport(TestCase): import_items = models.ImportItem.objects.filter(job=import_job).all() self.assertEqual(len(import_items), 2) self.assertEqual(import_items[0].index, 0) - self.assertEqual(import_items[0].data["Title"], "Always Coming Home") + self.assertEqual(import_items[0].normalized_data["title"], "Always Coming Home") self.assertEqual(import_items[1].index, 1) - self.assertEqual(import_items[1].data["Title"], "Subprime Attention Crisis") - self.assertEqual(import_items[1].data["My Rating"], 5.0) + self.assertEqual( + import_items[1].normalized_data["title"], "Subprime Attention Crisis" + ) + self.assertEqual(import_items[1].normalized_data["rating"], "5.0") def test_create_retry_job(self, *_): """trying again with items that didn't import""" @@ -75,9 +77,11 @@ class StorygraphImport(TestCase): retry_items = models.ImportItem.objects.filter(job=retry).all() self.assertEqual(len(retry_items), 2) self.assertEqual(retry_items[0].index, 0) - self.assertEqual(retry_items[0].data["Title"], "Always Coming Home") + self.assertEqual(retry_items[0].normalized_data["title"], "Always Coming Home") self.assertEqual(retry_items[1].index, 1) - self.assertEqual(retry_items[1].data["Title"], "Subprime Attention Crisis") + self.assertEqual( + retry_items[1].normalized_data["title"], "Subprime Attention Crisis" + ) def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" From 50ab4e8248bbcb740292d50e4b4459f8eddffa7b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 14:08:00 -0800 Subject: [PATCH 09/73] Updates model tests --- bookwyrm/tests/models/test_import_model.py | 154 ++++++++++++--------- 1 file changed, 87 insertions(+), 67 deletions(-) diff --git a/bookwyrm/tests/models/test_import_model.py b/bookwyrm/tests/models/test_import_model.py index 0e5d6760..f48143d0 100644 --- a/bookwyrm/tests/models/test_import_model.py +++ b/bookwyrm/tests/models/test_import_model.py @@ -18,83 +18,68 @@ class ImportJob(TestCase): def setUp(self): """data is from a goodreads export of The Raven Tower""" - read_data = { - "Book Id": 39395857, - "Title": "The Raven Tower", - "Author": "Ann Leckie", - "Author l-f": "Leckie, Ann", - "Additional Authors": "", - "ISBN": '="0356506991"', - "ISBN13": '="9780356506999"', - "My Rating": 0, - "Average Rating": 4.06, - "Publisher": "Orbit", - "Binding": "Hardcover", - "Number of Pages": 416, - "Year Published": 2019, - "Original Publication Year": 2019, - "Date Read": "2019/04/12", - "Date Added": "2019/04/09", - "Bookshelves": "", - "Bookshelves with positions": "", - "Exclusive Shelf": "read", - "My Review": "", - "Spoiler": "", - "Private Notes": "", - "Read Count": 1, - "Recommended For": "", - "Recommended By": "", - "Owned Copies": 0, - "Original Purchase Date": "", - "Original Purchase Location": "", - "Condition": "", - "Condition Description": "", - "BCID": "", - } - currently_reading_data = read_data.copy() - currently_reading_data["Exclusive Shelf"] = "currently-reading" - currently_reading_data["Date Read"] = "" - - unknown_read_data = currently_reading_data.copy() - unknown_read_data["Exclusive Shelf"] = "read" - unknown_read_data["Date Read"] = "" - with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" ): - user = models.User.objects.create_user( - "mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse" + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True ) - job = models.ImportJob.objects.create(user=user) - self.item_1 = models.ImportItem.objects.create( - job=job, index=1, data=currently_reading_data - ) - self.item_2 = models.ImportItem.objects.create(job=job, index=2, data=read_data) - self.item_3 = models.ImportItem.objects.create( - job=job, index=3, data=unknown_read_data - ) + self.job = models.ImportJob.objects.create(user=self.local_user, mappings={}) def test_isbn(self): """it unquotes the isbn13 field from data""" - expected = "9780356506999" - item = models.ImportItem.objects.get(index=1) - self.assertEqual(item.isbn, expected) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + }, + ) + self.assertEqual(item.isbn, "9780356506999") def test_shelf(self): """converts to the local shelf typology""" - expected = "reading" - self.assertEqual(self.item_1.shelf, expected) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + }, + ) + self.assertEqual(item.shelf, "reading") def test_date_added(self): """converts to the local shelf typology""" expected = datetime.datetime(2019, 4, 9, 0, 0, tzinfo=timezone.utc) - item = models.ImportItem.objects.get(index=1) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + "date_added": "2019/04/09", + }, + ) self.assertEqual(item.date_added, expected) def test_date_read(self): """converts to the local shelf typology""" expected = datetime.datetime(2019, 4, 12, 0, 0, tzinfo=timezone.utc) - item = models.ImportItem.objects.get(index=2) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + "date_added": "2019/04/09", + "date_finished": "2019/04/12", + }, + ) self.assertEqual(item.date_read, expected) def test_currently_reading_reads(self): @@ -104,31 +89,66 @@ class ImportJob(TestCase): start_date=datetime.datetime(2019, 4, 9, 0, 0, tzinfo=timezone.utc) ) ] - actual = models.ImportItem.objects.get(index=1) - self.assertEqual(actual.reads[0].start_date, expected[0].start_date) - self.assertEqual(actual.reads[0].finish_date, expected[0].finish_date) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + "date_added": "2019/04/09", + }, + ) + self.assertEqual(item.reads[0].start_date, expected[0].start_date) + self.assertIsNone(item.reads[0].finish_date) def test_read_reads(self): """infer read dates where available""" - actual = self.item_2 + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + "date_added": "2019/04/09", + "date_finished": "2019/04/12", + }, + ) self.assertEqual( - actual.reads[0].start_date, + item.reads[0].start_date, datetime.datetime(2019, 4, 9, 0, 0, tzinfo=timezone.utc), ) self.assertEqual( - actual.reads[0].finish_date, + item.reads[0].finish_date, datetime.datetime(2019, 4, 12, 0, 0, tzinfo=timezone.utc), ) def test_unread_reads(self): """handle books with no read dates""" expected = [] - actual = models.ImportItem.objects.get(index=3) - self.assertEqual(actual.reads, expected) + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + "shelf": "reading", + }, + ) + self.assertEqual(item.reads, expected) @responses.activate def test_get_book_from_isbn(self): """search and load books by isbn (9780356506999)""" + item = models.ImportItem.objects.create( + index=1, + job=self.job, + data={}, + normalized_data={ + "isbn_13": '="9780356506999"', + }, + ) connector_info = models.Connector.objects.create( identifier="openlibrary.org", name="OpenLibrary", @@ -177,6 +197,6 @@ class ImportJob(TestCase): with patch( "bookwyrm.connectors.openlibrary.Connector." "get_authors_from_data" ): - book = self.item_1.get_book_from_isbn() + book = item.get_book_from_isbn() self.assertEqual(book.title, "Sabriel") From be94818a10aa049ffc8e3ff0789dddd7a9706f72 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 14:27:29 -0800 Subject: [PATCH 10/73] Fixes views tests --- bookwyrm/tests/views/test_import.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bookwyrm/tests/views/test_import.py b/bookwyrm/tests/views/test_import.py index 1411a5a8..54f11f03 100644 --- a/bookwyrm/tests/views/test_import.py +++ b/bookwyrm/tests/views/test_import.py @@ -41,7 +41,7 @@ class ImportViews(TestCase): def test_import_status(self): """there are so many views, this just makes sure it LOADS""" view = views.ImportStatus.as_view() - import_job = models.ImportJob.objects.create(user=self.local_user) + import_job = models.ImportJob.objects.create(user=self.local_user, mappings={}) request = self.factory.get("") request.user = self.local_user with patch("bookwyrm.tasks.app.AsyncResult") as async_result: @@ -55,7 +55,7 @@ class ImportViews(TestCase): """retry failed items""" view = views.Import.as_view() form = forms.ImportForm() - form.data["source"] = "LibraryThing" + form.data["source"] = "Goodreads" form.data["privacy"] = "public" form.data["include_reviews"] = False csv_file = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") @@ -79,7 +79,7 @@ class ImportViews(TestCase): """retry failed items""" view = views.ImportStatus.as_view() import_job = models.ImportJob.objects.create( - user=self.local_user, privacy="unlisted" + user=self.local_user, privacy="unlisted", mappings={} ) request = self.factory.post("") request.user = self.local_user From f0ce236ffc4a17ea34d32f00065fdf727b6665c7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 14:28:01 -0800 Subject: [PATCH 11/73] Removes unused code --- bookwyrm/importers/importer.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index cf7841e6..69391e94 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -1,6 +1,5 @@ """ handle reading a csv from an external service, defaults are from Goodreads """ import csv -from dataclasses import dataclass import logging from django.utils import timezone @@ -202,23 +201,3 @@ def handle_imported_book(source, user, item, include_reviews, privacy): ) # only broadcast this review to other bookwyrm instances review.save(software="bookwyrm") - - -@dataclass -class ImportEntry: - """data extracted from a line in a csv""" - - title: str - authors: str = None - isbn_13: str = None - isbn_10: str = None - shelf: str = None - review_name: str = None - review_rating: float = None - review_body: str = None - review_cw: str = None - rating: float = None - date_added: str = None - date_started: str = None - date_finished: str = None - import_source: str = "Unknown" From 908c9dc689aefd70c278254b81611515e7ad67a9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 14:39:53 -0800 Subject: [PATCH 12/73] Use many small tasks instead of one big task --- bookwyrm/importers/importer.py | 49 +++++++++++++++++----------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 69391e94..8ad4cd2e 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -102,39 +102,40 @@ class Importer: def start_import(self, job): """initalizes a csv import job""" - result = import_data.delay(self.service, job.id) + result = start_import_task.delay(self.service, job.id) job.task_id = result.id job.save() @app.task(queue="low_priority") -def import_data(source, job_id): - """does the actual lookup work in a celery task""" +def start_import_task(source, job_id): + """trigger the child tasks for each row""" job = ImportJob.objects.get(id=job_id) + # these are sub-tasks so that one big task doesn't use up all the memory in celery + for item in job.items.values("id").all(): + import_item_task.delay(source, item.id) + + +@app.task(queue="low_priority") +def import_item_task(source, item_id): + """resolve a row into a book""" + item = models.ImportItem.objets.get(id=item_id) try: - for item in job.items.all(): - try: - item.resolve() - except Exception as err: # pylint: disable=broad-except - logger.exception(err) - item.fail_reason = _("Error loading book") - item.save() - continue + item.resolve() + except Exception as err: # pylint: disable=broad-except + logger.exception(err) + item.fail_reason = _("Error loading book") + item.save() + return - if item.book or item.book_guess: - item.save() + if item.book: + job = item.job + # shelves book and handles reviews + handle_imported_book(source, job.user, item, job.include_reviews, job.privacy) + else: + item.fail_reason = _("Could not find a match for book") - if item.book: - # shelves book and handles reviews - handle_imported_book( - source, job.user, item, job.include_reviews, job.privacy - ) - else: - item.fail_reason = _("Could not find a match for book") - item.save() - finally: - job.complete = True - job.save() + item.save() def handle_imported_book(source, user, item, include_reviews, privacy): From c33d791974df8a37de257baae37cf7fc86be88c9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 15:17:32 -0800 Subject: [PATCH 13/73] adds tests for new task system --- bookwyrm/importers/importer.py | 6 ++--- bookwyrm/tests/importers/test_importer.py | 30 ++++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 8ad4cd2e..d7f32c88 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -112,14 +112,14 @@ def start_import_task(source, job_id): """trigger the child tasks for each row""" job = ImportJob.objects.get(id=job_id) # these are sub-tasks so that one big task doesn't use up all the memory in celery - for item in job.items.values("id").all(): - import_item_task.delay(source, item.id) + for item in job.items.values_list("id", flat=True).all(): + import_item_task.delay(source, item) @app.task(queue="low_priority") def import_item_task(source, item_id): """resolve a row into a book""" - item = models.ImportItem.objets.get(id=item_id) + item = models.ImportItem.objects.get(id=item_id) try: item.resolve() except Exception as err: # pylint: disable=broad-except diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index b2f0284d..05377cce 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -10,7 +10,8 @@ import responses from bookwyrm import models from bookwyrm.importers import Importer -from bookwyrm.importers.importer import import_data, handle_imported_book +from bookwyrm.importers.importer import start_import_task, import_item_task +from bookwyrm.importers.importer import handle_imported_book def make_date(*args): @@ -105,29 +106,40 @@ class GenericImporter(TestCase): ) MockTask = namedtuple("Task", ("id")) mock_task = MockTask(7) - with patch("bookwyrm.importers.importer.import_data.delay") as start: + with patch("bookwyrm.importers.importer.start_import_task.delay") as start: start.return_value = mock_task self.importer.start_import(import_job) import_job.refresh_from_db() self.assertEqual(import_job.task_id, "7") @responses.activate - def test_import_data(self, *_): + def test_start_import_task(self, *_): """resolve entry""" import_job = self.importer.create_job( self.local_user, self.csv, False, "unlisted" ) - book = models.Edition.objects.create(title="Test Book") + with patch("bookwyrm.importers.importer.import_item_task.delay") as mock: + start_import_task(self.importer.service, import_job.id) + + self.assertEqual(mock.call_count, 4) + + @responses.activate + def test_import_item_task(self, *_): + """resolve entry""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) + + import_item = models.ImportItem.objects.get(job=import_job, index=0) with patch( "bookwyrm.models.import_job.ImportItem.get_book_from_isbn" ) as resolve: - resolve.return_value = book - with patch("bookwyrm.importers.importer.handle_imported_book"): - import_data(self.importer.service, import_job.id) + resolve.return_value = self.book + import_item_task(self.importer.service, import_item.id) + import_item.refresh_from_db() - import_item = models.ImportItem.objects.get(job=import_job, index=0) - self.assertEqual(import_item.book.id, book.id) + self.assertEqual(import_item.book.id, self.book.id) def test_handle_imported_book(self, *_): """import added a book, this adds related connections""" From 6aa57d4d34a441e38ccb00e435d0eb59beddca5a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 19:00:01 -0800 Subject: [PATCH 14/73] Set queue for broadcast task --- bookwyrm/models/activitypub_mixin.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index e1276d46..553b2a82 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -126,12 +126,15 @@ class ActivitypubMixin: # there OUGHT to be only one match return match.first() - def broadcast(self, activity, sender, software=None): + def broadcast(self, activity, sender, software=None, queue="medium_priority"): """send out an activity""" - broadcast_task.delay( - sender.id, - json.dumps(activity, cls=activitypub.ActivityEncoder), - self.get_recipients(software=software), + broadcast_task.apply_async( + args=( + sender.id, + json.dumps(activity, cls=activitypub.ActivityEncoder), + self.get_recipients(software=software), + ), + queue=queue, ) def get_recipients(self, software=None): @@ -374,9 +377,9 @@ class CollectionItemMixin(ActivitypubMixin): activity_serializer = activitypub.CollectionItem - def broadcast(self, activity, sender, software="bookwyrm"): + def broadcast(self, activity, sender, software="bookwyrm", queue="medium_priority"): """only send book collection updates to other bookwyrm instances""" - super().broadcast(activity, sender, software=software) + super().broadcast(activity, sender, software=software, queue=queue) @property def privacy(self): @@ -406,7 +409,7 @@ class CollectionItemMixin(ActivitypubMixin): # adding an obj to the collection activity = self.to_add_activity(self.user) - self.broadcast(activity, self.user) + self.broadcast(activity, self.user, queue="low_priority") def delete(self, *args, broadcast=True, **kwargs): """broadcast a remove activity""" From 9fee860b00899c56190c079e0f7bcb9134695295 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 19:10:22 -0800 Subject: [PATCH 15/73] Adds enum for queue names --- bookwyrm/models/activitypub_mixin.py | 27 ++++++++++++++------------- bookwyrm/tasks.py | 5 +++++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index 553b2a82..402cb040 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -20,7 +20,7 @@ from django.utils.http import http_date from bookwyrm import activitypub from bookwyrm.settings import USER_AGENT, PAGE_LENGTH from bookwyrm.signatures import make_signature, make_digest -from bookwyrm.tasks import app +from bookwyrm.tasks import app, MEDIUM from bookwyrm.models.fields import ImageField, ManyToManyField logger = logging.getLogger(__name__) @@ -29,7 +29,6 @@ logger = logging.getLogger(__name__) PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) - # pylint: disable=invalid-name def set_activity_from_property_field(activity, obj, field): """assign a model property value to the activity json""" @@ -126,7 +125,7 @@ class ActivitypubMixin: # there OUGHT to be only one match return match.first() - def broadcast(self, activity, sender, software=None, queue="medium_priority"): + def broadcast(self, activity, sender, software=None, queue=MEDIUM): """send out an activity""" broadcast_task.apply_async( args=( @@ -198,7 +197,7 @@ class ActivitypubMixin: class ObjectMixin(ActivitypubMixin): """add this mixin for object models that are AP serializable""" - def save(self, *args, created=None, software=None, **kwargs): + def save(self, *args, created=None, software=None, priority=MEDIUM, **kwargs): """broadcast created/updated/deleted objects as appropriate""" broadcast = kwargs.get("broadcast", True) # this bonus kwarg would cause an error in the base save method @@ -225,12 +224,14 @@ class ObjectMixin(ActivitypubMixin): # do we have a "pure" activitypub version of this for mastodon? if software != "bookwyrm" and hasattr(self, "pure_content"): pure_activity = self.to_create_activity(user, pure=True) - self.broadcast(pure_activity, user, software="other") + self.broadcast( + pure_activity, user, software="other", queue=priority + ) # set bookwyrm so that that type is also sent software = "bookwyrm" # sends to BW only if we just did a pure version for masto activity = self.to_create_activity(user) - self.broadcast(activity, user, software=software) + self.broadcast(activity, user, software=software, queue=priority) except AttributeError: # janky as heck, this catches the mutliple inheritence chain # for boosts and ignores this auxilliary broadcast @@ -254,7 +255,7 @@ class ObjectMixin(ActivitypubMixin): activity = self.to_delete_activity(user) else: activity = self.to_update_activity(user) - self.broadcast(activity, user) + self.broadcast(activity, user, queue=priority) def to_create_activity(self, user, **kwargs): """returns the object wrapped in a Create activity""" @@ -377,7 +378,7 @@ class CollectionItemMixin(ActivitypubMixin): activity_serializer = activitypub.CollectionItem - def broadcast(self, activity, sender, software="bookwyrm", queue="medium_priority"): + def broadcast(self, activity, sender, software="bookwyrm", queue=MEDIUM): """only send book collection updates to other bookwyrm instances""" super().broadcast(activity, sender, software=software, queue=queue) @@ -398,7 +399,7 @@ class CollectionItemMixin(ActivitypubMixin): return [] return [collection_field.user] - def save(self, *args, broadcast=True, **kwargs): + def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): """broadcast updated""" # first off, we want to save normally no matter what super().save(*args, **kwargs) @@ -409,7 +410,7 @@ class CollectionItemMixin(ActivitypubMixin): # adding an obj to the collection activity = self.to_add_activity(self.user) - self.broadcast(activity, self.user, queue="low_priority") + self.broadcast(activity, self.user, queue=priority) def delete(self, *args, broadcast=True, **kwargs): """broadcast a remove activity""" @@ -442,12 +443,12 @@ class CollectionItemMixin(ActivitypubMixin): class ActivityMixin(ActivitypubMixin): """add this mixin for models that are AP serializable""" - def save(self, *args, broadcast=True, **kwargs): + def save(self, *args, broadcast=True, priority=MEDIUM, **kwargs): """broadcast activity""" super().save(*args, **kwargs) user = self.user if hasattr(self, "user") else self.user_subject if broadcast and user.local: - self.broadcast(self.to_activity(), user) + self.broadcast(self.to_activity(), user, queue=priority) def delete(self, *args, broadcast=True, **kwargs): """nevermind, undo that activity""" @@ -504,7 +505,7 @@ def unfurl_related_field(related_field, sort_field=None): return related_field.remote_id -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def broadcast_task(sender_id, activity, recipients): """the celery task for broadcast""" user_model = apps.get_model("bookwyrm.User", require_ready=True) diff --git a/bookwyrm/tasks.py b/bookwyrm/tasks.py index b860e018..09e1d267 100644 --- a/bookwyrm/tasks.py +++ b/bookwyrm/tasks.py @@ -9,3 +9,8 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "celerywyrm.settings") app = Celery( "tasks", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND ) + +# priorities +LOW = "low_priority" +MEDIUM = "medium_priority" +HIGH = "high_priority" From 3190ef4346f3a1a763bbf15ba05a69c00a138fd3 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 11 Nov 2021 19:16:36 -0800 Subject: [PATCH 16/73] Deprioritize adding old statuses to timelines --- bookwyrm/activitystreams.py | 31 +++++++++++++++++++++---------- bookwyrm/importers/importer.py | 4 ++-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 4896e07d..a9ca17e2 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -7,7 +7,7 @@ from django.utils import timezone from bookwyrm import models from bookwyrm.redis_store import RedisStore, r -from bookwyrm.tasks import app +from bookwyrm.tasks import app, LOW, MEDIUM, HIGH class ActivityStream(RedisStore): @@ -277,7 +277,18 @@ def add_status_on_create(sender, instance, created, *args, **kwargs): def add_status_on_create_command(sender, instance, created): """runs this code only after the database commit completes""" - add_status_task.delay(instance.id, increment_unread=created) + priority=HIGH + # check if this is an old status, de-prioritize if so + # (this will happen if federation is very slow, or, more expectedly, on csv import) + one_day = 60 * 60 * 24 + if (instance.created_date - instance.published_date).seconds > one_day: + priority=LOW + + add_status_task.apply_async( + args=(instance.id,), + kwargs={"increment_unread": created}, + queue=priority, + ) if sender == models.Boost: handle_boost_task.delay(instance.id) @@ -409,7 +420,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs): # ---- TASKS -@app.task(queue="low_priority") +@app.task(queue=LOW) def add_book_statuses_task(user_id, book_id): """add statuses related to a book on shelve""" user = models.User.objects.get(id=user_id) @@ -417,7 +428,7 @@ def add_book_statuses_task(user_id, book_id): BooksStream().add_book_statuses(user, book) -@app.task(queue="low_priority") +@app.task(queue=LOW) def remove_book_statuses_task(user_id, book_id): """remove statuses about a book from a user's books feed""" user = models.User.objects.get(id=user_id) @@ -425,7 +436,7 @@ def remove_book_statuses_task(user_id, book_id): BooksStream().remove_book_statuses(user, book) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def populate_stream_task(stream, user_id): """background task for populating an empty activitystream""" user = models.User.objects.get(id=user_id) @@ -433,7 +444,7 @@ def populate_stream_task(stream, user_id): stream.populate_streams(user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def remove_status_task(status_ids): """remove a status from any stream it might be in""" # this can take an id or a list of ids @@ -446,7 +457,7 @@ def remove_status_task(status_ids): stream.remove_object_from_related_stores(status) -@app.task(queue="high_priority") +@app.task(queue=HIGH) def add_status_task(status_id, increment_unread=False): """add a status to any stream it should be in""" status = models.Status.objects.get(id=status_id) @@ -458,7 +469,7 @@ def add_status_task(status_id, increment_unread=False): stream.add_status(status, increment_unread=increment_unread) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def remove_user_statuses_task(viewer_id, user_id, stream_list=None): """remove all statuses by a user from a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -468,7 +479,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None): stream.remove_user_statuses(viewer, user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def add_user_statuses_task(viewer_id, user_id, stream_list=None): """add all statuses by a user to a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -478,7 +489,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None): stream.add_user_statuses(viewer, user) -@app.task(queue="medium_priority") +@app.task(queue=MEDIUM) def handle_boost_task(boost_id): """remove the original post and other, earlier boosts""" instance = models.Status.objects.get(id=boost_id) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index d7f32c88..3e0b0c28 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -7,7 +7,7 @@ from django.utils.translation import gettext_lazy as _ from bookwyrm import models from bookwyrm.models import ImportJob, ImportItem -from bookwyrm.tasks import app +from bookwyrm.tasks import app, LOW logger = logging.getLogger(__name__) @@ -201,4 +201,4 @@ def handle_imported_book(source, user, item, include_reviews, privacy): privacy=privacy, ) # only broadcast this review to other bookwyrm instances - review.save(software="bookwyrm") + review.save(software="bookwyrm", priority=LOW) From f71ef286b629c3e8744bb7d895b1a30f827ca484 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 08:55:47 -0800 Subject: [PATCH 17/73] Updates mocks --- bookwyrm/activitystreams.py | 4 ++-- bookwyrm/importers/importer.py | 4 ++-- bookwyrm/tests/importers/test_importer.py | 22 ++++++++++++------- .../importers/test_librarything_import.py | 6 ++--- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index a9ca17e2..e1a52d26 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -277,12 +277,12 @@ def add_status_on_create(sender, instance, created, *args, **kwargs): def add_status_on_create_command(sender, instance, created): """runs this code only after the database commit completes""" - priority=HIGH + priority = HIGH # check if this is an old status, de-prioritize if so # (this will happen if federation is very slow, or, more expectedly, on csv import) one_day = 60 * 60 * 24 if (instance.created_date - instance.published_date).seconds > one_day: - priority=LOW + priority = LOW add_status_task.apply_async( args=(instance.id,), diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 3e0b0c28..b32e2df7 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -151,9 +151,9 @@ def handle_imported_book(source, user, item, include_reviews, privacy): if item.shelf and not existing_shelf: desired_shelf = models.Shelf.objects.get(identifier=item.shelf, user=user) shelved_date = item.date_added or timezone.now() - models.ShelfBook.objects.create( + models.ShelfBook( book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date - ) + ).save(priority=LOW) for read in item.reads: # check for an existing readthrough with the same dates diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 05377cce..963eca54 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -136,7 +136,13 @@ class GenericImporter(TestCase): "bookwyrm.models.import_job.ImportItem.get_book_from_isbn" ) as resolve: resolve.return_value = self.book - import_item_task(self.importer.service, import_item.id) + + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: + import_item_task(self.importer.service, import_item.id) + kwargs = mock.call_args.kwargs + self.assertEqual(kwargs["queue"], "low_priority") import_item.refresh_from_db() self.assertEqual(import_item.book.id, self.book.id) @@ -153,7 +159,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -163,7 +169,7 @@ class GenericImporter(TestCase): def test_handle_imported_book_already_shelved(self, *_): """import added a book, this adds related connections""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = self.local_user.shelf_set.filter(identifier="to-read").first() models.ShelfBook.objects.create( shelf=shelf, @@ -179,7 +185,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -203,7 +209,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -223,7 +229,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): with patch("bookwyrm.models.Status.broadcast") as broadcast_mock: handle_imported_book( self.importer.service, @@ -249,7 +255,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, True, "unlisted" ) @@ -267,7 +273,7 @@ class GenericImporter(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "unlisted" ) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index e9352896..55bfef72 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -9,7 +9,7 @@ import responses from bookwyrm import models from bookwyrm.importers import LibrarythingImporter -from bookwyrm.importers.importer import import_data, handle_imported_book +from bookwyrm.importers.importer import start_import_task, handle_imported_book def make_date(*args): @@ -85,7 +85,7 @@ class LibrarythingImport(TestCase): self.assertEqual(retry_items[1].data["Book Id"], "5015319") @responses.activate - def test_import_data(self, *_): + def test_start_import_task(self, *_): """resolve entry""" import_job = self.importer.create_job( self.local_user, self.csv, False, "unlisted" @@ -97,7 +97,7 @@ class LibrarythingImport(TestCase): ) as resolve: resolve.return_value = book with patch("bookwyrm.importers.importer.handle_imported_book"): - import_data(self.importer.service, import_job.id) + start_import_task(self.importer.service, import_job.id) import_item = models.ImportItem.objects.get(job=import_job, index=0) self.assertEqual(import_item.book.id, book.id) From e6a251fdad509285139c5a0484c80182c7827856 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 09:17:00 -0800 Subject: [PATCH 18/73] Updates mocks across tests --- bookwyrm/models/book.py | 5 +- .../tests/activitypub/test_base_activity.py | 8 +-- .../activitystreams/test_abstractstream.py | 2 +- .../tests/activitystreams/test_booksstream.py | 2 +- .../tests/activitystreams/test_homestream.py | 2 +- .../tests/activitystreams/test_localstream.py | 2 +- .../tests/activitystreams/test_signals.py | 2 +- bookwyrm/tests/activitystreams/test_tasks.py | 10 +-- .../tests/importers/test_goodreads_import.py | 6 +- .../importers/test_librarything_import.py | 8 +-- .../tests/importers/test_storygraph_import.py | 4 +- .../tests/management/test_populate_streams.py | 2 +- .../tests/models/test_activitypub_mixin.py | 2 +- bookwyrm/tests/models/test_group.py | 6 +- bookwyrm/tests/models/test_list.py | 10 +-- .../tests/models/test_relationship_models.py | 20 +++--- bookwyrm/tests/models/test_shelf_model.py | 30 +++++---- bookwyrm/tests/models/test_user_model.py | 4 +- bookwyrm/tests/test_postgres.py | 2 +- bookwyrm/tests/test_suggested_users.py | 6 +- bookwyrm/tests/test_templatetags.py | 14 ++-- bookwyrm/tests/views/admin/test_reports.py | 6 +- bookwyrm/tests/views/admin/test_user_admin.py | 2 +- bookwyrm/tests/views/books/test_book.py | 8 +-- bookwyrm/tests/views/books/test_edit_book.py | 8 +-- bookwyrm/tests/views/books/test_editions.py | 4 +- .../tests/views/inbox/test_inbox_announce.py | 2 +- .../tests/views/inbox/test_inbox_block.py | 2 +- .../tests/views/inbox/test_inbox_create.py | 2 +- .../tests/views/inbox/test_inbox_follow.py | 24 ++++--- bookwyrm/tests/views/inbox/test_inbox_like.py | 2 +- .../tests/views/inbox/test_inbox_remove.py | 2 +- .../tests/views/inbox/test_inbox_update.py | 4 +- .../tests/views/preferences/test_block.py | 2 +- .../views/preferences/test_delete_user.py | 10 +-- .../tests/views/preferences/test_edit_user.py | 10 +-- bookwyrm/tests/views/shelf/test_shelf.py | 8 +-- .../tests/views/shelf/test_shelf_actions.py | 28 ++++---- bookwyrm/tests/views/test_author.py | 2 +- bookwyrm/tests/views/test_discover.py | 2 +- bookwyrm/tests/views/test_feed.py | 10 +-- bookwyrm/tests/views/test_follow.py | 16 +++-- bookwyrm/tests/views/test_get_started.py | 4 +- bookwyrm/tests/views/test_goal.py | 2 +- bookwyrm/tests/views/test_group.py | 2 +- bookwyrm/tests/views/test_helpers.py | 10 +-- bookwyrm/tests/views/test_interaction.py | 8 +-- bookwyrm/tests/views/test_list.py | 32 +++++---- bookwyrm/tests/views/test_list_actions.py | 66 +++++++++++-------- bookwyrm/tests/views/test_notifications.py | 2 +- bookwyrm/tests/views/test_outbox.py | 2 +- bookwyrm/tests/views/test_reading.py | 12 ++-- bookwyrm/tests/views/test_readthrough.py | 4 +- bookwyrm/tests/views/test_rss_feed.py | 2 +- bookwyrm/tests/views/test_search.py | 2 +- bookwyrm/tests/views/test_status.py | 8 +-- bookwyrm/tests/views/test_user.py | 8 ++- 57 files changed, 253 insertions(+), 212 deletions(-) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 8ae75baf..d97a1b8a 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -66,9 +66,10 @@ class BookDataModel(ObjectMixin, BookWyrmModel): self.remote_id = None return super().save(*args, **kwargs) - def broadcast(self, activity, sender, software="bookwyrm"): + # pylint: disable=arguments-differ + def broadcast(self, activity, sender, software="bookwyrm", **kwargs): """only send book data updates to other bookwyrm instances""" - super().broadcast(activity, sender, software=software) + super().broadcast(activity, sender, software=software, **kwargs) class Book(BookDataModel): diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index 7117eb52..b951c7ab 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -146,7 +146,7 @@ class BaseActivity(TestCase): self.user.avatar.file # pylint: disable=pointless-statement # this would trigger a broadcast because it's a local user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): activity.to_model(model=models.User, instance=self.user) self.assertIsNotNone(self.user.avatar.file) self.assertEqual(self.user.name, "New Name") @@ -154,7 +154,7 @@ class BaseActivity(TestCase): def test_to_model_many_to_many(self, *_): """annoying that these all need special handling""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create( content="test status", user=self.user, @@ -186,7 +186,7 @@ class BaseActivity(TestCase): def test_to_model_one_to_many(self, *_): """these are reversed relationships, where the secondary object keys the primary object but not vice versa""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create( content="test status", user=self.user, @@ -224,7 +224,7 @@ class BaseActivity(TestCase): @responses.activate def test_set_related_field(self, *_): """celery task to add back-references to created objects""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create( content="test status", user=self.user, diff --git a/bookwyrm/tests/activitystreams/test_abstractstream.py b/bookwyrm/tests/activitystreams/test_abstractstream.py index f9674286..17a1b587 100644 --- a/bookwyrm/tests/activitystreams/test_abstractstream.py +++ b/bookwyrm/tests/activitystreams/test_abstractstream.py @@ -4,7 +4,7 @@ from django.test import TestCase from bookwyrm import activitystreams, models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/activitystreams/test_booksstream.py b/bookwyrm/tests/activitystreams/test_booksstream.py index d6d94b73..347e7a94 100644 --- a/bookwyrm/tests/activitystreams/test_booksstream.py +++ b/bookwyrm/tests/activitystreams/test_booksstream.py @@ -4,7 +4,7 @@ from django.test import TestCase from bookwyrm import activitystreams, models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/activitystreams/test_homestream.py b/bookwyrm/tests/activitystreams/test_homestream.py index 42a73f3c..d48bdfba 100644 --- a/bookwyrm/tests/activitystreams/test_homestream.py +++ b/bookwyrm/tests/activitystreams/test_homestream.py @@ -4,7 +4,7 @@ from django.test import TestCase from bookwyrm import activitystreams, models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/activitystreams/test_localstream.py b/bookwyrm/tests/activitystreams/test_localstream.py index e6b05557..fa1a6741 100644 --- a/bookwyrm/tests/activitystreams/test_localstream.py +++ b/bookwyrm/tests/activitystreams/test_localstream.py @@ -4,7 +4,7 @@ from django.test import TestCase from bookwyrm import activitystreams, models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index eb70d28e..b3a99620 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -4,7 +4,7 @@ from django.test import TestCase from bookwyrm import activitystreams, models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class ActivitystreamsSignals(TestCase): """using redis to build activity streams""" diff --git a/bookwyrm/tests/activitystreams/test_tasks.py b/bookwyrm/tests/activitystreams/test_tasks.py index 80b0b771..f5750763 100644 --- a/bookwyrm/tests/activitystreams/test_tasks.py +++ b/bookwyrm/tests/activitystreams/test_tasks.py @@ -34,7 +34,7 @@ class Activitystreams(TestCase): ) work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.status = models.Status.objects.create( content="hi", user=self.local_user ) @@ -133,7 +133,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_another_timeline(self, *_): """boost from a non-follower doesn't remove original status from feed""" status = models.Status.objects.create(user=self.local_user, content="hi") @@ -155,7 +155,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_another_timeline_remote(self, *_): """boost from a remote non-follower doesn't remove original status from feed""" status = models.Status.objects.create(user=self.local_user, content="hi") @@ -177,7 +177,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_following_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" self.local_user.following.add(self.another_user) @@ -199,7 +199,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_same_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" status = models.Status.objects.create(user=self.local_user, content="hi") diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 87624a0e..12b5578b 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -91,7 +91,7 @@ class GoodreadsImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -115,7 +115,7 @@ class GoodreadsImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, True, "unlisted" ) @@ -135,7 +135,7 @@ class GoodreadsImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, True, "unlisted" ) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 55bfef72..ed081aa2 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -114,7 +114,7 @@ class LibrarythingImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -129,7 +129,7 @@ class LibrarythingImport(TestCase): def test_handle_imported_book_already_shelved(self, *_): """librarything import added a book, this adds related connections""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = self.local_user.shelf_set.filter(identifier="to-read").first() models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, book=self.book @@ -142,7 +142,7 @@ class LibrarythingImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -166,7 +166,7 @@ class LibrarythingImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, True, "unlisted" ) diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 2f0fd7ef..8002a3e1 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -95,7 +95,7 @@ class StorygraphImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, False, "public" ) @@ -116,7 +116,7 @@ class StorygraphImport(TestCase): import_item.book = self.book import_item.save() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): handle_imported_book( self.importer.service, self.local_user, import_item, True, "unlisted" ) diff --git a/bookwyrm/tests/management/test_populate_streams.py b/bookwyrm/tests/management/test_populate_streams.py index ca21b0ee..5be1774d 100644 --- a/bookwyrm/tests/management/test_populate_streams.py +++ b/bookwyrm/tests/management/test_populate_streams.py @@ -6,7 +6,7 @@ from bookwyrm import models from bookwyrm.management.commands.populate_streams import populate_streams -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class Activitystreams(TestCase): """using redis to build activity streams""" diff --git a/bookwyrm/tests/models/test_activitypub_mixin.py b/bookwyrm/tests/models/test_activitypub_mixin.py index 911cca5c..91a1fe7c 100644 --- a/bookwyrm/tests/models/test_activitypub_mixin.py +++ b/bookwyrm/tests/models/test_activitypub_mixin.py @@ -21,7 +21,7 @@ from bookwyrm.settings import PAGE_LENGTH # pylint: disable=invalid-name @patch("bookwyrm.activitystreams.add_status_task.delay") -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class ActivitypubMixins(TestCase): """functionality shared across models""" diff --git a/bookwyrm/tests/models/test_group.py b/bookwyrm/tests/models/test_group.py index 33341d19..2dd3cee1 100644 --- a/bookwyrm/tests/models/test_group.py +++ b/bookwyrm/tests/models/test_group.py @@ -5,7 +5,7 @@ from django.test import TestCase from bookwyrm import models, settings -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class Group(TestCase): """some activitypub oddness ahead""" @@ -87,7 +87,7 @@ class Group(TestCase): def test_group_members_can_see_followers_only_lists(self, _): """follower-only group booklists should not be excluded from group booklist listing for group members who do not follower list owner""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): followers_list = models.List.objects.create( name="Followers List", curation="group", @@ -107,7 +107,7 @@ class Group(TestCase): def test_group_members_can_see_private_lists(self, _): """private group booklists should not be excluded from group booklist listing for group members""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): private_list = models.List.objects.create( name="Private List", diff --git a/bookwyrm/tests/models/test_list.py b/bookwyrm/tests/models/test_list.py index 0d749453..254005bc 100644 --- a/bookwyrm/tests/models/test_list.py +++ b/bookwyrm/tests/models/test_list.py @@ -5,7 +5,7 @@ from django.test import TestCase from bookwyrm import models, settings -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class List(TestCase): """some activitypub oddness ahead""" @@ -22,7 +22,7 @@ class List(TestCase): def test_remote_id(self, _): """shelves use custom remote ids""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): book_list = models.List.objects.create( name="Test List", user=self.local_user ) @@ -31,7 +31,7 @@ class List(TestCase): def test_to_activity(self, _): """jsonify it""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): book_list = models.List.objects.create( name="Test List", user=self.local_user ) @@ -45,7 +45,7 @@ class List(TestCase): def test_list_item(self, _): """a list entry""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): book_list = models.List.objects.create( name="Test List", user=self.local_user, privacy="unlisted" ) @@ -63,7 +63,7 @@ class List(TestCase): def test_list_item_pending(self, _): """a list entry""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): book_list = models.List.objects.create( name="Test List", user=self.local_user ) diff --git a/bookwyrm/tests/models/test_relationship_models.py b/bookwyrm/tests/models/test_relationship_models.py index 04dbe1a3..2b388398 100644 --- a/bookwyrm/tests/models/test_relationship_models.py +++ b/bookwyrm/tests/models/test_relationship_models.py @@ -33,11 +33,13 @@ class Relationship(TestCase): def test_user_follows_from_request(self, _): """convert a follow request into a follow""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: request = models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user ) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Follow") self.assertEqual( request.remote_id, "http://local.com/user/mouse#follows/%d" % request.id @@ -54,7 +56,7 @@ class Relationship(TestCase): def test_user_follows_from_request_custom_remote_id(self, _): """store a specific remote id for a relationship provided by remote""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): request = models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user, @@ -69,19 +71,19 @@ class Relationship(TestCase): self.assertEqual(rel.user_subject, self.local_user) self.assertEqual(rel.user_object, self.remote_user) - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_follow_request_activity(self, broadcast_mock, _): """accept a request and make it a relationship""" models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user, ) - activity = json.loads(broadcast_mock.call_args[0][1]) + activity = json.loads(broadcast_mock.call_args[1]["args"][1]) self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"], self.remote_user.remote_id) self.assertEqual(activity["type"], "Follow") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_follow_request_accept(self, broadcast_mock, _): """accept a request and make it a relationship""" self.local_user.manually_approves_followers = True @@ -96,7 +98,7 @@ class Relationship(TestCase): ) request.accept() - activity = json.loads(broadcast_mock.call_args[0][1]) + activity = json.loads(broadcast_mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Accept") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], "https://www.hi.com/") @@ -107,7 +109,7 @@ class Relationship(TestCase): self.assertEqual(rel.user_subject, self.remote_user) self.assertEqual(rel.user_object, self.local_user) - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_follow_request_reject(self, broadcast_mock, _): """accept a request and make it a relationship""" self.local_user.manually_approves_followers = True @@ -120,7 +122,7 @@ class Relationship(TestCase): ) request.reject() - activity = json.loads(broadcast_mock.call_args[0][1]) + activity = json.loads(broadcast_mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Reject") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], request.remote_id) diff --git a/bookwyrm/tests/models/test_shelf_model.py b/bookwyrm/tests/models/test_shelf_model.py index fe179e88..0683fbef 100644 --- a/bookwyrm/tests/models/test_shelf_model.py +++ b/bookwyrm/tests/models/test_shelf_model.py @@ -27,7 +27,7 @@ class Shelf(TestCase): def test_remote_id(self, *_): """shelves use custom remote ids""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) @@ -36,7 +36,7 @@ class Shelf(TestCase): def test_to_activity(self, *_): """jsonify it""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) @@ -51,19 +51,23 @@ class Shelf(TestCase): def test_create_update_shelf(self, *_): """create and broadcast shelf creation""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Create") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["name"], "Test Shelf") shelf.name = "arthur russel" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: shelf.save() - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Update") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["name"], "arthur russel") @@ -71,27 +75,31 @@ class Shelf(TestCase): def test_shelve(self, *_): """create and broadcast shelf creation""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: shelf_book = models.ShelfBook.objects.create( shelf=shelf, user=self.local_user, book=self.book ) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], shelf_book.remote_id) self.assertEqual(activity["target"], shelf.remote_id) self.assertEqual(shelf.books.first(), self.book) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: shelf_book.delete() self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Remove") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], shelf_book.remote_id) diff --git a/bookwyrm/tests/models/test_user_model.py b/bookwyrm/tests/models/test_user_model.py index 528d3fdc..389928cd 100644 --- a/bookwyrm/tests/models/test_user_model.py +++ b/bookwyrm/tests/models/test_user_model.py @@ -165,12 +165,12 @@ class User(TestCase): """deactivate a user""" self.assertTrue(self.user.is_active) with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as broadcast_mock: self.user.delete() self.assertEqual(broadcast_mock.call_count, 1) - activity = json.loads(broadcast_mock.call_args[0][1]) + activity = json.loads(broadcast_mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.assertEqual(activity["object"], self.user.remote_id) self.assertFalse(self.user.is_active) diff --git a/bookwyrm/tests/test_postgres.py b/bookwyrm/tests/test_postgres.py index 70775d47..62451257 100644 --- a/bookwyrm/tests/test_postgres.py +++ b/bookwyrm/tests/test_postgres.py @@ -5,7 +5,7 @@ from django.test import TestCase from bookwyrm import models -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class PostgresTriggers(TestCase): """special migrations, fancy stuff ya know""" diff --git a/bookwyrm/tests/test_suggested_users.py b/bookwyrm/tests/test_suggested_users.py index f625ac10..dce5d770 100644 --- a/bookwyrm/tests/test_suggested_users.py +++ b/bookwyrm/tests/test_suggested_users.py @@ -9,7 +9,7 @@ from bookwyrm import models from bookwyrm.suggested_users import suggested_users, get_annotated_users -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") @@ -168,7 +168,7 @@ class SuggestedUsers(TestCase): remote_id="https://example.com/book/1", parent_work=work, ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): # 1 shared follow self.local_user.following.add(user_2) user_1.followers.add(user_2) @@ -213,7 +213,7 @@ class SuggestedUsers(TestCase): user.following.add(user_1) user.followers.add(self.local_user) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): for i in range(3): book = models.Edition.objects.create( title=i, diff --git a/bookwyrm/tests/test_templatetags.py b/bookwyrm/tests/test_templatetags.py index 5954ce27..ed4466f5 100644 --- a/bookwyrm/tests/test_templatetags.py +++ b/bookwyrm/tests/test_templatetags.py @@ -44,7 +44,7 @@ class TemplateTags(TestCase): def test_get_user_rating(self, *_): """get a user's most recent rating of a book""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.Review.objects.create(user=self.user, book=self.book, rating=3) self.assertEqual(bookwyrm_tags.get_user_rating(self.book, self.user), 3) @@ -63,7 +63,7 @@ class TemplateTags(TestCase): utilities.get_user_identifier(self.remote_user), "rat@example.com" ) - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_get_replies(self, *_): """direct replies to a status""" parent = models.Review.objects.create( @@ -90,7 +90,7 @@ class TemplateTags(TestCase): def test_get_parent(self, *_): """get the reply parent of a status""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): parent = models.Review.objects.create( user=self.user, book=self.book, content="hi" ) @@ -107,7 +107,7 @@ class TemplateTags(TestCase): status = models.Review.objects.create(user=self.remote_user, book=self.book) self.assertFalse(interaction.get_user_liked(self.user, status)) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.Favorite.objects.create(user=self.user, status=status) self.assertTrue(interaction.get_user_liked(self.user, status)) @@ -116,13 +116,13 @@ class TemplateTags(TestCase): status = models.Review.objects.create(user=self.remote_user, book=self.book) self.assertFalse(interaction.get_user_boosted(self.user, status)) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.Boost.objects.create(user=self.user, boosted_status=status) self.assertTrue(interaction.get_user_boosted(self.user, status)) def test_get_boosted(self, *_): """load a boosted status""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Review.objects.create(user=self.remote_user, book=self.book) boost = models.Boost.objects.create(user=self.user, boosted_status=status) boosted = status_display.get_boosted(boost) @@ -166,7 +166,7 @@ class TemplateTags(TestCase): def test_related_status(self, *_): """gets the subclass model for a notification status""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create(content="hi", user=self.user) notification = models.Notification.objects.create( user=self.user, notification_type="MENTION", related_status=status diff --git a/bookwyrm/tests/views/admin/test_reports.py b/bookwyrm/tests/views/admin/test_reports.py index 2b063446..8b9fe9f5 100644 --- a/bookwyrm/tests/views/admin/test_reports.py +++ b/bookwyrm/tests/views/admin/test_reports.py @@ -151,10 +151,12 @@ class ReportViews(TestCase): request.user.is_superuser = True # de-activate - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.moderator_delete_user(request, self.rat.id) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.rat.refresh_from_db() diff --git a/bookwyrm/tests/views/admin/test_user_admin.py b/bookwyrm/tests/views/admin/test_user_admin.py index ef35c220..486fe45e 100644 --- a/bookwyrm/tests/views/admin/test_user_admin.py +++ b/bookwyrm/tests/views/admin/test_user_admin.py @@ -67,7 +67,7 @@ class UserAdminViews(TestCase): request.user = self.local_user request.user.is_superuser = True - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): result = view(request, self.local_user.id) self.assertIsInstance(result, TemplateResponse) diff --git a/bookwyrm/tests/views/books/test_book.py b/bookwyrm/tests/views/books/test_book.py index a078f161..561e2192 100644 --- a/bookwyrm/tests/views/books/test_book.py +++ b/bookwyrm/tests/views/books/test_book.py @@ -78,7 +78,7 @@ class BookViews(TestCase): self.assertIsInstance(result, ActivitypubResponse) self.assertEqual(result.status_code, 200) - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") def test_book_page_statuses(self, *_): """there are so many views, this just makes sure it LOADS""" @@ -169,7 +169,7 @@ class BookViews(TestCase): request.user = self.local_user with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: views.upload_cover(request, self.book.id) self.assertEqual(delay_mock.call_count, 1) @@ -188,7 +188,7 @@ class BookViews(TestCase): request.user = self.local_user with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: views.upload_cover(request, self.book.id) self.assertEqual(delay_mock.call_count, 1) @@ -202,7 +202,7 @@ class BookViews(TestCase): request = self.factory.post("", {"description": "new description hi"}) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.add_description(request, self.book.id) self.book.refresh_from_db() diff --git a/bookwyrm/tests/views/books/test_edit_book.py b/bookwyrm/tests/views/books/test_edit_book.py index 7bf5708f..cd957858 100644 --- a/bookwyrm/tests/views/books/test_edit_book.py +++ b/bookwyrm/tests/views/books/test_edit_book.py @@ -79,7 +79,7 @@ class EditBookViews(TestCase): request = self.factory.post("", form.data) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, self.book.id) self.book.refresh_from_db() @@ -115,7 +115,7 @@ class EditBookViews(TestCase): request = self.factory.post("", form.data) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, self.book.id) self.book.refresh_from_db() @@ -136,7 +136,7 @@ class EditBookViews(TestCase): request = self.factory.post("", form.data) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, self.book.id) self.book.refresh_from_db() self.assertEqual(self.book.title, "New Title") @@ -207,7 +207,7 @@ class EditBookViews(TestCase): request.user = self.local_user with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: views.upload_cover(request, self.book.id) self.assertEqual(delay_mock.call_count, 1) diff --git a/bookwyrm/tests/views/books/test_editions.py b/bookwyrm/tests/views/books/test_editions.py index 2f41fe66..17f15654 100644 --- a/bookwyrm/tests/views/books/test_editions.py +++ b/bookwyrm/tests/views/books/test_editions.py @@ -111,7 +111,7 @@ class BookViews(TestCase): work = models.Work.objects.create(title="test work") edition1 = models.Edition.objects.create(title="first ed", parent_work=work) edition2 = models.Edition.objects.create(title="second ed", parent_work=work) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): shelf = models.Shelf.objects.create(name="Test Shelf", user=self.local_user) models.ShelfBook.objects.create( book=edition1, @@ -124,7 +124,7 @@ class BookViews(TestCase): self.assertEqual(models.ReadThrough.objects.get().book, edition1) request = self.factory.post("", {"edition": edition2.id}) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.switch_edition(request) self.assertEqual(models.ShelfBook.objects.get().book, edition2) diff --git a/bookwyrm/tests/views/inbox/test_inbox_announce.py b/bookwyrm/tests/views/inbox/test_inbox_announce.py index 3a108878..a291552d 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_announce.py +++ b/bookwyrm/tests/views/inbox/test_inbox_announce.py @@ -36,7 +36,7 @@ class InboxActivities(TestCase): outbox="https://example.com/users/rat/outbox", ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): with patch("bookwyrm.activitystreams.add_status_task.delay"): self.status = models.Status.objects.create( user=self.local_user, diff --git a/bookwyrm/tests/views/inbox/test_inbox_block.py b/bookwyrm/tests/views/inbox/test_inbox_block.py index ffd74dbd..f6898fc6 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_block.py +++ b/bookwyrm/tests/views/inbox/test_inbox_block.py @@ -40,7 +40,7 @@ class InboxBlock(TestCase): def test_handle_blocks(self): """create a "block" database entry from an activity""" self.local_user.followers.add(self.remote_user) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user ) diff --git a/bookwyrm/tests/views/inbox/test_inbox_create.py b/bookwyrm/tests/views/inbox/test_inbox_create.py index 76fd366c..53b17d68 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_create.py +++ b/bookwyrm/tests/views/inbox/test_inbox_create.py @@ -10,7 +10,7 @@ from bookwyrm.activitypub import ActivitySerializerError # pylint: disable=too-many-public-methods -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") class InboxCreate(TestCase): """readthrough tests""" diff --git a/bookwyrm/tests/views/inbox/test_inbox_follow.py b/bookwyrm/tests/views/inbox/test_inbox_follow.py index 6b629c2f..71f101ca 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_follow.py +++ b/bookwyrm/tests/views/inbox/test_inbox_follow.py @@ -49,10 +49,12 @@ class InboxRelationships(TestCase): } self.assertFalse(models.UserFollowRequest.objects.exists()) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.inbox.activity_task(activity) self.assertEqual(mock.call_count, 1) - response_activity = json.loads(mock.call_args[0][1]) + response_activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(response_activity["type"], "Accept") # notification created @@ -77,17 +79,19 @@ class InboxRelationships(TestCase): "object": "https://example.com/user/mouse", } - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.inbox.activity_task(activity) # the follow relationship should exist follow = models.UserFollows.objects.get(user_object=self.local_user) self.assertEqual(follow.user_subject, self.remote_user) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.inbox.activity_task(activity) self.assertEqual(mock.call_count, 1) - response_activity = json.loads(mock.call_args[0][1]) + response_activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(response_activity["type"], "Accept") # the follow relationship should STILL exist @@ -109,7 +113,7 @@ class InboxRelationships(TestCase): broadcast=False, update_fields=["manually_approves_followers"] ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.inbox.activity_task(activity) # notification created @@ -132,7 +136,7 @@ class InboxRelationships(TestCase): self.local_user.save( broadcast=False, update_fields=["manually_approves_followers"] ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): request = models.UserFollowRequest.objects.create( user_subject=self.remote_user, user_object=self.local_user ) @@ -160,7 +164,7 @@ class InboxRelationships(TestCase): def test_unfollow(self): """remove a relationship""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): rel = models.UserFollows.objects.create( user_subject=self.remote_user, user_object=self.local_user ) @@ -186,7 +190,7 @@ class InboxRelationships(TestCase): @patch("bookwyrm.activitystreams.add_user_statuses_task.delay") def test_follow_accept(self, _): """a remote user approved a follow request from local""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): rel = models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user ) @@ -217,7 +221,7 @@ class InboxRelationships(TestCase): def test_follow_reject(self): """turn down a follow request""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): rel = models.UserFollowRequest.objects.create( user_subject=self.local_user, user_object=self.remote_user ) diff --git a/bookwyrm/tests/views/inbox/test_inbox_like.py b/bookwyrm/tests/views/inbox/test_inbox_like.py index db8f1fca..2f1b6629 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_like.py +++ b/bookwyrm/tests/views/inbox/test_inbox_like.py @@ -35,7 +35,7 @@ class InboxActivities(TestCase): outbox="https://example.com/users/rat/outbox", ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): with patch("bookwyrm.activitystreams.add_status_task.delay"): self.status = models.Status.objects.create( user=self.local_user, diff --git a/bookwyrm/tests/views/inbox/test_inbox_remove.py b/bookwyrm/tests/views/inbox/test_inbox_remove.py index cb4e4a16..55cc8120 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_remove.py +++ b/bookwyrm/tests/views/inbox/test_inbox_remove.py @@ -75,7 +75,7 @@ class InboxRemove(TestCase): def test_handle_remove_book_from_list(self): """listing a book""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): booklist = models.List.objects.create( name="test list", user=self.local_user, diff --git a/bookwyrm/tests/views/inbox/test_inbox_update.py b/bookwyrm/tests/views/inbox/test_inbox_update.py index 0efeac0c..248c1ad2 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_update.py +++ b/bookwyrm/tests/views/inbox/test_inbox_update.py @@ -50,7 +50,7 @@ class InboxUpdate(TestCase): def test_update_list(self): """a new list""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): book_list = models.List.objects.create( name="hi", remote_id="https://example.com/list/22", user=self.local_user ) @@ -171,7 +171,7 @@ class InboxUpdate(TestCase): book = models.Work.objects.get(id=book.id) self.assertEqual(book.title, "Piranesi") - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") def test_update_status(self, *_): """edit a status""" diff --git a/bookwyrm/tests/views/preferences/test_block.py b/bookwyrm/tests/views/preferences/test_block.py index b23a6cbc..975142a1 100644 --- a/bookwyrm/tests/views/preferences/test_block.py +++ b/bookwyrm/tests/views/preferences/test_block.py @@ -9,7 +9,7 @@ from bookwyrm import models, views from bookwyrm.tests.validate_html import validate_html -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class BlockViews(TestCase): """view user and edit profile""" diff --git a/bookwyrm/tests/views/preferences/test_delete_user.py b/bookwyrm/tests/views/preferences/test_delete_user.py index 09f722f7..b6d87ccd 100644 --- a/bookwyrm/tests/views/preferences/test_delete_user.py +++ b/bookwyrm/tests/views/preferences/test_delete_user.py @@ -35,9 +35,9 @@ class DeleteUserViews(TestCase): self.book = models.Edition.objects.create( title="test", parent_work=models.Work.objects.create(title="test work") ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"), patch( - "bookwyrm.activitystreams.add_book_statuses_task.delay" - ): + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.activitystreams.add_book_statuses_task.delay"): models.ShelfBook.objects.create( book=self.book, user=self.local_user, @@ -70,11 +70,11 @@ class DeleteUserViews(TestCase): self.assertIsNone(self.local_user.name) with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: view(request) self.assertEqual(delay_mock.call_count, 1) - activity = json.loads(delay_mock.call_args[0][1]) + activity = json.loads(delay_mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual( diff --git a/bookwyrm/tests/views/preferences/test_edit_user.py b/bookwyrm/tests/views/preferences/test_edit_user.py index b52875a9..7a845fbe 100644 --- a/bookwyrm/tests/views/preferences/test_edit_user.py +++ b/bookwyrm/tests/views/preferences/test_edit_user.py @@ -38,9 +38,9 @@ class EditUserViews(TestCase): self.book = models.Edition.objects.create( title="test", parent_work=models.Work.objects.create(title="test work") ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"), patch( - "bookwyrm.activitystreams.add_book_statuses_task.delay" - ): + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.activitystreams.add_book_statuses_task.delay"): models.ShelfBook.objects.create( book=self.book, user=self.local_user, @@ -74,7 +74,7 @@ class EditUserViews(TestCase): self.assertIsNone(self.local_user.name) with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: view(request) self.assertEqual(delay_mock.call_count, 1) @@ -100,7 +100,7 @@ class EditUserViews(TestCase): request.user = self.local_user with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: view(request) self.assertEqual(delay_mock.call_count, 1) diff --git a/bookwyrm/tests/views/shelf/test_shelf.py b/bookwyrm/tests/views/shelf/test_shelf.py index 71df3631..ab88de0a 100644 --- a/bookwyrm/tests/views/shelf/test_shelf.py +++ b/bookwyrm/tests/views/shelf/test_shelf.py @@ -11,7 +11,7 @@ from bookwyrm.activitypub import ActivitypubResponse from bookwyrm.tests.validate_html import validate_html -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @@ -39,7 +39,7 @@ class ShelfViews(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) @@ -142,7 +142,7 @@ class ShelfViews(TestCase): "", {"privacy": "public", "user": self.local_user.id, "name": "cool name"} ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, request.user.username, shelf.identifier) shelf.refresh_from_db() @@ -159,7 +159,7 @@ class ShelfViews(TestCase): "", {"privacy": "public", "user": self.local_user.id, "name": "cool name"} ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, request.user.username, shelf.identifier) self.assertEqual(shelf.name, "To Read") diff --git a/bookwyrm/tests/views/shelf/test_shelf_actions.py b/bookwyrm/tests/views/shelf/test_shelf_actions.py index 3efae0f4..1a7d56fd 100644 --- a/bookwyrm/tests/views/shelf/test_shelf_actions.py +++ b/bookwyrm/tests/views/shelf/test_shelf_actions.py @@ -9,7 +9,7 @@ from django.test.client import RequestFactory from bookwyrm import forms, models, views -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @@ -37,7 +37,7 @@ class ShelfActionViews(TestCase): remote_id="https://example.com/book/1", parent_work=self.work, ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) @@ -49,11 +49,13 @@ class ShelfActionViews(TestCase): "", {"book": self.book.id, "shelf": self.shelf.identifier} ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.shelve(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") item = models.ShelfBook.objects.get() @@ -69,7 +71,7 @@ class ShelfActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.shelve(request) # make sure the book is on the shelf self.assertEqual(shelf.books.get(), self.book) @@ -82,7 +84,7 @@ class ShelfActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.shelve(request) # make sure the book is on the shelf self.assertEqual(shelf.books.get(), self.book) @@ -95,7 +97,7 @@ class ShelfActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.shelve(request) # make sure the book is on the shelf self.assertEqual(shelf.books.get(), self.book) @@ -118,7 +120,7 @@ class ShelfActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.shelve(request) # make sure the book is on the shelf self.assertEqual(shelf.books.get(), self.book) @@ -126,7 +128,7 @@ class ShelfActionViews(TestCase): def test_unshelve(self, *_): """remove a book from a shelf""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ShelfBook.objects.create( book=self.book, user=self.local_user, shelf=self.shelf ) @@ -136,9 +138,11 @@ class ShelfActionViews(TestCase): self.assertEqual(self.shelf.books.count(), 1) request = self.factory.post("", {"book": self.book.id, "shelf": self.shelf.id}) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.unshelve(request) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Remove") self.assertEqual(activity["object"]["id"], item.remote_id) self.assertEqual(self.shelf.books.count(), 0) @@ -192,7 +196,7 @@ class ShelfActionViews(TestCase): def test_delete_shelf_has_book(self, *_): """delete a brand new custom shelf""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ShelfBook.objects.create( book=self.book, user=self.local_user, shelf=self.shelf ) diff --git a/bookwyrm/tests/views/test_author.py b/bookwyrm/tests/views/test_author.py index ccbfe549..32b1565e 100644 --- a/bookwyrm/tests/views/test_author.py +++ b/bookwyrm/tests/views/test_author.py @@ -111,7 +111,7 @@ class AuthorViews(TestCase): request = self.factory.post("", form.data) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, author.id) author.refresh_from_db() self.assertEqual(author.name, "New Name") diff --git a/bookwyrm/tests/views/test_discover.py b/bookwyrm/tests/views/test_discover.py index 4b8927bc..b2a82241 100644 --- a/bookwyrm/tests/views/test_discover.py +++ b/bookwyrm/tests/views/test_discover.py @@ -41,7 +41,7 @@ class DiscoverViews(TestCase): self.assertEqual(result.status_code, 200) result.render() - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_status_task.delay") def test_discover_page(self, *_): """there are so many views, this just makes sure it LOADS""" diff --git a/bookwyrm/tests/views/test_feed.py b/bookwyrm/tests/views/test_feed.py index a6f220b5..5c6a4dd3 100644 --- a/bookwyrm/tests/views/test_feed.py +++ b/bookwyrm/tests/views/test_feed.py @@ -57,7 +57,7 @@ class FeedViews(TestCase): def test_status_page(self, *_): """there are so many views, this just makes sure it LOADS""" view = views.Status.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create(content="hi", user=self.local_user) request = self.factory.get("") request.user = self.local_user @@ -95,7 +95,7 @@ class FeedViews(TestCase): local=True, localname="rat", ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create(content="hi", user=another_user) request = self.factory.get("") @@ -115,7 +115,7 @@ class FeedViews(TestCase): image = Image.open(image_file) output = BytesIO() image.save(output, format=image.format) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Review.objects.create( content="hi", user=self.local_user, @@ -144,7 +144,7 @@ class FeedViews(TestCase): def test_replies_page(self, *_): """there are so many views, this just makes sure it LOADS""" view = views.Replies.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): status = models.Status.objects.create(content="hi", user=self.local_user) request = self.factory.get("") request.user = self.local_user @@ -171,7 +171,7 @@ class FeedViews(TestCase): result.render() self.assertEqual(result.status_code, 200) - @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") def test_get_suggested_book(self, *_): """gets books the ~*~ algorithm ~*~ thinks you want to post about""" diff --git a/bookwyrm/tests/views/test_follow.py b/bookwyrm/tests/views/test_follow.py index 947c55cf..25b5a014 100644 --- a/bookwyrm/tests/views/test_follow.py +++ b/bookwyrm/tests/views/test_follow.py @@ -59,7 +59,7 @@ class FollowViews(TestCase): request.user = self.local_user self.assertEqual(models.UserFollowRequest.objects.count(), 0) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.follow(request) rel = models.UserFollowRequest.objects.get() @@ -86,7 +86,7 @@ class FollowViews(TestCase): request.user = self.local_user self.assertEqual(models.UserFollowRequest.objects.count(), 0) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.follow(request) rel = models.UserFollowRequest.objects.get() @@ -111,7 +111,7 @@ class FollowViews(TestCase): request.user = self.local_user self.assertEqual(models.UserFollowRequest.objects.count(), 0) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.follow(request) rel = models.UserFollows.objects.get() @@ -127,10 +127,12 @@ class FollowViews(TestCase): request.user = self.local_user self.remote_user.followers.add(self.local_user) self.assertEqual(self.remote_user.followers.count(), 1) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.unfollow(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args_list[0][0][1]) + activity = json.loads(mock.call_args_list[0][1]["args"][1]) self.assertEqual(activity["type"], "Undo") self.assertEqual(self.remote_user.followers.count(), 0) @@ -147,7 +149,7 @@ class FollowViews(TestCase): user_subject=self.remote_user, user_object=self.local_user ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.accept_follow_request(request) # request should be deleted self.assertEqual(models.UserFollowRequest.objects.filter(id=rel.id).count(), 0) @@ -166,7 +168,7 @@ class FollowViews(TestCase): user_subject=self.remote_user, user_object=self.local_user ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.delete_follow_request(request) # request should be deleted self.assertEqual(models.UserFollowRequest.objects.filter(id=rel.id).count(), 0) diff --git a/bookwyrm/tests/views/test_get_started.py b/bookwyrm/tests/views/test_get_started.py index ff441b57..6d1819a4 100644 --- a/bookwyrm/tests/views/test_get_started.py +++ b/bookwyrm/tests/views/test_get_started.py @@ -56,7 +56,7 @@ class GetStartedViews(TestCase): self.assertIsNone(self.local_user.name) with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: view(request) self.assertEqual(delay_mock.call_count, 1) @@ -98,7 +98,7 @@ class GetStartedViews(TestCase): self.assertFalse(self.local_user.shelfbook_set.exists()) with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as delay_mock: view(request) self.assertEqual(delay_mock.call_count, 1) diff --git a/bookwyrm/tests/views/test_goal.py b/bookwyrm/tests/views/test_goal.py index 557510d7..73207240 100644 --- a/bookwyrm/tests/views/test_goal.py +++ b/bookwyrm/tests/views/test_goal.py @@ -123,7 +123,7 @@ class GoalViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, self.local_user.localname, self.year) goal = models.AnnualGoal.objects.get() diff --git a/bookwyrm/tests/views/test_group.py b/bookwyrm/tests/views/test_group.py index c7e0a0f7..b18ce6b4 100644 --- a/bookwyrm/tests/views/test_group.py +++ b/bookwyrm/tests/views/test_group.py @@ -10,7 +10,7 @@ from bookwyrm import models, views, forms from bookwyrm.tests.validate_html import validate_html -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class GroupViews(TestCase): """view group and edit details""" diff --git a/bookwyrm/tests/views/test_helpers.py b/bookwyrm/tests/views/test_helpers.py index 8f7b45a3..1aae830f 100644 --- a/bookwyrm/tests/views/test_helpers.py +++ b/bookwyrm/tests/views/test_helpers.py @@ -55,7 +55,7 @@ class ViewsHelpers(TestCase): datafile = pathlib.Path(__file__).parent.joinpath("../data/ap_user.json") self.userdata = json.loads(datafile.read_bytes()) del self.userdata["icon"] - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.shelf = models.Shelf.objects.create( name="Test Shelf", identifier="test-shelf", user=self.local_user ) @@ -166,7 +166,7 @@ class ViewsHelpers(TestCase): def test_handle_reading_status_to_read(self, *_): """posts shelve activities""" shelf = self.local_user.shelf_set.get(identifier="to-read") - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.helpers.handle_reading_status( self.local_user, shelf, self.book, "public" ) @@ -178,7 +178,7 @@ class ViewsHelpers(TestCase): def test_handle_reading_status_reading(self, *_): """posts shelve activities""" shelf = self.local_user.shelf_set.get(identifier="reading") - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.helpers.handle_reading_status( self.local_user, shelf, self.book, "public" ) @@ -190,7 +190,7 @@ class ViewsHelpers(TestCase): def test_handle_reading_status_read(self, *_): """posts shelve activities""" shelf = self.local_user.shelf_set.get(identifier="read") - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.helpers.handle_reading_status( self.local_user, shelf, self.book, "public" ) @@ -201,7 +201,7 @@ class ViewsHelpers(TestCase): def test_handle_reading_status_other(self, *_): """posts shelve activities""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.helpers.handle_reading_status( self.local_user, self.shelf, self.book, "public" ) diff --git a/bookwyrm/tests/views/test_interaction.py b/bookwyrm/tests/views/test_interaction.py index e8b9353e..aa402952 100644 --- a/bookwyrm/tests/views/test_interaction.py +++ b/bookwyrm/tests/views/test_interaction.py @@ -7,7 +7,7 @@ from django.test.client import RequestFactory from bookwyrm import models, views -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.remove_status_task.delay") class InteractionViews(TestCase): """viewing and creating statuses""" @@ -74,7 +74,7 @@ class InteractionViews(TestCase): self.assertEqual(models.Favorite.objects.count(), 1) self.assertEqual(models.Notification.objects.count(), 1) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, status.id) self.assertEqual(models.Favorite.objects.count(), 0) self.assertEqual(models.Notification.objects.count(), 0) @@ -110,12 +110,12 @@ class InteractionViews(TestCase): status = models.Status.objects.create(user=self.local_user, content="hi") with patch( - "bookwyrm.models.activitypub_mixin.broadcast_task.delay" + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as broadcast_mock: view(request, status.id) self.assertEqual(broadcast_mock.call_count, 1) - activity = json.loads(broadcast_mock.call_args[0][1]) + activity = json.loads(broadcast_mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Announce") boost = models.Boost.objects.get() diff --git a/bookwyrm/tests/views/test_list.py b/bookwyrm/tests/views/test_list.py index 35befec6..c670ad18 100644 --- a/bookwyrm/tests/views/test_list.py +++ b/bookwyrm/tests/views/test_list.py @@ -61,7 +61,7 @@ class ListViews(TestCase): parent_work=work_four, ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.list = models.List.objects.create( name="Test List", user=self.local_user ) @@ -73,7 +73,7 @@ class ListViews(TestCase): def test_lists_page(self): """there are so many views, this just makes sure it LOADS""" view = views.Lists.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.List.objects.create(name="Public list", user=self.local_user) models.List.objects.create( name="Private list", privacy="direct", user=self.local_user @@ -96,7 +96,7 @@ class ListViews(TestCase): def test_saved_lists_page(self): """there are so many views, this just makes sure it LOADS""" view = views.SavedLists.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): booklist = models.List.objects.create( name="Public list", user=self.local_user ) @@ -116,7 +116,7 @@ class ListViews(TestCase): def test_saved_lists_page_empty(self): """there are so many views, this just makes sure it LOADS""" view = views.SavedLists.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.List.objects.create(name="Public list", user=self.local_user) models.List.objects.create( name="Private list", privacy="direct", user=self.local_user @@ -153,11 +153,13 @@ class ListViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: result = view(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Create") self.assertEqual(activity["actor"], self.local_user.remote_id) @@ -172,7 +174,7 @@ class ListViews(TestCase): view = views.List.as_view() request = self.factory.get("") request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -191,7 +193,7 @@ class ListViews(TestCase): def test_list_page_sorted(self): """there are so many views, this just makes sure it LOADS""" view = views.List.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): for (i, book) in enumerate([self.book, self.book_two, self.book_three]): models.ListItem.objects.create( book_list=self.list, @@ -253,7 +255,7 @@ class ListViews(TestCase): def test_list_page_logged_out(self): """there are so many views, this just makes sure it LOADS""" view = views.List.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -276,7 +278,7 @@ class ListViews(TestCase): view = views.List.as_view() request = self.factory.get("") request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -320,11 +322,13 @@ class ListViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: result = view(request, self.list.id) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Update") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], self.list.remote_id) @@ -340,7 +344,7 @@ class ListViews(TestCase): def test_curate_page(self): """there are so many views, this just makes sure it LOADS""" view = views.Curate.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.List.objects.create(name="Public list", user=self.local_user) models.List.objects.create( name="Private list", privacy="direct", user=self.local_user @@ -360,7 +364,7 @@ class ListViews(TestCase): def test_user_lists_page(self): """there are so many views, this just makes sure it LOADS""" view = views.UserLists.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.List.objects.create(name="Public list", user=self.local_user) models.List.objects.create( name="Private list", privacy="direct", user=self.local_user diff --git a/bookwyrm/tests/views/test_list_actions.py b/bookwyrm/tests/views/test_list_actions.py index f7775d19..1d9f46b3 100644 --- a/bookwyrm/tests/views/test_list_actions.py +++ b/bookwyrm/tests/views/test_list_actions.py @@ -61,7 +61,7 @@ class ListActionViews(TestCase): parent_work=work_four, ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.list = models.List.objects.create( name="Test List", user=self.local_user ) @@ -71,7 +71,7 @@ class ListActionViews(TestCase): def test_delete_list(self): """delete an entire list""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -88,9 +88,11 @@ class ListActionViews(TestCase): ) request = self.factory.post("") request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.delete_list(request, self.list.id) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"]["id"], self.list.remote_id) @@ -110,7 +112,7 @@ class ListActionViews(TestCase): def test_curate_approve(self): """approve a pending item""" view = views.Curate.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): pending = models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -128,11 +130,13 @@ class ListActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: view(request, self.list.id) self.assertEqual(mock.call_count, 2) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["target"], self.list.remote_id) @@ -145,7 +149,7 @@ class ListActionViews(TestCase): def test_curate_reject(self): """approve a pending item""" view = views.Curate.as_view() - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): pending = models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -179,10 +183,12 @@ class ListActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.list.add_book(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["target"], self.list.remote_id) @@ -214,7 +220,7 @@ class ListActionViews(TestCase): }, ) request_two.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.add_book(request_one) views.list.add_book(request_two) @@ -256,7 +262,7 @@ class ListActionViews(TestCase): ) request_three.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.add_book(request_one) views.list.add_book(request_two) views.list.add_book(request_three) @@ -271,7 +277,7 @@ class ListActionViews(TestCase): remove_request = self.factory.post("", {"item": items[1].id}) remove_request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.remove_book(remove_request, self.list.id) items = self.list.listitem_set.order_by("order").all() self.assertEqual(items[0].book, self.book) @@ -293,7 +299,7 @@ class ListActionViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -330,7 +336,7 @@ class ListActionViews(TestCase): its order should be at the end of the approved books and before the remaining pending books. """ - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -370,7 +376,7 @@ class ListActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): view(request, self.list.id) items = self.list.listitem_set.order_by("order").all() @@ -422,7 +428,7 @@ class ListActionViews(TestCase): ) request_three.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.add_book(request_one) views.list.add_book(request_two) views.list.add_book(request_three) @@ -437,7 +443,7 @@ class ListActionViews(TestCase): set_position_request = self.factory.post("", {"position": 1}) set_position_request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.set_book_position(set_position_request, items[2].id) items = self.list.listitem_set.order_by("order").all() self.assertEqual(items[0].book, self.book_three) @@ -460,10 +466,12 @@ class ListActionViews(TestCase): ) request.user = self.rat - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.list.add_book(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.rat.remote_id) self.assertEqual(activity["target"], self.list.remote_id) @@ -486,11 +494,13 @@ class ListActionViews(TestCase): ) request.user = self.rat - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.list.add_book(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.rat.remote_id) @@ -516,10 +526,12 @@ class ListActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") as mock: + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ) as mock: views.list.add_book(request) self.assertEqual(mock.call_count, 1) - activity = json.loads(mock.call_args[0][1]) + activity = json.loads(mock.call_args[1]["args"][1]) self.assertEqual(activity["type"], "Add") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["target"], self.list.remote_id) @@ -532,7 +544,7 @@ class ListActionViews(TestCase): def test_remove_book(self): """take an item off a list""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): item = models.ListItem.objects.create( book_list=self.list, user=self.local_user, @@ -549,13 +561,13 @@ class ListActionViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.list.remove_book(request, self.list.id) self.assertFalse(self.list.listitem_set.exists()) def test_remove_book_unauthorized(self): """take an item off a list""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): item = models.ListItem.objects.create( book_list=self.list, user=self.local_user, book=self.book, order=1 ) diff --git a/bookwyrm/tests/views/test_notifications.py b/bookwyrm/tests/views/test_notifications.py index dd28a811..5df62b1d 100644 --- a/bookwyrm/tests/views/test_notifications.py +++ b/bookwyrm/tests/views/test_notifications.py @@ -25,7 +25,7 @@ class NotificationViews(TestCase): local=True, localname="mouse", ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.status = models.Status.objects.create( content="hi", user=self.local_user, diff --git a/bookwyrm/tests/views/test_outbox.py b/bookwyrm/tests/views/test_outbox.py index a1f62cc6..5c5d47b0 100644 --- a/bookwyrm/tests/views/test_outbox.py +++ b/bookwyrm/tests/views/test_outbox.py @@ -11,7 +11,7 @@ from bookwyrm.settings import USER_AGENT # pylint: disable=too-many-public-methods -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class OutboxView(TestCase): """sends out activities""" diff --git a/bookwyrm/tests/views/test_reading.py b/bookwyrm/tests/views/test_reading.py index 4e5206f6..4ec50165 100644 --- a/bookwyrm/tests/views/test_reading.py +++ b/bookwyrm/tests/views/test_reading.py @@ -64,7 +64,7 @@ class ReadingViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.ReadingStatus.as_view()(request, "start", self.book.id) self.assertEqual(shelf.books.get(), self.book) @@ -100,7 +100,7 @@ class ReadingViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.ReadingStatus.as_view()(request, "start", self.book.id) self.assertEqual(shelf.books.get(), self.book) @@ -124,7 +124,7 @@ class ReadingViews(TestCase): def test_start_reading_reshelve(self, *_): """begin a book""" to_read_shelf = self.local_user.shelf_set.get(identifier=models.Shelf.TO_READ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): models.ShelfBook.objects.create( shelf=to_read_shelf, book=self.book, user=self.local_user ) @@ -135,7 +135,7 @@ class ReadingViews(TestCase): request = self.factory.post("") request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.ReadingStatus.as_view()(request, "start", self.book.id) self.assertFalse(to_read_shelf.books.exists()) @@ -162,7 +162,7 @@ class ReadingViews(TestCase): ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.ReadingStatus.as_view()(request, "finish", self.book.id) self.assertEqual(shelf.books.get(), self.book) @@ -267,7 +267,7 @@ class ReadingViews(TestCase): }, ) request.user = self.local_user - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): views.update_progress(request, self.book.id) status = models.Comment.objects.get() diff --git a/bookwyrm/tests/views/test_readthrough.py b/bookwyrm/tests/views/test_readthrough.py index ef149ce3..5b554748 100644 --- a/bookwyrm/tests/views/test_readthrough.py +++ b/bookwyrm/tests/views/test_readthrough.py @@ -9,7 +9,7 @@ from bookwyrm import models @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.add_book_statuses_task.delay") @patch("bookwyrm.activitystreams.remove_book_statuses_task.delay") class ReadThrough(TestCase): @@ -32,7 +32,7 @@ class ReadThrough(TestCase): "cinco", "cinco@example.com", "seissiete", local=True, localname="cinco" ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): self.client.force_login(self.user) @patch("bookwyrm.activitystreams.remove_user_statuses_task.delay") diff --git a/bookwyrm/tests/views/test_rss_feed.py b/bookwyrm/tests/views/test_rss_feed.py index d4d11261..409c306d 100644 --- a/bookwyrm/tests/views/test_rss_feed.py +++ b/bookwyrm/tests/views/test_rss_feed.py @@ -6,7 +6,7 @@ from bookwyrm import models from bookwyrm.views import rss_feed -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") @patch("bookwyrm.activitystreams.ActivityStream.get_activity_stream") @patch("bookwyrm.activitystreams.add_status_task.delay") class RssFeedView(TestCase): diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 3299249a..f8045511 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -139,7 +139,7 @@ class Views(TestCase): def test_search_lists(self): """searches remote connectors""" - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): booklist = models.List.objects.create( user=self.local_user, name="test list" ) diff --git a/bookwyrm/tests/views/test_status.py b/bookwyrm/tests/views/test_status.py index db61b152..b5d7ac16 100644 --- a/bookwyrm/tests/views/test_status.py +++ b/bookwyrm/tests/views/test_status.py @@ -14,7 +14,7 @@ from bookwyrm.tests.validate_html import validate_html @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") @patch("bookwyrm.activitystreams.remove_status_task.delay") -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") class StatusViews(TestCase): """viewing and creating statuses""" @@ -310,7 +310,7 @@ http://www.fish.com/""" with patch("bookwyrm.activitystreams.remove_status_task.delay") as redis_mock: view(request, status.id) self.assertTrue(redis_mock.called) - activity = json.loads(mock.call_args_list[1][0][1]) + activity = json.loads(mock.call_args_list[1][1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.assertEqual(activity["object"]["type"], "Tombstone") status.refresh_from_db() @@ -344,7 +344,7 @@ http://www.fish.com/""" with patch("bookwyrm.activitystreams.remove_status_task.delay") as redis_mock: view(request, status.id) self.assertTrue(redis_mock.called) - activity = json.loads(mock.call_args_list[1][0][1]) + activity = json.loads(mock.call_args_list[1][1]["args"][1]) self.assertEqual(activity["type"], "Delete") self.assertEqual(activity["object"]["type"], "Tombstone") status.refresh_from_db() @@ -396,7 +396,7 @@ http://www.fish.com/""" request.user = self.local_user view(request, "comment", existing_status_id=status.id) - activity = json.loads(mock.call_args_list[1][0][1]) + activity = json.loads(mock.call_args_list[1][1]["args"][1]) self.assertEqual(activity["type"], "Update") self.assertEqual(activity["object"]["id"], status.remote_id) diff --git a/bookwyrm/tests/views/test_user.py b/bookwyrm/tests/views/test_user.py index 1183fa24..ddb029cc 100644 --- a/bookwyrm/tests/views/test_user.py +++ b/bookwyrm/tests/views/test_user.py @@ -34,9 +34,11 @@ class UserViews(TestCase): self.book = models.Edition.objects.create( title="test", parent_work=models.Work.objects.create(title="test work") ) - with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"), patch( - "bookwyrm.suggested_users.rerank_suggestions_task.delay" - ), patch("bookwyrm.activitystreams.add_book_statuses_task.delay"): + with patch( + "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" + ), patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.add_book_statuses_task.delay" + ): models.ShelfBook.objects.create( book=self.book, user=self.local_user, From cad5a128ac94b76e0b59763ba6e934ec1946100a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 10:19:13 -0800 Subject: [PATCH 19/73] Fixes call to add_status signal --- bookwyrm/tests/activitystreams/test_signals.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index b3a99620..34aeb947 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -53,11 +53,12 @@ class ActivitystreamsSignals(TestCase): status = models.Status.objects.create( user=self.remote_user, content="hi", privacy="public" ) - with patch("bookwyrm.activitystreams.add_status_task.delay") as mock: + with patch("bookwyrm.activitystreams.add_status_task.apply_async") as mock: activitystreams.add_status_on_create_command(models.Status, status, False) self.assertEqual(mock.call_count, 1) - args = mock.call_args[0] - self.assertEqual(args[0], status.id) + args = mock.call_args[1] + self.assertEqual(args["args"][0], status.id) + self.assertEqual(args["queue"], "high_priority") def test_populate_streams_on_account_create(self, _): """create streams for a user""" From 309d289a6515a1da8e12874127c5ce2505979877 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 10:49:49 -0800 Subject: [PATCH 20/73] A few more mocks --- .../importers/test_librarything_import.py | 18 ------------------ bookwyrm/tests/models/test_status_model.py | 17 ++++++----------- 2 files changed, 6 insertions(+), 29 deletions(-) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index ed081aa2..1ec94bbb 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -84,24 +84,6 @@ class LibrarythingImport(TestCase): self.assertEqual(retry_items[1].index, 1) self.assertEqual(retry_items[1].data["Book Id"], "5015319") - @responses.activate - def test_start_import_task(self, *_): - """resolve entry""" - import_job = self.importer.create_job( - self.local_user, self.csv, False, "unlisted" - ) - book = models.Edition.objects.create(title="Test Book") - - with patch( - "bookwyrm.models.import_job.ImportItem.get_book_from_isbn" - ) as resolve: - resolve.return_value = book - with patch("bookwyrm.importers.importer.handle_imported_book"): - start_import_task(self.importer.service, import_job.id) - - import_item = models.ImportItem.objects.get(job=import_job, index=0) - self.assertEqual(import_item.book.id, book.id) - def test_handle_imported_book(self, *_): """librarything import added a book, this adds related connections""" shelf = self.local_user.shelf_set.filter(identifier="read").first() diff --git a/bookwyrm/tests/models/test_status_model.py b/bookwyrm/tests/models/test_status_model.py index 7d0dd138..822d837a 100644 --- a/bookwyrm/tests/models/test_status_model.py +++ b/bookwyrm/tests/models/test_status_model.py @@ -362,19 +362,15 @@ class Status(TestCase): def test_favorite(self, *_): """fav a status""" - real_broadcast = models.Favorite.broadcast - - def fav_broadcast_mock(_, activity, user): - """ok""" - self.assertEqual(user.remote_id, self.local_user.remote_id) - self.assertEqual(activity["type"], "Like") - - models.Favorite.broadcast = fav_broadcast_mock - status = models.Status.objects.create( content="test content", user=self.local_user ) - fav = models.Favorite.objects.create(status=status, user=self.local_user) + + with patch("bookwyrm.models.Favorite.broadcast") as mock: + fav = models.Favorite.objects.create(status=status, user=self.local_user) + args = mock.call_args[0] + self.assertEqual(args[1].remote_id, self.local_user.remote_id) + self.assertEqual(args[0]["type"], "Like") # can't fav a status twice with self.assertRaises(IntegrityError): @@ -384,7 +380,6 @@ class Status(TestCase): self.assertEqual(activity["type"], "Like") self.assertEqual(activity["actor"], self.local_user.remote_id) self.assertEqual(activity["object"], status.remote_id) - models.Favorite.broadcast = real_broadcast def test_boost(self, *_): """boosting, this one's a bit fussy""" From 1e8269b6c9847df68ea12e1b5d361ac7898b92f2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 13:10:52 -0800 Subject: [PATCH 21/73] Refactors import status view --- bookwyrm/models/import_job.py | 2 +- bookwyrm/templates/import/import_status.html | 161 ++++++------------- bookwyrm/views/import_data.py | 36 +++-- 3 files changed, 75 insertions(+), 124 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 6bca57f8..61b818fe 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -33,7 +33,7 @@ class ImportJob(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) created_date = models.DateTimeField(default=timezone.now) - task_id = models.CharField(max_length=100, null=True) + task_id = models.CharField(max_length=100, null=True) # TODO: deprecated include_reviews = models.BooleanField(default=True) mappings = models.JSONField() complete = models.BooleanField(default=False) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 91e48e34..029b081d 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -6,116 +6,42 @@ {% block title %}{% trans "Import Status" %}{% endblock %} {% block content %}{% spaceless %} -
+

{% trans "Import Status" %}

{% trans "Back to imports" %} - {% if task.failed %} -
{% trans "TASK FAILED" %}
+
+
+
{% trans "Import started:" %}
+
{{ job.created_date | naturaltime }}
+
+
+ + {% if not complete %} +
+
+ + {% trans "In progress" %} + + {% trans "Refresh" %} + +
+
+ {{ percent }}% + {{ percent }}% +
+
{% endif %} - -
-
{% trans "Import started:" %}
-
{{ job.created_date | naturaltime }}
- - {% if job.complete %} -
{% trans "Import completed:" %}
-
{{ task.date_done | naturaltime }}
- {% endif %} -
-
+
- {% if not job.complete %} -

- {% trans "Import still in progress." %} -
- {% trans "(Hit reload to update!)" %} -

- {% endif %} -
- -{% if failed_items %} -
-

{% trans "Failed to load" %}

- {% if not job.retry %} -
- {% csrf_token %} - - {% with failed_count=failed_items|length %} - {% if failed_count > 10 %} -

- - {% blocktrans %}Jump to the bottom of the list to select the {{ failed_count }} items which failed to import.{% endblocktrans %} - -

- {% endif %} - {% endwith %} - -
-
    - {% for item in failed_items %} -
  • - - -
    • - {% endfor %} -
-
- -
- - - - - -
-
- -
- - {% else %} -
    - {% for item in failed_items %} -
  • -

    - Line {{ item.index }}: - {{ item.data.Title }} by - {{ item.data.Author }} -

    -

    - {{ item.fail_reason }}. -

    -
    • - {% endfor %} -
- {% endif %} -
-{% endif %} - -
- {% if job.complete %} -

{% trans "Successfully imported" %}

- {% else %} -

{% trans "Import Progress" %}

- {% endif %} +

+ {% trans "Your Import" %} +

+ {% for item in items %} + {% endfor %}
- {% trans "Book" %} + {% trans "Row" %} {% trans "Title" %} @@ -124,16 +50,16 @@ {% trans "Author" %} + {% trans "Book" %} + + {% trans "Status" %}
- {% if item.book %} - - {% include 'snippets/book_cover.html' with book=item.book cover_class='is-h-s' size='small' %} - - {% endif %} + {{ item.index }} {{ item.data.Title }} @@ -143,15 +69,34 @@ {% if item.book %} - - {% trans "Imported" %} + + {% include 'snippets/book_cover.html' with book=item.book cover_class='is-h-s' size='small' %} + + {% endif %} + + {% if item.book %} + + {% trans "Imported" %} + + {% elif item.fail_reason %} + + + {{ item.fail_reason }} + {% else %} + + {% trans "Pending" %} {% endif %}
+ +
+ {% include 'snippets/pagination.html' with page=items %} +
{% endspaceless %}{% endblock %} {% block scripts %} diff --git a/bookwyrm/views/import_data.py b/bookwyrm/views/import_data.py index 5e113be8..1dc9e6d8 100644 --- a/bookwyrm/views/import_data.py +++ b/bookwyrm/views/import_data.py @@ -1,8 +1,10 @@ """ import books from another app """ from io import TextIOWrapper +import math from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied +from django.core.paginator import Paginator from django.http import HttpResponseBadRequest from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse @@ -17,7 +19,7 @@ from bookwyrm.importers import ( GoodreadsImporter, StorygraphImporter, ) -from bookwyrm.tasks import app +from bookwyrm.settings import PAGE_LENGTH # pylint: disable= no-self-use @method_decorator(login_required, name="dispatch") @@ -82,21 +84,25 @@ class ImportStatus(View): if job.user != request.user: raise PermissionDenied() - try: - task = app.AsyncResult(job.task_id) - # triggers attribute error if the task won't load - task.status # pylint: disable=pointless-statement - except (ValueError, AttributeError): - task = None + items = job.items.order_by("index") + pending_items = items.filter(fail_reason__isnull=True, book__isnull=True) + item_count = items.count() or 1 - items = job.items.order_by("index").all() - failed_items = [i for i in items if i.fail_reason] - items = [i for i in items if not i.fail_reason] - return TemplateResponse( - request, - "import/import_status.html", - {"job": job, "items": items, "failed_items": failed_items, "task": task}, - ) + paginated = Paginator(items, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "job": job, + "items": page, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "complete": not pending_items.exists(), + "percent": math.floor( # pylint: disable=c-extension-no-member + (item_count - pending_items.count()) / item_count * 100 + ), + } + + return TemplateResponse(request, "import/import_status.html", data) def post(self, request, job_id): """retry lines from an import""" From 80c1954aa36398b096114816155b7d6ddbdb6afa Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 13:48:31 -0800 Subject: [PATCH 22/73] Fixes first_search_result behavior --- bookwyrm/book_search.py | 5 ++++- bookwyrm/models/import_job.py | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 6c89b61f..7b9b1230 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -82,6 +82,8 @@ def search_identifiers(query, *filters, return_first=False): *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) ).distinct() if results.count() <= 1: + if return_first: + return results.first() return results # when there are multiple editions of the same work, pick the default. @@ -124,8 +126,9 @@ def search_title_author(query, min_confidence, *filters, return_first=False): result = default else: result = editions.first() + if return_first: - return result + return result[0] list_results.append(result) return list_results diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 61b818fe..9f011f1e 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -6,7 +6,7 @@ from django.db import models from django.utils import timezone from bookwyrm.connectors import connector_manager -from bookwyrm.models import ReadThrough, User, Book +from bookwyrm.models import ReadThrough, User, Book, Edition from .fields import PrivacyLevels @@ -79,6 +79,10 @@ class ImportItem(models.Model): self.isbn, min_confidence=0.999 ) if search_result: + # it's already in the right format + if isinstance(search_result, Edition): + return search_result + # it's just a search result, book needs to be created # raises ConnectorException return search_result.connector.get_or_create_book(search_result.key) return None @@ -183,9 +187,7 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r}Item {!r}>".format( - self.normalized_data["import_source"], self.normalized_data["title"] - ) + return "<{!r}Item {!r}>".format(self.index, self.normalized_data["title"]) def __str__(self): # pylint: disable=consider-using-f-string From 2a84c0a370e9adcc09d113bdcc831f270e38d36d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 13:59:54 -0800 Subject: [PATCH 23/73] title author search already working correctly with return first --- bookwyrm/book_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 7b9b1230..e42a6d8c 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -128,7 +128,7 @@ def search_title_author(query, min_confidence, *filters, return_first=False): result = editions.first() if return_first: - return result[0] + return result list_results.append(result) return list_results From a65f07e0bff07493a8045a641fad75f2eda31cc0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:36:28 -0800 Subject: [PATCH 24/73] Adds retry page --- bookwyrm/templates/import/import_status.html | 16 ++++++++++ bookwyrm/templates/import/troubleshoot.html | 30 +++++++++++++++++++ bookwyrm/urls.py | 1 + bookwyrm/views/__init__.py | 2 +- bookwyrm/views/import_data.py | 31 ++++++++++++++++++-- 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 bookwyrm/templates/import/troubleshoot.html diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 029b081d..59bebb41 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -32,12 +32,28 @@ {% endif %} + + {% if complete and fail_count %} +
+ {% blocktrans trimmed count counter=fail_count with display_counter=fail_count|intcomma %} + {{ display_counter }} item failed to import. + {% plural %} + {{ display_counter }} items failed to import. + {% endblocktrans %} + + {% trans "View and troubleshoot failed items." %} + +
+ {% endif %}

+ {% block page_title %} {% trans "Your Import" %} + {% endblock %}

+ {% block actions %}{% endblock %} + @@ -82,10 +85,13 @@ {{ item.index }} + + + + + + + +{% endblock %} + +{% block table_row %} + + + + + + + + + + +{% endblock %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 061163a1..1004e30b 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -243,6 +243,11 @@ urlpatterns = [ views.ImportTroubleshoot.as_view(), name="import-troubleshoot", ), + re_path( + r"^import/(\d+)/review/?$", + views.ImportManualReview.as_view(), + name="import-review", + ), # users re_path(rf"{USER_PATH}\.json$", views.User.as_view()), re_path(rf"{USER_PATH}/?$", views.User.as_view(), name="user-feed"), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 21eeb39b..9fe09795 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -46,7 +46,8 @@ from .shelf.shelf_actions import shelve, unshelve # csv import from .imports.import_data import Import from .imports.import_status import ImportStatus -from .imports.troubelshoot import ImportTroubleshoot +from .imports.troubleshoot import ImportTroubleshoot +from .imports.manually_review import ImportManualReview # misc views from .author import Author, EditAuthor diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 64cefc7f..7f6a4d13 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -10,7 +10,11 @@ from django.utils.translation import gettext_lazy as _ from django.views import View from bookwyrm import forms, models -from bookwyrm.importers import LibrarythingImporter, GoodreadsImporter, StorygraphImporter +from bookwyrm.importers import ( + LibrarythingImporter, + GoodreadsImporter, + StorygraphImporter, +) # pylint: disable= no-self-use @method_decorator(login_required, name="dispatch") diff --git a/bookwyrm/views/imports/manually_review.py b/bookwyrm/views/imports/manually_review.py new file mode 100644 index 00000000..286251d4 --- /dev/null +++ b/bookwyrm/views/imports/manually_review.py @@ -0,0 +1,39 @@ +""" verify books we're unsure about """ +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied +from django.core.paginator import Paginator +from django.shortcuts import get_object_or_404 +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views import View + +from bookwyrm import models +from bookwyrm.settings import PAGE_LENGTH + +# pylint: disable= no-self-use +@method_decorator(login_required, name="dispatch") +class ImportManualReview(View): + """problems items in an existing import""" + + def get(self, request, job_id): + """status of an import job""" + job = get_object_or_404(models.ImportJob, id=job_id) + if job.user != request.user: + raise PermissionDenied() + + items = job.items.order_by("index").filter( + book__isnull=True, book_guess__isnull=False + ) + + paginated = Paginator(items, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "job": job, + "items": page, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "complete": True, + } + + return TemplateResponse(request, "import/manual_review.html", data) diff --git a/bookwyrm/views/imports/troubleshoot.py b/bookwyrm/views/imports/troubleshoot.py index 48f2b998..f637b966 100644 --- a/bookwyrm/views/imports/troubleshoot.py +++ b/bookwyrm/views/imports/troubleshoot.py @@ -23,7 +23,7 @@ class ImportTroubleshoot(View): raise PermissionDenied() items = job.items.order_by("index").filter( - fail_reason__isnull=False, book_guess__isnull=False + fail_reason__isnull=False, book_guess__isnull=True ) paginated = Paginator(items, PAGE_LENGTH) From 40fff02eec549fffe17b3073ff3cfe194f9abdd7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 17:10:47 -0800 Subject: [PATCH 37/73] Approve or delete import guesses --- bookwyrm/importers/importer.py | 24 +++++++------- bookwyrm/migrations/0114_importjob_source.py | 19 +++++++++++ bookwyrm/models/import_job.py | 6 ++++ bookwyrm/templates/import/import_status.html | 7 ++-- bookwyrm/templates/import/manual_review.html | 19 +++++++++-- bookwyrm/urls.py | 25 ++++++++++++-- bookwyrm/views/__init__.py | 6 +++- bookwyrm/views/imports/manually_review.py | 34 +++++++++++++++++++- 8 files changed, 118 insertions(+), 22 deletions(-) create mode 100644 bookwyrm/migrations/0114_importjob_source.py diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index b895d69a..6d0f6553 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) class Importer: """Generic class for csv data import from an outside service""" - service = "Unknown" + service = "Import" delimiter = "," encoding = "UTF-8" @@ -50,6 +50,7 @@ class Importer: include_reviews=include_reviews, privacy=privacy, mappings=self.create_row_mappings(csv_reader.fieldnames), + source=self.service, ) for index, entry in rows: @@ -108,16 +109,16 @@ class Importer: @app.task(queue="low_priority") -def start_import_task(source, job_id): +def start_import_task(job_id): """trigger the child tasks for each row""" job = ImportJob.objects.get(id=job_id) # these are sub-tasks so that one big task doesn't use up all the memory in celery for item in job.items.values_list("id", flat=True).all(): - import_item_task.delay(source, item) + import_item_task.delay(item) @app.task(queue="low_priority") -def import_item_task(source, item_id): +def import_item_task(item_id): """resolve a row into a book""" item = models.ImportItem.objects.get(id=item_id) try: @@ -128,17 +129,18 @@ def import_item_task(source, item_id): raise err if item.book: - job = item.job # shelves book and handles reviews - handle_imported_book(source, job.user, item, job.include_reviews, job.privacy) + handle_imported_book(item) else: item.fail_reason = _("Could not find a match for book") item.save() -def handle_imported_book(source, user, item, include_reviews, privacy): +def handle_imported_book(item): """process a csv and then post about it""" + job = item.job + user = job.user if isinstance(item.book, models.Work): item.book = item.book.default_edition if not item.book: @@ -167,7 +169,7 @@ def handle_imported_book(source, user, item, include_reviews, privacy): read.user = user read.save() - if include_reviews and (item.rating or item.review): + if job.include_reviews and (item.rating or item.review): # we don't know the publication date of the review, # but "now" is a bad guess published_date_guess = item.date_read or item.date_added @@ -176,7 +178,7 @@ def handle_imported_book(source, user, item, include_reviews, privacy): review_title = ( "Review of {!r} on {!r}".format( item.book.title, - source, + job.source, ) if item.review else "" @@ -188,7 +190,7 @@ def handle_imported_book(source, user, item, include_reviews, privacy): content=item.review, rating=item.rating, published_date=published_date_guess, - privacy=privacy, + privacy=job.privacy, ) else: # just a rating @@ -197,7 +199,7 @@ def handle_imported_book(source, user, item, include_reviews, privacy): book=item.book, rating=item.rating, published_date=published_date_guess, - privacy=privacy, + privacy=job.privacy, ) # only broadcast this review to other bookwyrm instances review.save(software="bookwyrm", priority=LOW) diff --git a/bookwyrm/migrations/0114_importjob_source.py b/bookwyrm/migrations/0114_importjob_source.py new file mode 100644 index 00000000..3ec1432e --- /dev/null +++ b/bookwyrm/migrations/0114_importjob_source.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.5 on 2021-11-13 00:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0113_auto_20211110_2104"), + ] + + operations = [ + migrations.AddField( + model_name="importjob", + name="source", + field=models.CharField(default="Import", max_length=100), + preserve_default=False, + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index aa86910c..6b8f0b46 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -37,6 +37,7 @@ class ImportJob(models.Model): include_reviews = models.BooleanField(default=True) mappings = models.JSONField() complete = models.BooleanField(default=False) + source = models.CharField(max_length=100) privacy = models.CharField( max_length=255, default="public", choices=PrivacyLevels.choices ) @@ -62,6 +63,11 @@ class ImportItem(models.Model): def resolve(self): """try various ways to lookup a book""" + # we might be calling this after manually adding the book, + # so no need to do searches + if self.book: + return + if self.isbn: self.book = self.get_book_from_isbn() else: diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index b2e21d22..12465bd7 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -40,10 +40,11 @@ {% if manual_review_count %}
{% blocktrans trimmed count counter=manual_review_count with display_counter=manual_review_count|intcomma %} - {{ display_counter }} item needs manual review. + {{ display_counter }} item needs manual approval. {% plural %} - {{ display_counter }} items need manual review. + {{ display_counter }} items need manual approval. {% endblocktrans %} + {% trans "Review items" %}
{% endif %} @@ -55,7 +56,7 @@ {{ display_counter }} items failed to import. {% endblocktrans %} - {% trans "View and troubleshoot failed items." %} + {% trans "View and troubleshoot failed items" %} {% endif %} diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index c9c03869..30274ca0 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -52,9 +52,22 @@ - diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 1004e30b..d6a79c20 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -237,17 +237,36 @@ urlpatterns = [ re_path(r"^search/?$", views.Search.as_view(), name="search"), # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), - re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"), re_path( - r"^import/(\d+)/failed/?$", + r"^import/(?P\d+)/?$", + views.ImportStatus.as_view(), + name="import-status", + ), + re_path( + r"^import/(?P\d+)/failed/?$", views.ImportTroubleshoot.as_view(), name="import-troubleshoot", ), re_path( - r"^import/(\d+)/review/?$", + r"^import/(?P\d+)/review/?$", views.ImportManualReview.as_view(), name="import-review", ), + re_path( + r"^import/(?P\d+)/review/?$", + views.ImportManualReview.as_view(), + name="import-review", + ), + re_path( + r"^import/(?P\d+)/review/(?P\d+)/approve/?$", + views.approve_import_item, + name="import-approve", + ), + re_path( + r"^import/(?P\d+)/review/(?P\d+)/delete/?$", + views.delete_import_item, + name="import-delete", + ), # users re_path(rf"{USER_PATH}\.json$", views.User.as_view()), re_path(rf"{USER_PATH}/?$", views.User.as_view(), name="user-feed"), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 9fe09795..1a6fbdc6 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -47,7 +47,11 @@ from .shelf.shelf_actions import shelve, unshelve from .imports.import_data import Import from .imports.import_status import ImportStatus from .imports.troubleshoot import ImportTroubleshoot -from .imports.manually_review import ImportManualReview +from .imports.manually_review import ( + ImportManualReview, + approve_import_item, + delete_import_item, +) # misc views from .author import Author, EditAuthor diff --git a/bookwyrm/views/imports/manually_review.py b/bookwyrm/views/imports/manually_review.py index 286251d4..877e3ffc 100644 --- a/bookwyrm/views/imports/manually_review.py +++ b/bookwyrm/views/imports/manually_review.py @@ -2,12 +2,14 @@ from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied from django.core.paginator import Paginator -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View +from django.views.decorators.http import require_POST from bookwyrm import models +from bookwyrm.importers.importer import import_item_task from bookwyrm.settings import PAGE_LENGTH # pylint: disable= no-self-use @@ -37,3 +39,33 @@ class ImportManualReview(View): } return TemplateResponse(request, "import/manual_review.html", data) + + +@login_required +@require_POST +# pylint: disable=unused-argument +def approve_import_item(request, job_id, item_id): + """we guessed right""" + item = get_object_or_404( + models.ImportItem, id=item_id, job__id=job_id, book_guess__isnull=False + ) + item.fail_reason = None + item.book = item.book_guess + item.save() + + # the good stuff - actually import the data + import_item_task.delay(item.id) + return redirect("import-review", job_id) + + +@login_required +@require_POST +# pylint: disable=unused-argument +def delete_import_item(request, job_id, item_id): + """we guessed right""" + item = get_object_or_404( + models.ImportItem, id=item_id, job__id=job_id, book_guess__isnull=False + ) + item.book_guess = None + item.save() + return redirect("import-review", job_id) From 08f4ad6cd47e1804c95259c19397c297e9857a19 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:02:42 -0800 Subject: [PATCH 38/73] Fixes call to import task --- bookwyrm/importers/importer.py | 4 ++-- bookwyrm/tests/importers/test_importer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 6d0f6553..4908e166 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -101,9 +101,9 @@ class Importer: self.create_item(job, item.index, item.data) return job - def start_import(self, job): + def start_import(self, job): # pylint: disable=no-self-use """initalizes a csv import job""" - result = start_import_task.delay(self.service, job.id) + result = start_import_task.delay(job.id) job.task_id = result.id job.save() diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 963eca54..b0863fbe 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -120,7 +120,7 @@ class GenericImporter(TestCase): ) with patch("bookwyrm.importers.importer.import_item_task.delay") as mock: - start_import_task(self.importer.service, import_job.id) + start_import_task(import_job.id) self.assertEqual(mock.call_count, 4) From c245ad09bba4350b92c572bc8fc08eca0faa0a0e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:07:50 -0800 Subject: [PATCH 39/73] Make sure book is in the right format --- bookwyrm/importers/importer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 4908e166..71f02231 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -145,6 +145,8 @@ def handle_imported_book(item): item.book = item.book.default_edition if not item.book: return + if not isinstance(item.book, models.Edition): + item.book = item.book.edition existing_shelf = models.ShelfBook.objects.filter(book=item.book, user=user).exists() From acc32d579e24c70ac1a46394d8c9bc2d8b11ce38 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:22:35 -0800 Subject: [PATCH 40/73] Preview review in import preview --- bookwyrm/templates/import/import_status.html | 7 ++++ bookwyrm/templates/import/manual_review.html | 39 ++++++++++++-------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 12465bd7..0a3c9b47 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -84,6 +84,9 @@ + @@ -107,6 +110,10 @@ + + @@ -52,22 +55,28 @@ - + From 60fb1ac2e6fd09da622de64cbecbc87126455408 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:34:10 -0800 Subject: [PATCH 41/73] More flexible templates --- bookwyrm/templates/import/import_status.html | 17 ++++++-- bookwyrm/templates/import/manual_review.html | 44 ++++---------------- 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 0a3c9b47..6c5560dc 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -70,7 +70,6 @@ {% block actions %}{% endblock %}
diff --git a/bookwyrm/templates/import/troubleshoot.html b/bookwyrm/templates/import/troubleshoot.html new file mode 100644 index 00000000..9b9b4827 --- /dev/null +++ b/bookwyrm/templates/import/troubleshoot.html @@ -0,0 +1,30 @@ +{% extends 'import/import_status.html' %} +{% load i18n %} + +{% block title %}{% trans "Import Troubleshooting" %}{% endblock %} + +{% block page_title %} +{% trans "Failed items" %} +{% endblock %} + +{% block actions %} +
+
+

+ {% trans "Re-trying an import can fix missing items in cases such as:" %} +

    +
  • {% trans "The book has been added to the instance since this import" %}
    • +
  • {% trans "A transient error or timeout caused the external data source to be unavailable." %}
    • +
  • {% trans "BookWyrm has been updated since this import with a bug fix" %}
    • +
+

+

+ {% trans "Contact your admin or open an issue if you are seeing unexpected failed items." %} +

+
+
+ {% csrf_token %} + +
+
+{% endblock %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 839d783f..350d6018 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -238,6 +238,7 @@ urlpatterns = [ # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"), + re_path(r"^import/(\d+)/failed/?$", views.ImportTroubleshoot.as_view(), name="import-troubleshoot"), # users re_path(rf"{USER_PATH}\.json$", views.User.as_view()), re_path(rf"{USER_PATH}/?$", views.User.as_view(), name="user-feed"), diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index e1dd8355..645b1307 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -62,7 +62,7 @@ from .group import ( accept_membership, reject_membership, ) -from .import_data import Import, ImportStatus +from .import_data import Import, ImportStatus, ImportTroubleshoot from .inbox import Inbox from .interaction import Favorite, Unfavorite, Boost, Unboost from .isbn import Isbn diff --git a/bookwyrm/views/import_data.py b/bookwyrm/views/import_data.py index 1dc9e6d8..e0e90c2d 100644 --- a/bookwyrm/views/import_data.py +++ b/bookwyrm/views/import_data.py @@ -93,6 +93,7 @@ class ImportStatus(View): data = { "job": job, "items": page, + "fail_count": items.filter(fail_reason__isnull=False).count(), "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), @@ -104,12 +105,36 @@ class ImportStatus(View): return TemplateResponse(request, "import/import_status.html", data) + +@method_decorator(login_required, name="dispatch") +class ImportTroubleshoot(View): + """problems items in an existing import""" + + def get(self, request, job_id): + """status of an import job""" + job = get_object_or_404(models.ImportJob, id=job_id) + if job.user != request.user: + raise PermissionDenied() + + items = job.items.order_by("index").filter(fail_reason__isnull=False) + + paginated = Paginator(items, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "job": job, + "items": page, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "complete": True, + } + + return TemplateResponse(request, "import/troubleshoot.html", data) + def post(self, request, job_id): """retry lines from an import""" job = get_object_or_404(models.ImportJob, id=job_id) - items = [] - for item in request.POST.getlist("import_item"): - items.append(get_object_or_404(models.ImportItem, id=item)) + items = job.items.filter(fail_reason__isnull=False) importer = Importer() job = importer.create_retry_job( From d8197cdcfab9bea1243d5a4d57bfad46c05dab01 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:38:41 -0800 Subject: [PATCH 25/73] Indicate retry on status page --- bookwyrm/templates/import/import_status.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 59bebb41..ac8fead0 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -7,7 +7,11 @@ {% block content %}{% spaceless %}
+ {% if job.retry %} +

{% trans "Retry Status" %}

+ {% else %}

{% trans "Import Status" %}

+ {% endif %} {% trans "Back to imports" %}
@@ -33,7 +37,7 @@
{% endif %} - {% if complete and fail_count %} + {% if complete and fail_count and not job.retry %}
{% blocktrans trimmed count counter=fail_count with display_counter=fail_count|intcomma %} {{ display_counter }} item failed to import. From 60c777ed495c4d37ea615c2c1b258926f1335bcd Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:41:25 -0800 Subject: [PATCH 26/73] Updates tests --- bookwyrm/tests/views/test_import.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bookwyrm/tests/views/test_import.py b/bookwyrm/tests/views/test_import.py index 54f11f03..4a6a8237 100644 --- a/bookwyrm/tests/views/test_import.py +++ b/bookwyrm/tests/views/test_import.py @@ -51,6 +51,19 @@ class ImportViews(TestCase): validate_html(result.render()) self.assertEqual(result.status_code, 200) + def test_import_troubleshoot_get(self): + """there are so many views, this just makes sure it LOADS""" + view = views.ImportTroubleshoot.as_view() + import_job = models.ImportJob.objects.create(user=self.local_user, mappings={}) + request = self.factory.get("") + request.user = self.local_user + with patch("bookwyrm.tasks.app.AsyncResult") as async_result: + async_result.return_value = [] + result = view(request, import_job.id) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + def test_start_import(self): """retry failed items""" view = views.Import.as_view() @@ -77,7 +90,7 @@ class ImportViews(TestCase): def test_retry_import(self): """retry failed items""" - view = views.ImportStatus.as_view() + view = views.ImportTroubleshoot.as_view() import_job = models.ImportJob.objects.create( user=self.local_user, privacy="unlisted", mappings={} ) From e09c02017c7efb041c410668b042e52a88e26e2e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:46:39 -0800 Subject: [PATCH 27/73] Fixes title/author search handling --- bookwyrm/models/import_job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 9f011f1e..aa86910c 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -94,6 +94,8 @@ class ImportItem(models.Model): search_term, min_confidence=0.1 ) if search_result: + if isinstance(search_result, Edition): + return (search_result, 1) # raises ConnectorException return ( search_result.connector.get_or_create_book(search_result.key), From 61eaf513105f7e413a81312d30c667e00f976b44 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:49:24 -0800 Subject: [PATCH 28/73] Fixes html validity --- bookwyrm/templates/import/troubleshoot.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/import/troubleshoot.html b/bookwyrm/templates/import/troubleshoot.html index 9b9b4827..0be68383 100644 --- a/bookwyrm/templates/import/troubleshoot.html +++ b/bookwyrm/templates/import/troubleshoot.html @@ -12,12 +12,12 @@

{% trans "Re-trying an import can fix missing items in cases such as:" %} -

    -
  • {% trans "The book has been added to the instance since this import" %}
    • -
  • {% trans "A transient error or timeout caused the external data source to be unavailable." %}
    • -
  • {% trans "BookWyrm has been updated since this import with a bug fix" %}
    • -

+
    +
  • {% trans "The book has been added to the instance since this import" %}
    • +
  • {% trans "A transient error or timeout caused the external data source to be unavailable." %}
    • +
  • {% trans "BookWyrm has been updated since this import with a bug fix" %}
    • +

{% trans "Contact your admin or open an issue if you are seeing unexpected failed items." %}

From 6161f60695dced0368c92482369df27f1b359281 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 14:54:20 -0800 Subject: [PATCH 29/73] Use normalized data in table --- bookwyrm/templates/import/import_status.html | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index ac8fead0..5674aace 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -66,6 +66,9 @@
{% trans "Title" %} + {% trans "ISBN" %} + {% trans "Author" %} - {{ item.data.Title }} + {{ item.normalized_data.title }} - {{ item.data.Author }} + {{ item.isbn }} + + {{ item.normalized_data.authors }} {% if item.book %} From 1d0f6d5243bca717e489dda97e20e921d89f9126 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:06:23 -0800 Subject: [PATCH 30/73] Python formatting --- bookwyrm/urls.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 350d6018..061163a1 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -238,7 +238,11 @@ urlpatterns = [ # imports re_path(r"^import/?$", views.Import.as_view(), name="import"), re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"), - re_path(r"^import/(\d+)/failed/?$", views.ImportTroubleshoot.as_view(), name="import-troubleshoot"), + re_path( + r"^import/(\d+)/failed/?$", + views.ImportTroubleshoot.as_view(), + name="import-troubleshoot", + ), # users re_path(rf"{USER_PATH}\.json$", views.User.as_view()), re_path(rf"{USER_PATH}/?$", views.User.as_view(), name="user-feed"), From b784dcdb46b70cf4f9e4567f27eae3dadea0aab7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:07:26 -0800 Subject: [PATCH 31/73] Removes uninformative test --- .../tests/importers/test_storygraph_import.py | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index 8002a3e1..fdad5c71 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -1,5 +1,4 @@ """ testing import """ -import csv import pathlib from unittest.mock import patch import datetime @@ -59,30 +58,6 @@ class StorygraphImport(TestCase): ) self.assertEqual(import_items[1].normalized_data["rating"], "5.0") - def test_create_retry_job(self, *_): - """trying again with items that didn't import""" - import_job = self.importer.create_job( - self.local_user, self.csv, False, "unlisted" - ) - import_items = models.ImportItem.objects.filter(job=import_job).all()[:2] - - retry = self.importer.create_retry_job( - self.local_user, import_job, import_items - ) - self.assertNotEqual(import_job, retry) - self.assertEqual(retry.user, self.local_user) - self.assertEqual(retry.include_reviews, False) - self.assertEqual(retry.privacy, "unlisted") - - retry_items = models.ImportItem.objects.filter(job=retry).all() - self.assertEqual(len(retry_items), 2) - self.assertEqual(retry_items[0].index, 0) - self.assertEqual(retry_items[0].normalized_data["title"], "Always Coming Home") - self.assertEqual(retry_items[1].index, 1) - self.assertEqual( - retry_items[1].normalized_data["title"], "Subprime Attention Crisis" - ) - def test_handle_imported_book(self, *_): """storygraph import added a book, this adds related connections""" shelf = self.local_user.shelf_set.filter(identifier="to-read").first() From 6a5a5983452769972c8a7a915060f7b9f67886c0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:17:01 -0800 Subject: [PATCH 32/73] Raise errors when import items fail This should make is way easier to debug --- bookwyrm/importers/importer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index b32e2df7..b895d69a 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -123,10 +123,9 @@ def import_item_task(source, item_id): try: item.resolve() except Exception as err: # pylint: disable=broad-except - logger.exception(err) item.fail_reason = _("Error loading book") item.save() - return + raise err if item.book: job = item.job From 5558ed810ead73e34fb41d9ae76a7d1016465c84 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:34:48 -0800 Subject: [PATCH 33/73] Show manual review flag --- bookwyrm/templates/import/import_status.html | 10 ++++++++++ bookwyrm/views/import_data.py | 11 +++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 5674aace..01a1fec6 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -37,6 +37,16 @@ {% endif %} + {% if manual_review_count %} +
+ {% blocktrans trimmed count counter=manual_review_count with display_counter=manual_review_count|intcomma %} + {{ display_counter }} item needs manual review. + {% plural %} + {{ display_counter }} items need manual review. + {% endblocktrans %} +
+ {% endif %} + {% if complete and fail_count and not job.retry %}
{% blocktrans trimmed count counter=fail_count with display_counter=fail_count|intcomma %} diff --git a/bookwyrm/views/import_data.py b/bookwyrm/views/import_data.py index e0e90c2d..907c77e2 100644 --- a/bookwyrm/views/import_data.py +++ b/bookwyrm/views/import_data.py @@ -93,7 +93,12 @@ class ImportStatus(View): data = { "job": job, "items": page, - "fail_count": items.filter(fail_reason__isnull=False).count(), + "manual_review_count": items.filter( + fail_reason__isnull=False, book_guess__isnull=False, book__isnull=True + ).count(), + "fail_count": items.filter( + fail_reason__isnull=False, book_guess__isnull=True + ).count(), "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), @@ -116,7 +121,9 @@ class ImportTroubleshoot(View): if job.user != request.user: raise PermissionDenied() - items = job.items.order_by("index").filter(fail_reason__isnull=False) + items = job.items.order_by("index").filter( + fail_reason__isnull=False, book_guess__isnull=False + ) paginated = Paginator(items, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) From be26e8363a7d9d5dada60b3fbda1bbaaf16f677f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:43:15 -0800 Subject: [PATCH 34/73] Create import directory --- bookwyrm/views/__init__.py | 4 +++- bookwyrm/views/imports/__init__.py | 0 bookwyrm/views/{ => imports}/import_data.py | 0 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/views/imports/__init__.py rename bookwyrm/views/{ => imports}/import_data.py (100%) diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 645b1307..d169d126 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -43,6 +43,9 @@ from .shelf.shelf import Shelf from .shelf.shelf_actions import create_shelf, delete_shelf from .shelf.shelf_actions import shelve, unshelve +# csv import +from .imports.import_data import Import, ImportStatus, ImportTroubleshoot + # misc views from .author import Author, EditAuthor from .directory import Directory @@ -62,7 +65,6 @@ from .group import ( accept_membership, reject_membership, ) -from .import_data import Import, ImportStatus, ImportTroubleshoot from .inbox import Inbox from .interaction import Favorite, Unfavorite, Boost, Unboost from .isbn import Isbn diff --git a/bookwyrm/views/imports/__init__.py b/bookwyrm/views/imports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bookwyrm/views/import_data.py b/bookwyrm/views/imports/import_data.py similarity index 100% rename from bookwyrm/views/import_data.py rename to bookwyrm/views/imports/import_data.py From 9bff27e61f2106ad9709e7907b0680fd4fdbe9e0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 15:50:33 -0800 Subject: [PATCH 35/73] Separate import classes into files --- bookwyrm/views/__init__.py | 4 +- bookwyrm/views/imports/import_data.py | 92 +------------------------ bookwyrm/views/imports/import_status.py | 50 ++++++++++++++ bookwyrm/views/imports/troubleshoot.py | 54 +++++++++++++++ 4 files changed, 109 insertions(+), 91 deletions(-) create mode 100644 bookwyrm/views/imports/import_status.py create mode 100644 bookwyrm/views/imports/troubleshoot.py diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index d169d126..21eeb39b 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -44,7 +44,9 @@ from .shelf.shelf_actions import create_shelf, delete_shelf from .shelf.shelf_actions import shelve, unshelve # csv import -from .imports.import_data import Import, ImportStatus, ImportTroubleshoot +from .imports.import_data import Import +from .imports.import_status import ImportStatus +from .imports.troubelshoot import ImportTroubleshoot # misc views from .author import Author, EditAuthor diff --git a/bookwyrm/views/imports/import_data.py b/bookwyrm/views/imports/import_data.py index 907c77e2..64cefc7f 100644 --- a/bookwyrm/views/imports/import_data.py +++ b/bookwyrm/views/imports/import_data.py @@ -1,25 +1,16 @@ """ import books from another app """ from io import TextIOWrapper -import math from django.contrib.auth.decorators import login_required -from django.core.exceptions import PermissionDenied -from django.core.paginator import Paginator from django.http import HttpResponseBadRequest -from django.shortcuts import get_object_or_404, redirect +from django.shortcuts import redirect from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.utils.translation import gettext_lazy as _ from django.views import View from bookwyrm import forms, models -from bookwyrm.importers import ( - Importer, - LibrarythingImporter, - GoodreadsImporter, - StorygraphImporter, -) -from bookwyrm.settings import PAGE_LENGTH +from bookwyrm.importers import LibrarythingImporter, GoodreadsImporter, StorygraphImporter # pylint: disable= no-self-use @method_decorator(login_required, name="dispatch") @@ -72,82 +63,3 @@ class Import(View): return redirect(f"/import/{job.id}") return HttpResponseBadRequest() - - -@method_decorator(login_required, name="dispatch") -class ImportStatus(View): - """status of an existing import""" - - def get(self, request, job_id): - """status of an import job""" - job = get_object_or_404(models.ImportJob, id=job_id) - if job.user != request.user: - raise PermissionDenied() - - items = job.items.order_by("index") - pending_items = items.filter(fail_reason__isnull=True, book__isnull=True) - item_count = items.count() or 1 - - paginated = Paginator(items, PAGE_LENGTH) - page = paginated.get_page(request.GET.get("page")) - data = { - "job": job, - "items": page, - "manual_review_count": items.filter( - fail_reason__isnull=False, book_guess__isnull=False, book__isnull=True - ).count(), - "fail_count": items.filter( - fail_reason__isnull=False, book_guess__isnull=True - ).count(), - "page_range": paginated.get_elided_page_range( - page.number, on_each_side=2, on_ends=1 - ), - "complete": not pending_items.exists(), - "percent": math.floor( # pylint: disable=c-extension-no-member - (item_count - pending_items.count()) / item_count * 100 - ), - } - - return TemplateResponse(request, "import/import_status.html", data) - - -@method_decorator(login_required, name="dispatch") -class ImportTroubleshoot(View): - """problems items in an existing import""" - - def get(self, request, job_id): - """status of an import job""" - job = get_object_or_404(models.ImportJob, id=job_id) - if job.user != request.user: - raise PermissionDenied() - - items = job.items.order_by("index").filter( - fail_reason__isnull=False, book_guess__isnull=False - ) - - paginated = Paginator(items, PAGE_LENGTH) - page = paginated.get_page(request.GET.get("page")) - data = { - "job": job, - "items": page, - "page_range": paginated.get_elided_page_range( - page.number, on_each_side=2, on_ends=1 - ), - "complete": True, - } - - return TemplateResponse(request, "import/troubleshoot.html", data) - - def post(self, request, job_id): - """retry lines from an import""" - job = get_object_or_404(models.ImportJob, id=job_id) - items = job.items.filter(fail_reason__isnull=False) - - importer = Importer() - job = importer.create_retry_job( - request.user, - job, - items, - ) - importer.start_import(job) - return redirect(f"/import/{job.id}") diff --git a/bookwyrm/views/imports/import_status.py b/bookwyrm/views/imports/import_status.py new file mode 100644 index 00000000..2d18d656 --- /dev/null +++ b/bookwyrm/views/imports/import_status.py @@ -0,0 +1,50 @@ +""" import books from another app """ +import math + +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied +from django.core.paginator import Paginator +from django.shortcuts import get_object_or_404 +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views import View + +from bookwyrm import models +from bookwyrm.settings import PAGE_LENGTH + +# pylint: disable= no-self-use +@method_decorator(login_required, name="dispatch") +class ImportStatus(View): + """status of an existing import""" + + def get(self, request, job_id): + """status of an import job""" + job = get_object_or_404(models.ImportJob, id=job_id) + if job.user != request.user: + raise PermissionDenied() + + items = job.items.order_by("index") + pending_items = items.filter(fail_reason__isnull=True, book__isnull=True) + item_count = items.count() or 1 + + paginated = Paginator(items, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "job": job, + "items": page, + "manual_review_count": items.filter( + fail_reason__isnull=False, book_guess__isnull=False, book__isnull=True + ).count(), + "fail_count": items.filter( + fail_reason__isnull=False, book_guess__isnull=True + ).count(), + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "complete": not pending_items.exists(), + "percent": math.floor( # pylint: disable=c-extension-no-member + (item_count - pending_items.count()) / item_count * 100 + ), + } + + return TemplateResponse(request, "import/import_status.html", data) diff --git a/bookwyrm/views/imports/troubleshoot.py b/bookwyrm/views/imports/troubleshoot.py new file mode 100644 index 00000000..48f2b998 --- /dev/null +++ b/bookwyrm/views/imports/troubleshoot.py @@ -0,0 +1,54 @@ +""" import books from another app """ +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied +from django.core.paginator import Paginator +from django.shortcuts import get_object_or_404, redirect +from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator +from django.views import View + +from bookwyrm import models +from bookwyrm.importers import Importer +from bookwyrm.settings import PAGE_LENGTH + +# pylint: disable= no-self-use +@method_decorator(login_required, name="dispatch") +class ImportTroubleshoot(View): + """problems items in an existing import""" + + def get(self, request, job_id): + """status of an import job""" + job = get_object_or_404(models.ImportJob, id=job_id) + if job.user != request.user: + raise PermissionDenied() + + items = job.items.order_by("index").filter( + fail_reason__isnull=False, book_guess__isnull=False + ) + + paginated = Paginator(items, PAGE_LENGTH) + page = paginated.get_page(request.GET.get("page")) + data = { + "job": job, + "items": page, + "page_range": paginated.get_elided_page_range( + page.number, on_each_side=2, on_ends=1 + ), + "complete": True, + } + + return TemplateResponse(request, "import/troubleshoot.html", data) + + def post(self, request, job_id): + """retry lines from an import""" + job = get_object_or_404(models.ImportJob, id=job_id) + items = job.items.filter(fail_reason__isnull=False) + + importer = Importer() + job = importer.create_retry_job( + request.user, + job, + items, + ) + importer.start_import(job) + return redirect(f"/import/{job.id}") From 221cde9be4a8e33d617326c508f52b309b545109 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Fri, 12 Nov 2021 16:23:56 -0800 Subject: [PATCH 36/73] Adds manual review view --- bookwyrm/templates/import/import_status.html | 4 ++ bookwyrm/templates/import/manual_review.html | 76 ++++++++++++++++++++ bookwyrm/urls.py | 5 ++ bookwyrm/views/__init__.py | 3 +- bookwyrm/views/imports/import_data.py | 6 +- bookwyrm/views/imports/manually_review.py | 39 ++++++++++ bookwyrm/views/imports/troubleshoot.py | 2 +- 7 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 bookwyrm/templates/import/manual_review.html create mode 100644 bookwyrm/views/imports/manually_review.py diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 01a1fec6..b2e21d22 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -69,6 +69,7 @@ {% block actions %}{% endblock %} + {% block table_headers %} + {% endblock %} {% for item in items %} + {% block table_row %} + {% endblock %} {% endfor %}
{% trans "Row" %} @@ -89,7 +90,9 @@ {% trans "Status" %}
{{ item.index }} @@ -126,6 +129,7 @@ {% endif %}
diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html new file mode 100644 index 00000000..c9c03869 --- /dev/null +++ b/bookwyrm/templates/import/manual_review.html @@ -0,0 +1,76 @@ +{% extends 'import/import_status.html' %} +{% load i18n %} +{% load utilities %} + +{% block title %}{% trans "Import Troubleshooting" %}{% endblock %} + +{% block page_title %} +{% trans "Review items" %} +{% endblock %} + +{% block actions %} +
+
+

+ {% trans "Approving a suggestion will permanently add the suggested book to your shelves and associate your reading dates, reviews, and ratings with that book." %} +

+
+
+{% endblock %} + +{% block table_headers %} +
+ {% trans "Row" %} + + {% trans "Title" %} + + {% trans "ISBN" %} + + {% trans "Author" %} + + {% trans "Actions" %} +
+ {{ item.index }} + + {{ item.normalized_data.title }} + + {{ item.isbn }} + + {{ item.normalized_data.authors }} + + + +
+
+ {% with guess=item.book_guess %} +
+ + {% include 'snippets/book_cover.html' with book=guess cover_class='is-h-s' size='small' %} + +
+
+ {% include 'snippets/book_titleby.html' with book=guess %} +
+ {% endwith %} +
+
{{ item.normalized_data.authors }} - - + +
+ {% csrf_token %} + +
+ +
+ {% csrf_token %} + +
{% trans "Author" %} + {% trans "Review" %} + {% trans "Book" %} {{ item.normalized_data.authors }} +

{% include 'snippets/stars.html' with rating=item.rating %}

+

{{ item.review|truncatechars:100 }}

+ 		{% if item.book %} diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index 30274ca0..b7c8e3b9 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -32,6 +32,9 @@ {% trans "Author" %} + {% trans "Review" %} + {% trans "Actions" %} {{ item.normalized_data.authors }} -
- {% csrf_token %} - -
+ 		+

{% include 'snippets/stars.html' with rating=item.rating %}

+

{{ item.review|truncatechars:100 }}

+ 		+
+
+ {% csrf_token %} + +
-
- {% csrf_token %} - -
+
+ {% csrf_token %} + +
+
- {% block table_headers %} + + {% block import_cols_headers %} + {% endblock %} - {% endblock %} {% for item in items %} - {% block table_row %} + {% block index_col %} + {% endblock %} @@ -110,10 +114,14 @@ + + {% block import_cols %} + {% endblock %} - {% endblock %} + {% block action_row %}{% endblock %} {% endfor %}
{% trans "Row" %} @@ -84,23 +83,28 @@ {% trans "Author" %} + {% trans "Shelf" %} + {% trans "Review" %} {% trans "Book" %} {% trans "Status" %}
{{ item.index }} {{ item.normalized_data.title }} {{ item.normalized_data.authors }} + {{ item.normalized_data.shelf }} +

{% include 'snippets/stars.html' with rating=item.rating %}

{{ item.review|truncatechars:100 }}

 {% if item.book %} @@ -136,8 +144,9 @@ {% trans "Pending" %} {% endif %}
diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index b7c8e3b9..833e9d7f 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -18,47 +18,19 @@ {% endblock %} -{% block table_headers %} - - - {% trans "Row" %} - - - {% trans "Title" %} - - - {% trans "ISBN" %} - - - {% trans "Author" %} - - - {% trans "Review" %} - +{% block import_cols_headers %} {% trans "Actions" %} - {% endblock %} -{% block table_row %} - +{% block index_col %} {{ item.index }} - - {{ item.normalized_data.title }} - - - {{ item.isbn }} - - - {{ item.normalized_data.authors }} - - -

{% include 'snippets/stars.html' with rating=item.rating %}

-

{{ item.review|truncatechars:100 }}

- +{% endblock %} + +{% block import_cols %}
@@ -78,9 +50,11 @@
- +{% endblock %} + +{% block action_row %} - +
{% with guess=item.book_guess %}
From 659d0f19ebdcbd4a5965abde30cefae5aab0d8b2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:43:52 -0800 Subject: [PATCH 42/73] Improves import table ui adds table container, plays with display a bit --- bookwyrm/templates/import/import_status.html | 164 ++++++++++--------- bookwyrm/templates/import/manual_review.html | 41 ++--- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 6c5560dc..5b33f394 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -69,86 +69,92 @@ {% endblock %} {% block actions %}{% endblock %} - - - - - - - - - {% block import_cols_headers %} - - - {% endblock %} - - {% for item in items %} - - {% block index_col %} - - {% endblock %} - - - - - - {% block import_cols %} - - + {% endblock %} + + {% block action_row %}{% endblock %} + {% endfor %} +
- {% trans "Row" %} - - {% trans "Title" %} - - {% trans "ISBN" %} - - {% trans "Author" %} - - {% trans "Shelf" %} - - {% trans "Review" %} - - {% trans "Book" %} - - {% trans "Status" %} -
- {{ item.index }} - - {{ item.normalized_data.title }} - - {{ item.isbn }} - - {{ item.normalized_data.authors }} - - {{ item.normalized_data.shelf }} - -

{% include 'snippets/stars.html' with rating=item.rating %}

-

{{ item.review|truncatechars:100 }}

- 		- {% if item.book %} - - {% include 'snippets/book_cover.html' with book=item.book cover_class='is-h-s' size='small' %} - - {% endif %} - - {% if item.book %} - - {% trans "Imported" %} +
+ + + + + + + + + {% block import_cols_headers %} + + + {% endblock %} + + {% for item in items %} + + {% block index_col %} + + {% endblock %} + + + + + + {% block import_cols %} + + - {% endblock %} - - {% block action_row %}{% endblock %} - {% endfor %} -
+ {% trans "Row" %} + + {% trans "Title" %} + + {% trans "ISBN" %} + + {% trans "Author" %} + + {% trans "Shelf" %} + + {% trans "Review" %} + + {% trans "Book" %} + + {% trans "Status" %} +
+ {{ item.index }} + + {{ item.normalized_data.title }} + + {{ item.isbn }} + + {{ item.normalized_data.authors }} + + {{ item.normalized_data.shelf }} + +

{% include 'snippets/stars.html' with rating=item.rating %}

+

{{ item.review|truncatechars:100 }}

+ 		+ {% if item.book %} + + {% include 'snippets/book_cover.html' with book=item.book cover_class='is-h-s' size='small' %} + + {% endif %} + + {% if item.book %} + + {% trans "Imported" %} - {% elif item.fail_reason %} - - - {{ item.fail_reason }} - - {% else %} - - {% trans "Pending" %} - {% endif %} -
+ {% elif item.fail_reason %} + + + {% if item.book_guess %} + {% trans "Needs manual review" %} + {% else %} + {{ item.fail_reason }} + {% endif %} + + {% else %} + + {% trans "Pending" %} + {% endif %} +
+
diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index 833e9d7f..4661520b 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -19,9 +19,6 @@ {% endblock %} {% block import_cols_headers %} - - {% trans "Actions" %} - {% endblock %} {% block index_col %} @@ -31,30 +28,11 @@ {% endblock %} {% block import_cols %} - -
-
- {% csrf_token %} - -
- -
- {% csrf_token %} - -
-
- {% endblock %} {% block action_row %} - +
{% with guess=item.book_guess %}
@@ -64,6 +42,23 @@
{% include 'snippets/book_titleby.html' with book=guess %} +
+
+ {% csrf_token %} + +
+ +
+ {% csrf_token %} + +
+
{% endwith %}
From d3f23b4a0a5700b22107b13e800966906978b006 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:50:56 -0800 Subject: [PATCH 43/73] Updates calls in tests --- .../tests/importers/test_goodreads_import.py | 14 +++++--------- .../tests/importers/test_librarything_import.py | 16 +++++----------- .../tests/importers/test_storygraph_import.py | 9 +++------ 3 files changed, 13 insertions(+), 26 deletions(-) diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 12b5578b..44605700 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -92,9 +92,7 @@ class GoodreadsImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -116,9 +114,8 @@ class GoodreadsImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, True, "unlisted" - ) + handle_imported_book(import_item) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "mixed feelings") self.assertEqual(review.rating, 2) @@ -136,9 +133,8 @@ class GoodreadsImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, True, "unlisted" - ) + handle_imported_book(import_item) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) self.assertEqual(review.rating, 3) diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 1ec94bbb..5745544e 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -5,11 +5,10 @@ import datetime import pytz from django.test import TestCase -import responses from bookwyrm import models from bookwyrm.importers import LibrarythingImporter -from bookwyrm.importers.importer import start_import_task, handle_imported_book +from bookwyrm.importers.importer import handle_imported_book def make_date(*args): @@ -97,9 +96,7 @@ class LibrarythingImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -125,9 +122,7 @@ class LibrarythingImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -149,9 +144,8 @@ class LibrarythingImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, True, "unlisted" - ) + handle_imported_book(import_item) + review = models.Review.objects.get(book=self.book, user=self.local_user) self.assertEqual(review.content, "chef d'oeuvre") self.assertEqual(review.rating, 4.5) diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index fdad5c71..d11a8d88 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -71,9 +71,7 @@ class StorygraphImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -92,9 +90,8 @@ class StorygraphImport(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, True, "unlisted" - ) + handle_imported_book(import_item) + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) self.assertEqual(review.rating, 5.0) From 3bdda973bc620cd1c2f15125830ccb6acb1643c4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 09:52:09 -0800 Subject: [PATCH 44/73] Creates subdirectory for import views tests --- bookwyrm/tests/views/imports/__init__.py | 1 + bookwyrm/tests/views/{ => imports}/test_import.py | 0 2 files changed, 1 insertion(+) create mode 100644 bookwyrm/tests/views/imports/__init__.py rename bookwyrm/tests/views/{ => imports}/test_import.py (100%) diff --git a/bookwyrm/tests/views/imports/__init__.py b/bookwyrm/tests/views/imports/__init__.py new file mode 100644 index 00000000..b6e690fd --- /dev/null +++ b/bookwyrm/tests/views/imports/__init__.py @@ -0,0 +1 @@ +from . import * diff --git a/bookwyrm/tests/views/test_import.py b/bookwyrm/tests/views/imports/test_import.py similarity index 100% rename from bookwyrm/tests/views/test_import.py rename to bookwyrm/tests/views/imports/test_import.py From 232e051dcb51976086204b63da0237ce4675ff9a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 10:16:05 -0800 Subject: [PATCH 45/73] Fixes import job creates in tests --- bookwyrm/tests/importers/test_goodreads_import.py | 6 ++++-- bookwyrm/tests/importers/test_librarything_import.py | 4 +++- bookwyrm/tests/importers/test_storygraph_import.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 44605700..b1600659 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -108,7 +108,9 @@ class GoodreadsImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """goodreads review import""" - import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) import_item = import_job.items.get(index=2) import_item.book = self.book import_item.save() @@ -126,7 +128,7 @@ class GoodreadsImport(TestCase): def test_handle_imported_book_rating(self, *_): """goodreads rating import""" import_job = self.importer.create_job( - self.local_user, self.csv, False, "public" + self.local_user, self.csv, True, "unlisted" ) import_item = import_job.items.filter(index=0).first() import_item.book = self.book diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 5745544e..804118ef 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -138,7 +138,9 @@ class LibrarythingImport(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """librarything review import""" - import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) import_item = import_job.items.filter(index=0).first() import_item.book = self.book import_item.save() diff --git a/bookwyrm/tests/importers/test_storygraph_import.py b/bookwyrm/tests/importers/test_storygraph_import.py index d11a8d88..09cf32dc 100644 --- a/bookwyrm/tests/importers/test_storygraph_import.py +++ b/bookwyrm/tests/importers/test_storygraph_import.py @@ -83,7 +83,7 @@ class StorygraphImport(TestCase): def test_handle_imported_book_rating(self, *_): """storygraph rating import""" import_job = self.importer.create_job( - self.local_user, self.csv, False, "public" + self.local_user, self.csv, True, "unlisted" ) import_item = import_job.items.filter(index=1).first() import_item.book = self.book From 628f104b13e52a571eb2b88d095f42b9c778fb54 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 10:16:25 -0800 Subject: [PATCH 46/73] Separates out imports views tests --- bookwyrm/tests/views/imports/test_import.py | 32 ---------- .../views/imports/test_import_troubleshoot.py | 59 +++++++++++++++++++ 2 files changed, 59 insertions(+), 32 deletions(-) create mode 100644 bookwyrm/tests/views/imports/test_import_troubleshoot.py diff --git a/bookwyrm/tests/views/imports/test_import.py b/bookwyrm/tests/views/imports/test_import.py index 4a6a8237..7de5e8c8 100644 --- a/bookwyrm/tests/views/imports/test_import.py +++ b/bookwyrm/tests/views/imports/test_import.py @@ -51,19 +51,6 @@ class ImportViews(TestCase): validate_html(result.render()) self.assertEqual(result.status_code, 200) - def test_import_troubleshoot_get(self): - """there are so many views, this just makes sure it LOADS""" - view = views.ImportTroubleshoot.as_view() - import_job = models.ImportJob.objects.create(user=self.local_user, mappings={}) - request = self.factory.get("") - request.user = self.local_user - with patch("bookwyrm.tasks.app.AsyncResult") as async_result: - async_result.return_value = [] - result = view(request, import_job.id) - self.assertIsInstance(result, TemplateResponse) - validate_html(result.render()) - self.assertEqual(result.status_code, 200) - def test_start_import(self): """retry failed items""" view = views.Import.as_view() @@ -87,22 +74,3 @@ class ImportViews(TestCase): job = models.ImportJob.objects.get() self.assertFalse(job.include_reviews) self.assertEqual(job.privacy, "public") - - def test_retry_import(self): - """retry failed items""" - view = views.ImportTroubleshoot.as_view() - import_job = models.ImportJob.objects.create( - user=self.local_user, privacy="unlisted", mappings={} - ) - request = self.factory.post("") - request.user = self.local_user - - with patch("bookwyrm.importers.Importer.start_import"): - view(request, import_job.id) - - self.assertEqual(models.ImportJob.objects.count(), 2) - retry_job = models.ImportJob.objects.last() - - self.assertTrue(retry_job.retry) - self.assertEqual(retry_job.user, self.local_user) - self.assertEqual(retry_job.privacy, "unlisted") diff --git a/bookwyrm/tests/views/imports/test_import_troubleshoot.py b/bookwyrm/tests/views/imports/test_import_troubleshoot.py new file mode 100644 index 00000000..5359cc1e --- /dev/null +++ b/bookwyrm/tests/views/imports/test_import_troubleshoot.py @@ -0,0 +1,59 @@ +""" test for app action functionality """ +from unittest.mock import patch +from django.template.response import TemplateResponse +from django.test import TestCase +from django.test.client import RequestFactory +from bookwyrm.tests.validate_html import validate_html + +from bookwyrm import models, views + + +class ImportTroubleshootViews(TestCase): + """goodreads import views""" + + def setUp(self): + """we need basic test data and mocks""" + self.factory = RequestFactory() + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse@local.com", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + ) + models.SiteSettings.objects.create() + + def test_import_troubleshoot_get(self): + """there are so many views, this just makes sure it LOADS""" + view = views.ImportTroubleshoot.as_view() + import_job = models.ImportJob.objects.create(user=self.local_user, mappings={}) + request = self.factory.get("") + request.user = self.local_user + with patch("bookwyrm.tasks.app.AsyncResult") as async_result: + async_result.return_value = [] + result = view(request, import_job.id) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + + def test_retry_import(self): + """retry failed items""" + view = views.ImportTroubleshoot.as_view() + import_job = models.ImportJob.objects.create( + user=self.local_user, privacy="unlisted", mappings={} + ) + request = self.factory.post("") + request.user = self.local_user + + with patch("bookwyrm.importers.Importer.start_import"): + view(request, import_job.id) + + self.assertEqual(models.ImportJob.objects.count(), 2) + retry_job = models.ImportJob.objects.last() + + self.assertTrue(retry_job.retry) + self.assertEqual(retry_job.user, self.local_user) + self.assertEqual(retry_job.privacy, "unlisted") From c3156a1de50e6d2376a749afd9e8830e965cd5fd Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 10:22:28 -0800 Subject: [PATCH 47/73] Fixes import path in test --- bookwyrm/tests/views/imports/test_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/views/imports/test_import.py b/bookwyrm/tests/views/imports/test_import.py index 7de5e8c8..b8b8b328 100644 --- a/bookwyrm/tests/views/imports/test_import.py +++ b/bookwyrm/tests/views/imports/test_import.py @@ -58,7 +58,7 @@ class ImportViews(TestCase): form.data["source"] = "Goodreads" form.data["privacy"] = "public" form.data["include_reviews"] = False - csv_file = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv") + csv_file = pathlib.Path(__file__).parent.joinpath("../../data/goodreads.csv") form.data["csv_file"] = SimpleUploadedFile( # pylint: disable=consider-using-with csv_file, From e77eea9c817ec9b836c370a632f2946b241677e2 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 10:52:11 -0800 Subject: [PATCH 48/73] Adds tests for import manual review --- .../tests/views/imports/test_import_review.py | 87 +++++++++++++++++++ bookwyrm/views/imports/manually_review.py | 1 + 2 files changed, 88 insertions(+) create mode 100644 bookwyrm/tests/views/imports/test_import_review.py diff --git a/bookwyrm/tests/views/imports/test_import_review.py b/bookwyrm/tests/views/imports/test_import_review.py new file mode 100644 index 00000000..2ab48468 --- /dev/null +++ b/bookwyrm/tests/views/imports/test_import_review.py @@ -0,0 +1,87 @@ +""" test for app action functionality """ +from unittest.mock import patch +from django.template.response import TemplateResponse +from django.test import TestCase +from django.test.client import RequestFactory +from bookwyrm.tests.validate_html import validate_html + +from bookwyrm import models, views + + +class ImportManualReviewViews(TestCase): + """goodreads import views""" + + def setUp(self): + """we need basic test data and mocks""" + self.factory = RequestFactory() + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse@local.com", + "mouse@mouse.mouse", + "password", + local=True, + localname="mouse", + ) + models.SiteSettings.objects.create() + self.job = models.ImportJob.objects.create(user=self.local_user, mappings={}) + + work = models.Work.objects.create(title="Test Work") + self.book = models.Edition.objects.create( + title="Example Edition", + remote_id="https://example.com/book/1", + parent_work=work, + ) + + def test_import_troubleshoot_get(self): + """there are so many views, this just makes sure it LOADS""" + view = views.ImportManualReview.as_view() + request = self.factory.get("") + request.user = self.local_user + with patch("bookwyrm.tasks.app.AsyncResult") as async_result: + async_result.return_value = [] + result = view(request, self.job.id) + self.assertIsInstance(result, TemplateResponse) + validate_html(result.render()) + self.assertEqual(result.status_code, 200) + + def test_approve_item(self): + """a guess is correct""" + import_item = models.ImportItem.objects.create( + index=0, + job=self.job, + book_guess=self.book, + fail_reason="no match", + data={}, + normalized_data={}, + ) + request = self.factory.post("") + request.user = self.local_user + + with patch("bookwyrm.importers.importer.import_item_task.delay") as mock: + views.approve_import_item(request, self.job.id, import_item.id) + self.assertEqual(mock.call_count, 1) + import_item.refresh_from_db() + self.assertIsNone(import_item.fail_reason) + self.assertIsNone(import_item.book_guess) + self.assertEqual(import_item.book.id, self.book.id) + + def test_delete_item(self): + """a guess is correct""" + import_item = models.ImportItem.objects.create( + index=0, + job=self.job, + book_guess=self.book, + fail_reason="no match", + data={}, + normalized_data={}, + ) + request = self.factory.post("") + request.user = self.local_user + + views.delete_import_item(request, self.job.id, import_item.id) + import_item.refresh_from_db() + self.assertEqual(import_item.fail_reason, "no match") + self.assertIsNone(import_item.book_guess) + self.assertIsNone(import_item.book) diff --git a/bookwyrm/views/imports/manually_review.py b/bookwyrm/views/imports/manually_review.py index 877e3ffc..723fd4bb 100644 --- a/bookwyrm/views/imports/manually_review.py +++ b/bookwyrm/views/imports/manually_review.py @@ -51,6 +51,7 @@ def approve_import_item(request, job_id, item_id): ) item.fail_reason = None item.book = item.book_guess + item.book_guess = None item.save() # the good stuff - actually import the data From 4dae851da0b2d85cc35a08c05f34ec9bcedec66d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:15:58 -0800 Subject: [PATCH 49/73] Adds breadcrumbs --- bookwyrm/templates/feed/layout.html | 2 +- bookwyrm/templates/import/import_status.html | 37 ++++++++++++++------ bookwyrm/templates/import/manual_review.html | 6 ++++ bookwyrm/templates/import/troubleshoot.html | 6 ++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/bookwyrm/templates/feed/layout.html b/bookwyrm/templates/feed/layout.html index 8d79781b..6e7ec849 100644 --- a/bookwyrm/templates/feed/layout.html +++ b/bookwyrm/templates/feed/layout.html @@ -9,7 +9,7 @@ {% if user.is_authenticated %}
-

{% trans "Your books" %}

+

{% trans "Your Books" %}

{% if not suggested_books %}

{% trans "There are no books here right now! Try searching for a book to get started" %}

{% else %} diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 5b33f394..f5f590e1 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -7,12 +7,32 @@ {% block content %}{% spaceless %}
- {% if job.retry %} -

{% trans "Retry Status" %}

- {% else %} -

{% trans "Import Status" %}

- {% endif %} - {% trans "Back to imports" %} +

+ {% block page_title %} + {% if job.retry %} + {% trans "Retry Status" %} + {% else %} + {% trans "Import Status" %} + {% endif %} + {% endblock %} +

+ +
@@ -63,11 +83,6 @@
-

- {% block page_title %} - {% trans "Your Import" %} - {% endblock %} -

{% block actions %}{% endblock %}
diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index 4661520b..53601fd1 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -8,6 +8,12 @@ {% trans "Review items" %} {% endblock %} +{% block breadcrumbs %} +
  • + {% trans "Review" %} +
    • +{% endblock %} + {% block actions %}
    diff --git a/bookwyrm/templates/import/troubleshoot.html b/bookwyrm/templates/import/troubleshoot.html index 0be68383..a96aaaea 100644 --- a/bookwyrm/templates/import/troubleshoot.html +++ b/bookwyrm/templates/import/troubleshoot.html @@ -7,6 +7,12 @@ {% trans "Failed items" %} {% endblock %} +{% block breadcrumbs %} +
  • + {% trans "Troubleshooting" %} +
    • +{% endblock %} + {% block actions %}
    From 644d9693300a73e66a40e4c9438bf7afbe411ea4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:22:07 -0800 Subject: [PATCH 50/73] Fixes importer tests --- bookwyrm/tests/importers/test_importer.py | 42 +++++++---------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index b0863fbe..45d87171 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -140,7 +140,7 @@ class GenericImporter(TestCase): with patch( "bookwyrm.models.activitypub_mixin.broadcast_task.apply_async" ) as mock: - import_item_task(self.importer.service, import_item.id) + import_item_task(import_item.id) kwargs = mock.call_args.kwargs self.assertEqual(kwargs["queue"], "low_priority") import_item.refresh_from_db() @@ -160,9 +160,7 @@ class GenericImporter(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -179,16 +177,14 @@ class GenericImporter(TestCase): ) import_job = self.importer.create_job( - self.local_user, self.csv, False, "unlisted" + self.local_user, self.csv, False, "public" ) import_item = import_job.items.first() import_item.book = self.book import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -210,12 +206,8 @@ class GenericImporter(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "public" - ) + handle_imported_book(import_item) + handle_imported_book(import_item) shelf.refresh_from_db() self.assertEqual(shelf.books.first(), self.book) @@ -224,20 +216,16 @@ class GenericImporter(TestCase): @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_review(self, *_): """review import""" - import_job = self.importer.create_job(self.local_user, self.csv, True, "public") + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) import_item = import_job.items.filter(index=3).first() import_item.book = self.book import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): with patch("bookwyrm.models.Status.broadcast") as broadcast_mock: - handle_imported_book( - self.importer.service, - self.local_user, - import_item, - True, - "unlisted", - ) + handle_imported_book(import_item) kwargs = broadcast_mock.call_args.kwargs self.assertEqual(kwargs["software"], "bookwyrm") review = models.Review.objects.get(book=self.book, user=self.local_user) @@ -249,16 +237,14 @@ class GenericImporter(TestCase): def test_handle_imported_book_rating(self, *_): """rating import""" import_job = self.importer.create_job( - self.local_user, self.csv, False, "public" + self.local_user, self.csv, True, "unlisted" ) import_item = import_job.items.filter(index=1).first() import_item.book = self.book import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, True, "unlisted" - ) + handle_imported_book(import_item) review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) self.assertIsInstance(review, models.ReviewRating) self.assertEqual(review.rating, 3.0) @@ -274,9 +260,7 @@ class GenericImporter(TestCase): import_item.save() with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): - handle_imported_book( - self.importer.service, self.local_user, import_item, False, "unlisted" - ) + handle_imported_book(import_item) self.assertFalse( models.Review.objects.filter(book=self.book, user=self.local_user).exists() ) From 1e2dca402b251fc9ae969516d26de7294fd72af5 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:40:19 -0800 Subject: [PATCH 51/73] Adds null value to breadcrumb links --- bookwyrm/templates/import/manual_review.html | 2 +- bookwyrm/templates/import/troubleshoot.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index 53601fd1..2c487563 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -10,7 +10,7 @@ {% block breadcrumbs %}
  • - {% trans "Review" %} + {% trans "Review" %}
    • {% endblock %} diff --git a/bookwyrm/templates/import/troubleshoot.html b/bookwyrm/templates/import/troubleshoot.html index a96aaaea..d73be6d0 100644 --- a/bookwyrm/templates/import/troubleshoot.html +++ b/bookwyrm/templates/import/troubleshoot.html @@ -9,7 +9,7 @@ {% block breadcrumbs %}
  • - {% trans "Troubleshooting" %} + {% trans "Troubleshooting" %}
    • {% endblock %} From 1e46de4c9d4a184dedf127a4245fb75254a21b3f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:44:05 -0800 Subject: [PATCH 52/73] Associate imported review with import item --- bookwyrm/importers/importer.py | 2 ++ .../0115_importitem_linked_review.py | 24 +++++++++++++++++++ bookwyrm/models/import_job.py | 3 +++ bookwyrm/tests/importers/test_importer.py | 6 +++++ 4 files changed, 35 insertions(+) create mode 100644 bookwyrm/migrations/0115_importitem_linked_review.py diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 71f02231..a1c35ef7 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -205,3 +205,5 @@ def handle_imported_book(item): ) # only broadcast this review to other bookwyrm instances review.save(software="bookwyrm", priority=LOW) + item.linked_review = review + item.save() diff --git a/bookwyrm/migrations/0115_importitem_linked_review.py b/bookwyrm/migrations/0115_importitem_linked_review.py new file mode 100644 index 00000000..8cff9b8c --- /dev/null +++ b/bookwyrm/migrations/0115_importitem_linked_review.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.5 on 2021-11-13 19:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0114_importjob_source"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="linked_review", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="bookwyrm.review", + ), + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 6b8f0b46..ddd9eaec 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -60,6 +60,9 @@ class ImportItem(models.Model): related_name="book_guess", ) fail_reason = models.TextField(null=True) + linked_review = models.ForeignKey( + "Review", on_delete=models.SET_NULL, null=True, blank=True + ) def resolve(self): """try various ways to lookup a book""" diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 45d87171..99cdcd28 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -233,6 +233,9 @@ class GenericImporter(TestCase): self.assertEqual(review.rating, 2.0) self.assertEqual(review.privacy, "unlisted") + import_item.refresh_from_db() + self.assertEqual(import_item.linked_review, review) + @patch("bookwyrm.activitystreams.add_status_task.delay") def test_handle_imported_book_rating(self, *_): """rating import""" @@ -250,6 +253,9 @@ class GenericImporter(TestCase): self.assertEqual(review.rating, 3.0) self.assertEqual(review.privacy, "unlisted") + import_item.refresh_from_db() + self.assertEqual(import_item.linked_review.id, review.id) + def test_handle_imported_book_reviews_disabled(self, *_): """review import""" import_job = self.importer.create_job( From 712d8ecfb437fcafafbfc971e796b8b01895c72b Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:52:08 -0800 Subject: [PATCH 53/73] Don't show empty stars when there's no review --- bookwyrm/templates/import/import_status.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index f5f590e1..4871a4c7 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -134,8 +134,15 @@ {{ item.normalized_data.shelf }}
    {% block import_cols %} {% endblock %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index d6a79c20..6f658016 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -242,6 +242,11 @@ urlpatterns = [ views.ImportStatus.as_view(), name="import-status", ), + re_path( + r"^import/(?P\d+)/retry/(?P\d+)/?$", + views.ImportStatus.as_view(), + name="import-item-retry", + ), re_path( r"^import/(?P\d+)/failed/?$", views.ImportTroubleshoot.as_view(), diff --git a/bookwyrm/views/imports/import_status.py b/bookwyrm/views/imports/import_status.py index 7e7d5179..54174082 100644 --- a/bookwyrm/views/imports/import_status.py +++ b/bookwyrm/views/imports/import_status.py @@ -4,12 +4,14 @@ import math from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied from django.core.paginator import Paginator -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse +from django.utils import timezone from django.utils.decorators import method_decorator from django.views import View from bookwyrm import models +from bookwyrm.importers.importer import import_item_task from bookwyrm.settings import PAGE_LENGTH # pylint: disable= no-self-use @@ -40,10 +42,19 @@ class ImportStatus(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "complete": not job.pending_items.exists(), "percent": math.floor( # pylint: disable=c-extension-no-member (item_count - job.pending_items.count()) / item_count * 100 ), + # hours since last import item update + "inactive_time": (job.updated_date - timezone.now()).seconds / 60 / 60, } return TemplateResponse(request, "import/import_status.html", data) + + def post(self, request, job_id, item_id): + """retry an item""" + item = get_object_or_404( + models.ImportItem, id=item_id, job__id=job_id, job__user=request.user + ) + import_item_task.delay(item.id) + return redirect("import-status", job_id) From 31f33518545453a5002c56e03a6da8968eff2633 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 10:22:26 -0800 Subject: [PATCH 67/73] Fixes bug comparing dates to nonetype --- bookwyrm/models/import_job.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 387261f0..97b93b97 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -205,7 +205,9 @@ class ImportItem(models.Model): if start_date and start_date is not None and not self.date_read: return [ReadThrough(start_date=start_date)] if self.date_read: - start_date = start_date if start_date < self.date_read else None + start_date = ( + start_date if start_date and start_date < self.date_read else None + ) return [ ReadThrough( start_date=start_date, From 9f6796bbf58b8831dc0e65b8ff472d98720c45f7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 10:29:12 -0800 Subject: [PATCH 68/73] Safer request for normalized data --- bookwyrm/models/import_job.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 97b93b97..c4679585 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -131,18 +131,18 @@ class ImportItem(models.Model): @property def title(self): """get the book title""" - return self.normalized_data["title"] + return self.normalized_data.get("title") @property def author(self): """get the book's authors""" - return self.normalized_data["authors"] + return self.normalized_data.get("authors") @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.normalized_data["isbn_13"]) or unquote_string( - self.normalized_data["isbn_10"] + return unquote_string(self.normalized_data.get("isbn_13")) or unquote_string( + self.normalized_data.get("isbn_10") ) @property @@ -153,13 +153,13 @@ class ImportItem(models.Model): @property def review(self): """a user-written review, to be imported with the book data""" - return self.normalized_data["review_body"] + return self.normalized_data.get("review_body") @property def rating(self): """x/5 star rating for a book""" if self.normalized_data.get("rating"): - return float(self.normalized_data["rating"]) + return float(self.normalized_data.get("rating")) return None @property @@ -167,7 +167,7 @@ class ImportItem(models.Model): """when the book was added to this dataset""" if self.normalized_data.get("date_added"): return timezone.make_aware( - dateutil.parser.parse(self.normalized_data["date_added"]) + dateutil.parser.parse(self.normalized_data.get("date_added")) ) return None @@ -176,7 +176,7 @@ class ImportItem(models.Model): """when the book was started""" if self.normalized_data.get("date_started"): return timezone.make_aware( - dateutil.parser.parse(self.normalized_data["date_started"]) + dateutil.parser.parse(self.normalized_data.get("date_started")) ) return None @@ -185,7 +185,7 @@ class ImportItem(models.Model): """the date a book was completed""" if self.normalized_data.get("date_finished"): return timezone.make_aware( - dateutil.parser.parse(self.normalized_data["date_finished"]) + dateutil.parser.parse(self.normalized_data.get("date_finished")) ) return None @@ -218,10 +218,10 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r} Item {!r}>".format(self.index, self.normalized_data["title"]) + return "<{!r} Item {!r}>".format(self.index, self.normalized_data.get("title")) def __str__(self): # pylint: disable=consider-using-f-string return "{} by {}".format( - self.normalized_data["title"], self.normalized_data["authors"] + self.normalized_data.get("title"), self.normalized_data.get("authors") ) From 14e2960d063a71471b52edbd8df9548d429d5559 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 10:58:46 -0800 Subject: [PATCH 69/73] Update legacy jobs --- bookwyrm/importers/importer.py | 14 ++++++++++ bookwyrm/templates/import/import_status.html | 29 ++++++++++++++++++-- bookwyrm/urls.py | 2 +- bookwyrm/views/__init__.py | 2 +- bookwyrm/views/imports/import_status.py | 26 ++++++++++++++---- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 438ff7db..94e6734e 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -57,6 +57,20 @@ class Importer: self.create_item(job, index, entry) return job + def update_legacy_job(self, job): + """patch up a job that was in the old format""" + items = job.items + headers = list(items.first().data.keys()) + job.mappings = self.create_row_mappings(headers) + job.updated_date = timezone.now() + job.save() + + for item in items.all(): + normalized = self.normalize_row(item.data, job.mappings) + normalized["shelf"] = self.get_shelf(normalized) + item.normalized_data = normalized + item.save() + def create_row_mappings(self, headers): """guess what the headers mean""" mappings = {} diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 6c7d54b9..6370b866 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -57,7 +57,7 @@ {% endif %} - {% if manual_review_count %} + {% if manual_review_count and not legacy %}
    {% blocktrans trimmed count counter=manual_review_count with display_counter=manual_review_count|intcomma %} {{ display_counter }} item needs manual approval. @@ -68,7 +68,7 @@
    {% endif %} - {% if complete and fail_count and not job.retry %} + {% if job.complete and fail_count and not job.retry and not legacy %}
    {% blocktrans trimmed count counter=fail_count with display_counter=fail_count|intcomma %} {{ display_counter }} item failed to import. @@ -114,6 +114,15 @@ {% endblock %}
    + {% if legacy %} + + + + {% else %} {% for item in items %} {% block index_col %} @@ -171,7 +180,7 @@ {% trans "Pending" %} {# retry option if an item appears to be hanging #} - {% if job.created_date != job.updated_date and inactive_time > 0.24 %} + {% if job.created_date != job.updated_date and inactive_time > 24 %} {% csrf_token %} @@ -184,13 +193,27 @@ {% block action_row %}{% endblock %} {% endfor %} + {% endif %}
    + {% if item.rating %}

    {% include 'snippets/stars.html' with rating=item.rating %}

    + {% endif %} + {% if item.review %}

    {{ item.review|truncatechars:100 }}

    + {% endif %} + {% if item.linked_review %} + {% trans "View imported review" %} + {% endif %}     		From 8b7720c8b85264e533bc160991f36e533ed46461 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 11:54:28 -0800 Subject: [PATCH 54/73] Use "reject" instead of "delete" on review page --- bookwyrm/templates/import/manual_review.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/templates/import/manual_review.html b/bookwyrm/templates/import/manual_review.html index 2c487563..b6c2b6b2 100644 --- a/bookwyrm/templates/import/manual_review.html +++ b/bookwyrm/templates/import/manual_review.html @@ -61,7 +61,7 @@ {% csrf_token %} From a9622942cd71ebbc8ac2bf048ecd81e8e3bcb9db Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 12:11:07 -0800 Subject: [PATCH 55/73] Test correctly adding goodreads isbns --- bookwyrm/tests/importers/test_goodreads_import.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index b1600659..a51eca89 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -52,6 +52,11 @@ class GoodreadsImport(TestCase): self.assertEqual(len(import_items), 3) self.assertEqual(import_items[0].index, 0) self.assertEqual(import_items[0].data["Book Id"], "42036538") + self.assertEqual( + import_items[0].normalized_data["isbn_13"], '=""9781250313195"' + ) + self.assertEqual(import_items[0].normalized_data["isbn_10"], '=""1250313198"') + self.assertEqual(import_items[1].index, 1) self.assertEqual(import_items[1].data["Book Id"], "52691223") self.assertEqual(import_items[2].index, 2) From fb91c33682c2e42d8b365a24209aa41d7678301f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 12:24:16 -0800 Subject: [PATCH 56/73] Fixes isbn assignment for goodreads --- bookwyrm/importers/importer.py | 30 +++++++++---------- bookwyrm/models/import_job.py | 4 ++- bookwyrm/templates/import/import_status.html | 2 +- .../tests/importers/test_goodreads_import.py | 4 +-- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 71f02231..db13b652 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -20,20 +20,20 @@ class Importer: encoding = "UTF-8" # these are from Goodreads - row_mappings_guesses = { - "id": ["id", "book id"], - "title": ["title"], - "authors": ["author", "authors", "primary author"], - "isbn_13": ["isbn13", "isbn"], - "isbn_10": ["isbn10", "isbn"], - "shelf": ["shelf", "exclusive shelf", "read status"], - "review_name": ["review name"], - "review_body": ["my review", "review"], - "rating": ["my rating", "rating", "star rating"], - "date_added": ["date added", "entry date", "added"], - "date_started": ["date started", "started"], - "date_finished": ["date finished", "last date read", "date read", "finished"], - } + row_mappings_guesses = [ + ("id", ["id", "book id"]), + ("title", ["title"]), + ("authors", ["author", "authors", "primary author"]), + ("isbn_10", ["isbn10", "isbn"]), + ("isbn_13", ["isbn13", "isbn"]), + ("shelf", ["shelf", "exclusive shelf", "read status"]), + ("review_name", ["review name"]), + ("review_body", ["my review", "review"]), + ("rating", ["my rating", "rating", "star rating"]), + ("date_added", ["date added", "entry date", "added"]), + ("date_started", ["date started", "started"]), + ("date_finished", ["date finished", "last date read", "date read", "finished"]), + ] date_fields = ["date_added", "date_started", "date_finished"] shelf_mapping_guesses = { "to-read": ["to-read"], @@ -60,7 +60,7 @@ class Importer: def create_row_mappings(self, headers): """guess what the headers mean""" mappings = {} - for (key, guesses) in self.row_mappings_guesses.items(): + for (key, guesses) in self.row_mappings_guesses: value = [h for h in headers if h.lower() in guesses] value = value[0] if len(value) else None if value: diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 6b8f0b46..18565017 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -12,6 +12,8 @@ from .fields import PrivacyLevels def unquote_string(text): """resolve csv quote weirdness""" + if not text: + return None match = re.match(r'="([^"]*)"', text) if match: return match.group(1) @@ -122,7 +124,7 @@ class ImportItem(models.Model): @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.normalized_data["isbn_13"]) + return unquote_string(self.normalized_data["isbn_13"]) or unquote_string(self.normalized_data["isbn_10"]) @property def shelf(self): diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index f5f590e1..8208a2fa 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -125,7 +125,7 @@ {{ item.normalized_data.title }} - {{ item.isbn }} + {{ item.isbn|default:'' }} {{ item.normalized_data.authors }} diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index a51eca89..4a043b59 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -53,9 +53,9 @@ class GoodreadsImport(TestCase): self.assertEqual(import_items[0].index, 0) self.assertEqual(import_items[0].data["Book Id"], "42036538") self.assertEqual( - import_items[0].normalized_data["isbn_13"], '=""9781250313195"' + import_items[0].normalized_data["isbn_13"], '="9781250313195"' ) - self.assertEqual(import_items[0].normalized_data["isbn_10"], '=""1250313198"') + self.assertEqual(import_items[0].normalized_data["isbn_10"], '="1250313198"') self.assertEqual(import_items[1].index, 1) self.assertEqual(import_items[1].data["Book Id"], "52691223") From 7f06ee3844d0704bfa3947c0e70f623b70556dec Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 12:46:27 -0800 Subject: [PATCH 57/73] Fixes getting isbn for librarything imports --- bookwyrm/importers/importer.py | 2 +- bookwyrm/importers/librarything_import.py | 10 ++++++---- bookwyrm/tests/importers/test_librarything_import.py | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index db13b652..ca63ae4a 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -25,7 +25,7 @@ class Importer: ("title", ["title"]), ("authors", ["author", "authors", "primary author"]), ("isbn_10", ["isbn10", "isbn"]), - ("isbn_13", ["isbn13", "isbn"]), + ("isbn_13", ["isbn13", "isbn", "isbns"]), ("shelf", ["shelf", "exclusive shelf", "read status"]), ("review_name", ["review name"]), ("review_body", ["my review", "review"]), diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index d6426de6..cff6ba7d 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -12,10 +12,12 @@ class LibrarythingImporter(Importer): def normalize_row(self, entry, mappings): # pylint: disable=no-self-use """use the dataclass to create the formatted row of data""" - normalized = {k: entry.get(v) for k, v in mappings.items()} - for date_field in self.date_fields: - date = normalized[date_field] - normalized[date_field] = re.sub(r"\[|\]", "", date) + remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None + normalized = { + k: remove_brackets(entry.get(v)) for k, v in mappings.items() + } + isbn_13 = normalized["isbn_13"].split(', ') + normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None return normalized def get_shelf(self, normalized_row): diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index 804118ef..f5d8d669 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -56,6 +56,9 @@ class LibrarythingImport(TestCase): self.assertEqual(len(import_items), 3) self.assertEqual(import_items[0].index, 0) self.assertEqual(import_items[0].data["Book Id"], "5498194") + self.assertEqual(import_items[0].normalized_data["isbn_13"], "9782070291342") + self.assertEqual(import_items[0].normalized_data["isbn_10"], "2070291340") + self.assertEqual(import_items[1].index, 1) self.assertEqual(import_items[1].data["Book Id"], "5015319") self.assertEqual(import_items[2].index, 2) From 32d0d8d0274150f5e10e28f81e7f73cd06265f26 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 13 Nov 2021 13:04:54 -0800 Subject: [PATCH 58/73] Expand librarything csv processing tests --- bookwyrm/tests/data/librarything.tsv | 2 +- bookwyrm/tests/importers/test_librarything_import.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bookwyrm/tests/data/librarything.tsv b/bookwyrm/tests/data/librarything.tsv index a707f2a9..68bbe48e 100644 --- a/bookwyrm/tests/data/librarything.tsv +++ b/bookwyrm/tests/data/librarything.tsv @@ -1,4 +1,4 @@ Book Id Title Sort Character Primary Author Primary Author Role Secondary Author Secondary Author Roles Publication Date Review Rating Comment Private Comment Summary Media Physical Description Weight Height Thickness Length Dimensions Page Count LCCN Acquired Date Started Date Read Barcode BCID Tags Collections Languages Original Languages LC Classification ISBN ISBNs Subjects Dewey Decimal Dewey Wording Other Call Number Copies Source Entry Date From Where OCLC Work id Lending Patron Lending Status Lending Start Lending End -5498194 Marelle 1 Cortázar, Julio Gallimard (1979), Poche 1979 chef d'oeuvre 4.5 Marelle by Julio Cortázar (1979) Broché 590 p.; 7.24 inches 1.28 pounds 7.24 inches 1.26 inches 4.96 inches 7.24 x 4.96 x 1.26 inches 590 [2007-04-16] [2007-05-08] roman, espagnol, expérimental, bohème, philosophie Your library French Spanish PQ7797 .C7145 [2070291340] 2070291340, 9782070291342 Cortâazar, Julio. Rayuela 863 Literature > Spanish And Portuguese > Spanish fiction 1 Amazon.fr [2006-08-09] 57814 +5498194 Marelle 1 Cortazar, Julio Gallimard (1979), Poche 1979 chef d'oeuvre 4.5 Marelle by Julio Cortázar (1979) Broché 590 p.; 7.24 inches 1.28 pounds 7.24 inches 1.26 inches 4.96 inches 7.24 x 4.96 x 1.26 inches 590 [2007-04-16] [2007-05-08] roman, espagnol, expérimental, bohème, philosophie Your library French Spanish PQ7797 .C7145 [2070291340] 2070291340, 9782070291342 Cortâazar, Julio. Rayuela 863 Literature > Spanish And Portuguese > Spanish fiction 1 Amazon.fr [2006-08-09] 57814 5015319 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) 1 Roubaud, Jacques Seuil (1989), Unknown Binding 1989 5 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) by Jacques Roubaud (1989) Broché 411 p.; 7.72 inches 0.88 pounds 7.72 inches 1.02 inches 5.43 inches 7.72 x 5.43 x 1.02 inches 411 Your library English PQ2678 .O77 [2020104725] 2020104725, 9782020104722 Autobiographical fiction|Roubaud, Jacques > Fiction 813 American And Canadian > Fiction > Literature 1 Amazon.com [2006-07-25] 478910 5015399 Le Maître et Marguerite 1 Boulgakov, Mikhaïl Pocket (1994), Poche 1994 Le Maître et Marguerite by Mikhaïl Boulgakov (1994) Broché 579 p.; 7.09 inches 0.66 pounds 7.09 inches 1.18 inches 4.33 inches 7.09 x 4.33 x 1.18 inches 579 Your library French PG3476 .B78 [2266062328] 2266062328, 9782266062329 Allegories|Bulgakov|Good and evil > Fiction|Humanities|Jerusalem > Fiction|Jesus Christ > Fiction|Literature|Mental illness > Fiction|Moscow (Russia) > Fiction|Novel|Pilate, Pontius, 1st cent. > Fiction|Political fiction|Russia > Fiction|Russian fiction|Russian publications (Form Entry)|Soviet Union > History > 1925-1953 > Fiction|literature 891.7342 1917-1945 > 1917-1991 (USSR) > Literature > Literature of other Indo-European languages > Other Languages > Russian > Russian Fiction 1 Amazon.fr [2006-07-25] 10151 diff --git a/bookwyrm/tests/importers/test_librarything_import.py b/bookwyrm/tests/importers/test_librarything_import.py index f5d8d669..49354b36 100644 --- a/bookwyrm/tests/importers/test_librarything_import.py +++ b/bookwyrm/tests/importers/test_librarything_import.py @@ -58,6 +58,11 @@ class LibrarythingImport(TestCase): self.assertEqual(import_items[0].data["Book Id"], "5498194") self.assertEqual(import_items[0].normalized_data["isbn_13"], "9782070291342") self.assertEqual(import_items[0].normalized_data["isbn_10"], "2070291340") + self.assertEqual(import_items[0].normalized_data["title"], "Marelle") + self.assertEqual(import_items[0].normalized_data["authors"], "Cortazar, Julio") + self.assertEqual(import_items[0].normalized_data["date_added"], "2006-08-09") + self.assertEqual(import_items[0].normalized_data["date_started"], "2007-04-16") + self.assertEqual(import_items[0].normalized_data["date_finished"], "2007-05-08") self.assertEqual(import_items[1].index, 1) self.assertEqual(import_items[1].data["Book Id"], "5015319") From 66ad8c3b25114977134269c8dcb0ce3d3bdc14f4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 07:11:33 -0800 Subject: [PATCH 59/73] Updates locale --- locale/en_US/LC_MESSAGES/django.po | 248 +++++++++++++++++++---------- 1 file changed, 161 insertions(+), 87 deletions(-) diff --git a/locale/en_US/LC_MESSAGES/django.po b/locale/en_US/LC_MESSAGES/django.po index 7d8fc801..14bbb1b9 100644 --- a/locale/en_US/LC_MESSAGES/django.po +++ b/locale/en_US/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: 0.0.1\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-10-24 14:09+0000\n" +"POT-Creation-Date: 2021-11-14 15:08+0000\n" "PO-Revision-Date: 2021-02-28 17:19-0800\n" "Last-Translator: Mouse Reeve \n" "Language-Team: English \n" @@ -73,15 +73,16 @@ msgstr "" msgid "Descending" msgstr "" -#: bookwyrm/importers/importer.py:75 +#: bookwyrm/importers/importer.py:127 msgid "Error loading book" msgstr "" -#: bookwyrm/importers/importer.py:88 +#: bookwyrm/importers/importer.py:135 msgid "Could not find a match for book" msgstr "" #: bookwyrm/models/base_model.py:17 +#: bookwyrm/templates/import/import_status.html:171 msgid "Pending" msgstr "" @@ -101,23 +102,23 @@ msgstr "" msgid "Domain block" msgstr "" -#: bookwyrm/models/book.py:232 +#: bookwyrm/models/book.py:233 msgid "Audiobook" msgstr "" -#: bookwyrm/models/book.py:233 +#: bookwyrm/models/book.py:234 msgid "eBook" msgstr "" -#: bookwyrm/models/book.py:234 +#: bookwyrm/models/book.py:235 msgid "Graphic novel" msgstr "" -#: bookwyrm/models/book.py:235 +#: bookwyrm/models/book.py:236 msgid "Hardcover" msgstr "" -#: bookwyrm/models/book.py:236 +#: bookwyrm/models/book.py:237 msgid "Paperback" msgstr "" @@ -134,21 +135,21 @@ msgstr "" msgid "Blocked" msgstr "" -#: bookwyrm/models/fields.py:27 +#: bookwyrm/models/fields.py:29 #, python-format msgid "%(value)s is not a valid remote_id" msgstr "" -#: bookwyrm/models/fields.py:36 bookwyrm/models/fields.py:45 +#: bookwyrm/models/fields.py:38 bookwyrm/models/fields.py:47 #, python-format msgid "%(value)s is not a valid username" msgstr "" -#: bookwyrm/models/fields.py:181 bookwyrm/templates/layout.html:171 +#: bookwyrm/models/fields.py:183 bookwyrm/templates/layout.html:171 msgid "username" msgstr "" -#: bookwyrm/models/fields.py:186 +#: bookwyrm/models/fields.py:188 msgid "A user with that username already exists." msgstr "" @@ -893,22 +894,37 @@ msgstr "" msgid "All known users" msgstr "" -#: bookwyrm/templates/discover/card-header.html:9 +#: bookwyrm/templates/discover/card-header.html:8 #, python-format -msgid "%(username)s rated %(book_title)s" +msgid "%(username)s wants to read %(book_title)s" msgstr "" #: bookwyrm/templates/discover/card-header.html:13 #, python-format +msgid "%(username)s finished reading %(book_title)s" +msgstr "" + +#: bookwyrm/templates/discover/card-header.html:18 +#, python-format +msgid "%(username)s started reading %(book_title)s" +msgstr "" + +#: bookwyrm/templates/discover/card-header.html:23 +#, python-format +msgid "%(username)s rated %(book_title)s" +msgstr "" + +#: bookwyrm/templates/discover/card-header.html:27 +#, python-format msgid "%(username)s reviewed %(book_title)s" msgstr "" -#: bookwyrm/templates/discover/card-header.html:17 +#: bookwyrm/templates/discover/card-header.html:31 #, python-format msgid "%(username)s commented on %(book_title)s" msgstr "" -#: bookwyrm/templates/discover/card-header.html:21 +#: bookwyrm/templates/discover/card-header.html:35 #, python-format msgid "%(username)s quoted %(book_title)s" msgstr "" @@ -1059,9 +1075,8 @@ msgstr "" msgid "Updates" msgstr "" -#: bookwyrm/templates/feed/layout.html:12 -#: bookwyrm/templates/user/books_header.html:3 -msgid "Your books" +#: bookwyrm/templates/feed/layout.html:12 bookwyrm/templates/layout.html:106 +msgid "Your Books" msgstr "" #: bookwyrm/templates/feed/layout.html:14 @@ -1070,11 +1085,13 @@ msgstr "" #: bookwyrm/templates/feed/layout.html:25 #: bookwyrm/templates/shelf/shelf.html:38 +#: bookwyrm/templates/user/books_header.html:4 msgid "To Read" msgstr "" #: bookwyrm/templates/feed/layout.html:26 #: bookwyrm/templates/shelf/shelf.html:40 +#: bookwyrm/templates/user/books_header.html:6 msgid "Currently Reading" msgstr "" @@ -1082,6 +1099,7 @@ msgstr "" #: bookwyrm/templates/shelf/shelf.html:42 #: bookwyrm/templates/snippets/shelve_button/shelve_button_dropdown_options.html:23 #: bookwyrm/templates/snippets/shelve_button/shelve_button_options.html:12 +#: bookwyrm/templates/user/books_header.html:8 msgid "Read" msgstr "" @@ -1367,88 +1385,161 @@ msgid "No recent imports" msgstr "" #: bookwyrm/templates/import/import_status.html:6 -#: bookwyrm/templates/import/import_status.html:10 +#: bookwyrm/templates/import/import_status.html:15 +#: bookwyrm/templates/import/import_status.html:29 msgid "Import Status" msgstr "" -#: bookwyrm/templates/import/import_status.html:11 -msgid "Back to imports" +#: bookwyrm/templates/import/import_status.html:13 +#: bookwyrm/templates/import/import_status.html:27 +msgid "Retry Status" msgstr "" -#: bookwyrm/templates/import/import_status.html:15 +#: bookwyrm/templates/import/import_status.html:22 +msgid "Imports" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:39 msgid "Import started:" msgstr "" -#: bookwyrm/templates/import/import_status.html:20 -msgid "Import completed:" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:24 -msgid "TASK FAILED" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:32 -msgid "Import still in progress." -msgstr "" - -#: bookwyrm/templates/import/import_status.html:34 -msgid "(Hit reload to update!)" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:41 -msgid "Failed to load" +#: bookwyrm/templates/import/import_status.html:48 +msgid "In progress" msgstr "" #: bookwyrm/templates/import/import_status.html:50 -#, python-format -msgid "Jump to the bottom of the list to select the %(failed_count)s items which failed to import." +msgid "Refresh" msgstr "" #: bookwyrm/templates/import/import_status.html:62 #, python-format -msgid "Line %(index)s: %(title)s by %(author)s" +msgid "%(display_counter)s item needs manual approval." +msgid_plural "%(display_counter)s items need manual approval." +msgstr[0] "" +msgstr[1] "" + +#: bookwyrm/templates/import/import_status.html:67 +#: bookwyrm/templates/import/manual_review.html:8 +msgid "Review items" msgstr "" -#: bookwyrm/templates/import/import_status.html:82 -msgid "Select all" +#: bookwyrm/templates/import/import_status.html:73 +#, python-format +msgid "%(display_counter)s item failed to import." +msgid_plural "%(display_counter)s items failed to import." +msgstr[0] "" +msgstr[1] "" + +#: bookwyrm/templates/import/import_status.html:79 +msgid "View and troubleshoot failed items" msgstr "" -#: bookwyrm/templates/import/import_status.html:85 -msgid "Retry items" +#: bookwyrm/templates/import/import_status.html:91 +msgid "Row" msgstr "" -#: bookwyrm/templates/import/import_status.html:112 -msgid "Successfully imported" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:114 -msgid "Import Progress" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:119 -msgid "Book" -msgstr "" - -#: bookwyrm/templates/import/import_status.html:122 +#: bookwyrm/templates/import/import_status.html:94 #: bookwyrm/templates/shelf/shelf.html:141 #: bookwyrm/templates/shelf/shelf.html:163 msgid "Title" msgstr "" -#: bookwyrm/templates/import/import_status.html:125 +#: bookwyrm/templates/import/import_status.html:97 +msgid "ISBN" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:100 #: bookwyrm/templates/shelf/shelf.html:142 #: bookwyrm/templates/shelf/shelf.html:166 msgid "Author" msgstr "" -#: bookwyrm/templates/import/import_status.html:148 +#: bookwyrm/templates/import/import_status.html:103 +msgid "Shelf" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:106 +#: bookwyrm/templates/import/manual_review.html:13 +#: bookwyrm/templates/snippets/create_status.html:17 +msgid "Review" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:110 +msgid "Book" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:113 +#: bookwyrm/templates/settings/announcements/announcements.html:38 +#: bookwyrm/templates/settings/federation/instance_list.html:46 +#: bookwyrm/templates/settings/invites/manage_invite_requests.html:44 +#: bookwyrm/templates/settings/invites/status_filter.html:5 +#: bookwyrm/templates/settings/users/user_admin.html:34 +#: bookwyrm/templates/settings/users/user_info.html:20 +msgid "Status" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:144 +msgid "View imported review" +msgstr "" + +#: bookwyrm/templates/import/import_status.html:158 msgid "Imported" msgstr "" +#: bookwyrm/templates/import/import_status.html:164 +msgid "Needs manual review" +msgstr "" + +#: bookwyrm/templates/import/manual_review.html:5 +#: bookwyrm/templates/import/troubleshoot.html:4 +msgid "Import Troubleshooting" +msgstr "" + +#: bookwyrm/templates/import/manual_review.html:21 +msgid "Approving a suggestion will permanently add the suggested book to your shelves and associate your reading dates, reviews, and ratings with that book." +msgstr "" + +#: bookwyrm/templates/import/manual_review.html:56 +#: bookwyrm/templates/lists/curate.html:57 +msgid "Approve" +msgstr "" + +#: bookwyrm/templates/import/manual_review.html:64 +msgid "Reject" +msgstr "" + #: bookwyrm/templates/import/tooltip.html:6 msgid "You can download your Goodreads data from the Import/Export page of your Goodreads account." msgstr "" +#: bookwyrm/templates/import/troubleshoot.html:7 +msgid "Failed items" +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:12 +msgid "Troubleshooting" +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:20 +msgid "Re-trying an import can fix missing items in cases such as:" +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:23 +msgid "The book has been added to the instance since this import" +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:24 +msgid "A transient error or timeout caused the external data source to be unavailable." +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:25 +msgid "BookWyrm has been updated since this import with a bug fix" +msgstr "" + +#: bookwyrm/templates/import/troubleshoot.html:28 +msgid "Contact your admin or open an issue if you are seeing unexpected failed items." +msgstr "" + #: bookwyrm/templates/landing/about.html:7 bookwyrm/templates/layout.html:230 #, python-format msgid "About %(site_name)s" @@ -1580,10 +1671,6 @@ msgstr "" msgid "Feed" msgstr "" -#: bookwyrm/templates/layout.html:106 -msgid "Your Books" -msgstr "" - #: bookwyrm/templates/layout.html:116 msgid "Settings" msgstr "" @@ -1683,10 +1770,6 @@ msgstr "" msgid "Suggested by" msgstr "" -#: bookwyrm/templates/lists/curate.html:57 -msgid "Approve" -msgstr "" - #: bookwyrm/templates/lists/curate.html:63 msgid "Discard" msgstr "" @@ -2239,15 +2322,6 @@ msgstr "" msgid "End date" msgstr "" -#: bookwyrm/templates/settings/announcements/announcements.html:38 -#: bookwyrm/templates/settings/federation/instance_list.html:46 -#: bookwyrm/templates/settings/invites/manage_invite_requests.html:44 -#: bookwyrm/templates/settings/invites/status_filter.html:5 -#: bookwyrm/templates/settings/users/user_admin.html:34 -#: bookwyrm/templates/settings/users/user_info.html:20 -msgid "Status" -msgstr "" - #: bookwyrm/templates/settings/announcements/announcements.html:48 msgid "active" msgstr "" @@ -3096,10 +3170,6 @@ msgstr "" msgid "Un-boost" msgstr "" -#: bookwyrm/templates/snippets/create_status.html:17 -msgid "Review" -msgstr "" - #: bookwyrm/templates/snippets/create_status.html:39 msgid "Quote" msgstr "" @@ -3526,7 +3596,7 @@ msgstr "" msgid "commented on %(book)s" msgstr "" -#: bookwyrm/templates/snippets/status/headers/note.html:15 +#: bookwyrm/templates/snippets/status/headers/note.html:8 #, python-format msgid "replied to %(username)s's status" msgstr "" @@ -3605,7 +3675,11 @@ msgstr "" msgid "Show less" msgstr "" -#: bookwyrm/templates/user/books_header.html:5 +#: bookwyrm/templates/user/books_header.html:10 +msgid "Your books" +msgstr "" + +#: bookwyrm/templates/user/books_header.html:15 #, python-format msgid "%(username)s's books" msgstr "" @@ -3749,7 +3823,7 @@ msgstr "" msgid "%(title)s: %(subtitle)s" msgstr "" -#: bookwyrm/views/import_data.py:67 +#: bookwyrm/views/imports/import_data.py:64 msgid "Not a valid csv file" msgstr "" From bdc3f6828ba476e4855f0b7922bd599240490182 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 07:11:48 -0800 Subject: [PATCH 60/73] Python formatting --- bookwyrm/importers/librarything_import.py | 6 ++---- bookwyrm/models/import_job.py | 6 ++++-- bookwyrm/tests/importers/test_goodreads_import.py | 4 +--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bookwyrm/importers/librarything_import.py b/bookwyrm/importers/librarything_import.py index cff6ba7d..1b61a6f1 100644 --- a/bookwyrm/importers/librarything_import.py +++ b/bookwyrm/importers/librarything_import.py @@ -13,10 +13,8 @@ class LibrarythingImporter(Importer): def normalize_row(self, entry, mappings): # pylint: disable=no-self-use """use the dataclass to create the formatted row of data""" remove_brackets = lambda v: re.sub(r"\[|\]", "", v) if v else None - normalized = { - k: remove_brackets(entry.get(v)) for k, v in mappings.items() - } - isbn_13 = normalized["isbn_13"].split(', ') + normalized = {k: remove_brackets(entry.get(v)) for k, v in mappings.items()} + isbn_13 = normalized["isbn_13"].split(", ") normalized["isbn_13"] = isbn_13[1] if len(isbn_13) > 0 else None return normalized diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index b47379bb..753662d6 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -127,7 +127,9 @@ class ImportItem(models.Model): @property def isbn(self): """pulls out the isbn13 field from the csv line data""" - return unquote_string(self.normalized_data["isbn_13"]) or unquote_string(self.normalized_data["isbn_10"]) + return unquote_string(self.normalized_data["isbn_13"]) or unquote_string( + self.normalized_data["isbn_10"] + ) @property def shelf(self): @@ -200,7 +202,7 @@ class ImportItem(models.Model): def __repr__(self): # pylint: disable=consider-using-f-string - return "<{!r}Item {!r}>".format(self.index, self.normalized_data["title"]) + return "<{!r} Item {!r}>".format(self.index, self.normalized_data["title"]) def __str__(self): # pylint: disable=consider-using-f-string diff --git a/bookwyrm/tests/importers/test_goodreads_import.py b/bookwyrm/tests/importers/test_goodreads_import.py index 4a043b59..0a421df4 100644 --- a/bookwyrm/tests/importers/test_goodreads_import.py +++ b/bookwyrm/tests/importers/test_goodreads_import.py @@ -52,9 +52,7 @@ class GoodreadsImport(TestCase): self.assertEqual(len(import_items), 3) self.assertEqual(import_items[0].index, 0) self.assertEqual(import_items[0].data["Book Id"], "42036538") - self.assertEqual( - import_items[0].normalized_data["isbn_13"], '="9781250313195"' - ) + self.assertEqual(import_items[0].normalized_data["isbn_13"], '="9781250313195"') self.assertEqual(import_items[0].normalized_data["isbn_10"], '="1250313198"') self.assertEqual(import_items[1].index, 1) From 2748e0a8249fd6c04594093339d6faf22cb1141f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 07:50:19 -0800 Subject: [PATCH 61/73] Check for existing reviews/ratings on import items --- bookwyrm/importers/importer.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 1d350c6d..657ede05 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -171,21 +171,25 @@ def handle_imported_book(item): read.user = user read.save() - if job.include_reviews and (item.rating or item.review): + if job.include_reviews and (item.rating or item.review) and not item.linked_review: # we don't know the publication date of the review, # but "now" is a bad guess published_date_guess = item.date_read or item.date_added if item.review: # pylint: disable=consider-using-f-string - review_title = ( - "Review of {!r} on {!r}".format( - item.book.title, - job.source, - ) - if item.review - else "" + review_title = "Review of {!r} on {!r}".format( + item.book.title, + job.source, ) - review = models.Review( + existing = models.Review.objects.filter( + user=user, + book=item.book, + name=review_title, + rating=item.rating, + published_date=published_date_guess, + ).first() + + review = existing or models.Review( user=user, book=item.book, name=review_title, @@ -196,13 +200,20 @@ def handle_imported_book(item): ) else: # just a rating - review = models.ReviewRating( + existing = models.ReviewRating.objects.filter( + user=user, + book=item.book, + published_date=published_date_guess, + rating=item.rating, + ).first() + review = existing or models.ReviewRating( user=user, book=item.book, rating=item.rating, published_date=published_date_guess, privacy=job.privacy, ) + # only broadcast this review to other bookwyrm instances review.save(software="bookwyrm", priority=LOW) item.linked_review = review From 6cca3f97724dbab6eae49bb2d0a55c02b39b1347 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 07:57:13 -0800 Subject: [PATCH 62/73] Updates test data --- bookwyrm/tests/data/generic.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/data/generic.csv b/bookwyrm/tests/data/generic.csv index 9c5b6f02..470ce7a8 100644 --- a/bookwyrm/tests/data/generic.csv +++ b/bookwyrm/tests/data/generic.csv @@ -1,4 +1,4 @@ -id,title,author,ISBN,rating,shelf,review,added,finished +id,title,author,ISBN13,rating,shelf,review,added,finished 38,Gideon the Ninth,Tamsyn Muir,"9781250313195",,read,,2021-11-10,2021-11-11 48,Harrow the Ninth,Tamsyn Muir,,3,read,,2021-11-10 23,Subcutanean,Aaron A. Reed,,,read,,2021-11-10 From 9e673834dc9294262acda328cba48cf371c022f6 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 08:23:26 -0800 Subject: [PATCH 63/73] Check for duplicates --- bookwyrm/importers/importer.py | 44 ++++++++++---------- bookwyrm/tests/importers/test_importer.py | 49 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 657ede05..05550429 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -181,40 +181,42 @@ def handle_imported_book(item): item.book.title, job.source, ) - existing = models.Review.objects.filter( + review = models.Review.objects.filter( user=user, book=item.book, name=review_title, rating=item.rating, published_date=published_date_guess, ).first() - - review = existing or models.Review( - user=user, - book=item.book, - name=review_title, - content=item.review, - rating=item.rating, - published_date=published_date_guess, - privacy=job.privacy, - ) + if not review: + review = models.Review( + user=user, + book=item.book, + name=review_title, + content=item.review, + rating=item.rating, + published_date=published_date_guess, + privacy=job.privacy, + ) + review.save(software="bookwyrm", priority=LOW) else: # just a rating - existing = models.ReviewRating.objects.filter( + review = models.ReviewRating.objects.filter( user=user, book=item.book, published_date=published_date_guess, rating=item.rating, ).first() - review = existing or models.ReviewRating( - user=user, - book=item.book, - rating=item.rating, - published_date=published_date_guess, - privacy=job.privacy, - ) + if not review: + review = models.ReviewRating( + user=user, + book=item.book, + rating=item.rating, + published_date=published_date_guess, + privacy=job.privacy, + ) + review.save(software="bookwyrm", priority=LOW) # only broadcast this review to other bookwyrm instances - review.save(software="bookwyrm", priority=LOW) item.linked_review = review - item.save() + item.save() diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 99cdcd28..6996a92b 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -256,6 +256,55 @@ class GenericImporter(TestCase): import_item.refresh_from_db() self.assertEqual(import_item.linked_review.id, review.id) + @patch("bookwyrm.activitystreams.add_status_task.delay") + def test_handle_imported_book_rating_duplicate_with_link(self, *_): + """rating import twice""" + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) + import_item = import_job.items.filter(index=1).first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + handle_imported_book(import_item) + + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) + self.assertIsInstance(review, models.ReviewRating) + self.assertEqual(review.rating, 3.0) + self.assertEqual(review.privacy, "unlisted") + + import_item.refresh_from_db() + self.assertEqual(import_item.linked_review.id, review.id) + + @patch("bookwyrm.activitystreams.add_status_task.delay") + def test_handle_imported_book_rating_duplicate_without_link(self, *_): + """rating import twice""" + import_job = self.importer.create_job( + self.local_user, self.csv, True, "unlisted" + ) + import_item = import_job.items.filter(index=1).first() + import_item.book = self.book + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + import_item.refresh_from_db() + import_item.linked_review = None + import_item.save() + + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"): + handle_imported_book(import_item) + + review = models.ReviewRating.objects.get(book=self.book, user=self.local_user) + self.assertIsInstance(review, models.ReviewRating) + self.assertEqual(review.rating, 3.0) + self.assertEqual(review.privacy, "unlisted") + + import_item.refresh_from_db() + self.assertEqual(import_item.linked_review.id, review.id) + def test_handle_imported_book_reviews_disabled(self, *_): """review import""" import_job = self.importer.create_job( From 47b98ad0d9e87f05eb569cb278772a2c3a91042f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 09:04:12 -0800 Subject: [PATCH 64/73] Track completed items on job --- bookwyrm/importers/importer.py | 4 +++ .../migrations/0116_auto_20211114_1700.py | 32 +++++++++++++++++++ bookwyrm/models/import_job.py | 10 ++++-- 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 bookwyrm/migrations/0116_auto_20211114_1700.py diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 05550429..438ff7db 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -126,6 +126,7 @@ def import_item_task(item_id): except Exception as err: # pylint: disable=broad-except item.fail_reason = _("Error loading book") item.save() + item.update_job() raise err if item.book: @@ -135,6 +136,7 @@ def import_item_task(item_id): item.fail_reason = _("Could not find a match for book") item.save() + item.update_job() def handle_imported_book(item): @@ -144,6 +146,8 @@ def handle_imported_book(item): if isinstance(item.book, models.Work): item.book = item.book.default_edition if not item.book: + item.fail_reason = _("Error loading book") + item.save() return if not isinstance(item.book, models.Edition): item.book = item.book.edition diff --git a/bookwyrm/migrations/0116_auto_20211114_1700.py b/bookwyrm/migrations/0116_auto_20211114_1700.py new file mode 100644 index 00000000..ff71b89c --- /dev/null +++ b/bookwyrm/migrations/0116_auto_20211114_1700.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.5 on 2021-11-14 17:00 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0115_importitem_linked_review'), + ] + + operations = [ + migrations.RemoveField( + model_name='importjob', + name='complete', + ), + migrations.RemoveField( + model_name='importjob', + name='task_id', + ), + migrations.AddField( + model_name='importjob', + name='completed_count', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='importjob', + name='updated_date', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 753662d6..953ae394 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -35,10 +35,10 @@ class ImportJob(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) created_date = models.DateTimeField(default=timezone.now) - task_id = models.CharField(max_length=100, null=True) # TODO: deprecated include_reviews = models.BooleanField(default=True) mappings = models.JSONField() - complete = models.BooleanField(default=False) + updated_date = models.DateTimeField(default=timezone.now) + completed_count = models.IntegerField(default=0) source = models.CharField(max_length=100) privacy = models.CharField( max_length=255, default="public", choices=PrivacyLevels.choices @@ -66,6 +66,12 @@ class ImportItem(models.Model): "Review", on_delete=models.SET_NULL, null=True, blank=True ) + def update_job(self): + """this user is here! they are doing things!""" + self.job.completed_count += 1 + self.job.updated_date = timezone.now() + self.job.save() + def resolve(self): """try various ways to lookup a book""" # we might be calling this after manually adding the book, From f92863ad3ec562feed8baf88e9bcac4db463fbd1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 09:56:23 -0800 Subject: [PATCH 65/73] Notify when import completes --- .../migrations/0116_auto_20211114_1700.py | 32 ----------------- .../migrations/0116_auto_20211114_1734.py | 23 ++++++++++++ bookwyrm/models/import_job.py | 18 +++++++--- bookwyrm/models/notification.py | 7 ++-- bookwyrm/tests/importers/test_importer.py | 35 ++++++++++++++++++- bookwyrm/views/imports/import_status.py | 5 ++- 6 files changed, 77 insertions(+), 43 deletions(-) delete mode 100644 bookwyrm/migrations/0116_auto_20211114_1700.py create mode 100644 bookwyrm/migrations/0116_auto_20211114_1734.py diff --git a/bookwyrm/migrations/0116_auto_20211114_1700.py b/bookwyrm/migrations/0116_auto_20211114_1700.py deleted file mode 100644 index ff71b89c..00000000 --- a/bookwyrm/migrations/0116_auto_20211114_1700.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 3.2.5 on 2021-11-14 17:00 - -from django.db import migrations, models -import django.utils.timezone - - -class Migration(migrations.Migration): - - dependencies = [ - ('bookwyrm', '0115_importitem_linked_review'), - ] - - operations = [ - migrations.RemoveField( - model_name='importjob', - name='complete', - ), - migrations.RemoveField( - model_name='importjob', - name='task_id', - ), - migrations.AddField( - model_name='importjob', - name='completed_count', - field=models.IntegerField(default=0), - ), - migrations.AddField( - model_name='importjob', - name='updated_date', - field=models.DateTimeField(default=django.utils.timezone.now), - ), - ] diff --git a/bookwyrm/migrations/0116_auto_20211114_1734.py b/bookwyrm/migrations/0116_auto_20211114_1734.py new file mode 100644 index 00000000..1da001bd --- /dev/null +++ b/bookwyrm/migrations/0116_auto_20211114_1734.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.5 on 2021-11-14 17:34 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0115_importitem_linked_review"), + ] + + operations = [ + migrations.RemoveField( + model_name="importjob", + name="task_id", + ), + migrations.AddField( + model_name="importjob", + name="updated_date", + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 953ae394..fbec88ca 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -38,13 +38,18 @@ class ImportJob(models.Model): include_reviews = models.BooleanField(default=True) mappings = models.JSONField() updated_date = models.DateTimeField(default=timezone.now) - completed_count = models.IntegerField(default=0) + complete = models.BooleanField(default=False) source = models.CharField(max_length=100) privacy = models.CharField( max_length=255, default="public", choices=PrivacyLevels.choices ) retry = models.BooleanField(default=False) + @property + def pending_items(self): + """items that haven't been processed yet""" + return self.items.filter(fail_reason__isnull=True, book__isnull=True) + class ImportItem(models.Model): """a single line of a csv being imported""" @@ -67,10 +72,13 @@ class ImportItem(models.Model): ) def update_job(self): - """this user is here! they are doing things!""" - self.job.completed_count += 1 - self.job.updated_date = timezone.now() - self.job.save() + """let the job know when the items get work done""" + job = self.job + job.updated_date = timezone.now() + job.save() + if not job.pending_items.exists() and not job.complete: + job.complete = True + job.save(update_fields=["complete"]) def resolve(self): """try various ways to lookup a book""" diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 2f1aae4f..417bf759 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -157,9 +157,12 @@ def notify_user_on_unboost(sender, instance, *args, **kwargs): @receiver(models.signals.post_save, sender=ImportJob) # pylint: disable=unused-argument -def notify_user_on_import_complete(sender, instance, *args, **kwargs): +def notify_user_on_import_complete( + sender, instance, *args, update_fields=None, **kwargs +): """we imported your books! aren't you proud of us""" - if not instance.complete: + update_fields = update_fields or [] + if not instance.complete or "complete" not in update_fields: return Notification.objects.create( user=instance.user, diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 6996a92b..3fbfa264 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -145,7 +145,40 @@ class GenericImporter(TestCase): self.assertEqual(kwargs["queue"], "low_priority") import_item.refresh_from_db() - self.assertEqual(import_item.book.id, self.book.id) + def test_complete_job(self, *_): + """test notification""" + import_job = self.importer.create_job( + self.local_user, self.csv, False, "unlisted" + ) + item = import_job.items[0] + item.update_job() + self.assertFalse( + models.Notification.objects.filter( + user=self.local_user, + related_import=import_job, + notification_type="IMPORT", + ).exists() + ) + + item = import_job.items[1] + item.update_job() + self.assertFalse( + models.Notification.objects.filter( + user=self.local_user, + related_import=import_job, + notification_type="IMPORT", + ).exists() + ) + + item = import_job.items[2] + item.update_job() + self.assertTrue( + models.Notification.objects.filter( + user=self.local_user, + related_import=import_job, + notification_type="IMPORT", + ).exists() + ) def test_handle_imported_book(self, *_): """import added a book, this adds related connections""" diff --git a/bookwyrm/views/imports/import_status.py b/bookwyrm/views/imports/import_status.py index 2d18d656..7e7d5179 100644 --- a/bookwyrm/views/imports/import_status.py +++ b/bookwyrm/views/imports/import_status.py @@ -24,7 +24,6 @@ class ImportStatus(View): raise PermissionDenied() items = job.items.order_by("index") - pending_items = items.filter(fail_reason__isnull=True, book__isnull=True) item_count = items.count() or 1 paginated = Paginator(items, PAGE_LENGTH) @@ -41,9 +40,9 @@ class ImportStatus(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "complete": not pending_items.exists(), + "complete": not job.pending_items.exists(), "percent": math.floor( # pylint: disable=c-extension-no-member - (item_count - pending_items.count()) / item_count * 100 + (item_count - job.pending_items.count()) / item_count * 100 ), } From 8cede05d32895d2a4c73e3f208b13841651a881f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 10:20:14 -0800 Subject: [PATCH 66/73] Retry hanging items --- bookwyrm/models/import_job.py | 2 +- bookwyrm/templates/import/import_status.html | 15 ++++++++++++--- bookwyrm/urls.py | 5 +++++ bookwyrm/views/imports/import_status.py | 15 +++++++++++++-- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index fbec88ca..387261f0 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -35,9 +35,9 @@ class ImportJob(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) created_date = models.DateTimeField(default=timezone.now) + updated_date = models.DateTimeField(default=timezone.now) include_reviews = models.BooleanField(default=True) mappings = models.JSONField() - updated_date = models.DateTimeField(default=timezone.now) complete = models.BooleanField(default=False) source = models.CharField(max_length=100) privacy = models.CharField( diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 61b7b5cf..6c7d54b9 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -41,7 +41,7 @@ - {% if not complete %} + {% if not job.complete %}
    @@ -167,8 +167,17 @@ {% endif %} {% else %} - - {% trans "Pending" %} +
    + + {% trans "Pending" %} + {# retry option if an item appears to be hanging #} + {% if job.created_date != job.updated_date and inactive_time > 0.24 %} +
    + {% csrf_token %} + +
    + {% endif %} +
    {% endif %}
    +

    + {% trans "Import preview unavailable." %} +

    +
    + {% if legacy %} +
    + + {% csrf_token %} +

    + {% trans "This import is in an old format that is no longer supported. If you would like to troubleshoot missing items from this import, click the button below to update the import format." %} +

    + + +
    + {% endif %}
    +{% if not legacy %}
    {% include 'snippets/pagination.html' with page=items %}
    +{% endif %} {% endspaceless %}{% endblock %} {% block scripts %} diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 6f658016..514bb7e6 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -244,7 +244,7 @@ urlpatterns = [ ), re_path( r"^import/(?P\d+)/retry/(?P\d+)/?$", - views.ImportStatus.as_view(), + views.retry_item, name="import-item-retry", ), re_path( diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 1a6fbdc6..d79de424 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -45,7 +45,7 @@ from .shelf.shelf_actions import shelve, unshelve # csv import from .imports.import_data import Import -from .imports.import_status import ImportStatus +from .imports.import_status import ImportStatus, retry_item from .imports.troubleshoot import ImportTroubleshoot from .imports.manually_review import ( ImportManualReview, diff --git a/bookwyrm/views/imports/import_status.py b/bookwyrm/views/imports/import_status.py index 54174082..8e07a171 100644 --- a/bookwyrm/views/imports/import_status.py +++ b/bookwyrm/views/imports/import_status.py @@ -9,8 +9,10 @@ from django.template.response import TemplateResponse from django.utils import timezone from django.utils.decorators import method_decorator from django.views import View +from django.views.decorators.http import require_POST from bookwyrm import models +from bookwyrm.importers import GoodreadsImporter from bookwyrm.importers.importer import import_item_task from bookwyrm.settings import PAGE_LENGTH @@ -47,14 +49,26 @@ class ImportStatus(View): ), # hours since last import item update "inactive_time": (job.updated_date - timezone.now()).seconds / 60 / 60, + "legacy": not job.mappings, } return TemplateResponse(request, "import/import_status.html", data) - def post(self, request, job_id, item_id): - """retry an item""" - item = get_object_or_404( - models.ImportItem, id=item_id, job__id=job_id, job__user=request.user - ) - import_item_task.delay(item.id) + def post(self, request, job_id): + """bring a legacy import into the latest format""" + job = get_object_or_404(models.ImportJob, id=job_id) + if job.user != request.user: + raise PermissionDenied() + GoodreadsImporter().update_legacy_job(job) return redirect("import-status", job_id) + + +@login_required +@require_POST +def retry_item(request, job_id, item_id): + """retry an item""" + item = get_object_or_404( + models.ImportItem, id=item_id, job__id=job_id, job__user=request.user + ) + import_item_task.delay(item.id) + return redirect("import-status", job_id) From 8612cf654d2edec8b06abbf837bb4094d4673507 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 14 Nov 2021 11:31:47 -0800 Subject: [PATCH 70/73] Invalid href --- bookwyrm/templates/import/import_status.html | 2 +- bookwyrm/tests/importers/test_importer.py | 46 ++++++++------------ 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 6370b866..ddeeeb31 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -47,7 +47,7 @@ {% trans "In progress" %} - {% trans "Refresh" %} + {% trans "Refresh" %}
    diff --git a/bookwyrm/tests/importers/test_importer.py b/bookwyrm/tests/importers/test_importer.py index 3fbfa264..5c3b2031 100644 --- a/bookwyrm/tests/importers/test_importer.py +++ b/bookwyrm/tests/importers/test_importer.py @@ -1,5 +1,4 @@ """ testing import """ -from collections import namedtuple import pathlib from unittest.mock import patch import datetime @@ -104,13 +103,9 @@ class GenericImporter(TestCase): import_job = self.importer.create_job( self.local_user, self.csv, False, "unlisted" ) - MockTask = namedtuple("Task", ("id")) - mock_task = MockTask(7) - with patch("bookwyrm.importers.importer.start_import_task.delay") as start: - start.return_value = mock_task + with patch("bookwyrm.importers.importer.start_import_task.delay") as mock: self.importer.start_import(import_job) - import_job.refresh_from_db() - self.assertEqual(import_job.task_id, "7") + self.assertEqual(mock.call_count, 1) @responses.activate def test_start_import_task(self, *_): @@ -150,28 +145,25 @@ class GenericImporter(TestCase): import_job = self.importer.create_job( self.local_user, self.csv, False, "unlisted" ) - item = import_job.items[0] - item.update_job() - self.assertFalse( - models.Notification.objects.filter( - user=self.local_user, - related_import=import_job, - notification_type="IMPORT", - ).exists() - ) + items = import_job.items.all() + for item in items[:3]: + item.fail_reason = "hello" + item.save() + item.update_job() + self.assertFalse( + models.Notification.objects.filter( + user=self.local_user, + related_import=import_job, + notification_type="IMPORT", + ).exists() + ) - item = import_job.items[1] - item.update_job() - self.assertFalse( - models.Notification.objects.filter( - user=self.local_user, - related_import=import_job, - notification_type="IMPORT", - ).exists() - ) - - item = import_job.items[2] + item = items[3] + item.fail_reason = "hello" + item.save() item.update_job() + import_job.refresh_from_db() + self.assertTrue(import_job.complete) self.assertTrue( models.Notification.objects.filter( user=self.local_user, From 77ee1147d52133a02e4e0a9647fb892f6859bfba Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 15 Nov 2021 09:03:00 -0800 Subject: [PATCH 71/73] Adds return_first tests to book_search --- bookwyrm/tests/test_book_search.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bookwyrm/tests/test_book_search.py b/bookwyrm/tests/test_book_search.py index 4b9a0681..16435fff 100644 --- a/bookwyrm/tests/test_book_search.py +++ b/bookwyrm/tests/test_book_search.py @@ -57,12 +57,24 @@ class BookSearch(TestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0], self.second_edition) + def test_search_identifiers_return_first(self): + """search by unique identifiers""" + result = book_search.search_identifiers("hello", return_first=True) + self.assertEqual(result, self.second_edition) + def test_search_title_author(self): """search by unique identifiers""" results = book_search.search_title_author("Another", min_confidence=0) self.assertEqual(len(results), 1) self.assertEqual(results[0], self.second_edition) + def test_search_title_author_return_first(self): + """search by unique identifiers""" + results = book_search.search_title_author( + "Another", min_confidence=0, return_first=True + ) + self.assertEqual(results, self.second_edition) + def test_format_search_result(self): """format a search result""" result = book_search.format_search_result(self.first_edition) From 83e468a4f817ffcb00324ef099a8c8db1fe7e240 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 15 Nov 2021 09:34:36 -0800 Subject: [PATCH 72/73] Fixes "indeterminate" state of progress indicator on screen reader --- bookwyrm/templates/import/import_status.html | 2 +- bookwyrm/views/imports/import_status.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index ddeeeb31..61763ec2 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -51,7 +51,7 @@
    - {{ percent }}% + {{ percent }} % {{ percent }}%
    diff --git a/bookwyrm/views/imports/import_status.py b/bookwyrm/views/imports/import_status.py index 8e07a171..26ff8cde 100644 --- a/bookwyrm/views/imports/import_status.py +++ b/bookwyrm/views/imports/import_status.py @@ -32,20 +32,25 @@ class ImportStatus(View): paginated = Paginator(items, PAGE_LENGTH) page = paginated.get_page(request.GET.get("page")) + manual_review_count = items.filter( + fail_reason__isnull=False, book_guess__isnull=False, book__isnull=True + ).count() + fail_count = items.filter( + fail_reason__isnull=False, book_guess__isnull=True + ).count() + pending_item_count = job.pending_items.count() data = { "job": job, "items": page, - "manual_review_count": items.filter( - fail_reason__isnull=False, book_guess__isnull=False, book__isnull=True - ).count(), - "fail_count": items.filter( - fail_reason__isnull=False, book_guess__isnull=True - ).count(), + "manual_review_count": manual_review_count, + "fail_count": fail_count, "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), + "item_count": item_count, + "complete_count": item_count - pending_item_count, "percent": math.floor( # pylint: disable=c-extension-no-member - (item_count - job.pending_items.count()) / item_count * 100 + (item_count - pending_item_count) / item_count * 100 ), # hours since last import item update "inactive_time": (job.updated_date - timezone.now()).seconds / 60 / 60, From 30afe42b3ad0b6c78615187bc246d17b9e90f9f4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Mon, 15 Nov 2021 09:41:05 -0800 Subject: [PATCH 73/73] Removes extra space in progress bar --- bookwyrm/templates/import/import_status.html | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bookwyrm/templates/import/import_status.html b/bookwyrm/templates/import/import_status.html index 61763ec2..9b437969 100644 --- a/bookwyrm/templates/import/import_status.html +++ b/bookwyrm/templates/import/import_status.html @@ -51,7 +51,16 @@
    - {{ percent }} % + + {{ percent }} % + {{ percent }}%