From c4b8e7949d661c0e6d6a4d78d83e7712c82f037a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 30 Jan 2022 11:41:06 -0800 Subject: [PATCH 1/5] Add more book identifier fields --- bookwyrm/activitypub/book.py | 1 + .../migrations/0132_auto_20220130_1938.py | 42 +++++++++++++++++++ bookwyrm/models/book.py | 6 +++ 3 files changed, 49 insertions(+) create mode 100644 bookwyrm/migrations/0132_auto_20220130_1938.py diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 2238e3a8..7401a134 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -16,6 +16,7 @@ class BookData(ActivityObject): librarythingKey: str = None goodreadsKey: str = None bnfId: str = None + viaf: str = None lastEditedBy: str = None links: List[str] = field(default_factory=lambda: []) fileLinks: List[str] = field(default_factory=lambda: []) diff --git a/bookwyrm/migrations/0132_auto_20220130_1938.py b/bookwyrm/migrations/0132_auto_20220130_1938.py new file mode 100644 index 00000000..a83ac290 --- /dev/null +++ b/bookwyrm/migrations/0132_auto_20220130_1938.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.10 on 2022-01-30 19:38 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0131_merge_20220125_1644"), + ] + + operations = [ + migrations.AddField( + model_name="author", + name="viaf", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + migrations.AddField( + model_name="author", + name="wikidata", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + migrations.AddField( + model_name="book", + name="viaf", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + migrations.AddField( + model_name="book", + name="wikidata", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), + ), + ] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index ffc03d3e..c796b7d2 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -46,6 +46,12 @@ class BookDataModel(ObjectMixin, BookWyrmModel): bnf_id = fields.CharField( # Bibliothèque nationale de France max_length=255, blank=True, null=True, deduplication_field=True ) + viaf = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) + wikidata = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) search_vector = SearchVectorField(null=True) last_edited_by = fields.ForeignKey( From 44dad43f366ed6a014fdbcc4f86913e2765e16d1 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 30 Jan 2022 11:41:33 -0800 Subject: [PATCH 2/5] Load new fields via connector --- bookwyrm/connectors/openlibrary.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index c15277f8..5a94a16c 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -68,7 +68,26 @@ class Connector(AbstractConnector): Mapping("born", remote_field="birth_date"), Mapping("died", remote_field="death_date"), Mapping("bio", formatter=get_description), - Mapping("isni", remote_field="remote_ids", formatter=get_isni), + Mapping( + "isni", + remote_field="remote_ids", + formatter=lambda b: get_dict_field(b, "isni"), + ), + Mapping( + "asin", + remote_field="remote_ids", + formatter=lambda b: get_dict_field(b, "amazon"), + ), + Mapping( + "viaf", + remote_field="remote_ids", + formatter=lambda b: get_dict_field(b, "viaf"), + ), + Mapping( + "wikidata", + remote_field="remote_ids", + formatter=lambda b: get_dict_field(b, "wikidata"), + ), ] def get_book_data(self, remote_id): @@ -227,11 +246,11 @@ def get_languages(language_blob): return langs -def get_isni(remote_ids_blob): +def get_dict_field(blob, field_name): """extract the isni from the remote id data for the author""" - if not remote_ids_blob or not isinstance(remote_ids_blob, dict): + if not blob or not isinstance(blob, dict): return None - return remote_ids_blob.get("isni") + return blob.get(field_name) def pick_default_edition(options): From 9611815b44267682bd482ec8ce8b6f5069aa10e4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 30 Jan 2022 12:02:18 -0800 Subject: [PATCH 3/5] Extract wikipedia and inventaire ids --- bookwyrm/connectors/openlibrary.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 5a94a16c..118222a1 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -88,6 +88,10 @@ class Connector(AbstractConnector): remote_field="remote_ids", formatter=lambda b: get_dict_field(b, "wikidata"), ), + Mapping( + "wikipedia_link", remote_field="links", formatter=get_wikipedia_link + ), + Mapping("inventaire_id", remote_field="links", formatter=get_inventaire_id), ] def get_book_data(self, remote_id): @@ -253,6 +257,33 @@ def get_dict_field(blob, field_name): return blob.get(field_name) +def get_wikipedia_link(links): + """extract wikipedia links""" + if not isinstance(links, list): + return None + + for link in links: + if not isinstance(link, dict): + continue + if link.get("title") == "wikipedia": + return link.get("url") + return None + + +def get_inventaire_id(links): + """extract and format inventaire ids""" + if not isinstance(links, list): + return None + + for link in links: + if not isinstance(link, dict): + continue + if link.get("title") == "inventaire.io": + iv_link = link.get("url") + return iv_link.split("/")[-1] + return None + + def pick_default_edition(options): """favor physical copies with covers in english""" if not options: From a2c4dd4f9f4393bfe53cf5d6fc6b4629d48b1e4a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 17 Feb 2022 08:25:01 -0800 Subject: [PATCH 4/5] Updates migration and database fields --- bookwyrm/activitypub/book.py | 7 +-- .../migrations/0132_auto_20220130_1938.py | 42 ---------------- .../migrations/0135_auto_20220217_1624.py | 48 +++++++++++++++++++ bookwyrm/models/author.py | 3 -- bookwyrm/models/book.py | 6 +-- 5 files changed, 55 insertions(+), 51 deletions(-) delete mode 100644 bookwyrm/migrations/0132_auto_20220130_1938.py create mode 100644 bookwyrm/migrations/0135_auto_20220217_1624.py diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 7401a134..e6a01b35 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -17,6 +17,8 @@ class BookData(ActivityObject): goodreadsKey: str = None bnfId: str = None viaf: str = None + wikidata: str = None + asin: str = None lastEditedBy: str = None links: List[str] = field(default_factory=lambda: []) fileLinks: List[str] = field(default_factory=lambda: []) @@ -28,8 +30,8 @@ class Book(BookData): """serializes an edition or work, abstract""" title: str - sortTitle: str = "" - subtitle: str = "" + sortTitle: str = None + subtitle: str = None description: str = "" languages: List[str] = field(default_factory=lambda: []) series: str = "" @@ -54,7 +56,6 @@ class Edition(Book): isbn10: str = "" isbn13: str = "" oclcNumber: str = "" - asin: str = "" pages: int = None physicalFormat: str = "" physicalFormatDetail: str = "" diff --git a/bookwyrm/migrations/0132_auto_20220130_1938.py b/bookwyrm/migrations/0132_auto_20220130_1938.py deleted file mode 100644 index a83ac290..00000000 --- a/bookwyrm/migrations/0132_auto_20220130_1938.py +++ /dev/null @@ -1,42 +0,0 @@ -# Generated by Django 3.2.10 on 2022-01-30 19:38 - -import bookwyrm.models.fields -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ("bookwyrm", "0131_merge_20220125_1644"), - ] - - operations = [ - migrations.AddField( - model_name="author", - name="viaf", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - migrations.AddField( - model_name="author", - name="wikidata", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - migrations.AddField( - model_name="book", - name="viaf", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - migrations.AddField( - model_name="book", - name="wikidata", - field=bookwyrm.models.fields.CharField( - blank=True, max_length=255, null=True - ), - ), - ] diff --git a/bookwyrm/migrations/0135_auto_20220217_1624.py b/bookwyrm/migrations/0135_auto_20220217_1624.py new file mode 100644 index 00000000..793cb580 --- /dev/null +++ b/bookwyrm/migrations/0135_auto_20220217_1624.py @@ -0,0 +1,48 @@ +# Generated by Django 3.2.12 on 2022-02-17 16:24 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0134_announcement_display_type'), + ] + + operations = [ + migrations.RenameField( + model_name='author', + old_name='viaf_id', + new_name='viaf', + ), + migrations.RemoveField( + model_name='edition', + name='asin', + ), + migrations.AddField( + model_name='author', + name='asin', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='wikidata', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='asin', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='viaf', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='wikidata', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 5edac57d..78d153a2 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -21,9 +21,6 @@ class Author(BookDataModel): isni = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) - viaf_id = fields.CharField( - max_length=255, blank=True, null=True, deduplication_field=True - ) gutenberg_id = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index c796b7d2..3ea8e1a8 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -52,6 +52,9 @@ class BookDataModel(ObjectMixin, BookWyrmModel): wikidata = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) + asin = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True + ) search_vector = SearchVectorField(null=True) last_edited_by = fields.ForeignKey( @@ -277,9 +280,6 @@ class Edition(Book): oclc_number = fields.CharField( max_length=255, blank=True, null=True, deduplication_field=True ) - asin = fields.CharField( - max_length=255, blank=True, null=True, deduplication_field=True - ) pages = fields.IntegerField(blank=True, null=True) physical_format = fields.CharField( max_length=255, choices=FormatChoices, null=True, blank=True From c04d2d285b8e334f8704274347e1c07e629a081f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 17 Feb 2022 08:42:12 -0800 Subject: [PATCH 5/5] Python formatting --- .../migrations/0135_auto_20220217_1624.py | 52 +++++++++++-------- dev-tools/requirements.txt | 2 +- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/bookwyrm/migrations/0135_auto_20220217_1624.py b/bookwyrm/migrations/0135_auto_20220217_1624.py index 793cb580..b557fb03 100644 --- a/bookwyrm/migrations/0135_auto_20220217_1624.py +++ b/bookwyrm/migrations/0135_auto_20220217_1624.py @@ -7,42 +7,52 @@ from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('bookwyrm', '0134_announcement_display_type'), + ("bookwyrm", "0134_announcement_display_type"), ] operations = [ migrations.RenameField( - model_name='author', - old_name='viaf_id', - new_name='viaf', + model_name="author", + old_name="viaf_id", + new_name="viaf", ), migrations.RemoveField( - model_name='edition', - name='asin', + model_name="edition", + name="asin", ), migrations.AddField( - model_name='author', - name='asin', - field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + model_name="author", + name="asin", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), ), migrations.AddField( - model_name='author', - name='wikidata', - field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + model_name="author", + name="wikidata", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), ), migrations.AddField( - model_name='book', - name='asin', - field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + model_name="book", + name="asin", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), ), migrations.AddField( - model_name='book', - name='viaf', - field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + model_name="book", + name="viaf", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), ), migrations.AddField( - model_name='book', - name='wikidata', - field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + model_name="book", + name="wikidata", + field=bookwyrm.models.fields.CharField( + blank=True, max_length=255, null=True + ), ), ] diff --git a/dev-tools/requirements.txt b/dev-tools/requirements.txt index c2014be7..5719da1c 100644 --- a/dev-tools/requirements.txt +++ b/dev-tools/requirements.txt @@ -1 +1 @@ -black==21.4b2 +black==22.1.0