- {{ result.title }}
+ {{ result.title }}
{% if result.author %}
{% blocktrans with author=result.author %}by {{ author }}{% endblocktrans %}{% endif %}{% if result.year %} ({{ result.year }})
From 22ebe60c0abb2badaf0d354190eee125d5278514 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 12:29:06 -0700
Subject: [PATCH 05/37] Use custom data extractor for inventaire connector
---
bookwyrm/connectors/abstract_connector.py | 16 ++++++++++++----
bookwyrm/connectors/inventaire.py | 9 ++++++---
2 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 378f23f7..8007bb1a 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -44,7 +44,7 @@ class AbstractMinimalConnector(ABC):
if min_confidence:
params["min_confidence"] = min_confidence
- data = get_data(
+ data = self.get_search_data(
"%s%s" % (self.search_url, query),
params=params,
)
@@ -57,7 +57,7 @@ class AbstractMinimalConnector(ABC):
def isbn_search(self, query):
""" isbn search """
params = {}
- data = get_data(
+ data = self.get_search_data(
"%s%s" % (self.isbn_search_url, query),
params=params,
)
@@ -117,7 +117,7 @@ class AbstractConnector(AbstractMinimalConnector):
return existing
# load the json
- data = get_data(remote_id)
+ data = self.get_book_data(remote_id)
mapped_data = dict_from_mappings(data, self.book_mappings)
if self.is_work_data(data):
try:
@@ -150,6 +150,14 @@ class AbstractConnector(AbstractMinimalConnector):
load_more_data.delay(self.connector.id, work.id)
return edition
+ def get_book_data(self, remote_id):
+ """ this allows connectors to override the default behavior """
+ return get_data(remote_id)
+
+ def get_search_data(self, remote_id):
+ """ this allows connectors to override the default behavior """
+ return get_data(remote_id)
+
def create_edition_from_data(self, work, edition_data):
""" if we already have the work, we're ready """
mapped_data = dict_from_mappings(edition_data, self.book_mappings)
@@ -176,7 +184,7 @@ class AbstractConnector(AbstractMinimalConnector):
if existing:
return existing
- data = get_data(remote_id)
+ data = self.get_book_data(remote_id)
mapped_data = dict_from_mappings(data, self.author_mappings)
activity = activitypub.Author(**mapped_data)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 748259eb..72ed8221 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -1,8 +1,6 @@
""" inventaire data connector """
-from bookwyrm import models
from .abstract_connector import AbstractConnector, SearchResult, Mapping
from .abstract_connector import get_data
-from .connector_manager import ConnectorException
class Connector(AbstractConnector):
@@ -26,6 +24,11 @@ class Connector(AbstractConnector):
Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_claim),
]
+ def get_book_data(self, remote_id):
+ data = get_data(remote_id)
+ extracted = list(data.get("entities").values())
+ return extracted[0] if extracted else {}
+
def parse_search_data(self, data):
return data.get('results')
@@ -49,7 +52,7 @@ class Connector(AbstractConnector):
""" beep bloop """
def is_work_data(self, data):
- return True
+ return data.get("type") == "work"
def get_edition_from_work_data(self, data):
return {}
From 5149c7e8c2da5766ba16c1b033e39d3da3844a3d Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 13:03:22 -0700
Subject: [PATCH 06/37] Expands mappings for inventaire/wikidata properties
---
bookwyrm/connectors/inventaire.py | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 72ed8221..b3c49772 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -9,11 +9,14 @@ class Connector(AbstractConnector):
def __init__(self, identifier):
super().__init__(identifier)
- get_remote_id = lambda a, *args: self.base_url + a
+ get_remote_id = lambda a, *args: self.books_url + a
+ get_remote_id_list = lambda a, *_: [get_remote_id(v) for v in a]
self.book_mappings = [
Mapping("title", remote_field="wdt:P1476", formatter=get_claim),
- Mapping("id", remote_field="key", formatter=get_remote_id),
- Mapping("inventaireId", remote_field="id"),
+ Mapping("subtitle", remote_field="wdt:P1680", formatter=get_claim),
+ Mapping("id", remote_field="uri", formatter=get_remote_id),
+ Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list),
+ Mapping("inventaireId", remote_field="uri"),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
Mapping("isbn13", remote_field="wdt:P212", formatter=get_claim),
Mapping("isbn10", remote_field="wdt:P957", formatter=get_claim),
@@ -22,7 +25,14 @@ class Connector(AbstractConnector):
Mapping("publishedDate", remote_field="wdt:P577", formatter=get_claim),
Mapping("pages", remote_field="wdt:P1104", formatter=get_claim),
Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_claim),
+ Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_claim),
+ Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key),
+ Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key),
+ Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_claim),
+ Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_claim),
+ Mapping("asin", remote_field="wdt:P5749", formatter=get_claim),
]
+ # TODO: P136: genre, P268 bnf id, P674 characters, P950 bne
def get_book_data(self, remote_id):
data = get_data(remote_id)
From 3158701075bf1328be9d7c5851afdf0ed1b63edf Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 13:39:10 -0700
Subject: [PATCH 07/37] Gets editions for works
---
bookwyrm/connectors/inventaire.py | 33 ++++++++++++++++++++++++-------
1 file changed, 26 insertions(+), 7 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index b3c49772..9987079e 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -1,6 +1,7 @@
""" inventaire data connector """
from .abstract_connector import AbstractConnector, SearchResult, Mapping
from .abstract_connector import get_data
+from .connector_manager import ConnectorException
class Connector(AbstractConnector):
@@ -9,12 +10,11 @@ class Connector(AbstractConnector):
def __init__(self, identifier):
super().__init__(identifier)
- get_remote_id = lambda a, *args: self.books_url + a
- get_remote_id_list = lambda a, *_: [get_remote_id(v) for v in a]
+ get_remote_id_list = lambda a, *_: [self.get_remote_id(v) for v in a]
self.book_mappings = [
Mapping("title", remote_field="wdt:P1476", formatter=get_claim),
Mapping("subtitle", remote_field="wdt:P1680", formatter=get_claim),
- Mapping("id", remote_field="uri", formatter=get_remote_id),
+ Mapping("id", remote_field="uri", formatter=self.get_remote_id),
Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list),
Mapping("inventaireId", remote_field="uri"),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
@@ -34,6 +34,10 @@ class Connector(AbstractConnector):
]
# TODO: P136: genre, P268 bnf id, P674 characters, P950 bne
+ def get_remote_id(self, value, *_):
+ """ convert an id/uri into a url """
+ return '{:s}?action=by-uris&uris={:s}'.format(self.books_url, value)
+
def get_book_data(self, remote_id):
data = get_data(remote_id)
extracted = list(data.get("entities").values())
@@ -49,7 +53,9 @@ class Connector(AbstractConnector):
) if images else None
return SearchResult(
title=search_result.get("label"),
- key="{:s}{:s}".format(self.books_url, search_result.get("uri")),
+ key="{:s}?action=by-uris&uris={:s}".format(
+ self.books_url, search_result.get("uri")
+ ),
view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
cover=cover,
connector=self,
@@ -65,11 +71,24 @@ class Connector(AbstractConnector):
return data.get("type") == "work"
def get_edition_from_work_data(self, data):
- return {}
+ value = data.get("uri")
+ url = "{:s}?action=reverse-claims&property=P629&value={:s}".format(
+ self.books_url, value
+ )
+ data = get_data(url)
+ try:
+ uri = data["uris"][0]
+ except KeyError:
+ raise ConnectorException("Invalid book data")
+ return self.get_book_data(self.get_remote_id(uri))
def get_work_from_edition_data(self, data):
- # P629
- return {}
+ try:
+ uri = data["claims"]["wdt:P629"]
+ except KeyError:
+ raise ConnectorException("Invalid book data")
+ return self.get_book_data(self.get_remote_id(uri))
+
def get_authors_from_data(self, data):
return []
From e594cd0a3619015091c654ab85de9ba85ab14a07 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 13:53:58 -0700
Subject: [PATCH 08/37] Load simple fields from inventaire
---
bookwyrm/connectors/inventaire.py | 64 +++++++++++++++++--------------
1 file changed, 36 insertions(+), 28 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 9987079e..1ff834bf 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -10,47 +10,59 @@ class Connector(AbstractConnector):
def __init__(self, identifier):
super().__init__(identifier)
+ get_first = lambda a, *args: a[0]
get_remote_id_list = lambda a, *_: [self.get_remote_id(v) for v in a]
self.book_mappings = [
- Mapping("title", remote_field="wdt:P1476", formatter=get_claim),
- Mapping("subtitle", remote_field="wdt:P1680", formatter=get_claim),
+ Mapping("title", remote_field="wdt:P1476", formatter=get_first),
+ Mapping("subtitle", remote_field="wdt:P1680", formatter=get_first),
Mapping("id", remote_field="uri", formatter=self.get_remote_id),
- Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list),
+ # Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list),
Mapping("inventaireId", remote_field="uri"),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
- Mapping("isbn13", remote_field="wdt:P212", formatter=get_claim),
- Mapping("isbn10", remote_field="wdt:P957", formatter=get_claim),
- Mapping("languages", remote_field="wdt:P407", formatter=get_language),
- Mapping("publishers", remote_field="wdt:P123", formatter=resolve_key),
- Mapping("publishedDate", remote_field="wdt:P577", formatter=get_claim),
- Mapping("pages", remote_field="wdt:P1104", formatter=get_claim),
- Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_claim),
- Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_claim),
- Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key),
- Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key),
- Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_claim),
- Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_claim),
- Mapping("asin", remote_field="wdt:P5749", formatter=get_claim),
+ Mapping("isbn13", remote_field="wdt:P212", formatter=get_first),
+ Mapping("isbn10", remote_field="wdt:P957", formatter=get_first),
+ # Mapping("languages", remote_field="wdt:P407", formatter=get_language),
+ # Mapping("publishers", remote_field="wdt:P123", formatter=resolve_key),
+ Mapping("publishedDate", remote_field="wdt:P577", formatter=get_first),
+ Mapping("pages", remote_field="wdt:P1104", formatter=get_first),
+ Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first),
+ Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first),
+ # Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key),
+ # Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key),
+ Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first),
+ Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
+ Mapping("asin", remote_field="wdt:P5749", formatter=get_first),
]
# TODO: P136: genre, P268 bnf id, P674 characters, P950 bne
def get_remote_id(self, value, *_):
""" convert an id/uri into a url """
- return '{:s}?action=by-uris&uris={:s}'.format(self.books_url, value)
+ return "{:s}?action=by-uris&uris={:s}".format(self.books_url, value)
def get_book_data(self, remote_id):
data = get_data(remote_id)
extracted = list(data.get("entities").values())
- return extracted[0] if extracted else {}
+ try:
+ data = extracted[0]
+ except KeyError:
+ raise ConnectorException("Invalid book data")
+ # flatten the data so that images, uri, and claims are on the same level
+ return {
+ **data.get("claims"),
+ "uri": data.get("uri"),
+ "image": data.get("image"),
+ }
def parse_search_data(self, data):
- return data.get('results')
+ return data.get("results")
def format_search_result(self, search_result):
images = search_result.get("image")
- cover = "{:s}/img/entities/{:s}".format(
- self.covers_url, images[0]
- ) if images else None
+ cover = (
+ "{:s}/img/entities/{:s}".format(self.covers_url, images[0])
+ if images
+ else None
+ )
return SearchResult(
title=search_result.get("label"),
key="{:s}?action=by-uris&uris={:s}".format(
@@ -89,7 +101,6 @@ class Connector(AbstractConnector):
raise ConnectorException("Invalid book data")
return self.get_book_data(self.get_remote_id(uri))
-
def get_authors_from_data(self, data):
return []
@@ -97,7 +108,7 @@ class Connector(AbstractConnector):
return
def get_cover_url(self, cover_blob, *_):
- """ format the relative cover url into an absolute one:
+ """format the relative cover url into an absolute one:
{"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
"""
cover_id = cover_blob[0].get("url")
@@ -106,14 +117,11 @@ class Connector(AbstractConnector):
return "%s%s" % (self.covers_url, cover_id)
-def get_claim(data, claim_key):
- """ all the metadata is in a "claims" dict with a buncha wikidata keys """
- return data.get('claims', {}).get(claim_key)
-
def get_language(wikidata_key, *_):
""" who here speaks "wd:Q150" """
return wikidata_key # TODO
+
def resolve_key(wikidata_key, *_):
""" cool, it's "wd:Q3156592" now what the heck does that mean """
return wikidata_key # TODO
From bfdcf611e7966ea54d051a313115c40cf35eb40b Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 13:54:20 -0700
Subject: [PATCH 09/37] Adds inventaire identifier to book data fields
---
.../migrations/0062_auto_20210406_1731.py | 20 +++++++++----
.../migrations/0063_auto_20210406_1738.py | 28 +++++++++++++++++++
2 files changed, 42 insertions(+), 6 deletions(-)
create mode 100644 bookwyrm/migrations/0063_auto_20210406_1738.py
diff --git a/bookwyrm/migrations/0062_auto_20210406_1731.py b/bookwyrm/migrations/0062_auto_20210406_1731.py
index 9ff26af3..5db176ec 100644
--- a/bookwyrm/migrations/0062_auto_20210406_1731.py
+++ b/bookwyrm/migrations/0062_auto_20210406_1731.py
@@ -6,17 +6,25 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
- ('bookwyrm', '0061_auto_20210402_1435'),
+ ("bookwyrm", "0061_auto_20210402_1435"),
]
operations = [
migrations.RemoveConstraint(
- model_name='connector',
- name='connector_file_valid',
+ model_name="connector",
+ name="connector_file_valid",
),
migrations.AlterField(
- model_name='connector',
- name='connector_file',
- field=models.CharField(choices=[('openlibrary', 'Openlibrary'), ('inventaire', 'Inventaire'), ('self_connector', 'Self Connector'), ('bookwyrm_connector', 'Bookwyrm Connector')], max_length=255),
+ model_name="connector",
+ name="connector_file",
+ field=models.CharField(
+ choices=[
+ ("openlibrary", "Openlibrary"),
+ ("inventaire", "Inventaire"),
+ ("self_connector", "Self Connector"),
+ ("bookwyrm_connector", "Bookwyrm Connector"),
+ ],
+ max_length=255,
+ ),
),
]
diff --git a/bookwyrm/migrations/0063_auto_20210406_1738.py b/bookwyrm/migrations/0063_auto_20210406_1738.py
new file mode 100644
index 00000000..c9f07b3c
--- /dev/null
+++ b/bookwyrm/migrations/0063_auto_20210406_1738.py
@@ -0,0 +1,28 @@
+# Generated by Django 3.1.6 on 2021-04-06 17:38
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0062_auto_20210406_1731"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="author",
+ name="inventaire_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ migrations.AddField(
+ model_name="book",
+ name="inventaire_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
+ ),
+ ]
From 82c2f2eeb1b0d8a9fa1133c2bd1c626214dfa1d0 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 17:46:06 -0700
Subject: [PATCH 10/37] Adds more author identifier fields
---
bookwyrm/connectors/abstract_connector.py | 8 +--
bookwyrm/connectors/inventaire.py | 47 ++++++++++++------
bookwyrm/connectors/openlibrary.py | 10 ++--
.../migrations/0063_auto_20210406_1738.py | 28 -----------
.../migrations/0063_auto_20210407_0045.py | 49 +++++++++++++++++++
bookwyrm/models/author.py | 9 ++++
bookwyrm/models/book.py | 3 ++
7 files changed, 103 insertions(+), 51 deletions(-)
delete mode 100644 bookwyrm/migrations/0063_auto_20210406_1738.py
create mode 100644 bookwyrm/migrations/0063_auto_20210407_0045.py
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 8007bb1a..f66cca7f 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -150,11 +150,11 @@ class AbstractConnector(AbstractMinimalConnector):
load_more_data.delay(self.connector.id, work.id)
return edition
- def get_book_data(self, remote_id):
+ def get_book_data(self, remote_id): # pylint: disable=no-self-use
""" this allows connectors to override the default behavior """
return get_data(remote_id)
- def get_search_data(self, remote_id):
+ def get_search_data(self, remote_id): # pylint: disable=no-self-use
""" this allows connectors to override the default behavior """
return get_data(remote_id)
@@ -293,7 +293,7 @@ class Mapping:
""" associate a local database field with a field in an external dataset """
def __init__(self, local_field, remote_field=None, formatter=None):
- noop = lambda x, *_: x
+ noop = lambda x: x
self.local_field = local_field
self.remote_field = remote_field or local_field
@@ -305,6 +305,6 @@ class Mapping:
if not value:
return None
try:
- return self.formatter(value, self.remote_field)
+ return self.formatter(value)
except: # pylint: disable=bare-except
return None
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 1ff834bf..6eff6508 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -10,32 +10,45 @@ class Connector(AbstractConnector):
def __init__(self, identifier):
super().__init__(identifier)
- get_first = lambda a, *args: a[0]
- get_remote_id_list = lambda a, *_: [self.get_remote_id(v) for v in a]
+ get_first = lambda a: a[0]
+ shared_mappings = [
+ Mapping("id", remote_field="uri", formatter=self.get_remote_id),
+ Mapping("bnfId", remote_field="wdt:P268", formatter=get_first),
+ Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
+ Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first),
+ ]
self.book_mappings = [
Mapping("title", remote_field="wdt:P1476", formatter=get_first),
Mapping("subtitle", remote_field="wdt:P1680", formatter=get_first),
- Mapping("id", remote_field="uri", formatter=self.get_remote_id),
- # Mapping("authors", remote_field="wdt:P50", formatter=get_remote_id_list),
Mapping("inventaireId", remote_field="uri"),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
Mapping("isbn13", remote_field="wdt:P212", formatter=get_first),
Mapping("isbn10", remote_field="wdt:P957", formatter=get_first),
+ Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first),
+ Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first),
# Mapping("languages", remote_field="wdt:P407", formatter=get_language),
# Mapping("publishers", remote_field="wdt:P123", formatter=resolve_key),
Mapping("publishedDate", remote_field="wdt:P577", formatter=get_first),
Mapping("pages", remote_field="wdt:P1104", formatter=get_first),
- Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first),
- Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first),
# Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key),
# Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key),
- Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first),
- Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
Mapping("asin", remote_field="wdt:P5749", formatter=get_first),
- ]
- # TODO: P136: genre, P268 bnf id, P674 characters, P950 bne
+ ] + shared_mappings
+ # TODO: P136: genre, P674 characters, P950 bne
- def get_remote_id(self, value, *_):
+ self.author_mappings = [
+ Mapping("id", remote_field="uri", formatter=self.get_remote_id),
+ Mapping("name", remote_field="label", formatter=get_language_code),
+ Mapping("goodreadsKey", remote_field="wdt:P2963", formatter=get_first),
+ Mapping("isni", remote_field="wdt:P213", formatter=get_first),
+ Mapping("viafId", remote_field="wdt:P214", formatter=get_first),
+ Mapping("gutenberg_id", remote_field="wdt:P1938", formatter=get_first),
+ Mapping("born", remote_field="wdt:P569", formatter=get_first),
+ Mapping("died", remote_field="wdt:P570", formatter=get_first),
+ ] + shared_mappings
+
+
+ def get_remote_id(self, value):
""" convert an id/uri into a url """
return "{:s}?action=by-uris&uris={:s}".format(self.books_url, value)
@@ -102,7 +115,9 @@ class Connector(AbstractConnector):
return self.get_book_data(self.get_remote_id(uri))
def get_authors_from_data(self, data):
- return []
+ authors = data.get("wdt:P50")
+ for author in authors:
+ yield self.get_or_create_author(self.get_remote_id(author))
def expand_book_data(self, book):
return
@@ -117,11 +132,15 @@ class Connector(AbstractConnector):
return "%s%s" % (self.covers_url, cover_id)
-def get_language(wikidata_key, *_):
+def get_language(wikidata_key):
""" who here speaks "wd:Q150" """
return wikidata_key # TODO
-def resolve_key(wikidata_key, *_):
+def resolve_key(wikidata_key):
""" cool, it's "wd:Q3156592" now what the heck does that mean """
return wikidata_key # TODO
+
+def get_language_code(options, code="en"):
+ """ when there are a bunch of translation but we need a single field """
+ return options.get(code)
diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py
index 34357726..466bf1e5 100644
--- a/bookwyrm/connectors/openlibrary.py
+++ b/bookwyrm/connectors/openlibrary.py
@@ -95,7 +95,7 @@ class Connector(AbstractConnector):
url = "%s%s" % (self.base_url, author_id)
yield self.get_or_create_author(url)
- def get_cover_url(self, cover_blob, *_, size="L"):
+ def get_cover_url(self, cover_blob, size="L"):
""" ask openlibrary for the cover """
if not cover_blob:
return None
@@ -181,19 +181,19 @@ def ignore_edition(edition_data):
return True
-def get_description(description_blob, *_):
+def get_description(description_blob):
""" descriptions can be a string or a dict """
if isinstance(description_blob, dict):
return description_blob.get("value")
return description_blob
-def get_openlibrary_key(key, *_):
+def get_openlibrary_key(key):
""" convert /books/OL27320736M into OL27320736M """
return key.split("/")[-1]
-def get_languages(language_blob, *_):
+def get_languages(language_blob):
""" /language/eng -> English """
langs = []
for lang in language_blob:
@@ -201,7 +201,7 @@ def get_languages(language_blob, *_):
return langs
-def pick_default_edition(options, *_):
+def pick_default_edition(options):
""" favor physical copies with covers in english """
if not options:
return None
diff --git a/bookwyrm/migrations/0063_auto_20210406_1738.py b/bookwyrm/migrations/0063_auto_20210406_1738.py
deleted file mode 100644
index c9f07b3c..00000000
--- a/bookwyrm/migrations/0063_auto_20210406_1738.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Generated by Django 3.1.6 on 2021-04-06 17:38
-
-import bookwyrm.models.fields
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
- dependencies = [
- ("bookwyrm", "0062_auto_20210406_1731"),
- ]
-
- operations = [
- migrations.AddField(
- model_name="author",
- name="inventaire_id",
- field=bookwyrm.models.fields.CharField(
- blank=True, max_length=255, null=True
- ),
- ),
- migrations.AddField(
- model_name="book",
- name="inventaire_id",
- field=bookwyrm.models.fields.CharField(
- blank=True, max_length=255, null=True
- ),
- ),
- ]
diff --git a/bookwyrm/migrations/0063_auto_20210407_0045.py b/bookwyrm/migrations/0063_auto_20210407_0045.py
new file mode 100644
index 00000000..2543193b
--- /dev/null
+++ b/bookwyrm/migrations/0063_auto_20210407_0045.py
@@ -0,0 +1,49 @@
+# Generated by Django 3.1.6 on 2021-04-07 00:45
+
+import bookwyrm.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bookwyrm', '0062_auto_20210406_1731'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='author',
+ name='bnf_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='author',
+ name='gutenberg_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='author',
+ name='inventaire_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='author',
+ name='isni',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='author',
+ name='viaf_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='book',
+ name='bnf_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ migrations.AddField(
+ model_name='book',
+ name='inventaire_id',
+ field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ ),
+ ]
diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py
index 4c5fe6c8..f7740b1d 100644
--- a/bookwyrm/models/author.py
+++ b/bookwyrm/models/author.py
@@ -14,6 +14,15 @@ class Author(BookDataModel):
wikipedia_link = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
+ isni = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
+ viaf_id = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
+ gutenberg_id = fields.CharField(
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
# idk probably other keys would be useful here?
born = fields.DateTimeField(blank=True, null=True)
died = fields.DateTimeField(blank=True, null=True)
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index ccbec9eb..94bbe330 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -28,6 +28,9 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
goodreads_key = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
+ bnf_id = fields.CharField( # Bibliothèque nationale de France
+ max_length=255, blank=True, null=True, deduplication_field=True
+ )
last_edited_by = models.ForeignKey("User", on_delete=models.PROTECT, null=True)
From 4112862924abedb0362649b4592405b99837371c Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:00:54 -0700
Subject: [PATCH 11/37] Fixes search data and new activitypub fields
---
bookwyrm/activitypub/book.py | 26 +++++++++++++----------
bookwyrm/connectors/abstract_connector.py | 8 +++----
bookwyrm/connectors/inventaire.py | 7 +++---
3 files changed, 22 insertions(+), 19 deletions(-)
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index 7a427927..17e5b686 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -7,7 +7,17 @@ from .image import Image
@dataclass(init=False)
-class Book(ActivityObject):
+class BookData(ActivityObject):
+ """ shared fields for all book data and authors"""
+ openlibraryKey: str = None
+ inventaireId: str = None
+ librarythingKey: str = None
+ goodreadsKey: str = None
+ bnfId: str = None
+
+
+@dataclass(init=False)
+class Book(BookData):
""" serializes an edition or work, abstract """
title: str
@@ -24,11 +34,6 @@ class Book(ActivityObject):
firstPublishedDate: str = ""
publishedDate: str = ""
- openlibraryKey: str = None
- inventaireId: str = None
- librarythingKey: str = None
- goodreadsKey: str = None
-
cover: Image = None
type: str = "Book"
@@ -61,17 +66,16 @@ class Work(Book):
@dataclass(init=False)
-class Author(ActivityObject):
+class Author(BookData):
""" author of a book """
name: str
+ isni: str = None
+ viafId: str = None
+ gutenbergId: str = None
born: str = None
died: str = None
aliases: List[str] = field(default_factory=lambda: [])
bio: str = ""
- openlibraryKey: str = None
- inventaireId: str = None
- librarythingKey: str = None
- goodreadsKey: str = None
wikipediaLink: str = ""
type: str = "Author"
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index f66cca7f..43cd6aad 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -68,6 +68,10 @@ class AbstractMinimalConnector(ABC):
results.append(self.format_isbn_search_result(doc))
return results
+ def get_search_data(self, remote_id, **kwargs): # pylint: disable=no-self-use
+ """ this allows connectors to override the default behavior """
+ return get_data(remote_id, **kwargs)
+
@abstractmethod
def get_or_create_book(self, remote_id):
""" pull up a book record by whatever means possible """
@@ -154,10 +158,6 @@ class AbstractConnector(AbstractMinimalConnector):
""" this allows connectors to override the default behavior """
return get_data(remote_id)
- def get_search_data(self, remote_id): # pylint: disable=no-self-use
- """ this allows connectors to override the default behavior """
- return get_data(remote_id)
-
def create_edition_from_data(self, work, edition_data):
""" if we already have the work, we're ready """
mapped_data = dict_from_mappings(edition_data, self.book_mappings)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 6eff6508..a16bb0fd 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -14,7 +14,6 @@ class Connector(AbstractConnector):
shared_mappings = [
Mapping("id", remote_field="uri", formatter=self.get_remote_id),
Mapping("bnfId", remote_field="wdt:P268", formatter=get_first),
- Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
Mapping("openlibraryKey", remote_field="wdt:P648", formatter=get_first),
]
self.book_mappings = [
@@ -24,6 +23,7 @@ class Connector(AbstractConnector):
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
Mapping("isbn13", remote_field="wdt:P212", formatter=get_first),
Mapping("isbn10", remote_field="wdt:P957", formatter=get_first),
+ Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first),
Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first),
# Mapping("languages", remote_field="wdt:P407", formatter=get_language),
@@ -38,7 +38,7 @@ class Connector(AbstractConnector):
self.author_mappings = [
Mapping("id", remote_field="uri", formatter=self.get_remote_id),
- Mapping("name", remote_field="label", formatter=get_language_code),
+ Mapping("name", remote_field="labels", formatter=get_language_code),
Mapping("goodreadsKey", remote_field="wdt:P2963", formatter=get_first),
Mapping("isni", remote_field="wdt:P213", formatter=get_first),
Mapping("viafId", remote_field="wdt:P214", formatter=get_first),
@@ -62,8 +62,7 @@ class Connector(AbstractConnector):
# flatten the data so that images, uri, and claims are on the same level
return {
**data.get("claims"),
- "uri": data.get("uri"),
- "image": data.get("image"),
+ **{k: data.get(k) for k in ["uri", "image", "labels"]},
}
def parse_search_data(self, data):
From f21aca1211a10b9efc66469c4bb8bad023bf7da0 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:10:42 -0700
Subject: [PATCH 12/37] Load remote keys
---
bookwyrm/connectors/inventaire.py | 28 ++++++++++++++++------------
1 file changed, 16 insertions(+), 12 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index a16bb0fd..4be0dd5c 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -26,12 +26,14 @@ class Connector(AbstractConnector):
Mapping("oclcNumber", remote_field="wdt:P5331", formatter=get_first),
Mapping("goodreadsKey", remote_field="wdt:P2969", formatter=get_first),
Mapping("librarythingKey", remote_field="wdt:P1085", formatter=get_first),
- # Mapping("languages", remote_field="wdt:P407", formatter=get_language),
- # Mapping("publishers", remote_field="wdt:P123", formatter=resolve_key),
+ Mapping("languages", remote_field="wdt:P407", formatter=self.resolve_keys),
+ Mapping("publishers", remote_field="wdt:P123", formatter=self.resolve_keys),
Mapping("publishedDate", remote_field="wdt:P577", formatter=get_first),
Mapping("pages", remote_field="wdt:P1104", formatter=get_first),
- # Mapping("subjectPlaces", remote_field="wdt:P840", formatter=resolve_key),
- # Mapping("subjects", remote_field="wdt:P921", formatter=resolve_key),
+ Mapping(
+ "subjectPlaces", remote_field="wdt:P840", formatter=self.resolve_keys
+ ),
+ Mapping("subjects", remote_field="wdt:P921", formatter=self.resolve_keys),
Mapping("asin", remote_field="wdt:P5749", formatter=get_first),
] + shared_mappings
# TODO: P136: genre, P674 characters, P950 bne
@@ -130,15 +132,17 @@ class Connector(AbstractConnector):
return None
return "%s%s" % (self.covers_url, cover_id)
+ def resolve_keys(self, keys):
+ """ cool, it's "wd:Q3156592" now what the heck does that mean """
+ results = []
+ for uri in keys:
+ try:
+ data = self.get_book_data(self.get_remote_id(uri))
+ except ConnectorException:
+ continue
+ results.append(get_language_code(data.get("labels")))
+ return results
-def get_language(wikidata_key):
- """ who here speaks "wd:Q150" """
- return wikidata_key # TODO
-
-
-def resolve_key(wikidata_key):
- """ cool, it's "wd:Q3156592" now what the heck does that mean """
- return wikidata_key # TODO
def get_language_code(options, code="en"):
""" when there are a bunch of translation but we need a single field """
From ac27111f0555fbd96ec95503d5434b05be53ee32 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:13:33 -0700
Subject: [PATCH 13/37] Adds inventaire to default connector list
---
bookwyrm/management/commands/initdb.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py
index d6101c87..45c81089 100644
--- a/bookwyrm/management/commands/initdb.py
+++ b/bookwyrm/management/commands/initdb.py
@@ -94,6 +94,18 @@ def init_connectors():
priority=2,
)
+ Connector.objects.create(
+ identifier="inventaire.io",
+ name="Inventaire",
+ connector_file="inventaire",
+ base_url="https://inventaire.io/entity/",
+ books_url="https://inventaire.io/api/entities",
+ covers_url="https://inventaire.io",
+ search_url="https://inventaire.io/api/search?types=works&types=works&search=",
+ isbn_search_url="https://inventaire.io/api/entities?action=by-uris&uris=isbn%3A",
+ priority=3,
+ )
+
Connector.objects.create(
identifier="openlibrary.org",
name="OpenLibrary",
From fec3d63e4623f7a47526d1648cd0232f1ef6c531 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:17:33 -0700
Subject: [PATCH 14/37] Python formatting
---
bookwyrm/activitypub/book.py | 1 +
bookwyrm/connectors/inventaire.py | 1 -
.../migrations/0063_auto_20210407_0045.py | 58 ++++++++++++-------
3 files changed, 37 insertions(+), 23 deletions(-)
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index 17e5b686..ca4d69da 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -9,6 +9,7 @@ from .image import Image
@dataclass(init=False)
class BookData(ActivityObject):
""" shared fields for all book data and authors"""
+
openlibraryKey: str = None
inventaireId: str = None
librarythingKey: str = None
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 4be0dd5c..2a4569b3 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -49,7 +49,6 @@ class Connector(AbstractConnector):
Mapping("died", remote_field="wdt:P570", formatter=get_first),
] + shared_mappings
-
def get_remote_id(self, value):
""" convert an id/uri into a url """
return "{:s}?action=by-uris&uris={:s}".format(self.books_url, value)
diff --git a/bookwyrm/migrations/0063_auto_20210407_0045.py b/bookwyrm/migrations/0063_auto_20210407_0045.py
index 2543193b..cd87dd97 100644
--- a/bookwyrm/migrations/0063_auto_20210407_0045.py
+++ b/bookwyrm/migrations/0063_auto_20210407_0045.py
@@ -7,43 +7,57 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
- ('bookwyrm', '0062_auto_20210406_1731'),
+ ("bookwyrm", "0062_auto_20210406_1731"),
]
operations = [
migrations.AddField(
- model_name='author',
- name='bnf_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="author",
+ name="bnf_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='author',
- name='gutenberg_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="author",
+ name="gutenberg_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='author',
- name='inventaire_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="author",
+ name="inventaire_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='author',
- name='isni',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="author",
+ name="isni",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='author',
- name='viaf_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="author",
+ name="viaf_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='book',
- name='bnf_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="book",
+ name="bnf_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
migrations.AddField(
- model_name='book',
- name='inventaire_id',
- field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True),
+ model_name="book",
+ name="inventaire_id",
+ field=bookwyrm.models.fields.CharField(
+ blank=True, max_length=255, null=True
+ ),
),
]
From 29e7659b76ec094f9bb3ff8659e97cdf01346080 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:34:55 -0700
Subject: [PATCH 15/37] Expand inventaire book data
---
bookwyrm/connectors/inventaire.py | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 2a4569b3..465268b2 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -1,4 +1,5 @@
""" inventaire data connector """
+from bookwyrm import models
from .abstract_connector import AbstractConnector, SearchResult, Mapping
from .abstract_connector import get_data
from .connector_manager import ConnectorException
@@ -95,12 +96,15 @@ class Connector(AbstractConnector):
def is_work_data(self, data):
return data.get("type") == "work"
- def get_edition_from_work_data(self, data):
- value = data.get("uri")
+ def load_edition_data(self, work_uri):
+ """ get a list of editions for a work """
url = "{:s}?action=reverse-claims&property=P629&value={:s}".format(
- self.books_url, value
+ self.books_url, work_uri
)
- data = get_data(url)
+ return get_data(url)
+
+ def get_edition_from_work_data(self, data):
+ data = self.load_edition_data(data.get("uri"))
try:
uri = data["uris"][0]
except KeyError:
@@ -120,7 +124,20 @@ class Connector(AbstractConnector):
yield self.get_or_create_author(self.get_remote_id(author))
def expand_book_data(self, book):
- return
+ work = book
+ # go from the edition to the work, if necessary
+ if isinstance(book, models.Edition):
+ work = book.parent_work
+
+ try:
+ edition_options = self.load_edition_data(work.inventaire_id)
+ except ConnectorException:
+ # who knows, man
+ return
+
+ for edition_uri in edition_options.get("uris"):
+ remote_id = self.get_remote_id(edition_uri)
+ self.get_or_create_book(remote_id)
def get_cover_url(self, cover_blob, *_):
"""format the relative cover url into an absolute one:
From 922428cab760ba324619eb64dd2a89bfe352d15c Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Tue, 6 Apr 2021 18:51:43 -0700
Subject: [PATCH 16/37] Fixes error in reverse path
---
bookwyrm/connectors/inventaire.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 465268b2..594fe810 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -98,7 +98,7 @@ class Connector(AbstractConnector):
def load_edition_data(self, work_uri):
""" get a list of editions for a work """
- url = "{:s}?action=reverse-claims&property=P629&value={:s}".format(
+ url = "{:s}?action=reverse-claims&property=wdt:P629&value={:s}".format(
self.books_url, work_uri
)
return get_data(url)
@@ -119,7 +119,7 @@ class Connector(AbstractConnector):
return self.get_book_data(self.get_remote_id(uri))
def get_authors_from_data(self, data):
- authors = data.get("wdt:P50")
+ authors = data.get("wdt:P50", [])
for author in authors:
yield self.get_or_create_author(self.get_remote_id(author))
From 85297426e07917c20d5d5d7fed344380af5b6608 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Mon, 26 Apr 2021 14:33:16 -0700
Subject: [PATCH 17/37] Adds merge migration
---
...63_auto_20210407_0045_0070_auto_20210423_0121.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
create mode 100644 bookwyrm/migrations/0071_merge_0063_auto_20210407_0045_0070_auto_20210423_0121.py
diff --git a/bookwyrm/migrations/0071_merge_0063_auto_20210407_0045_0070_auto_20210423_0121.py b/bookwyrm/migrations/0071_merge_0063_auto_20210407_0045_0070_auto_20210423_0121.py
new file mode 100644
index 00000000..b6489b80
--- /dev/null
+++ b/bookwyrm/migrations/0071_merge_0063_auto_20210407_0045_0070_auto_20210423_0121.py
@@ -0,0 +1,13 @@
+# Generated by Django 3.2 on 2021-04-26 21:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0063_auto_20210407_0045"),
+ ("bookwyrm", "0070_auto_20210423_0121"),
+ ]
+
+ operations = []
From 7b65291a59b6f3b5f7a64360227ce41451aec5b5 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Mon, 26 Apr 2021 14:43:29 -0700
Subject: [PATCH 18/37] Python formatting for the new Black standard
---
bookwyrm/activitypub/book.py | 2 +-
bookwyrm/connectors/abstract_connector.py | 4 ++--
bookwyrm/connectors/inventaire.py | 14 +++++++-------
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index dfe14b6f..597d7a66 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -8,7 +8,7 @@ from .image import Document
@dataclass(init=False)
class BookData(ActivityObject):
- """ shared fields for all book data and authors"""
+ """shared fields for all book data and authors"""
openlibraryKey: str = None
inventaireId: str = None
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 14fe3cb7..fd2b2707 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -69,7 +69,7 @@ class AbstractMinimalConnector(ABC):
return results
def get_search_data(self, remote_id, **kwargs): # pylint: disable=no-self-use
- """ this allows connectors to override the default behavior """
+ """this allows connectors to override the default behavior"""
return get_data(remote_id, **kwargs)
@abstractmethod
@@ -155,7 +155,7 @@ class AbstractConnector(AbstractMinimalConnector):
return edition
def get_book_data(self, remote_id): # pylint: disable=no-self-use
- """ this allows connectors to override the default behavior """
+ """this allows connectors to override the default behavior"""
return get_data(remote_id)
def create_edition_from_data(self, work, edition_data):
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index 594fe810..ae6fb862 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -6,7 +6,7 @@ from .connector_manager import ConnectorException
class Connector(AbstractConnector):
- """ instantiate a connector for OL """
+ """instantiate a connector for OL"""
def __init__(self, identifier):
super().__init__(identifier)
@@ -51,7 +51,7 @@ class Connector(AbstractConnector):
] + shared_mappings
def get_remote_id(self, value):
- """ convert an id/uri into a url """
+ """convert an id/uri into a url"""
return "{:s}?action=by-uris&uris={:s}".format(self.books_url, value)
def get_book_data(self, remote_id):
@@ -88,16 +88,16 @@ class Connector(AbstractConnector):
)
def parse_isbn_search_data(self, data):
- """ boop doop """
+ """boop doop"""
def format_isbn_search_result(self, search_result):
- """ beep bloop """
+ """beep bloop"""
def is_work_data(self, data):
return data.get("type") == "work"
def load_edition_data(self, work_uri):
- """ get a list of editions for a work """
+ """get a list of editions for a work"""
url = "{:s}?action=reverse-claims&property=wdt:P629&value={:s}".format(
self.books_url, work_uri
)
@@ -149,7 +149,7 @@ class Connector(AbstractConnector):
return "%s%s" % (self.covers_url, cover_id)
def resolve_keys(self, keys):
- """ cool, it's "wd:Q3156592" now what the heck does that mean """
+ """cool, it's "wd:Q3156592" now what the heck does that mean"""
results = []
for uri in keys:
try:
@@ -161,5 +161,5 @@ class Connector(AbstractConnector):
def get_language_code(options, code="en"):
- """ when there are a bunch of translation but we need a single field """
+ """when there are a bunch of translation but we need a single field"""
return options.get(code)
From 3faacffacaaa3fdd474f12c1768544393aa22970 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 13:28:07 -0700
Subject: [PATCH 19/37] Associated expanded editions with correct work
---
bookwyrm/connectors/abstract_connector.py | 27 +++++++++++++----------
bookwyrm/connectors/inventaire.py | 2 +-
2 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index fd2b2707..48c05182 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -73,7 +73,7 @@ class AbstractMinimalConnector(ABC):
return get_data(remote_id, **kwargs)
@abstractmethod
- def get_or_create_book(self, remote_id):
+ def get_or_create_book(self, remote_id, work=None):
"""pull up a book record by whatever means possible"""
@abstractmethod
@@ -109,7 +109,7 @@ class AbstractConnector(AbstractMinimalConnector):
return False
return True
- def get_or_create_book(self, remote_id):
+ def get_or_create_book(self, remote_id, work=None):
"""translate arbitrary json into an Activitypub dataclass"""
# first, check if we have the origin_id saved
existing = models.Edition.find_existing_by_remote_id(
@@ -123,6 +123,7 @@ class AbstractConnector(AbstractMinimalConnector):
# load the json
data = self.get_book_data(remote_id)
mapped_data = dict_from_mappings(data, self.book_mappings)
+ work_data = edition_data = None
if self.is_work_data(data):
try:
edition_data = self.get_edition_from_work_data(data)
@@ -132,21 +133,23 @@ class AbstractConnector(AbstractMinimalConnector):
edition_data = data
work_data = mapped_data
else:
- try:
- work_data = self.get_work_from_edition_data(data)
- work_data = dict_from_mappings(work_data, self.book_mappings)
- except (KeyError, ConnectorException):
- work_data = mapped_data
edition_data = data
+ if not work:
+ try:
+ work_data = self.get_work_from_edition_data(data)
+ work_data = dict_from_mappings(work_data, self.book_mappings)
+ except (KeyError, ConnectorException):
+ work_data = mapped_data
- if not work_data or not edition_data:
+ if (not work_data and not work) or not edition_data:
raise ConnectorException("Unable to load book data: %s" % remote_id)
with transaction.atomic():
- # create activitypub object
- work_activity = activitypub.Work(**work_data)
- # this will dedupe automatically
- work = work_activity.to_model(model=models.Work)
+ if not work:
+ # create activitypub object
+ work_activity = activitypub.Work(**work_data)
+ # this will dedupe automatically
+ work = work_activity.to_model(model=models.Work)
for author in self.get_authors_from_data(data):
work.authors.add(author)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index ae6fb862..c91ac937 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -137,7 +137,7 @@ class Connector(AbstractConnector):
for edition_uri in edition_options.get("uris"):
remote_id = self.get_remote_id(edition_uri)
- self.get_or_create_book(remote_id)
+ self.get_or_create_book(remote_id, work=work)
def get_cover_url(self, cover_blob, *_):
"""format the relative cover url into an absolute one:
From ccf10e801279dbc8946c67337a0269758c06643d Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 13:50:47 -0700
Subject: [PATCH 20/37] Fixes cover load logic
---
bookwyrm/connectors/inventaire.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index c91ac937..e8416c37 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -143,7 +143,9 @@ class Connector(AbstractConnector):
"""format the relative cover url into an absolute one:
{"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
"""
- cover_id = cover_blob[0].get("url")
+ if isinstance(cover_blob, list) and len(cover_blob) > 0:
+ cover_blob = cover_blob[0]
+ cover_id = cover_blob.get("url")
if not cover_id:
return None
return "%s%s" % (self.covers_url, cover_id)
From 6551c59c45d74b7920c58198d4a045b7afc0dfd3 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 15:19:24 -0700
Subject: [PATCH 21/37] Don't store default edition in the dataase
---
bookwyrm/activitypub/book.py | 1 -
bookwyrm/connectors/abstract_connector.py | 8 ++---
bookwyrm/connectors/bookwyrm_connector.py | 8 ++---
bookwyrm/connectors/self_connector.py | 29 +++++++++++++++----
.../0072_remove_work_default_edition.py | 17 +++++++++++
bookwyrm/models/book.py | 29 ++++---------------
.../connectors/test_connector_manager.py | 2 --
.../tests/connectors/test_self_connector.py | 6 ++--
bookwyrm/tests/models/test_book_model.py | 6 ----
.../tests/models/test_readthrough_model.py | 4 +--
bookwyrm/tests/views/test_readthrough.py | 2 --
bookwyrm/views/author.py | 2 +-
bookwyrm/views/books.py | 2 +-
bookwyrm/views/helpers.py | 2 +-
14 files changed, 56 insertions(+), 62 deletions(-)
create mode 100644 bookwyrm/migrations/0072_remove_work_default_edition.py
diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py
index 597d7a66..1599b408 100644
--- a/bookwyrm/activitypub/book.py
+++ b/bookwyrm/activitypub/book.py
@@ -62,7 +62,6 @@ class Work(Book):
"""work instance of a book object"""
lccn: str = ""
- defaultEdition: str = ""
editions: List[str] = field(default_factory=lambda: [])
type: str = "Work"
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 48c05182..08a0175a 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -116,8 +116,8 @@ class AbstractConnector(AbstractMinimalConnector):
remote_id
) or models.Work.find_existing_by_remote_id(remote_id)
if existing:
- if hasattr(existing, "get_default_editon"):
- return existing.get_default_editon()
+ if hasattr(existing, "default_edition"):
+ return existing.default_edition
return existing
# load the json
@@ -170,10 +170,6 @@ class AbstractConnector(AbstractMinimalConnector):
edition.connector = self.connector
edition.save()
- if not work.default_edition:
- work.default_edition = edition
- work.save()
-
for author in self.get_authors_from_data(edition_data):
edition.authors.add(author)
if not edition.authors.exists() and work.authors.exists():
diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py
index 640a0bca..6b1d2f8c 100644
--- a/bookwyrm/connectors/bookwyrm_connector.py
+++ b/bookwyrm/connectors/bookwyrm_connector.py
@@ -6,12 +6,8 @@ from .abstract_connector import AbstractMinimalConnector, SearchResult
class Connector(AbstractMinimalConnector):
"""this is basically just for search"""
- def get_or_create_book(self, remote_id):
- edition = activitypub.resolve_remote_id(remote_id, model=models.Edition)
- work = edition.parent_work
- work.default_edition = work.get_default_edition()
- work.save()
- return edition
+ def get_or_create_book(self, remote_id, work=None):
+ return activitypub.resolve_remote_id(remote_id, model=models.Edition)
def parse_search_data(self, data):
return data
diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py
index 0dc922a5..6b1b349f 100644
--- a/bookwyrm/connectors/self_connector.py
+++ b/bookwyrm/connectors/self_connector.py
@@ -3,7 +3,7 @@ from functools import reduce
import operator
from django.contrib.postgres.search import SearchRank, SearchVector
-from django.db.models import Count, F, Q
+from django.db.models import Count, OuterRef, Subquery, F, Q
from bookwyrm import models
from .abstract_connector import AbstractConnector, SearchResult
@@ -47,7 +47,16 @@ class Connector(AbstractConnector):
# when there are multiple editions of the same work, pick the default.
# it would be odd for this to happen.
- results = results.filter(parent_work__default_edition__id=F("id")) or results
+
+ default_editions = models.Edition.objects.filter(
+ parent_work=OuterRef("parent_work")
+ ).order_by("-edition_rank")
+ results = (
+ results.annotate(
+ default_id=Subquery(default_editions.values("id")[:1])
+ ).filter(default_id=F("id"))
+ or results
+ )
search_results = []
for result in results:
@@ -112,7 +121,15 @@ def search_identifiers(query, *filters):
# when there are multiple editions of the same work, pick the default.
# it would be odd for this to happen.
- return results.filter(parent_work__default_edition__id=F("id")) or results
+ default_editions = models.Edition.objects.filter(
+ parent_work=OuterRef("parent_work")
+ ).order_by("-edition_rank")
+ return (
+ results.annotate(default_id=Subquery(default_editions.values("id")[:1])).filter(
+ default_id=F("id")
+ )
+ or results
+ )
def search_title_author(query, min_confidence, *filters):
@@ -140,10 +157,10 @@ def search_title_author(query, min_confidence, *filters):
for work_id in set(editions_of_work):
editions = results.filter(parent_work=work_id)
- default = editions.filter(parent_work__default_edition=F("id"))
- default_rank = default.first().rank if default.exists() else 0
+ default = editions.order_by("-edition_rank").first()
+ default_rank = default.rank if default else 0
# if mutliple books have the top rank, pick the default edition
if default_rank == editions.first().rank:
- yield default.first()
+ yield default
else:
yield editions.first()
diff --git a/bookwyrm/migrations/0072_remove_work_default_edition.py b/bookwyrm/migrations/0072_remove_work_default_edition.py
new file mode 100644
index 00000000..1c05c95e
--- /dev/null
+++ b/bookwyrm/migrations/0072_remove_work_default_edition.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.2 on 2021-04-28 22:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("bookwyrm", "0071_merge_0063_auto_20210407_0045_0070_auto_20210423_0121"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="work",
+ name="default_edition",
+ ),
+ ]
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 10ebb317..0bbae033 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -1,7 +1,7 @@
""" database schema for books and shelves """
import re
-from django.db import models, transaction
+from django.db import models
from model_utils.managers import InheritanceManager
from bookwyrm import activitypub
@@ -143,10 +143,6 @@ class Work(OrderedCollectionPageMixin, Book):
lccn = fields.CharField(
max_length=255, blank=True, null=True, deduplication_field=True
)
- # this has to be nullable but should never be null
- default_edition = fields.ForeignKey(
- "Edition", on_delete=models.PROTECT, null=True, load_remote=False
- )
def save(self, *args, **kwargs):
"""set some fields on the edition object"""
@@ -155,18 +151,10 @@ class Work(OrderedCollectionPageMixin, Book):
edition.save()
return super().save(*args, **kwargs)
- def get_default_edition(self):
+ @property
+ def default_edition(self):
"""in case the default edition is not set"""
- return self.default_edition or self.editions.order_by("-edition_rank").first()
-
- @transaction.atomic()
- def reset_default_edition(self):
- """sets a new default edition based on computed rank"""
- self.default_edition = None
- # editions are re-ranked implicitly
- self.save()
- self.default_edition = self.get_default_edition()
- self.save()
+ return self.editions.order_by("-edition_rank").first()
def to_edition_list(self, **kwargs):
"""an ordered collection of editions"""
@@ -220,15 +208,8 @@ class Edition(Book):
activity_serializer = activitypub.Edition
name_field = "title"
- def get_rank(self, ignore_default=False):
+ def get_rank(self):
"""calculate how complete the data is on this edition"""
- if (
- not ignore_default
- and self.parent_work
- and self.parent_work.default_edition == self
- ):
- # default edition has the highest rank
- return 20
rank = 0
rank += int(bool(self.cover)) * 3
rank += int(bool(self.isbn_13))
diff --git a/bookwyrm/tests/connectors/test_connector_manager.py b/bookwyrm/tests/connectors/test_connector_manager.py
index feded616..34abbeaf 100644
--- a/bookwyrm/tests/connectors/test_connector_manager.py
+++ b/bookwyrm/tests/connectors/test_connector_manager.py
@@ -17,8 +17,6 @@ class ConnectorManager(TestCase):
self.edition = models.Edition.objects.create(
title="Example Edition", parent_work=self.work, isbn_10="0000000000"
)
- self.work.default_edition = self.edition
- self.work.save()
self.connector = models.Connector.objects.create(
identifier="test_connector",
diff --git a/bookwyrm/tests/connectors/test_self_connector.py b/bookwyrm/tests/connectors/test_self_connector.py
index eee7d00c..db97b65a 100644
--- a/bookwyrm/tests/connectors/test_self_connector.py
+++ b/bookwyrm/tests/connectors/test_self_connector.py
@@ -84,11 +84,11 @@ class SelfConnector(TestCase):
title="Edition 1 Title", parent_work=work
)
edition_2 = models.Edition.objects.create(
- title="Edition 2 Title", parent_work=work
+ title="Edition 2 Title",
+ parent_work=work,
+ edition_rank=20, # that's default babey
)
edition_3 = models.Edition.objects.create(title="Fish", parent_work=work)
- work.default_edition = edition_2
- work.save()
# pick the best edition
results = self.connector.search("Edition 1 Title")
diff --git a/bookwyrm/tests/models/test_book_model.py b/bookwyrm/tests/models/test_book_model.py
index c80cc4a8..cad00d43 100644
--- a/bookwyrm/tests/models/test_book_model.py
+++ b/bookwyrm/tests/models/test_book_model.py
@@ -84,9 +84,3 @@ class Book(TestCase):
self.first_edition.description = "hi"
self.first_edition.save()
self.assertEqual(self.first_edition.edition_rank, 1)
-
- # default edition
- self.work.default_edition = self.first_edition
- self.work.save()
- self.first_edition.refresh_from_db()
- self.assertEqual(self.first_edition.edition_rank, 20)
diff --git a/bookwyrm/tests/models/test_readthrough_model.py b/bookwyrm/tests/models/test_readthrough_model.py
index 93e9e654..986b739b 100644
--- a/bookwyrm/tests/models/test_readthrough_model.py
+++ b/bookwyrm/tests/models/test_readthrough_model.py
@@ -2,7 +2,7 @@
from django.test import TestCase
from django.core.exceptions import ValidationError
-from bookwyrm import models, settings
+from bookwyrm import models
class ReadThrough(TestCase):
@@ -19,8 +19,6 @@ class ReadThrough(TestCase):
self.edition = models.Edition.objects.create(
title="Example Edition", parent_work=self.work
)
- self.work.default_edition = self.edition
- self.work.save()
self.readthrough = models.ReadThrough.objects.create(
user=self.user, book=self.edition
diff --git a/bookwyrm/tests/views/test_readthrough.py b/bookwyrm/tests/views/test_readthrough.py
index c9ebf216..882c7929 100644
--- a/bookwyrm/tests/views/test_readthrough.py
+++ b/bookwyrm/tests/views/test_readthrough.py
@@ -20,8 +20,6 @@ class ReadThrough(TestCase):
self.edition = models.Edition.objects.create(
title="Example Edition", parent_work=self.work
)
- self.work.default_edition = self.edition
- self.work.save()
self.user = models.User.objects.create_user(
"cinco", "cinco@example.com", "seissiete", local=True, localname="cinco"
diff --git a/bookwyrm/views/author.py b/bookwyrm/views/author.py
index 0bd7b0e0..41298161 100644
--- a/bookwyrm/views/author.py
+++ b/bookwyrm/views/author.py
@@ -27,7 +27,7 @@ class Author(View):
).distinct()
data = {
"author": author,
- "books": [b.get_default_edition() for b in books],
+ "books": [b.default_edition for b in books],
}
return TemplateResponse(request, "author.html", data)
diff --git a/bookwyrm/views/books.py b/bookwyrm/views/books.py
index 448cf992..ee1a5e06 100644
--- a/bookwyrm/views/books.py
+++ b/bookwyrm/views/books.py
@@ -39,7 +39,7 @@ class Book(View):
return ActivitypubResponse(book.to_activity())
if isinstance(book, models.Work):
- book = book.get_default_edition()
+ book = book.default_edition
if not book or not book.parent_work:
return HttpResponseNotFound()
diff --git a/bookwyrm/views/helpers.py b/bookwyrm/views/helpers.py
index 8a60b54c..540b578f 100644
--- a/bookwyrm/views/helpers.py
+++ b/bookwyrm/views/helpers.py
@@ -123,7 +123,7 @@ def get_edition(book_id):
"""look up a book in the db and return an edition"""
book = models.Book.objects.select_subclasses().get(id=book_id)
if isinstance(book, models.Work):
- book = book.get_default_edition()
+ book = book.default_edition
return book
From 8eec3eca5ba6c748c7d98a488cebf30c1d5ae5cf Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 16:33:40 -0700
Subject: [PATCH 22/37] Don't spin out infinite load data tasks
---
bookwyrm/connectors/abstract_connector.py | 27 ++++++++++-------------
bookwyrm/connectors/inventaire.py | 7 +++++-
2 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py
index 08a0175a..76718823 100644
--- a/bookwyrm/connectors/abstract_connector.py
+++ b/bookwyrm/connectors/abstract_connector.py
@@ -73,7 +73,7 @@ class AbstractMinimalConnector(ABC):
return get_data(remote_id, **kwargs)
@abstractmethod
- def get_or_create_book(self, remote_id, work=None):
+ def get_or_create_book(self, remote_id):
"""pull up a book record by whatever means possible"""
@abstractmethod
@@ -109,7 +109,7 @@ class AbstractConnector(AbstractMinimalConnector):
return False
return True
- def get_or_create_book(self, remote_id, work=None):
+ def get_or_create_book(self, remote_id):
"""translate arbitrary json into an Activitypub dataclass"""
# first, check if we have the origin_id saved
existing = models.Edition.find_existing_by_remote_id(
@@ -123,7 +123,6 @@ class AbstractConnector(AbstractMinimalConnector):
# load the json
data = self.get_book_data(remote_id)
mapped_data = dict_from_mappings(data, self.book_mappings)
- work_data = edition_data = None
if self.is_work_data(data):
try:
edition_data = self.get_edition_from_work_data(data)
@@ -134,22 +133,20 @@ class AbstractConnector(AbstractMinimalConnector):
work_data = mapped_data
else:
edition_data = data
- if not work:
- try:
- work_data = self.get_work_from_edition_data(data)
- work_data = dict_from_mappings(work_data, self.book_mappings)
- except (KeyError, ConnectorException):
- work_data = mapped_data
+ try:
+ work_data = self.get_work_from_edition_data(data)
+ work_data = dict_from_mappings(work_data, self.book_mappings)
+ except (KeyError, ConnectorException):
+ work_data = mapped_data
- if (not work_data and not work) or not edition_data:
+ if not work_data or not edition_data:
raise ConnectorException("Unable to load book data: %s" % remote_id)
with transaction.atomic():
- if not work:
- # create activitypub object
- work_activity = activitypub.Work(**work_data)
- # this will dedupe automatically
- work = work_activity.to_model(model=models.Work)
+ # create activitypub object
+ work_activity = activitypub.Work(**work_data)
+ # this will dedupe automatically
+ work = work_activity.to_model(model=models.Work)
for author in self.get_authors_from_data(data):
work.authors.add(author)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index e8416c37..cd14c253 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -137,7 +137,12 @@ class Connector(AbstractConnector):
for edition_uri in edition_options.get("uris"):
remote_id = self.get_remote_id(edition_uri)
- self.get_or_create_book(remote_id, work=work)
+ try:
+ data = self.get_book_data(remote_id)
+ except ConnectorException:
+ # who, indeed, knows
+ continue
+ self.create_edition_from_data(work, data)
def get_cover_url(self, cover_blob, *_):
"""format the relative cover url into an absolute one:
From 95fce963d1ae7555daec75afcaa794dd23784f6e Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 17:18:14 -0700
Subject: [PATCH 23/37] Images may already be absolute paths
---
bookwyrm/connectors/inventaire.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index cd14c253..ebbe8d51 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -1,4 +1,6 @@
""" inventaire data connector """
+import re
+
from bookwyrm import models
from .abstract_connector import AbstractConnector, SearchResult, Mapping
from .abstract_connector import get_data
@@ -148,11 +150,15 @@ class Connector(AbstractConnector):
"""format the relative cover url into an absolute one:
{"url": "/img/entities/e794783f01b9d4f897a1ea9820b96e00d346994f"}
"""
+ # covers may or may not be a list
if isinstance(cover_blob, list) and len(cover_blob) > 0:
cover_blob = cover_blob[0]
cover_id = cover_blob.get("url")
if not cover_id:
return None
+ # cover may or may not be an absolute url already
+ if re.match(r"^http", cover_id):
+ return cover_id
return "%s%s" % (self.covers_url, cover_id)
def resolve_keys(self, keys):
From 5e5392c007ea746b7714aa2ef16e65085376cf87 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Wed, 28 Apr 2021 17:20:14 -0700
Subject: [PATCH 24/37] Adds test file
---
.../connectors/test_inventaire_connector.py | 47 +++++++++++++++++++
1 file changed, 47 insertions(+)
create mode 100644 bookwyrm/tests/connectors/test_inventaire_connector.py
diff --git a/bookwyrm/tests/connectors/test_inventaire_connector.py b/bookwyrm/tests/connectors/test_inventaire_connector.py
new file mode 100644
index 00000000..a124f35b
--- /dev/null
+++ b/bookwyrm/tests/connectors/test_inventaire_connector.py
@@ -0,0 +1,47 @@
+""" testing book data connectors """
+from django.test import TestCase
+import responses
+
+from bookwyrm import models
+from bookwyrm.connectors.inventaire import Connector
+
+
+class Inventaire(TestCase):
+ """test loading data from inventaire.io"""
+
+ def setUp(self):
+ """creates the connector we'll use"""
+ models.Connector.objects.create(
+ identifier="inventaire.io",
+ name="Inventaire",
+ connector_file="inventaire",
+ base_url="https://inventaire.io",
+ books_url="https://inventaire.io",
+ covers_url="https://covers.inventaire.io",
+ search_url="https://inventaire.io/search?q=",
+ isbn_search_url="https://inventaire.io/isbn",
+ )
+ self.connector = Connector("inventaire.io")
+
+ @responses.activate
+ def test_get_book_data(self):
+ """flattens the default structure to make it easier to parse"""
+ responses.add(
+ responses.GET,
+ "https://test.url/ok",
+ json={
+ "entities": {
+ "isbn:9780375757853": {
+ "claims": {
+ "wdt:P31": ["wd:Q3331189"],
+ },
+ "uri": "isbn:9780375757853",
+ }
+ },
+ "redirects": {},
+ },
+ )
+
+ result = self.connector.get_book_data("https://test.url/ok")
+ self.assertEqual(result["wdt:P31"], ["wd:Q3331189"])
+ self.assertEqual(result["uri"], "isbn:9780375757853")
From fc095a087c5637d58cdd645ad9505cc60ecfcc22 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 08:35:37 -0700
Subject: [PATCH 25/37] Adds search result test
---
.../connectors/test_inventaire_connector.py | 21 +++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/bookwyrm/tests/connectors/test_inventaire_connector.py b/bookwyrm/tests/connectors/test_inventaire_connector.py
index a124f35b..2b1d9bdf 100644
--- a/bookwyrm/tests/connectors/test_inventaire_connector.py
+++ b/bookwyrm/tests/connectors/test_inventaire_connector.py
@@ -1,4 +1,6 @@
""" testing book data connectors """
+import json
+import pathlib
from django.test import TestCase
import responses
@@ -45,3 +47,22 @@ class Inventaire(TestCase):
result = self.connector.get_book_data("https://test.url/ok")
self.assertEqual(result["wdt:P31"], ["wd:Q3331189"])
self.assertEqual(result["uri"], "isbn:9780375757853")
+
+ def test_format_search_result(self):
+ """json to search result objs"""
+ search_file = pathlib.Path(__file__).parent.joinpath(
+ "../data/inventaire_search.json"
+ )
+ search_results = json.loads(search_file.read_bytes())
+
+ results = self.connector.parse_search_data(search_results)
+ formatted = self.connector.format_search_result(results[0])
+
+ self.assertEqual(formatted.title, "The Stories of Vladimir Nabokov")
+ self.assertEqual(
+ formatted.key, "https://inventaire.io?action=by-uris&uris=wd:Q7766679"
+ )
+ self.assertEqual(
+ formatted.cover,
+ "https://covers.inventaire.io/img/entities/ddb32e115a28dcc0465023869ba19f6868ec4042",
+ )
From 0a415035725a2bd5f9f7ec76f57e47b67ebaa062 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 09:06:17 -0700
Subject: [PATCH 26/37] Adds a couple more tests and test data
---
.../connectors/test_inventaire_connector.py | 70 ++++++++
bookwyrm/tests/data/inventaire_edition.json | 45 +++++
bookwyrm/tests/data/inventaire_search.json | 111 +++++++++++++
bookwyrm/tests/data/inventaire_work.json | 155 ++++++++++++++++++
4 files changed, 381 insertions(+)
create mode 100644 bookwyrm/tests/data/inventaire_edition.json
create mode 100644 bookwyrm/tests/data/inventaire_search.json
create mode 100644 bookwyrm/tests/data/inventaire_work.json
diff --git a/bookwyrm/tests/connectors/test_inventaire_connector.py b/bookwyrm/tests/connectors/test_inventaire_connector.py
index 2b1d9bdf..a37382c5 100644
--- a/bookwyrm/tests/connectors/test_inventaire_connector.py
+++ b/bookwyrm/tests/connectors/test_inventaire_connector.py
@@ -66,3 +66,73 @@ class Inventaire(TestCase):
formatted.cover,
"https://covers.inventaire.io/img/entities/ddb32e115a28dcc0465023869ba19f6868ec4042",
)
+
+ def test_get_cover_url(self):
+ """figure out where the cover image is"""
+ cover_blob = {"url": "/img/entities/d46a8"}
+ result = self.connector.get_cover_url(cover_blob)
+ self.assertEqual(result, "https://covers.inventaire.io/img/entities/d46a8")
+
+ cover_blob = {
+ "url": "https://commons.wikimedia.org/wiki/Special:FilePath/The%20Moonstone%201st%20ed.jpg?width=1000",
+ "file": "The Moonstone 1st ed.jpg",
+ "credits": {
+ "text": "Wikimedia Commons",
+ "url": "https://commons.wikimedia.org/wiki/File:The Moonstone 1st ed.jpg",
+ },
+ }
+
+ result = self.connector.get_cover_url(cover_blob)
+ self.assertEqual(
+ result,
+ "https://commons.wikimedia.org/wiki/Special:FilePath/The%20Moonstone%201st%20ed.jpg?width=1000",
+ )
+
+ @responses.activate
+ def test_resolve_keys(self):
+ """makes an http request"""
+ responses.add(
+ responses.GET,
+ "https://inventaire.io?action=by-uris&uris=wd:Q465821",
+ json={
+ "entities": {
+ "wd:Q465821": {
+ "type": "genre",
+ "labels": {
+ "nl": "briefroman",
+ "en": "epistolary novel",
+ "de-ch": "Briefroman",
+ "en-ca": "Epistolary novel",
+ "nb": "brev- og dagbokroman",
+ },
+ "descriptions": {
+ "en": "novel written as a series of documents",
+ "es": "novela escrita como una serie de documentos",
+ "eo": "romano en la formo de serio de leteroj",
+ },
+ },
+ "redirects": {},
+ }
+ },
+ )
+ responses.add(
+ responses.GET,
+ "https://inventaire.io?action=by-uris&uris=wd:Q208505",
+ json={
+ "entities": {
+ "wd:Q208505": {
+ "type": "genre",
+ "labels": {
+ "en": "crime novel",
+ },
+ },
+ }
+ },
+ )
+
+ keys = [
+ "wd:Q465821",
+ "wd:Q208505",
+ ]
+ result = self.connector.resolve_keys(keys)
+ self.assertEqual(result, ["epistolary novel", "crime novel"])
diff --git a/bookwyrm/tests/data/inventaire_edition.json b/bookwyrm/tests/data/inventaire_edition.json
new file mode 100644
index 00000000..1150bc9b
--- /dev/null
+++ b/bookwyrm/tests/data/inventaire_edition.json
@@ -0,0 +1,45 @@
+{
+ "entities": {
+ "isbn:9780375757853": {
+ "_id": "7beee121a8d9ac345cdf4e9128577723",
+ "_rev": "2-ac318b04b953ca3894deb77fee28211c",
+ "type": "edition",
+ "labels": {},
+ "claims": {
+ "wdt:P31": [
+ "wd:Q3331189"
+ ],
+ "wdt:P212": [
+ "978-0-375-75785-3"
+ ],
+ "wdt:P957": [
+ "0-375-75785-6"
+ ],
+ "wdt:P407": [
+ "wd:Q1860"
+ ],
+ "wdt:P1476": [
+ "The Moonstone"
+ ],
+ "wdt:P577": [
+ "2001"
+ ],
+ "wdt:P629": [
+ "wd:Q2362563"
+ ],
+ "invp:P2": [
+ "d46a8eac7555afa479b8bbb5149f35858e8e19c4"
+ ]
+ },
+ "created": 1495452670475,
+ "updated": 1541032981834,
+ "version": 3,
+ "uri": "isbn:9780375757853",
+ "originalLang": "en",
+ "image": {
+ "url": "/img/entities/d46a8eac7555afa479b8bbb5149f35858e8e19c4"
+ }
+ }
+ },
+ "redirects": {}
+}
diff --git a/bookwyrm/tests/data/inventaire_search.json b/bookwyrm/tests/data/inventaire_search.json
new file mode 100644
index 00000000..e80e593a
--- /dev/null
+++ b/bookwyrm/tests/data/inventaire_search.json
@@ -0,0 +1,111 @@
+{
+ "results": [
+ {
+ "id": "Q7766679",
+ "type": "works",
+ "uri": "wd:Q7766679",
+ "label": "The Stories of Vladimir Nabokov",
+ "description": "book by Vladimir Nabokov",
+ "image": [
+ "ddb32e115a28dcc0465023869ba19f6868ec4042"
+ ],
+ "_score": 25.180836,
+ "_popularity": 4
+ },
+ {
+ "id": "Q47407212",
+ "type": "works",
+ "uri": "wd:Q47407212",
+ "label": "Conversations with Vladimir Nabokov",
+ "description": "book edited by Robert Golla",
+ "image": [],
+ "_score": 24.41498,
+ "_popularity": 2
+ },
+ {
+ "id": "Q6956987",
+ "type": "works",
+ "uri": "wd:Q6956987",
+ "label": "Nabokov's Congeries",
+ "description": "book by Vladimir Nabokov",
+ "image": [],
+ "_score": 22.343866,
+ "_popularity": 2
+ },
+ {
+ "id": "Q6956986",
+ "type": "works",
+ "uri": "wd:Q6956986",
+ "label": "Nabokov's Butterflies",
+ "description": "book by Brian Boyd",
+ "image": [],
+ "_score": 22.343866,
+ "_popularity": 2
+ },
+ {
+ "id": "Q47472170",
+ "type": "works",
+ "uri": "wd:Q47472170",
+ "label": "A Reader's Guide to Nabokov's \"Lolita\"",
+ "description": "book by Julian W. Connolly",
+ "image": [],
+ "_score": 19.482553,
+ "_popularity": 2
+ },
+ {
+ "id": "Q7936323",
+ "type": "works",
+ "uri": "wd:Q7936323",
+ "label": "Visiting Mrs Nabokov: And Other Excursions",
+ "description": "book by Martin Amis",
+ "image": [],
+ "_score": 18.684965,
+ "_popularity": 2
+ },
+ {
+ "id": "1732d81bf7376e04da27568a778561a4",
+ "type": "works",
+ "uri": "inv:1732d81bf7376e04da27568a778561a4",
+ "label": "Nabokov's Dark Cinema",
+ "image": [
+ "7512805a53da569b11bf29cc3fb272c969619749"
+ ],
+ "_score": 16.56681,
+ "_popularity": 1
+ },
+ {
+ "id": "00f118336b02219e1bddc8fa93c56050",
+ "type": "works",
+ "uri": "inv:00f118336b02219e1bddc8fa93c56050",
+ "label": "The Cambridge Companion to Nabokov",
+ "image": [
+ "0683a059fb95430cfa73334f9eff2ef377f3ae3d"
+ ],
+ "_score": 15.502292,
+ "_popularity": 1
+ },
+ {
+ "id": "6e59f968a1cd00dbedeb1964dec47507",
+ "type": "works",
+ "uri": "inv:6e59f968a1cd00dbedeb1964dec47507",
+ "label": "Vladimir Nabokov : selected letters, 1940-1977",
+ "image": [
+ "e3ce8c0ee89d576adf2651a6e5ce55fc6d9f8cb3"
+ ],
+ "_score": 15.019735,
+ "_popularity": 1
+ },
+ {
+ "id": "Q127149",
+ "type": "works",
+ "uri": "wd:Q127149",
+ "label": "Lolita",
+ "description": "novel by Vladimir Nabokov",
+ "image": [
+ "51cbfdbf7257b1a6bb3ea3fbb167dbce1fb44a0e"
+ ],
+ "_score": 13.458428,
+ "_popularity": 32
+ }
+ ]
+}
diff --git a/bookwyrm/tests/data/inventaire_work.json b/bookwyrm/tests/data/inventaire_work.json
new file mode 100644
index 00000000..635c52f3
--- /dev/null
+++ b/bookwyrm/tests/data/inventaire_work.json
@@ -0,0 +1,155 @@
+{
+ "entities": {
+ "wd:Q2362563": {
+ "type": "work",
+ "labels": {
+ "zh-hans": "月亮宝石",
+ "zh-hant": "月亮寶石",
+ "zh-hk": "月光石",
+ "zh-tw": "月光石",
+ "cy": "The Moonstone",
+ "ml": "ദ മൂൺസ്റ്റോൺ",
+ "ja": "月長石",
+ "te": "ది మూన్ స్టోన్",
+ "ru": "Лунный камень",
+ "fr": "La Pierre de lune",
+ "en": "The Moonstone",
+ "es": "La piedra lunar",
+ "it": "La Pietra di Luna",
+ "zh": "月亮宝石",
+ "pl": "Kamień Księżycowy",
+ "sr": "2 Јн",
+ "ta": "moon stone",
+ "ar": "حجر القمر",
+ "fa": "ماهالماس",
+ "uk": "Місячний камінь",
+ "nl": "The Moonstone",
+ "de": "Der Monddiamant",
+ "sl": "Diamant",
+ "sv": "Månstenen",
+ "he": "אבן הירח",
+ "eu": "Ilargi-harriak",
+ "bg": "Лунният камък",
+ "ka": "მთვარის ქვა",
+ "eo": "La Lunŝtono",
+ "hy": "Լուսնաքար",
+ "ro": "Piatra Lunii",
+ "ca": "The Moonstone",
+ "is": "The Moonstone"
+ },
+ "descriptions": {
+ "it": "romanzo scritto da Wilkie Collins",
+ "en": "novel by Wilkie Collins",
+ "de": "Buch von Wilkie Collins",
+ "nl": "boek van Wilkie Collins",
+ "ru": "роман Уилки Коллинза",
+ "he": "רומן מאת וילקי קולינס",
+ "ar": "رواية من تأليف ويلكي كولينز",
+ "fr": "livre de Wilkie Collins",
+ "es": "libro de Wilkie Collins",
+ "bg": "роман на Уилки Колинс",
+ "ka": "უილკი კოლინსის რომანი",
+ "eo": "angalingva romano far Wilkie Collins",
+ "ro": "roman de Wilkie Collins"
+ },
+ "aliases": {
+ "zh": [
+ "月光石"
+ ],
+ "ml": [
+ "The Moonstone"
+ ],
+ "fr": [
+ "The Moonstone"
+ ],
+ "it": [
+ "Il diamante indiano",
+ "La pietra della luna",
+ "La maledizione del diamante indiano"
+ ],
+ "ro": [
+ "The Moonstone"
+ ]
+ },
+ "claims": {
+ "wdt:P18": [
+ "The Moonstone 1st ed.jpg"
+ ],
+ "wdt:P31": [
+ "wd:Q7725634"
+ ],
+ "wdt:P50": [
+ "wd:Q210740"
+ ],
+ "wdt:P123": [
+ "wd:Q4457856"
+ ],
+ "wdt:P136": [
+ "wd:Q465821",
+ "wd:Q208505",
+ "wd:Q10992055"
+ ],
+ "wdt:P156": [
+ "wd:Q7228798"
+ ],
+ "wdt:P268": [
+ "12496407z"
+ ],
+ "wdt:P407": [
+ "wd:Q7979"
+ ],
+ "wdt:P577": [
+ "1868"
+ ],
+ "wdt:P1433": [
+ "wd:Q21"
+ ],
+ "wdt:P1476": [
+ "The Moonstone"
+ ],
+ "wdt:P1680": [
+ "A Romance"
+ ],
+ "wdt:P2034": [
+ "155"
+ ]
+ },
+ "sitelinks": {
+ "arwiki": "حجر القمر (رواية)",
+ "bgwiki": "Лунният камък (роман)",
+ "cywiki": "The Moonstone",
+ "dewiki": "Der Monddiamant",
+ "enwiki": "The Moonstone",
+ "enwikisource": "The Moonstone",
+ "eswiki": "La piedra lunar",
+ "euwiki": "Ilargi-harria",
+ "fawiki": "ماهالماس",
+ "frwiki": "La Pierre de lune (roman de Wilkie Collins)",
+ "hewiki": "אבן הירח",
+ "hywiki": "Լուսնաքար",
+ "iswiki": "The Moonstone",
+ "itwiki": "La pietra di Luna",
+ "jawiki": "月長石 (小説)",
+ "mlwiki": "ദ മൂൺസ്റ്റോൺ",
+ "plwiki": "Kamień Księżycowy (powieść)",
+ "ruwiki": "Лунный камень (роман)",
+ "slwiki": "Diamant (roman)",
+ "srwikisource": "Нови завјет (Караџић) / 2. Јованова",
+ "svwiki": "Månstenen",
+ "tewiki": "ది మూన్స్టోన్",
+ "ukwiki": "Місячний камінь (роман)",
+ "zhwiki": "月亮宝石"
+ },
+ "uri": "wd:Q2362563",
+ "image": {
+ "url": "https://commons.wikimedia.org/wiki/Special:FilePath/The%20Moonstone%201st%20ed.jpg?width=1000",
+ "file": "The Moonstone 1st ed.jpg",
+ "credits": {
+ "text": "Wikimedia Commons",
+ "url": "https://commons.wikimedia.org/wiki/File:The Moonstone 1st ed.jpg"
+ }
+ }
+ }
+ },
+ "redirects": {}
+}
From f4800307b44ac8f0c55b659dcc64e1aef4139b34 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 09:54:42 -0700
Subject: [PATCH 27/37] Adds isbn search
---
bookwyrm/connectors/inventaire.py | 22 +++++++--
.../connectors/test_inventaire_connector.py | 19 ++++++++
.../tests/data/inventaire_isbn_search.json | 48 +++++++++++++++++++
bookwyrm/views/books.py | 1 -
4 files changed, 84 insertions(+), 6 deletions(-)
create mode 100644 bookwyrm/tests/data/inventaire_isbn_search.json
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index ebbe8d51..fe01ddda 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -81,19 +81,31 @@ class Connector(AbstractConnector):
)
return SearchResult(
title=search_result.get("label"),
- key="{:s}?action=by-uris&uris={:s}".format(
- self.books_url, search_result.get("uri")
- ),
+ key=self.get_remote_id(search_result.get("uri")),
view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
cover=cover,
connector=self,
)
def parse_isbn_search_data(self, data):
- """boop doop"""
+ """got some daaaata"""
+ results = data.get('entities')
+ if not results:
+ return []
+ return list(results.values())
def format_isbn_search_result(self, search_result):
- """beep bloop"""
+ """totally different format than a regular search result"""
+ title = search_result.get("claims", {}).get("wdt:P1476", [])
+ if not title:
+ return None
+ return SearchResult(
+ title=title[0],
+ key=self.get_remote_id(search_result.get("uri")),
+ view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
+ cover=self.get_cover_url(search_result.get("image")),
+ connector=self
+ )
def is_work_data(self, data):
return data.get("type") == "work"
diff --git a/bookwyrm/tests/connectors/test_inventaire_connector.py b/bookwyrm/tests/connectors/test_inventaire_connector.py
index a37382c5..90018534 100644
--- a/bookwyrm/tests/connectors/test_inventaire_connector.py
+++ b/bookwyrm/tests/connectors/test_inventaire_connector.py
@@ -136,3 +136,22 @@ class Inventaire(TestCase):
]
result = self.connector.resolve_keys(keys)
self.assertEqual(result, ["epistolary novel", "crime novel"])
+
+ def test_isbn_search(self):
+ """ another search type """
+ search_file = pathlib.Path(__file__).parent.joinpath(
+ "../data/inventaire_isbn_search.json"
+ )
+ search_results = json.loads(search_file.read_bytes())
+
+ results = self.connector.parse_isbn_search_data(search_results)
+ formatted = self.connector.format_isbn_search_result(results[0])
+
+ self.assertEqual(formatted.title, "L'homme aux cercles bleus")
+ self.assertEqual(
+ formatted.key, "https://inventaire.io?action=by-uris&uris=isbn:9782290349229"
+ )
+ self.assertEqual(
+ formatted.cover,
+ "https://covers.inventaire.io/img/entities/12345",
+ )
diff --git a/bookwyrm/tests/data/inventaire_isbn_search.json b/bookwyrm/tests/data/inventaire_isbn_search.json
new file mode 100644
index 00000000..7328a78f
--- /dev/null
+++ b/bookwyrm/tests/data/inventaire_isbn_search.json
@@ -0,0 +1,48 @@
+{
+ "entities": {
+ "isbn:9782290349229": {
+ "_id": "d59e3e64f92c6340fbb10c5dcf7c0abf",
+ "_rev": "3-079ed51158a001dc74caafb21cff1c22",
+ "type": "edition",
+ "labels": {},
+ "claims": {
+ "wdt:P31": [
+ "wd:Q3331189"
+ ],
+ "wdt:P212": [
+ "978-2-290-34922-9"
+ ],
+ "wdt:P957": [
+ "2-290-34922-4"
+ ],
+ "wdt:P407": [
+ "wd:Q150"
+ ],
+ "wdt:P1476": [
+ "L'homme aux cercles bleus"
+ ],
+ "wdt:P629": [
+ "wd:Q3203603"
+ ],
+ "wdt:P123": [
+ "wd:Q3156592"
+ ],
+ "invp:P2": [
+ "57883743aa7c6ad25885a63e6e94349ec4f71562"
+ ],
+ "wdt:P577": [
+ "2005-05-01"
+ ]
+ },
+ "created": 1485023383338,
+ "updated": 1609171008418,
+ "version": 5,
+ "uri": "isbn:9782290349229",
+ "originalLang": "fr",
+ "image": {
+ "url": "/img/entities/12345"
+ }
+ }
+ },
+ "redirects": {}
+}
diff --git a/bookwyrm/views/books.py b/bookwyrm/views/books.py
index ee1a5e06..6005c9fd 100644
--- a/bookwyrm/views/books.py
+++ b/bookwyrm/views/books.py
@@ -156,7 +156,6 @@ class EditBook(View):
),
}
)
- print(data["author_matches"])
# we're creating a new book
if not book:
From 8d38d1c9d1bd51b7d4622845f6d3e242ba6e9c1d Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 09:56:35 -0700
Subject: [PATCH 28/37] Python formatting
---
bookwyrm/connectors/inventaire.py | 4 ++--
bookwyrm/tests/connectors/test_inventaire_connector.py | 5 +++--
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index fe01ddda..e972950f 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -89,7 +89,7 @@ class Connector(AbstractConnector):
def parse_isbn_search_data(self, data):
"""got some daaaata"""
- results = data.get('entities')
+ results = data.get("entities")
if not results:
return []
return list(results.values())
@@ -104,7 +104,7 @@ class Connector(AbstractConnector):
key=self.get_remote_id(search_result.get("uri")),
view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
cover=self.get_cover_url(search_result.get("image")),
- connector=self
+ connector=self,
)
def is_work_data(self, data):
diff --git a/bookwyrm/tests/connectors/test_inventaire_connector.py b/bookwyrm/tests/connectors/test_inventaire_connector.py
index 90018534..4058b067 100644
--- a/bookwyrm/tests/connectors/test_inventaire_connector.py
+++ b/bookwyrm/tests/connectors/test_inventaire_connector.py
@@ -138,7 +138,7 @@ class Inventaire(TestCase):
self.assertEqual(result, ["epistolary novel", "crime novel"])
def test_isbn_search(self):
- """ another search type """
+ """another search type"""
search_file = pathlib.Path(__file__).parent.joinpath(
"../data/inventaire_isbn_search.json"
)
@@ -149,7 +149,8 @@ class Inventaire(TestCase):
self.assertEqual(formatted.title, "L'homme aux cercles bleus")
self.assertEqual(
- formatted.key, "https://inventaire.io?action=by-uris&uris=isbn:9782290349229"
+ formatted.key,
+ "https://inventaire.io?action=by-uris&uris=isbn:9782290349229",
)
self.assertEqual(
formatted.cover,
From cfd2c05ae2b5638cdc26ca7aadb0c613983b8fa9 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 10:12:56 -0700
Subject: [PATCH 29/37] Safely handle absent claims field
---
bookwyrm/connectors/inventaire.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index e972950f..e6fb6826 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -65,7 +65,7 @@ class Connector(AbstractConnector):
raise ConnectorException("Invalid book data")
# flatten the data so that images, uri, and claims are on the same level
return {
- **data.get("claims"),
+ **data.get("claims", {}),
**{k: data.get(k) for k in ["uri", "image", "labels"]},
}
From ca86af22ce3a968212ce21e04ba619b406e4537c Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 10:30:38 -0700
Subject: [PATCH 30/37] Get inventaire entity descriptions
---
bookwyrm/connectors/inventaire.py | 20 ++++++++++++++++++--
bookwyrm/management/commands/initdb.py | 2 +-
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index e6fb6826..a50a2584 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -23,6 +23,7 @@ class Connector(AbstractConnector):
Mapping("title", remote_field="wdt:P1476", formatter=get_first),
Mapping("subtitle", remote_field="wdt:P1680", formatter=get_first),
Mapping("inventaireId", remote_field="uri"),
+ Mapping("description", remote_field="sitelinks", formatter=self.get_description),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
Mapping("isbn13", remote_field="wdt:P212", formatter=get_first),
Mapping("isbn10", remote_field="wdt:P957", formatter=get_first),
@@ -44,6 +45,7 @@ class Connector(AbstractConnector):
self.author_mappings = [
Mapping("id", remote_field="uri", formatter=self.get_remote_id),
Mapping("name", remote_field="labels", formatter=get_language_code),
+ Mapping("bio", remote_field="sitelinks", formatter=self.get_description),
Mapping("goodreadsKey", remote_field="wdt:P2963", formatter=get_first),
Mapping("isni", remote_field="wdt:P213", formatter=get_first),
Mapping("viafId", remote_field="wdt:P214", formatter=get_first),
@@ -82,7 +84,7 @@ class Connector(AbstractConnector):
return SearchResult(
title=search_result.get("label"),
key=self.get_remote_id(search_result.get("uri")),
- view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
+ view_link="{:s}/entity/{:s}".format(self.base_url, search_result.get("uri")),
cover=cover,
connector=self,
)
@@ -102,7 +104,7 @@ class Connector(AbstractConnector):
return SearchResult(
title=title[0],
key=self.get_remote_id(search_result.get("uri")),
- view_link="{:s}{:s}".format(self.base_url, search_result.get("uri")),
+ view_link="{:s}/entity/{:s}".format(self.base_url, search_result.get("uri")),
cover=self.get_cover_url(search_result.get("image")),
connector=self,
)
@@ -184,6 +186,20 @@ class Connector(AbstractConnector):
results.append(get_language_code(data.get("labels")))
return results
+ def get_description(self, links):
+ """ grab an extracted except from wikipedia """
+ link = links.get("enwiki")
+ if not link:
+ return ""
+ url = "{:s}/api/data?action=wp-extract&lang=en&title={:s}".format(
+ self.base_url, link
+ )
+ try:
+ data = get_data(url)
+ except ConnectorException:
+ return ""
+ return data.get("extract")
+
def get_language_code(options, code="en"):
"""when there are a bunch of translation but we need a single field"""
diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py
index 07071a2e..9033249d 100644
--- a/bookwyrm/management/commands/initdb.py
+++ b/bookwyrm/management/commands/initdb.py
@@ -98,7 +98,7 @@ def init_connectors():
identifier="inventaire.io",
name="Inventaire",
connector_file="inventaire",
- base_url="https://inventaire.io/entity/",
+ base_url="https://inventaire.io",
books_url="https://inventaire.io/api/entities",
covers_url="https://inventaire.io",
search_url="https://inventaire.io/api/search?types=works&types=works&search=",
From b1c38d291c432f196107a27af8dc26bc00179c10 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 10:40:49 -0700
Subject: [PATCH 31/37] Set preferred language
---
.env.dev.example | 3 +++
.env.prod.example | 3 +++
bookwyrm/connectors/inventaire.py | 14 ++++++++++----
bookwyrm/models/book.py | 10 +++++++++-
bookwyrm/settings.py | 1 +
5 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/.env.dev.example b/.env.dev.example
index 5e605d74..538d1611 100644
--- a/.env.dev.example
+++ b/.env.dev.example
@@ -7,6 +7,9 @@ DEBUG=true
DOMAIN=your.domain.here
#EMAIL=your@email.here
+# Used for deciding which editions to prefer
+DEFAULT_LANGUAGE="English"
+
## Leave unset to allow all hosts
# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
diff --git a/.env.prod.example b/.env.prod.example
index 0013bf9d..ac9fe70f 100644
--- a/.env.prod.example
+++ b/.env.prod.example
@@ -7,6 +7,9 @@ DEBUG=false
DOMAIN=your.domain.here
EMAIL=your@email.here
+# Used for deciding which editions to prefer
+DEFAULT_LANGUAGE="English"
+
## Leave unset to allow all hosts
# ALLOWED_HOSTS="localhost,127.0.0.1,[::1]"
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index a50a2584..ca84338d 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -23,7 +23,9 @@ class Connector(AbstractConnector):
Mapping("title", remote_field="wdt:P1476", formatter=get_first),
Mapping("subtitle", remote_field="wdt:P1680", formatter=get_first),
Mapping("inventaireId", remote_field="uri"),
- Mapping("description", remote_field="sitelinks", formatter=self.get_description),
+ Mapping(
+ "description", remote_field="sitelinks", formatter=self.get_description
+ ),
Mapping("cover", remote_field="image", formatter=self.get_cover_url),
Mapping("isbn13", remote_field="wdt:P212", formatter=get_first),
Mapping("isbn10", remote_field="wdt:P957", formatter=get_first),
@@ -84,7 +86,9 @@ class Connector(AbstractConnector):
return SearchResult(
title=search_result.get("label"),
key=self.get_remote_id(search_result.get("uri")),
- view_link="{:s}/entity/{:s}".format(self.base_url, search_result.get("uri")),
+ view_link="{:s}/entity/{:s}".format(
+ self.base_url, search_result.get("uri")
+ ),
cover=cover,
connector=self,
)
@@ -104,7 +108,9 @@ class Connector(AbstractConnector):
return SearchResult(
title=title[0],
key=self.get_remote_id(search_result.get("uri")),
- view_link="{:s}/entity/{:s}".format(self.base_url, search_result.get("uri")),
+ view_link="{:s}/entity/{:s}".format(
+ self.base_url, search_result.get("uri")
+ ),
cover=self.get_cover_url(search_result.get("image")),
connector=self,
)
@@ -187,7 +193,7 @@ class Connector(AbstractConnector):
return results
def get_description(self, links):
- """ grab an extracted except from wikipedia """
+ """grab an extracted except from wikipedia"""
link = links.get("enwiki")
if not link:
return ""
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 0bbae033..10af9b5b 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -5,7 +5,7 @@ from django.db import models
from model_utils.managers import InheritanceManager
from bookwyrm import activitypub
-from bookwyrm.settings import DOMAIN
+from bookwyrm.settings import DOMAIN, DEFAULT_LANGUAGE
from .activitypub_mixin import OrderedCollectionPageMixin, ObjectMixin
from .base_model import BookWyrmModel
@@ -211,7 +211,15 @@ class Edition(Book):
def get_rank(self):
"""calculate how complete the data is on this edition"""
rank = 0
+ # big ups for havinga cover
rank += int(bool(self.cover)) * 3
+ # is it in the instance's preferred language?
+ rank += int(bool(DEFAULT_LANGUAGE in self.languages))
+ print(DEFAULT_LANGUAGE)
+ # prefer print editions
+ rank += int(bool(self.physical_format.lower() in ['paperback', 'hardcover']))
+
+ # does it have metadata?
rank += int(bool(self.isbn_13))
rank += int(bool(self.isbn_10))
rank += int(bool(self.oclc_number))
diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py
index b679e2d4..1bc3c587 100644
--- a/bookwyrm/settings.py
+++ b/bookwyrm/settings.py
@@ -11,6 +11,7 @@ DOMAIN = env("DOMAIN")
VERSION = "0.0.1"
PAGE_LENGTH = env("PAGE_LENGTH", 15)
+DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
# celery
CELERY_BROKER = env("CELERY_BROKER")
From 7853610a205a5d6e8ed42d766df65f617972cf3a Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 10:54:36 -0700
Subject: [PATCH 32/37] Load descriptions correctly
---
bookwyrm/connectors/inventaire.py | 4 ++--
bookwyrm/models/book.py | 3 +--
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py
index ca84338d..896f84af 100644
--- a/bookwyrm/connectors/inventaire.py
+++ b/bookwyrm/connectors/inventaire.py
@@ -70,7 +70,7 @@ class Connector(AbstractConnector):
# flatten the data so that images, uri, and claims are on the same level
return {
**data.get("claims", {}),
- **{k: data.get(k) for k in ["uri", "image", "labels"]},
+ **{k: data.get(k) for k in ["uri", "image", "labels", "sitelinks"]},
}
def parse_search_data(self, data):
@@ -193,7 +193,7 @@ class Connector(AbstractConnector):
return results
def get_description(self, links):
- """grab an extracted except from wikipedia"""
+ """grab an extracted excerpt from wikipedia"""
link = links.get("enwiki")
if not link:
return ""
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index 10af9b5b..e7c95759 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -215,9 +215,8 @@ class Edition(Book):
rank += int(bool(self.cover)) * 3
# is it in the instance's preferred language?
rank += int(bool(DEFAULT_LANGUAGE in self.languages))
- print(DEFAULT_LANGUAGE)
# prefer print editions
- rank += int(bool(self.physical_format.lower() in ['paperback', 'hardcover']))
+ rank += int(bool(self.physical_format.lower() in ["paperback", "hardcover"]))
# does it have metadata?
rank += int(bool(self.isbn_13))
From d1b5f3b6f0d98fc0076da1f16a5fc465d4877433 Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 11:04:14 -0700
Subject: [PATCH 33/37] Fixes ranks and normalizes isbns
---
bookwyrm/models/book.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py
index e7c95759..0539414a 100644
--- a/bookwyrm/models/book.py
+++ b/bookwyrm/models/book.py
@@ -216,7 +216,10 @@ class Edition(Book):
# is it in the instance's preferred language?
rank += int(bool(DEFAULT_LANGUAGE in self.languages))
# prefer print editions
- rank += int(bool(self.physical_format.lower() in ["paperback", "hardcover"]))
+ if self.physical_format:
+ rank += int(
+ bool(self.physical_format.lower() in ["paperback", "hardcover"])
+ )
# does it have metadata?
rank += int(bool(self.isbn_13))
@@ -236,6 +239,10 @@ class Edition(Book):
if self.isbn_10 and not self.isbn_13:
self.isbn_13 = isbn_10_to_13(self.isbn_10)
+ # normalize isbn format
+ self.isbn_10 = re.sub(r"[^0-9X]", "", self.isbn_10)
+ self.isbn_13 = re.sub(r"[^0-9X]", "", self.isbn_13)
+
# set rank
self.edition_rank = self.get_rank()
From d1b788b61fbfbaa129c7f8d19cea4eeaa154214b Mon Sep 17 00:00:00 2001
From: Mouse Reeve
Date: Thu, 29 Apr 2021 11:14:20 -0700
Subject: [PATCH 34/37] Adds inventaire link to book page
---
bookwyrm/templates/book/book.html | 3 +++
1 file changed, 3 insertions(+)
diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html
index 97f105bf..abe50145 100644
--- a/bookwyrm/templates/book/book.html
+++ b/bookwyrm/templates/book/book.html
@@ -81,6 +81,9 @@
{% if book.openlibrary_key %}
{% trans "View on OpenLibrary" %}
{% endif %}
+ {% if book.inventaire_id %}
+ {% trans "View on Inventaire" %}
+ {% endif %}
From 095b60bff107ab36370ce1743fb585a38598e904 Mon Sep 17 00:00:00 2001
From: Mouse Reeve