diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..5662d1d5 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: bookwyrm +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/README.md b/README.md index 5e16597c..2dfedebb 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Social reading and reviewing, decentralized with ActivityPub - [The role of federation](#the-role-of-federation) - [Features](#features) - [Setting up the developer environment](#setting-up-the-developer-environment) + - [Installing in Production](#installing-in-production) - [Project structure](#project-structure) - [Book data](#book-data) - [Contributing](#contributing) @@ -59,8 +60,6 @@ cp .env.example .env For most testing, you'll want to use ngrok. Remember to set the DOMAIN in `.env` to your ngrok domain. - -#### With Docker You'll have to install the Docker and docker-compose. When you're ready, run: ```bash @@ -69,33 +68,7 @@ docker-compose run --rm web python manage.py migrate docker-compose run --rm web python manage.py initdb ``` -### Without Docker -You will need postgres installed and running on your computer. - -``` bash -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -createdb bookwyrm -``` - -Create the psql user in `psql bookwyrm`: -``` psql -CREATE ROLE bookwyrm WITH LOGIN PASSWORD 'bookwyrm'; -GRANT ALL PRIVILEGES ON DATABASE bookwyrm TO bookwyrm; -``` - -Initialize the database (or, more specifically, delete the existing database, run migrations, and start fresh): -``` bash -./rebuilddb.sh -``` -This creates two users, `mouse` with password `password123` and `rat` with password `ratword`. - -The application uses Celery and Redis for task management, which must also be installed and configured. - -And go to the app at `localhost:8000` - - +Once the build is complete, you can access the instance at `localhost:1333` ## Installing in Production @@ -114,7 +87,7 @@ This project is still young and isn't, at the momoment, very stable, so please p `cp .env.example .env` - Add your domain, email address, mailgun credentials - Set a secure redis password and secret key - - Update your nginx configuration in `nginx/defautl.conf` + - Update your nginx configuration in `nginx/default.conf` - Replace `your-domain.com` with your domain name - Run the application (this should also set up a Certbot ssl cert for your domain) `docker-compose up --build` @@ -124,13 +97,13 @@ This project is still young and isn't, at the momoment, very stable, so please p - Run docker-compose in the background `docker-compose up -d` - Initialize the database - `./fr-dev initdb` + `./bw-dev initdb` - Congrats! You did it, go to your domain and enjoy the fruits of your labors ### Configure your instance - Register a user account in the applcation UI - Make your account a superuser (warning: do *not* use django's `createsuperuser` command) - On your server, open the django shell - `./fr-dev shell` + `./bw-dev shell` - Load your user and make it a superuser ```python from bookwyrm import models diff --git a/bookwyrm/activitypub/__init__.py b/bookwyrm/activitypub/__init__.py index 85245929..a4fef41e 100644 --- a/bookwyrm/activitypub/__init__.py +++ b/bookwyrm/activitypub/__init__.py @@ -2,20 +2,20 @@ import inspect import sys -from .base_activity import ActivityEncoder, PublicKey, Signature +from .base_activity import ActivityEncoder, Signature from .base_activity import Link, Mention -from .base_activity import ActivitySerializerError -from .base_activity import tag_formatter +from .base_activity import ActivitySerializerError, resolve_remote_id from .image import Image from .note import Note, GeneratedNote, Article, Comment, Review, Quotation from .note import Tombstone from .interaction import Boost, Like from .ordered_collection import OrderedCollection, OrderedCollectionPage -from .person import Person +from .person import Person, PublicKey +from .response import ActivitypubResponse from .book import Edition, Work, Author from .verbs import Create, Delete, Undo, Update from .verbs import Follow, Accept, Reject -from .verbs import Add, Remove +from .verbs import Add, AddBook, Remove # this creates a list of all the Activity types that we can serialize, # so when an Activity comes in from outside, we can check if it's known diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index caa4aeb8..7ef0920f 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -1,18 +1,12 @@ ''' basics for an activitypub serializer ''' from dataclasses import dataclass, fields, MISSING from json import JSONEncoder -from uuid import uuid4 -from django.core.files.base import ContentFile -from django.db import transaction -from django.db.models.fields.related_descriptors \ - import ForwardManyToOneDescriptor, ManyToManyDescriptor, \ - ReverseManyToOneDescriptor -from django.db.models.fields.files import ImageFileDescriptor -import requests - -from bookwyrm import books_manager, models +from django.apps import apps +from django.db import IntegrityError, transaction +from bookwyrm.connectors import ConnectorException, get_data +from bookwyrm.tasks import app class ActivitySerializerError(ValueError): ''' routine problems serializing activitypub json ''' @@ -25,26 +19,19 @@ class ActivityEncoder(JSONEncoder): @dataclass -class Link(): +class Link: ''' for tagging a book in a status ''' href: str name: str type: str = 'Link' + @dataclass class Mention(Link): ''' a subtype of Link for mentioning an actor ''' type: str = 'Mention' -@dataclass -class PublicKey: - ''' public key block ''' - id: str - owner: str - publicKeyPem: str - - @dataclass class Signature: ''' public key block ''' @@ -76,88 +63,67 @@ class ActivityObject: setattr(self, field.name, value) - def to_model(self, model, instance=None): + def to_model(self, model, instance=None, save=True): ''' convert from an activity to a model instance ''' if not isinstance(self, model.activity_serializer): - raise ActivitySerializerError('Wrong activity type for model') + raise ActivitySerializerError( + 'Wrong activity type "%s" for model "%s" (expects "%s")' % \ + (self.__class__, + model.__name__, + model.activity_serializer) + ) + + if hasattr(model, 'ignore_activity') and model.ignore_activity(self): + return instance # check for an existing instance, if we're not updating a known obj - if not instance: - try: - return model.objects.get(remote_id=self.id) - except model.DoesNotExist: - pass + instance = instance or model.find_existing(self.serialize()) or model() - model_fields = [m.name for m in model._meta.get_fields()] - mapped_fields = {} - many_to_many_fields = {} - one_to_many_fields = {} - image_fields = {} + for field in instance.simple_fields: + field.set_field_from_activity(instance, self) - for mapping in model.activity_mappings: - if mapping.model_key not in model_fields: - continue - # value is None if there's a default that isn't supplied - # in the activity but is supplied in the formatter - value = None - if mapping.activity_key: - value = getattr(self, mapping.activity_key) - model_field = getattr(model, mapping.model_key) + # image fields have to be set after other fields because they can save + # too early and jank up users + for field in instance.image_fields: + field.set_field_from_activity(instance, self, save=save) - formatted_value = mapping.model_formatter(value) - if isinstance(model_field, ForwardManyToOneDescriptor) and \ - formatted_value: - # foreign key remote id reolver (work on Edition, for example) - fk_model = model_field.field.related_model - reference = resolve_foreign_key(fk_model, formatted_value) - mapped_fields[mapping.model_key] = reference - elif isinstance(model_field, ManyToManyDescriptor): - # status mentions book/users - many_to_many_fields[mapping.model_key] = formatted_value - elif isinstance(model_field, ReverseManyToOneDescriptor): - # attachments on Status, for example - one_to_many_fields[mapping.model_key] = formatted_value - elif isinstance(model_field, ImageFileDescriptor): - # image fields need custom handling - image_fields[mapping.model_key] = formatted_value - else: - mapped_fields[mapping.model_key] = formatted_value + if not save: + return instance with transaction.atomic(): - if instance: - # updating an existing model isntance - for k, v in mapped_fields.items(): - setattr(instance, k, v) + # we can't set many to many and reverse fields on an unsaved object + try: instance.save() - else: - # creating a new model instance - instance = model.objects.create(**mapped_fields) + except IntegrityError as e: + raise ActivitySerializerError(e) - # add images - for (model_key, value) in image_fields.items(): - formatted_value = image_formatter(value) - if not formatted_value: - continue - getattr(instance, model_key).save(*formatted_value, save=True) + # add many to many fields, which have to be set post-save + for field in instance.many_to_many_fields: + # mention books/users, for example + field.set_field_from_activity(instance, self) - for (model_key, values) in many_to_many_fields.items(): - # mention books, mention users - getattr(instance, model_key).set(values) + # reversed relationships in the models + for (model_field_name, activity_field_name) in \ + instance.deserialize_reverse_fields: + # attachments on Status, for example + values = getattr(self, activity_field_name) + if values is None or values is MISSING: + continue - # add one to many fields - for (model_key, values) in one_to_many_fields.items(): - if values == MISSING: - continue - model_field = getattr(instance, model_key) - model = model_field.model - for item in values: - item = model.activity_serializer(**item) - field_name = instance.__class__.__name__.lower() - with transaction.atomic(): - item = item.to_model(model) - setattr(item, field_name, instance) - item.save() + model_field = getattr(model, model_field_name) + # creating a Work, model_field is 'editions' + # creating a User, model field is 'key_pair' + related_model = model_field.field.model + related_field_name = model_field.field.name + for item in values: + set_related_field.delay( + related_model.__name__, + instance.__class__.__name__, + related_field_name, + instance.remote_id, + item + ) return instance @@ -168,66 +134,72 @@ class ActivityObject: return data -def resolve_foreign_key(model, remote_id): - ''' look up the remote_id on an activity json field ''' - if model in [models.Edition, models.Work, models.Book]: - return books_manager.get_or_create_book(remote_id) +@app.task +@transaction.atomic +def set_related_field( + model_name, origin_model_name, related_field_name, + related_remote_id, data): + ''' load reverse related fields (editions, attachments) without blocking ''' + model = apps.get_model('bookwyrm.%s' % model_name, require_ready=True) + origin_model = apps.get_model( + 'bookwyrm.%s' % origin_model_name, + require_ready=True + ) - result = model.objects - if hasattr(model.objects, 'select_subclasses'): - result = result.select_subclasses() + with transaction.atomic(): + if isinstance(data, str): + existing = model.find_existing_by_remote_id(data) + if existing: + data = existing.to_activity() + else: + data = get_data(data) + activity = model.activity_serializer(**data) - result = result.filter( - remote_id=remote_id - ).first() + # this must exist because it's the object that triggered this function + instance = origin_model.find_existing_by_remote_id(related_remote_id) + if not instance: + raise ValueError( + 'Invalid related remote id: %s' % related_remote_id) - if not result: - raise ActivitySerializerError( - 'Could not resolve remote_id in %s model: %s' % \ - (model.__name__, remote_id)) - return result + # set the origin's remote id on the activity so it will be there when + # the model instance is created + # edition.parentWork = instance, for example + model_field = getattr(model, related_field_name) + if hasattr(model_field, 'activitypub_field'): + setattr( + activity, + getattr(model_field, 'activitypub_field'), + instance.remote_id + ) + item = activity.to_model(model) + + # if the related field isn't serialized (attachments on Status), then + # we have to set it post-creation + if not hasattr(model_field, 'activitypub_field'): + setattr(item, related_field_name, instance) + item.save() -def tag_formatter(tags, tag_type): - ''' helper function to extract foreign keys from tag activity json ''' - if not isinstance(tags, list): - return [] - items = [] - types = { - 'Book': models.Book, - 'Mention': models.User, - } - for tag in [t for t in tags if t.get('type') == tag_type]: - if not tag_type in types: - continue - remote_id = tag.get('href') - try: - item = resolve_foreign_key(types[tag_type], remote_id) - except ActivitySerializerError: - continue - items.append(item) - return items +def resolve_remote_id(model, remote_id, refresh=False, save=True): + ''' take a remote_id and return an instance, creating if necessary ''' + result = model.find_existing_by_remote_id(remote_id) + if result and not refresh: + return result - -def image_formatter(image_slug): - ''' helper function to load images and format them for a model ''' - # when it's an inline image (User avatar/icon, Book cover), it's a json - # blob, but when it's an attached image, it's just a url - if isinstance(image_slug, dict): - url = image_slug.get('url') - elif isinstance(image_slug, str): - url = image_slug - else: - return None - if not url: - return None + # load the data and create the object try: - response = requests.get(url) - except ConnectionError: - return None - if not response.ok: - return None + data = get_data(remote_id) + except (ConnectorException, ConnectionError): + raise ActivitySerializerError( + 'Could not connect to host for remote_id in %s model: %s' % \ + (model.__name__, remote_id)) - image_name = str(uuid4()) + '.' + url.split('.')[-1] - image_content = ContentFile(response.content) - return [image_name, image_content] + # check for existing items with shared unique identifiers + if not result: + result = model.find_existing(data) + if result and not refresh: + return result + + item = model.activity_serializer(**data) + # if we're refreshing, "result" will be set and we'll update it + return item.to_model(model, instance=result, save=save) diff --git a/bookwyrm/activitypub/book.py b/bookwyrm/activitypub/book.py index 02cab281..68036559 100644 --- a/bookwyrm/activitypub/book.py +++ b/bookwyrm/activitypub/book.py @@ -12,13 +12,13 @@ class Book(ActivityObject): sortTitle: str = '' subtitle: str = '' description: str = '' - languages: List[str] + languages: List[str] = field(default_factory=lambda: []) series: str = '' seriesNumber: str = '' - subjects: List[str] - subjectPlaces: List[str] + subjects: List[str] = field(default_factory=lambda: []) + subjectPlaces: List[str] = field(default_factory=lambda: []) - authors: List[str] + authors: List[str] = field(default_factory=lambda: []) firstPublishedDate: str = '' publishedDate: str = '' @@ -33,23 +33,25 @@ class Book(ActivityObject): @dataclass(init=False) class Edition(Book): ''' Edition instance of a book object ''' - isbn10: str - isbn13: str - oclcNumber: str - asin: str - pages: str - physicalFormat: str - publishers: List[str] - work: str + isbn10: str = '' + isbn13: str = '' + oclcNumber: str = '' + asin: str = '' + pages: int = None + physicalFormat: str = '' + publishers: List[str] = field(default_factory=lambda: []) + editionRank: int = 0 + type: str = 'Edition' @dataclass(init=False) class Work(Book): ''' work instance of a book object ''' - lccn: str - editions: List[str] + lccn: str = '' + defaultEdition: str = '' + editions: List[str] = field(default_factory=lambda: []) type: str = 'Work' @@ -57,10 +59,12 @@ class Work(Book): class Author(ActivityObject): ''' author of a book ''' name: str - born: str = '' - died: str = '' - aliases: str = '' + born: str = None + died: str = None + aliases: List[str] = field(default_factory=lambda: []) bio: str = '' openlibraryKey: str = '' + librarythingKey: str = '' + goodreadsKey: str = '' wikipediaLink: str = '' type: str = 'Person' diff --git a/bookwyrm/activitypub/note.py b/bookwyrm/activitypub/note.py index aeb078dc..72fbe5fc 100644 --- a/bookwyrm/activitypub/note.py +++ b/bookwyrm/activitypub/note.py @@ -8,7 +8,6 @@ from .image import Image @dataclass(init=False) class Tombstone(ActivityObject): ''' the placeholder for a deleted status ''' - url: str published: str deleted: str type: str = 'Tombstone' @@ -17,14 +16,14 @@ class Tombstone(ActivityObject): @dataclass(init=False) class Note(ActivityObject): ''' Note activity ''' - url: str - inReplyTo: str published: str attributedTo: str - to: List[str] - cc: List[str] content: str - replies: Dict + to: List[str] = field(default_factory=lambda: []) + cc: List[str] = field(default_factory=lambda: []) + replies: Dict = field(default_factory=lambda: {}) + inReplyTo: str = '' + summary: str = '' tag: List[Link] = field(default_factory=lambda: []) attachment: List[Image] = field(default_factory=lambda: []) sensitive: bool = False @@ -54,8 +53,8 @@ class Comment(Note): @dataclass(init=False) class Review(Comment): ''' a full book review ''' - name: str - rating: int + name: str = None + rating: int = None type: str = 'Review' diff --git a/bookwyrm/activitypub/ordered_collection.py b/bookwyrm/activitypub/ordered_collection.py index efd23d5a..9aeaf664 100644 --- a/bookwyrm/activitypub/ordered_collection.py +++ b/bookwyrm/activitypub/ordered_collection.py @@ -12,6 +12,7 @@ class OrderedCollection(ActivityObject): first: str last: str = '' name: str = '' + owner: str = '' type: str = 'OrderedCollection' diff --git a/bookwyrm/activitypub/person.py b/bookwyrm/activitypub/person.py index e7d720ec..7e7d027e 100644 --- a/bookwyrm/activitypub/person.py +++ b/bookwyrm/activitypub/person.py @@ -2,20 +2,29 @@ from dataclasses import dataclass, field from typing import Dict -from .base_activity import ActivityObject, PublicKey +from .base_activity import ActivityObject from .image import Image + +@dataclass(init=False) +class PublicKey(ActivityObject): + ''' public key block ''' + owner: str + publicKeyPem: str + type: str = 'PublicKey' + + @dataclass(init=False) class Person(ActivityObject): ''' actor activitypub json ''' preferredUsername: str - name: str inbox: str outbox: str followers: str - summary: str publicKey: PublicKey endpoints: Dict + name: str = None + summary: str = None icon: Image = field(default_factory=lambda: {}) bookwyrmUser: bool = False manuallyApprovesFollowers: str = False diff --git a/bookwyrm/activitypub/response.py b/bookwyrm/activitypub/response.py new file mode 100644 index 00000000..bbc44c4d --- /dev/null +++ b/bookwyrm/activitypub/response.py @@ -0,0 +1,18 @@ +from django.http import JsonResponse + +from .base_activity import ActivityEncoder + +class ActivitypubResponse(JsonResponse): + """ + A class to be used in any place that's serializing responses for + Activitypub enabled clients. Uses JsonResponse under the hood, but already + configures some stuff beforehand. Made to be a drop-in replacement of + JsonResponse. + """ + def __init__(self, data, encoder=ActivityEncoder, safe=True, + json_dumps_params=None, **kwargs): + + if 'content_type' not in kwargs: + kwargs['content_type'] = 'application/activity+json' + + super().__init__(data, encoder, safe, json_dumps_params, **kwargs) diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py index eb166260..7c627927 100644 --- a/bookwyrm/activitypub/verbs.py +++ b/bookwyrm/activitypub/verbs.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import List from .base_activity import ActivityObject, Signature +from .book import Edition @dataclass(init=False) class Verb(ActivityObject): @@ -69,6 +70,13 @@ class Add(Verb): type: str = 'Add' +@dataclass(init=False) +class AddBook(Verb): + '''Add activity that's aware of the book obj ''' + target: Edition + type: str = 'Add' + + @dataclass(init=False) class Remove(Verb): '''Remove activity ''' diff --git a/bookwyrm/broadcast.py b/bookwyrm/broadcast.py index a1eebaee..f4186c4d 100644 --- a/bookwyrm/broadcast.py +++ b/bookwyrm/broadcast.py @@ -3,7 +3,7 @@ import json from django.utils.http import http_date import requests -from bookwyrm import models +from bookwyrm import models, settings from bookwyrm.activitypub import ActivityEncoder from bookwyrm.tasks import app from bookwyrm.signatures import make_signature, make_digest @@ -65,7 +65,7 @@ def sign_and_send(sender, data, destination): ''' crpyto whatever and http junk ''' now = http_date() - if not sender.private_key: + if not sender.key_pair.private_key: # this shouldn't happen. it would be bad if it happened. raise ValueError('No private key found for sender') @@ -79,6 +79,7 @@ def sign_and_send(sender, data, destination): 'Digest': digest, 'Signature': make_signature(sender, destination, now, digest), 'Content-Type': 'application/activity+json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, }, ) if not response.ok: diff --git a/bookwyrm/connectors/__init__.py b/bookwyrm/connectors/__init__.py index b5d93b47..cfafd286 100644 --- a/bookwyrm/connectors/__init__.py +++ b/bookwyrm/connectors/__init__.py @@ -1,3 +1,6 @@ ''' bring connectors into the namespace ''' from .settings import CONNECTORS from .abstract_connector import ConnectorException +from .abstract_connector import get_data, get_image + +from .connector_manager import search, local_search, first_search_result diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index d709b075..ab9900a7 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -1,35 +1,25 @@ ''' functionality outline for a book data connector ''' from abc import ABC, abstractmethod -from dataclasses import dataclass -import pytz +from dataclasses import asdict, dataclass +import logging from urllib3.exceptions import RequestError from django.db import transaction -from dateutil import parser import requests -from requests import HTTPError +from requests.exceptions import SSLError -from bookwyrm import models +from bookwyrm import activitypub, models, settings +from .connector_manager import load_more_data, ConnectorException -class ConnectorException(HTTPError): - ''' when the connector can't do what was asked ''' - - -class AbstractConnector(ABC): - ''' generic book data connector ''' - +logger = logging.getLogger(__name__) +class AbstractMinimalConnector(ABC): + ''' just the bare bones, for other bookwyrm instances ''' def __init__(self, identifier): # load connector settings info = models.Connector.objects.get(identifier=identifier) self.connector = info - self.key_mappings = [] - - # fields we want to look for in book data to copy over - # title we handle separately. - self.book_mappings = [] - # the things in the connector model to copy over self_fields = [ 'base_url', @@ -44,6 +34,54 @@ class AbstractConnector(ABC): for field in self_fields: setattr(self, field, getattr(info, field)) + def search(self, query, min_confidence=None): + ''' free text search ''' + params = {} + if min_confidence: + params['min_confidence'] = min_confidence + + resp = requests.get( + '%s%s' % (self.search_url, query), + params=params, + headers={ + 'Accept': 'application/json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, + }, + ) + if not resp.ok: + resp.raise_for_status() + try: + data = resp.json() + except ValueError as e: + logger.exception(e) + raise ConnectorException('Unable to parse json response', e) + results = [] + + for doc in self.parse_search_data(data)[:10]: + results.append(self.format_search_result(doc)) + return results + + @abstractmethod + def get_or_create_book(self, remote_id): + ''' pull up a book record by whatever means possible ''' + + @abstractmethod + def parse_search_data(self, data): + ''' turn the result json from a search into a list ''' + + @abstractmethod + def format_search_result(self, search_result): + ''' create a SearchResult obj from json ''' + + +class AbstractConnector(AbstractMinimalConnector): + ''' generic book data connector ''' + def __init__(self, identifier): + super().__init__(identifier) + # fields we want to look for in book data to copy over + # title we handle separately. + self.book_mappings = [] + def is_available(self): ''' check if you're allowed to use this connector ''' @@ -53,248 +91,113 @@ class AbstractConnector(ABC): return True - def search(self, query, min_confidence=None): - ''' free text search ''' - resp = requests.get( - '%s%s' % (self.search_url, query), - headers={ - 'Accept': 'application/json; charset=utf-8', - }, - ) - if not resp.ok: - resp.raise_for_status() - data = resp.json() - results = [] - - for doc in self.parse_search_data(data)[:10]: - results.append(self.format_search_result(doc)) - return results - - def get_or_create_book(self, remote_id): - ''' pull up a book record by whatever means possible ''' - # try to load the book - book = models.Book.objects.select_subclasses().filter( - origin_id=remote_id - ).first() - if book: - if isinstance(book, models.Work): - return book.default_edition - return book + ''' translate arbitrary json into an Activitypub dataclass ''' + # first, check if we have the origin_id saved + existing = models.Edition.find_existing_by_remote_id(remote_id) or \ + models.Work.find_existing_by_remote_id(remote_id) + if existing: + if hasattr(existing, 'get_default_editon'): + return existing.get_default_editon() + return existing - # no book was found, so we start creating a new one + # load the json data = get_data(remote_id) - - work = None - edition = None + mapped_data = dict_from_mappings(data, self.book_mappings) if self.is_work_data(data): - work_data = data - # if we requested a work and there's already an edition, we're set - work = self.match_from_mappings(work_data, models.Work) - if work and work.default_edition: - return work.default_edition - - # no such luck, we need more information. try: - edition_data = self.get_edition_from_work_data(work_data) + edition_data = self.get_edition_from_work_data(data) except KeyError: # hack: re-use the work data as the edition data # this is why remote ids aren't necessarily unique edition_data = data + work_data = mapped_data else: - edition_data = data - edition = self.match_from_mappings(edition_data, models.Edition) - # no need to figure out about the work if we already know about it - if edition and edition.parent_work: - return edition - - # no such luck, we need more information. try: - work_data = self.get_work_from_edition_date(edition_data) + work_data = self.get_work_from_edition_data(data) + work_data = dict_from_mappings(work_data, self.book_mappings) except KeyError: - # remember this hack: re-use the work data as the edition data - work_data = data + work_data = mapped_data + edition_data = data if not work_data or not edition_data: raise ConnectorException('Unable to load book data: %s' % remote_id) - # at this point, we need to figure out the work, edition, or both - # atomic so that we don't save a work with no edition for vice versa with transaction.atomic(): - if not work: - work_key = self.get_remote_id_from_data(work_data) - work = self.create_book(work_key, work_data, models.Work) + # create activitypub object + work_activity = activitypub.Work(**work_data) + # this will dedupe automatically + work = work_activity.to_model(models.Work) + for author in self.get_authors_from_data(data): + work.authors.add(author) - if not edition: - ed_key = self.get_remote_id_from_data(edition_data) - edition = self.create_book(ed_key, edition_data, models.Edition) - edition.parent_work = work - edition.save() - work.default_edition = edition - work.save() + edition = self.create_edition_from_data(work, edition_data) + load_more_data.delay(self.connector.id, work.id) + return edition - # now's our change to fill in author gaps + + def create_edition_from_data(self, work, edition_data): + ''' if we already have the work, we're ready ''' + mapped_data = dict_from_mappings(edition_data, self.book_mappings) + mapped_data['work'] = work.remote_id + edition_activity = activitypub.Edition(**mapped_data) + edition = edition_activity.to_model(models.Edition) + edition.connector = self.connector + edition.save() + + work.default_edition = edition + work.save() + + for author in self.get_authors_from_data(edition_data): + edition.authors.add(author) if not edition.authors.exists() and work.authors.exists(): edition.authors.set(work.authors.all()) - edition.author_text = work.author_text - edition.save() - - if not edition: - raise ConnectorException('Unable to create book: %s' % remote_id) return edition - def create_book(self, remote_id, data, model): - ''' create a work or edition from data ''' - book = model.objects.create( - origin_id=remote_id, - title=data['title'], - connector=self.connector, - ) - return self.update_book_from_data(book, data) + def get_or_create_author(self, remote_id): + ''' load that author ''' + existing = models.Author.find_existing_by_remote_id(remote_id) + if existing: + return existing + data = get_data(remote_id) - def update_book_from_data(self, book, data, update_cover=True): - ''' for creating a new book or syncing with data ''' - book = self.update_from_mappings(book, data, self.book_mappings) - - author_text = [] - for author in self.get_authors_from_data(data): - book.authors.add(author) - if author.display_name: - author_text.append(author.display_name) - book.author_text = ', '.join(author_text) - book.save() - - if not update_cover: - return book - - cover = self.get_cover_from_data(data) - if cover: - book.cover.save(*cover, save=True) - return book - - - def update_book(self, book, data=None): - ''' load new data ''' - if not book.sync and not book.sync_cover: - return book - - if not data: - key = getattr(book, self.key_name) - data = self.load_book_data(key) - - if book.sync: - book = self.update_book_from_data( - book, data, update_cover=book.sync_cover) - else: - cover = self.get_cover_from_data(data) - if cover: - book.cover.save(*cover, save=True) - - return book - - - def match_from_mappings(self, data, model): - ''' try to find existing copies of this book using various keys ''' - relevent_mappings = [m for m in self.key_mappings if \ - not m.model or model == m.model] - for mapping in relevent_mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue - - # extract the value in the right format - value = mapping.formatter(value) - - # search our database for a matching book - kwargs = {mapping.local_field: value} - match = model.objects.filter(**kwargs).first() - if match: - return match - return None - - - @abstractmethod - def get_remote_id_from_data(self, data): - ''' otherwise we won't properly set the remote_id in the db ''' + mapped_data = dict_from_mappings(data, self.author_mappings) + activity = activitypub.Author(**mapped_data) + # this will dedupe + return activity.to_model(models.Author) @abstractmethod def is_work_data(self, data): ''' differentiate works and editions ''' - @abstractmethod def get_edition_from_work_data(self, data): ''' every work needs at least one edition ''' - @abstractmethod - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): ''' every edition needs a work ''' - @abstractmethod def get_authors_from_data(self, data): ''' load author data ''' - - @abstractmethod - def get_cover_from_data(self, data): - ''' load cover ''' - - - @abstractmethod - def parse_search_data(self, data): - ''' turn the result json from a search into a list ''' - - - @abstractmethod - def format_search_result(self, search_result): - ''' create a SearchResult obj from json ''' - - @abstractmethod def expand_book_data(self, book): ''' get more info on a book ''' - def update_from_mappings(self, obj, data, mappings): - ''' assign data to model with mappings ''' - for mapping in mappings: - # check if this field is present in the data - value = data.get(mapping.remote_field) - if not value: - continue - - # extract the value in the right format - try: - value = mapping.formatter(value) - except: - continue - - # assign the formatted value to the model - obj.__setattr__(mapping.local_field, value) - return obj - - -def get_date(date_string): - ''' helper function to try to interpret dates ''' - if not date_string: - return None - - try: - return pytz.utc.localize(parser.parse(date_string)) - except ValueError: - pass - - try: - return parser.parse(date_string) - except ValueError: - return None +def dict_from_mappings(data, mappings): + ''' create a dict in Activitypub format, using mappings supplies by + the subclass ''' + result = {} + for mapping in mappings: + result[mapping.local_field] = mapping.get_value(data) + return result def get_data(url): @@ -304,16 +207,37 @@ def get_data(url): url, headers={ 'Accept': 'application/json; charset=utf-8', + 'User-Agent': settings.USER_AGENT, }, ) - except RequestError: + except (RequestError, SSLError): raise ConnectorException() if not resp.ok: resp.raise_for_status() - data = resp.json() + try: + data = resp.json() + except ValueError: + raise ConnectorException() + return data +def get_image(url): + ''' wrapper for requesting an image ''' + try: + resp = requests.get( + url, + headers={ + 'User-Agent': settings.USER_AGENT, + }, + ) + except (RequestError, SSLError): + return None + if not resp.ok: + return None + return resp + + @dataclass class SearchResult: ''' standardized search result object ''' @@ -321,20 +245,35 @@ class SearchResult: key: str author: str year: str + connector: object confidence: int = 1 def __repr__(self): return "".format( self.key, self.title, self.author) + def json(self): + ''' serialize a connector for json response ''' + serialized = asdict(self) + del serialized['connector'] + return serialized + class Mapping: ''' associate a local database field with a field in an external dataset ''' - def __init__( - self, local_field, remote_field=None, formatter=None, model=None): + def __init__(self, local_field, remote_field=None, formatter=None): noop = lambda x: x self.local_field = local_field self.remote_field = remote_field or local_field self.formatter = formatter or noop - self.model = model + + def get_value(self, data): + ''' pull a field from incoming json and return the formatted version ''' + value = data.get(self.remote_field) + if not value: + return None + try: + return self.formatter(value) + except:# pylint: disable=bare-except + return None diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 1bc81450..1f877993 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -1,83 +1,21 @@ ''' using another bookwyrm instance as a source of book data ''' -from django.db import transaction - from bookwyrm import activitypub, models -from .abstract_connector import AbstractConnector, SearchResult -from .abstract_connector import get_data +from .abstract_connector import AbstractMinimalConnector, SearchResult -class Connector(AbstractConnector): - ''' interact with other instances ''' - - def update_from_mappings(self, obj, data, mappings): - ''' serialize book data into a model ''' - if self.is_work_data(data): - work_data = activitypub.Work(**data) - return work_data.to_model(models.Work, instance=obj) - edition_data = activitypub.Edition(**data) - return edition_data.to_model(models.Edition, instance=obj) - - - def get_remote_id_from_data(self, data): - return data.get('id') - - - def is_work_data(self, data): - return data.get('type') == 'Work' - - - def get_edition_from_work_data(self, data): - ''' we're served a list of edition urls ''' - path = data['editions'][0] - return get_data(path) - - - def get_work_from_edition_date(self, data): - return get_data(data['work']) - - - def get_authors_from_data(self, data): - ''' load author data ''' - for author_id in data.get('authors', []): - try: - yield models.Author.objects.get(origin_id=author_id) - except models.Author.DoesNotExist: - pass - data = get_data(author_id) - author_data = activitypub.Author(**data) - author = author_data.to_model(models.Author) - yield author - - - def get_cover_from_data(self, data): - pass +class Connector(AbstractMinimalConnector): + ''' this is basically just for search ''' + def get_or_create_book(self, remote_id): + edition = activitypub.resolve_remote_id(models.Edition, remote_id) + work = edition.parent_work + work.default_edition = work.get_default_edition() + work.save() + return edition def parse_search_data(self, data): return data - def format_search_result(self, search_result): + search_result['connector'] = self return SearchResult(**search_result) - - - def expand_book_data(self, book): - work = book - # go from the edition to the work, if necessary - if isinstance(book, models.Edition): - work = book.parent_work - - # it may be that we actually want to request this url - editions_url = '%s/editions?page=true' % work.remote_id - edition_options = get_data(editions_url) - for edition_data in edition_options['orderedItems']: - with transaction.atomic(): - edition = self.create_book( - edition_data['id'], - edition_data, - models.Edition - ) - edition.parent_work = work - edition.save() - if not edition.authors.exists() and work.authors.exists(): - edition.authors.set(work.authors.all()) diff --git a/bookwyrm/books_manager.py b/bookwyrm/connectors/connector_manager.py similarity index 73% rename from bookwyrm/books_manager.py rename to bookwyrm/connectors/connector_manager.py index 461017a0..d3b01f7a 100644 --- a/bookwyrm/books_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -1,4 +1,4 @@ -''' select and call a connector for whatever book task needs doing ''' +''' interface with whatever connectors the app has ''' import importlib from urllib.parse import urlparse @@ -8,60 +8,8 @@ from bookwyrm import models from bookwyrm.tasks import app -def get_edition(book_id): - ''' look up a book in the db and return an edition ''' - book = models.Book.objects.select_subclasses().get(id=book_id) - if isinstance(book, models.Work): - book = book.default_edition - return book - - -def get_or_create_book(remote_id): - ''' pull up a book record by whatever means possible ''' - book = models.Book.objects.select_subclasses().filter( - remote_id=remote_id - ).first() - if book: - return book - - connector = get_or_create_connector(remote_id) - - # raises ConnectorException - book = connector.get_or_create_book(remote_id) - if book: - load_more_data.delay(book.id) - return book - - -def get_or_create_connector(remote_id): - ''' get the connector related to the author's server ''' - url = urlparse(remote_id) - identifier = url.netloc - if not identifier: - raise ValueError('Invalid remote id') - - try: - connector_info = models.Connector.objects.get(identifier=identifier) - except models.Connector.DoesNotExist: - connector_info = models.Connector.objects.create( - identifier=identifier, - connector_file='bookwyrm_connector', - base_url='https://%s' % identifier, - books_url='https://%s/book' % identifier, - covers_url='https://%s/images/covers' % identifier, - search_url='https://%s/search?q=' % identifier, - priority=2 - ) - - return load_connector(connector_info) - - -@app.task -def load_more_data(book_id): - ''' background the work of getting all 10,000 editions of LoTR ''' - book = models.Book.objects.select_subclasses().get(id=book_id) - connector = load_connector(book.connector) - connector.expand_book_data(book) +class ConnectorException(HTTPError): + ''' when the connector can't do what was asked ''' def search(query, min_confidence=0.1): @@ -72,7 +20,7 @@ def search(query, min_confidence=0.1): for connector in get_connectors(): try: result_set = connector.search(query, min_confidence=min_confidence) - except HTTPError: + except (HTTPError, ConnectorException): continue result_set = [r for r in result_set \ @@ -102,18 +50,44 @@ def first_search_result(query, min_confidence=0.1): return None -def update_book(book, data=None): - ''' re-sync with the original data source ''' - connector = load_connector(book.connector) - connector.update_book(book, data=data) - - def get_connectors(): ''' load all connectors ''' for info in models.Connector.objects.order_by('priority').all(): yield load_connector(info) +def get_or_create_connector(remote_id): + ''' get the connector related to the author's server ''' + url = urlparse(remote_id) + identifier = url.netloc + if not identifier: + raise ValueError('Invalid remote id') + + try: + connector_info = models.Connector.objects.get(identifier=identifier) + except models.Connector.DoesNotExist: + connector_info = models.Connector.objects.create( + identifier=identifier, + connector_file='bookwyrm_connector', + base_url='https://%s' % identifier, + books_url='https://%s/book' % identifier, + covers_url='https://%s/images/covers' % identifier, + search_url='https://%s/search?q=' % identifier, + priority=2 + ) + + return load_connector(connector_info) + + +@app.task +def load_more_data(connector_id, book_id): + ''' background the work of getting all 10,000 editions of LoTR ''' + connector_info = models.Connector.objects.get(id=connector_id) + connector = load_connector(connector_info) + book = models.Book.objects.select_subclasses().get(id=book_id) + connector.expand_book_data(book) + + def load_connector(connector_info): ''' instantiate the connector class ''' connector = importlib.import_module( diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 5e18616d..55355131 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -1,13 +1,10 @@ ''' openlibrary data connector ''' import re -import requests - -from django.core.files.base import ContentFile from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult, Mapping -from .abstract_connector import ConnectorException -from .abstract_connector import get_date, get_data +from .abstract_connector import get_data +from .connector_manager import ConnectorException from .openlibrary_languages import languages @@ -17,66 +14,62 @@ class Connector(AbstractConnector): super().__init__(identifier) get_first = lambda a: a[0] - self.key_mappings = [ - Mapping('isbn_13', model=models.Edition, formatter=get_first), - Mapping('isbn_10', model=models.Edition, formatter=get_first), - Mapping('lccn', model=models.Work, formatter=get_first), + get_remote_id = lambda a: self.base_url + a + self.book_mappings = [ + Mapping('title'), + Mapping('id', remote_field='key', formatter=get_remote_id), Mapping( - 'oclc_number', - remote_field='oclc_numbers', - model=models.Edition, - formatter=get_first - ), - Mapping( - 'openlibrary_key', - remote_field='key', - formatter=get_openlibrary_key - ), - Mapping('goodreads_key'), - Mapping('asin'), - ] - - self.book_mappings = self.key_mappings + [ - Mapping('sort_title'), + 'cover', remote_field='covers', formatter=self.get_cover_url), + Mapping('sortTitle', remote_field='sort_title'), Mapping('subtitle'), Mapping('description', formatter=get_description), Mapping('languages', formatter=get_languages), Mapping('series', formatter=get_first), - Mapping('series_number'), + Mapping('seriesNumber', remote_field='series_number'), Mapping('subjects'), - Mapping('subject_places'), + Mapping('subjectPlaces'), + Mapping('isbn13', formatter=get_first), + Mapping('isbn10', formatter=get_first), + Mapping('lccn', formatter=get_first), Mapping( - 'first_published_date', - remote_field='first_publish_date', - formatter=get_date + 'oclcNumber', remote_field='oclc_numbers', + formatter=get_first ), Mapping( - 'published_date', - remote_field='publish_date', - formatter=get_date + 'openlibraryKey', remote_field='key', + formatter=get_openlibrary_key ), + Mapping('goodreadsKey', remote_field='goodreads_key'), + Mapping('asin'), Mapping( - 'pages', - model=models.Edition, - remote_field='number_of_pages' + 'firstPublishedDate', remote_field='first_publish_date', ), - Mapping('physical_format', model=models.Edition), + Mapping('publishedDate', remote_field='publish_date'), + Mapping('pages', remote_field='number_of_pages'), + Mapping('physicalFormat', remote_field='physical_format'), Mapping('publishers'), ] self.author_mappings = [ - Mapping('born', remote_field='birth_date', formatter=get_date), - Mapping('died', remote_field='death_date', formatter=get_date), + Mapping('id', remote_field='key', formatter=get_remote_id), + Mapping('name'), + Mapping( + 'openlibraryKey', remote_field='key', + formatter=get_openlibrary_key + ), + Mapping('born', remote_field='birth_date'), + Mapping('died', remote_field='death_date'), Mapping('bio', formatter=get_description), ] def get_remote_id_from_data(self, data): + ''' format a url from an openlibrary id field ''' try: key = data['key'] except KeyError: raise ConnectorException('Invalid book data') - return '%s/%s' % (self.books_url, key) + return '%s%s' % (self.books_url, key) def is_work_data(self, data): @@ -88,17 +81,17 @@ class Connector(AbstractConnector): key = data['key'] except KeyError: raise ConnectorException('Invalid book data') - url = '%s/%s/editions' % (self.books_url, key) + url = '%s%s/editions' % (self.books_url, key) data = get_data(url) return pick_default_edition(data['entries']) - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): try: key = data['works'][0]['key'] except (IndexError, KeyError): raise ConnectorException('No work found for edition') - url = '%s/%s' % (self.books_url, key) + url = '%s%s' % (self.books_url, key) return get_data(url) @@ -106,24 +99,17 @@ class Connector(AbstractConnector): ''' parse author json and load or create authors ''' for author_blob in data.get('authors', []): author_blob = author_blob.get('author', author_blob) - # this id is "/authors/OL1234567A" and we want just "OL1234567A" - author_id = author_blob['key'].split('/')[-1] - yield self.get_or_create_author(author_id) + # this id is "/authors/OL1234567A" + author_id = author_blob['key'] + url = '%s%s' % (self.base_url, author_id) + yield self.get_or_create_author(url) - def get_cover_from_data(self, data): + def get_cover_url(self, cover_blob): ''' ask openlibrary for the cover ''' - if not data.get('covers'): - return None - - cover_id = data.get('covers')[0] - image_name = '%s-M.jpg' % cover_id - url = '%s/b/id/%s' % (self.covers_url, image_name) - response = requests.get(url) - if not response.ok: - response.raise_for_status() - image_content = ContentFile(response.content) - return [image_name, image_content] + cover_id = cover_blob[0] + image_name = '%s-L.jpg' % cover_id + return '%s/b/id/%s' % (self.covers_url, image_name) def parse_search_data(self, data): @@ -138,13 +124,14 @@ class Connector(AbstractConnector): title=search_result.get('title'), key=key, author=', '.join(author), + connector=self, year=search_result.get('first_publish_year'), ) def load_edition_data(self, olkey): ''' query openlibrary for editions of a work ''' - url = '%s/works/%s/editions.json' % (self.books_url, olkey) + url = '%s/works/%s/editions' % (self.books_url, olkey) return get_data(url) @@ -157,49 +144,14 @@ class Connector(AbstractConnector): # we can mass download edition data from OL to avoid repeatedly querying edition_options = self.load_edition_data(work.openlibrary_key) for edition_data in edition_options.get('entries'): - olkey = edition_data.get('key').split('/')[-1] - # make sure the edition isn't already in the database - if models.Edition.objects.filter(openlibrary_key=olkey).count(): - continue - - # creates and populates the book from the data - edition = self.create_book(olkey, edition_data, models.Edition) - # ensures that the edition is associated with the work - edition.parent_work = work - edition.save() - # get author data from the work if it's missing from the edition - if not edition.authors and work.authors: - edition.authors.set(work.authors.all()) - - - def get_or_create_author(self, olkey): - ''' load that author ''' - if not re.match(r'^OL\d+A$', olkey): - raise ValueError('Invalid OpenLibrary author ID') - author = models.Author.objects.filter(openlibrary_key=olkey).first() - if author: - return author - - url = '%s/authors/%s.json' % (self.base_url, olkey) - data = get_data(url) - - author = models.Author(openlibrary_key=olkey) - author = self.update_from_mappings(author, data, self.author_mappings) - name = data.get('name') - # TODO this is making some BOLD assumption - if name: - author.last_name = name.split(' ')[-1] - author.first_name = ' '.join(name.split(' ')[:-1]) - author.save() - - return author + self.create_edition_from_data(work, edition_data) def get_description(description_blob): ''' descriptions can be a string or a dict ''' if isinstance(description_blob, dict): return description_blob.get('value') - return description_blob + return description_blob def get_openlibrary_key(key): @@ -224,7 +176,7 @@ def pick_default_edition(options): if len(options) == 1: return options[0] - options = [e for e in options if e.get('cover')] or options + options = [e for e in options if e.get('covers')] or options options = [e for e in options if \ '/languages/eng' in str(e.get('languages'))] or options formats = ['paperback', 'hardcover', 'mass market paperback'] diff --git a/bookwyrm/connectors/self_connector.py b/bookwyrm/connectors/self_connector.py index 80d3a67d..c5d58a59 100644 --- a/bookwyrm/connectors/self_connector.py +++ b/bookwyrm/connectors/self_connector.py @@ -1,6 +1,9 @@ ''' using a bookwyrm instance as a source of book data ''' +from functools import reduce +import operator + from django.contrib.postgres.search import SearchRank, SearchVector -from django.db.models import F +from django.db.models import Count, F, Q from bookwyrm import models from .abstract_connector import AbstractConnector, SearchResult @@ -9,38 +12,20 @@ from .abstract_connector import AbstractConnector, SearchResult class Connector(AbstractConnector): ''' instantiate a connector ''' def search(self, query, min_confidence=0.1): - ''' right now you can't search bookwyrm sorry, but when - that gets implemented it will totally rule ''' - vector = SearchVector('title', weight='A') +\ - SearchVector('subtitle', weight='B') +\ - SearchVector('author_text', weight='C') +\ - SearchVector('isbn_13', weight='A') +\ - SearchVector('isbn_10', weight='A') +\ - SearchVector('openlibrary_key', weight='C') +\ - SearchVector('goodreads_key', weight='C') +\ - SearchVector('asin', weight='C') +\ - SearchVector('oclc_number', weight='C') +\ - SearchVector('remote_id', weight='C') +\ - SearchVector('description', weight='D') +\ - SearchVector('series', weight='D') - - results = models.Edition.objects.annotate( - search=vector - ).annotate( - rank=SearchRank(vector, query) - ).filter( - rank__gt=min_confidence - ).order_by('-rank') - - # remove non-default editions, if possible - results = results.filter(parent_work__default_edition__id=F('id')) \ - or results - + ''' search your local database ''' + if not query: + return [] + # first, try searching unqiue identifiers + results = search_identifiers(query) + if not results: + # then try searching title/author + results = search_title_author(query, min_confidence) search_results = [] - for book in results[:10]: - search_results.append( - self.format_search_result(book) - ) + for result in results: + search_results.append(self.format_search_result(result)) + if len(search_results) >= 10: + break + search_results.sort(key=lambda r: r.confidence, reverse=True) return search_results @@ -51,31 +36,74 @@ class Connector(AbstractConnector): author=search_result.author_text, year=search_result.published_date.year if \ search_result.published_date else None, - confidence=search_result.rank, + connector=self, + confidence=search_result.rank if \ + hasattr(search_result, 'rank') else 1, ) - def get_remote_id_from_data(self, data): - pass - def is_work_data(self, data): pass def get_edition_from_work_data(self, data): pass - def get_work_from_edition_date(self, data): + def get_work_from_edition_data(self, data): pass def get_authors_from_data(self, data): return None - def get_cover_from_data(self, data): - return None - def parse_search_data(self, data): ''' it's already in the right format, don't even worry about it ''' return data def expand_book_data(self, book): pass + + +def search_identifiers(query): + ''' tries remote_id, isbn; defined as dedupe fields on the model ''' + filters = [{f.name: query} for f in models.Edition._meta.get_fields() \ + if hasattr(f, 'deduplication_field') and f.deduplication_field] + results = models.Edition.objects.filter( + reduce(operator.or_, (Q(**f) for f in filters)) + ).distinct() + + # when there are multiple editions of the same work, pick the default. + # it would be odd for this to happen. + return results.filter(parent_work__default_edition__id=F('id')) \ + or results + + +def search_title_author(query, min_confidence): + ''' searches for title and author ''' + vector = SearchVector('title', weight='A') +\ + SearchVector('subtitle', weight='B') +\ + SearchVector('authors__name', weight='C') +\ + SearchVector('series', weight='D') + + results = models.Edition.objects.annotate( + search=vector + ).annotate( + rank=SearchRank(vector, query) + ).filter( + rank__gt=min_confidence + ).order_by('-rank') + + # when there are multiple editions of the same work, pick the closest + editions_of_work = results.values( + 'parent_work' + ).annotate( + Count('parent_work') + ).values_list('parent_work') + + for work_id in set(editions_of_work): + editions = results.filter(parent_work=work_id) + default = editions.filter(parent_work__default_edition=F('id')) + default_rank = default.first().rank if default.exists() else 0 + # if mutliple books have the top rank, pick the default edition + if default_rank == editions.first().rank: + yield default.first() + else: + yield editions.first() diff --git a/bookwyrm/context_processors.py b/bookwyrm/context_processors.py new file mode 100644 index 00000000..a1471ac4 --- /dev/null +++ b/bookwyrm/context_processors.py @@ -0,0 +1,8 @@ +''' customize the info available in context for rendering templates ''' +from bookwyrm import models + +def site_settings(request):# pylint: disable=unused-argument + ''' include the custom info about the site ''' + return { + 'site': models.SiteSettings.objects.get() + } diff --git a/bookwyrm/forms.py b/bookwyrm/forms.py index 784f1038..eafbe407 100644 --- a/bookwyrm/forms.py +++ b/bookwyrm/forms.py @@ -31,10 +31,11 @@ class CustomForm(ModelForm): visible.field.widget.attrs['class'] = css_classes[input_type] +# pylint: disable=missing-class-docstring class LoginForm(CustomForm): class Meta: model = models.User - fields = ['username', 'password'] + fields = ['localname', 'password'] help_texts = {f: None for f in fields} widgets = { 'password': PasswordInput(), @@ -44,7 +45,7 @@ class LoginForm(CustomForm): class RegisterForm(CustomForm): class Meta: model = models.User - fields = ['username', 'email', 'password'] + fields = ['localname', 'email', 'password'] help_texts = {f: None for f in fields} widgets = { 'password': PasswordInput() @@ -60,25 +61,36 @@ class RatingForm(CustomForm): class ReviewForm(CustomForm): class Meta: model = models.Review - fields = ['user', 'book', 'name', 'content', 'rating', 'privacy'] + fields = [ + 'user', 'book', + 'name', 'content', 'rating', + 'content_warning', 'sensitive', + 'privacy'] class CommentForm(CustomForm): class Meta: model = models.Comment - fields = ['user', 'book', 'content', 'privacy'] + fields = [ + 'user', 'book', 'content', + 'content_warning', 'sensitive', + 'privacy'] class QuotationForm(CustomForm): class Meta: model = models.Quotation - fields = ['user', 'book', 'quote', 'content', 'privacy'] + fields = [ + 'user', 'book', 'quote', 'content', + 'content_warning', 'sensitive', 'privacy'] class ReplyForm(CustomForm): class Meta: model = models.Status - fields = ['user', 'content', 'reply_parent', 'privacy'] + fields = [ + 'user', 'content', 'content_warning', 'sensitive', + 'reply_parent', 'privacy'] class EditUserForm(CustomForm): @@ -110,14 +122,14 @@ class EditionForm(CustomForm): model = models.Edition exclude = [ 'remote_id', + 'origin_id', 'created_date', 'updated_date', - 'last_sync_date', + 'edition_rank', 'authors',# TODO 'parent_work', 'shelves', - 'misc_identifiers', 'subjects',# TODO 'subject_places',# TODO @@ -125,12 +137,23 @@ class EditionForm(CustomForm): 'connector', ] +class AuthorForm(CustomForm): + class Meta: + model = models.Author + exclude = [ + 'remote_id', + 'origin_id', + 'created_date', + 'updated_date', + ] + class ImportForm(forms.Form): csv_file = forms.FileField() class ExpiryWidget(widgets.Select): def value_from_datadict(self, data, files, name): + ''' human-readable exiration time buckets ''' selected_string = super().value_from_datadict(data, files, name) if selected_string == 'day': diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index 3fd330ab..5c982764 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -8,8 +8,6 @@ from bookwyrm.models import ImportJob, ImportItem from bookwyrm.status import create_notification logger = logging.getLogger(__name__) -# TODO: remove or increase once we're confident it's not causing problems. -MAX_ENTRIES = 500 def create_job(user, csv_file, include_reviews, privacy): @@ -19,12 +17,13 @@ def create_job(user, csv_file, include_reviews, privacy): include_reviews=include_reviews, privacy=privacy ) - for index, entry in enumerate(list(csv.DictReader(csv_file))[:MAX_ENTRIES]): + for index, entry in enumerate(list(csv.DictReader(csv_file))): if not all(x in entry for x in ('ISBN13', 'Title', 'Author')): raise ValueError('Author, title, and isbn must be in data.') ImportItem(job=job, index=index, data=entry).save() return job + def create_retry_job(user, original_job, items): ''' retry items that didn't import ''' job = ImportJob.objects.create( @@ -37,6 +36,7 @@ def create_retry_job(user, original_job, items): ImportItem(job=job, index=item.index, data=item.data).save() return job + def start_import(job): ''' initalizes a csv import job ''' result = import_data.delay(job.id) @@ -49,11 +49,10 @@ def import_data(job_id): ''' does the actual lookup work in a celery task ''' job = ImportJob.objects.get(id=job_id) try: - results = [] for item in job.items.all(): try: item.resolve() - except Exception as e: + except Exception as e:# pylint: disable=broad-except logger.exception(e) item.fail_reason = 'Error loading book' item.save() @@ -61,7 +60,6 @@ def import_data(job_id): if item.book: item.save() - results.append(item) # shelves book and handles reviews outgoing.handle_imported_book( @@ -71,3 +69,5 @@ def import_data(job_id): item.save() finally: create_notification(job.user, 'IMPORT', related_import=job) + job.complete = True + job.save() diff --git a/bookwyrm/incoming.py b/bookwyrm/incoming.py index 0e7c1856..920e99c5 100644 --- a/bookwyrm/incoming.py +++ b/bookwyrm/incoming.py @@ -1,26 +1,24 @@ ''' handles all of the activity coming in to the server ''' import json -from urllib.parse import urldefrag, unquote_plus +from urllib.parse import urldefrag import django.db.utils from django.http import HttpResponse from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST import requests -from bookwyrm import activitypub, books_manager, models, outgoing +from bookwyrm import activitypub, models, outgoing from bookwyrm import status as status_builder -from bookwyrm.remote_user import get_or_create_remote_user, refresh_remote_user from bookwyrm.tasks import app from bookwyrm.signatures import Signature @csrf_exempt +@require_POST def inbox(request, username): ''' incoming activitypub events ''' - # TODO: should do some kind of checking if the user accepts - # this action from the sender probably? idk - # but this will just throw a 404 if the user doesn't exist try: models.User.objects.get(localname=username) except models.User.DoesNotExist: @@ -30,11 +28,9 @@ def inbox(request, username): @csrf_exempt +@require_POST def shared_inbox(request): ''' incoming activitypub events ''' - if request.method == 'GET': - return HttpResponseNotFound() - try: resp = request.body activity = json.loads(resp) @@ -60,9 +56,7 @@ def shared_inbox(request): 'Like': handle_favorite, 'Announce': handle_boost, 'Add': { - 'Tag': handle_tag, - 'Edition': handle_shelve, - 'Work': handle_shelve, + 'Edition': handle_add, }, 'Undo': { 'Follow': handle_unfollow, @@ -71,8 +65,8 @@ def shared_inbox(request): }, 'Update': { 'Person': handle_update_user, - 'Edition': handle_update_book, - 'Work': handle_update_book, + 'Edition': handle_update_edition, + 'Work': handle_update_work, }, } activity_type = activity['type'] @@ -97,16 +91,20 @@ def has_valid_signature(request, activity): if key_actor != activity.get('actor'): raise ValueError("Wrong actor created signature.") - remote_user = get_or_create_remote_user(key_actor) + remote_user = activitypub.resolve_remote_id(models.User, key_actor) + if not remote_user: + return False try: - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except ValueError: - old_key = remote_user.public_key - refresh_remote_user(remote_user) - if remote_user.public_key == old_key: + old_key = remote_user.key_pair.public_key + remote_user = activitypub.resolve_remote_id( + models.User, remote_user.remote_id, refresh=True + ) + if remote_user.key_pair.public_key == old_key: raise # Key unchanged. - signature.verify(remote_user.public_key, request) + signature.verify(remote_user.key_pair.public_key, request) except (ValueError, requests.exceptions.HTTPError): return False return True @@ -115,26 +113,10 @@ def has_valid_signature(request, activity): @app.task def handle_follow(activity): ''' someone wants to follow a local user ''' - # figure out who they want to follow -- not using get_or_create because - # we only care if you want to follow local users try: - to_follow = models.User.objects.get(remote_id=activity['object']) - except models.User.DoesNotExist: - # some rando, who cares - return - if not to_follow.local: - # just ignore follow alerts about other servers. maybe they should be - # handled. maybe they shouldn't be sent at all. - return - - # figure out who the actor is - actor = get_or_create_remote_user(activity['actor']) - try: - relationship = models.UserFollowRequest.objects.create( - user_subject=actor, - user_object=to_follow, - remote_id=activity['id'] - ) + relationship = activitypub.Follow( + **activity + ).to_model(models.UserFollowRequest) except django.db.utils.IntegrityError as err: if err.__cause__.diag.constraint_name != 'userfollowrequest_unique': raise @@ -143,27 +125,22 @@ def handle_follow(activity): ) # send the accept normally for a duplicate request - if not to_follow.manually_approves_followers: - status_builder.create_notification( - to_follow, - 'FOLLOW', - related_user=actor - ) + manually_approves = relationship.user_object.manually_approves_followers + + status_builder.create_notification( + relationship.user_object, + 'FOLLOW_REQUEST' if manually_approves else 'FOLLOW', + related_user=relationship.user_subject + ) + if not manually_approves: outgoing.handle_accept(relationship) - else: - # Accept will be triggered manually - status_builder.create_notification( - to_follow, - 'FOLLOW_REQUEST', - related_user=actor - ) @app.task def handle_unfollow(activity): ''' unfollow a local user ''' obj = activity['object'] - requester = get_or_create_remote_user(obj['actor']) + requester = activitypub.resolve_remote_id(models.User, obj['actor']) to_unfollow = models.User.objects.get(remote_id=obj['object']) # raises models.User.DoesNotExist @@ -176,7 +153,7 @@ def handle_follow_accept(activity): # figure out who they want to follow requester = models.User.objects.get(remote_id=activity['object']['actor']) # figure out who they are - accepter = get_or_create_remote_user(activity['actor']) + accepter = activitypub.resolve_remote_id(models.User, activity['actor']) try: request = models.UserFollowRequest.objects.get( @@ -193,7 +170,7 @@ def handle_follow_accept(activity): def handle_follow_reject(activity): ''' someone is rejecting a follow request ''' requester = models.User.objects.get(remote_id=activity['object']['actor']) - rejecter = get_or_create_remote_user(activity['actor']) + rejecter = activitypub.resolve_remote_id(models.User, activity['actor']) request = models.UserFollowRequest.objects.get( user_subject=requester, @@ -206,33 +183,49 @@ def handle_follow_reject(activity): @app.task def handle_create(activity): ''' someone did something, good on them ''' - if activity['object'].get('type') not in \ - ['Note', 'Comment', 'Quotation', 'Review', 'GeneratedNote']: - # if it's an article or unknown type, ignore it - return - - user = get_or_create_remote_user(activity['actor']) - if user.local: - # we really oughtn't even be sending in this case - return - # deduplicate incoming activities - status_id = activity['object']['id'] + activity = activity['object'] + status_id = activity.get('id') if models.Status.objects.filter(remote_id=status_id).count(): return - status = status_builder.create_status(activity['object']) + try: + serializer = activitypub.activity_objects[activity['type']] + except KeyError: + return + + activity = serializer(**activity) + try: + model = models.activity_models[activity.type] + except KeyError: + # not a type of status we are prepared to deserialize + return + + status = activity.to_model(model) if not status: + # it was discarded because it's not a bookwyrm type return # create a notification if this is a reply + notified = [] if status.reply_parent and status.reply_parent.user.local: + notified.append(status.reply_parent.user) status_builder.create_notification( status.reply_parent.user, 'REPLY', related_user=status.user, related_status=status, ) + if status.mention_users.exists(): + for mentioned_user in status.mention_users.all(): + if not mentioned_user.local or mentioned_user in notified: + continue + status_builder.create_notification( + mentioned_user, + 'MENTION', + related_user=status.user, + related_status=status, + ) @app.task @@ -245,11 +238,12 @@ def handle_delete_status(activity): # is trying to delete a user. return try: - status = models.Status.objects.select_subclasses().get( + status = models.Status.objects.get( remote_id=status_id ) except models.Status.DoesNotExist: return + models.Notification.objects.filter(related_status=status).all().delete() status_builder.delete_status(status) @@ -257,17 +251,18 @@ def handle_delete_status(activity): def handle_favorite(activity): ''' approval of your good good post ''' fav = activitypub.Like(**activity) - - liker = get_or_create_remote_user(activity['actor']) - if liker.local: + # we dont know this status, we don't care about this status + if not models.Status.objects.filter(remote_id=fav.object).exists(): return fav = fav.to_model(models.Favorite) + if fav.user.local: + return status_builder.create_notification( fav.status.user, 'FAVORITE', - related_user=liker, + related_user=fav.user, related_status=fav.status, ) @@ -312,35 +307,13 @@ def handle_unboost(activity): @app.task -def handle_tag(activity): - ''' someone is tagging a book ''' - user = get_or_create_remote_user(activity['actor']) - if not user.local: - # ordered collection weirndess so we can't just to_model - book = books_manager.get_or_create_book(activity['object']['id']) - name = activity['object']['target'].split('/')[-1] - name = unquote_plus(name) - models.Tag.objects.get_or_create( - user=user, - book=book, - name=name - ) - - -@app.task -def handle_shelve(activity): +def handle_add(activity): ''' putting a book on a shelf ''' - user = get_or_create_remote_user(activity['actor']) - book = books_manager.get_or_create_book(activity['object']) + #this is janky as heck but I haven't thought of a better solution try: - shelf = models.Shelf.objects.get(remote_id=activity['target']) - except models.Shelf.DoesNotExist: - return - if shelf.user != user: - # this doesn't add up. - return - shelf.books.add(book) - shelf.save() + activitypub.AddBook(**activity).to_model(models.ShelfBook) + except activitypub.ActivitySerializerError: + activitypub.AddBook(**activity).to_model(models.Tag) @app.task @@ -358,15 +331,12 @@ def handle_update_user(activity): @app.task -def handle_update_book(activity): +def handle_update_edition(activity): ''' a remote instance changed a book (Document) ''' - document = activity['object'] - # check if we have their copy and care about their updates - book = models.Book.objects.select_subclasses().filter( - remote_id=document['id'], - sync=True, - ).first() - if not book: - return + activitypub.Edition(**activity['object']).to_model(models.Edition) - books_manager.update_book(book, data=document) + +@app.task +def handle_update_work(activity): + ''' a remote instance changed a book (Document) ''' + activitypub.Work(**activity['object']).to_model(models.Work) diff --git a/bookwyrm/management/commands/deduplicate_book_data.py b/bookwyrm/management/commands/deduplicate_book_data.py new file mode 100644 index 00000000..044b2a98 --- /dev/null +++ b/bookwyrm/management/commands/deduplicate_book_data.py @@ -0,0 +1,83 @@ +''' PROCEED WITH CAUTION: uses deduplication fields to permanently +merge book data objects ''' +from django.core.management.base import BaseCommand +from django.db.models import Count +from bookwyrm import models + + +def update_related(canonical, obj): + ''' update all the models with fk to the object being removed ''' + # move related models to canonical + related_models = [ + (r.remote_field.name, r.related_model) for r in \ + canonical._meta.related_objects] + for (related_field, related_model) in related_models: + related_objs = related_model.objects.filter( + **{related_field: obj}) + for related_obj in related_objs: + print( + 'replacing in', + related_model.__name__, + related_field, + related_obj.id + ) + try: + setattr(related_obj, related_field, canonical) + related_obj.save() + except TypeError: + getattr(related_obj, related_field).add(canonical) + getattr(related_obj, related_field).remove(obj) + + +def copy_data(canonical, obj): + ''' try to get the most data possible ''' + for data_field in obj._meta.get_fields(): + if not hasattr(data_field, 'activitypub_field'): + continue + data_value = getattr(obj, data_field.name) + if not data_value: + continue + if not getattr(canonical, data_field.name): + print('setting data field', data_field.name, data_value) + setattr(canonical, data_field.name, data_value) + canonical.save() + + +def dedupe_model(model): + ''' combine duplicate editions and update related models ''' + fields = model._meta.get_fields() + dedupe_fields = [f for f in fields if \ + hasattr(f, 'deduplication_field') and f.deduplication_field] + for field in dedupe_fields: + dupes = model.objects.values(field.name).annotate( + Count(field.name) + ).filter(**{'%s__count__gt' % field.name: 1}) + + for dupe in dupes: + value = dupe[field.name] + if not value or value == '': + continue + print('----------') + print(dupe) + objs = model.objects.filter( + **{field.name: value} + ).order_by('id') + canonical = objs.first() + print('keeping', canonical.remote_id) + for obj in objs[1:]: + print(obj.remote_id) + copy_data(canonical, obj) + update_related(canonical, obj) + # remove the outdated entry + obj.delete() + + +class Command(BaseCommand): + ''' dedplucate allllll the book data models ''' + help = 'merges duplicate book data' + # pylint: disable=no-self-use,unused-argument + def handle(self, *args, **options): + ''' run deudplications ''' + dedupe_model(models.Edition) + dedupe_model(models.Work) + dedupe_model(models.Author) diff --git a/bookwyrm/management/commands/initdb.py b/bookwyrm/management/commands/initdb.py index f29ed102..9fd11787 100644 --- a/bookwyrm/management/commands/initdb.py +++ b/bookwyrm/management/commands/initdb.py @@ -2,7 +2,7 @@ from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import Group, Permission from django.contrib.contenttypes.models import ContentType -from bookwyrm.models import Connector, User +from bookwyrm.models import Connector, SiteSettings, User from bookwyrm.settings import DOMAIN def init_groups(): @@ -73,7 +73,7 @@ def init_connectors(): identifier='bookwyrm.social', name='BookWyrm dot Social', connector_file='bookwyrm_connector', - base_url='https://bookwyrm.social' , + base_url='https://bookwyrm.social', books_url='https://bookwyrm.social/book', covers_url='https://bookwyrm.social/images/covers', search_url='https://bookwyrm.social/search?q=', @@ -91,10 +91,14 @@ def init_connectors(): priority=3, ) +def init_settings(): + SiteSettings.objects.create() + class Command(BaseCommand): help = 'Initializes the database with starter data' def handle(self, *args, **options): init_groups() init_permissions() - init_connectors() \ No newline at end of file + init_connectors() + init_settings() diff --git a/bookwyrm/migrations/0001_initial.py b/bookwyrm/migrations/0001_initial.py index b1aba7df..347057e1 100644 --- a/bookwyrm/migrations/0001_initial.py +++ b/bookwyrm/migrations/0001_initial.py @@ -7,7 +7,7 @@ import django.core.validators from django.db import migrations, models import django.db.models.deletion import django.utils.timezone -import bookwyrm.utils.fields +from django.contrib.postgres.fields import JSONField class Migration(migrations.Migration): @@ -62,7 +62,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], options={ 'abstract': False, @@ -75,7 +75,7 @@ class Migration(migrations.Migration): ('content', models.TextField(blank=True, null=True)), ('created_date', models.DateTimeField(auto_now_add=True)), ('openlibrary_key', models.CharField(max_length=255, unique=True)), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ('cover', models.ImageField(blank=True, null=True, upload_to='covers/')), ('added_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), ('authors', models.ManyToManyField(to='bookwyrm.Author')), diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py index 13cb1406..6a149ab5 100644 --- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py +++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py @@ -2,7 +2,6 @@ import bookwyrm.models.connector import bookwyrm.models.site -import bookwyrm.utils.fields from django.conf import settings import django.contrib.postgres.operations import django.core.validators @@ -10,6 +9,7 @@ from django.db import migrations, models import django.db.models.deletion import django.db.models.expressions import django.utils.timezone +from django.contrib.postgres.fields import JSONField, ArrayField import uuid @@ -148,7 +148,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='misc_identifiers', - field=bookwyrm.utils.fields.JSONField(null=True), + field=JSONField(null=True), ), migrations.AddField( model_name='book', @@ -226,7 +226,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='author', name='aliases', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='user', @@ -394,17 +394,17 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='subject_places', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='book', name='subjects', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', name='publishers', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AlterField( model_name='connector', @@ -578,7 +578,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='book', name='languages', - field=bookwyrm.utils.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), ), migrations.AddField( model_name='edition', @@ -676,7 +676,7 @@ class Migration(migrations.Migration): name='ImportItem', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data', bookwyrm.utils.fields.JSONField()), + ('data', JSONField()), ], ), migrations.CreateModel( diff --git a/bookwyrm/migrations/0016_auto_20201129_0304.py b/bookwyrm/migrations/0016_auto_20201129_0304.py new file mode 100644 index 00000000..1e715969 --- /dev/null +++ b/bookwyrm/migrations/0016_auto_20201129_0304.py @@ -0,0 +1,62 @@ +# Generated by Django 3.0.7 on 2020-11-29 03:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +from django.contrib.postgres.fields import ArrayField + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0015_auto_20201128_0349'), + ] + + operations = [ + migrations.AlterField( + model_name='book', + name='subject_places', + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=models.CharField(max_length=100, unique=True), + ), + migrations.AlterUniqueTogether( + name='tag', + unique_together=set(), + ), + migrations.RemoveField( + model_name='tag', + name='book', + ), + migrations.RemoveField( + model_name='tag', + name='user', + ), + migrations.CreateModel( + name='UserTag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', models.CharField(max_length=255, null=True)), + ('book', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition')), + ('tag', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'book', 'tag')}, + }, + ), + ] diff --git a/bookwyrm/migrations/0016_auto_20201211_2026.py b/bookwyrm/migrations/0016_auto_20201211_2026.py new file mode 100644 index 00000000..46b6140c --- /dev/null +++ b/bookwyrm/migrations/0016_auto_20201211_2026.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.7 on 2020-12-11 20:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0015_auto_20201128_0349'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='admin_email', + field=models.EmailField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='sitesettings', + name='support_link', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='sitesettings', + name='support_title', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/bookwyrm/migrations/0017_auto_20201130_1819.py b/bookwyrm/migrations/0017_auto_20201130_1819.py new file mode 100644 index 00000000..ce9f1cc7 --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201130_1819.py @@ -0,0 +1,189 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:19 + +import bookwyrm.models.base_model +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + +def copy_rsa_keys(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + keypair = app_registry.get_model('bookwyrm', 'KeyPair') + for user in users.objects.using(db_alias): + if user.public_key or user.private_key: + user.key_pair = keypair.objects.create( + remote_id='%s/#main-key' % user.remote_id, + private_key=user.private_key, + public_key=user.public_key + ) + user.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201129_0304'), + ] + operations = [ + migrations.CreateModel( + name='KeyPair', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_date', models.DateTimeField(auto_now_add=True)), + ('updated_date', models.DateTimeField(auto_now=True)), + ('remote_id', bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id])), + ('private_key', models.TextField(blank=True, null=True)), + ('public_key', bookwyrm.models.fields.TextField(blank=True, null=True)), + ], + options={ + 'abstract': False, + }, + bases=(bookwyrm.models.base_model.ActivitypubMixin, models.Model), + ), + migrations.AddField( + model_name='user', + name='followers', + field=bookwyrm.models.fields.ManyToManyField(related_name='following', through='bookwyrm.UserFollows', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='author', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='book', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='connector', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='favorite', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='federatedserver', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='image', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='notification', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='readthrough', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelf', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='shelfbook', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='status', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='tag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='avatar', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='avatars/'), + ), + migrations.AlterField( + model_name='user', + name='bookwyrm_user', + field=bookwyrm.models.fields.BooleanField(default=True), + ), + migrations.AlterField( + model_name='user', + name='inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='local', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='manually_approves_followers', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='user', + name='outbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='shared_inbox', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='user', + name='username', + field=bookwyrm.models.fields.UsernameField(), + ), + migrations.AlterField( + model_name='userblocks', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='userfollows', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AlterField( + model_name='usertag', + name='remote_id', + field=bookwyrm.models.fields.RemoteIdField(max_length=255, null=True, validators=[bookwyrm.models.fields.validate_remote_id]), + ), + migrations.AddField( + model_name='user', + name='key_pair', + field=bookwyrm.models.fields.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owner', to='bookwyrm.KeyPair'), + ), + migrations.RunPython(copy_rsa_keys), + ] diff --git a/bookwyrm/migrations/0017_auto_20201212_0059.py b/bookwyrm/migrations/0017_auto_20201212_0059.py new file mode 100644 index 00000000..c9e3fcf4 --- /dev/null +++ b/bookwyrm/migrations/0017_auto_20201212_0059.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-12 00:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0016_auto_20201211_2026'), + ] + + operations = [ + migrations.AlterField( + model_name='readthrough', + name='book', + field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + ] diff --git a/bookwyrm/migrations/0018_auto_20201130_1832.py b/bookwyrm/migrations/0018_auto_20201130_1832.py new file mode 100644 index 00000000..278446cf --- /dev/null +++ b/bookwyrm/migrations/0018_auto_20201130_1832.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.7 on 2020-11-30 18:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201130_1819'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='following', + ), + migrations.RemoveField( + model_name='user', + name='private_key', + ), + migrations.RemoveField( + model_name='user', + name='public_key', + ), + ] diff --git a/bookwyrm/migrations/0019_auto_20201130_1939.py b/bookwyrm/migrations/0019_auto_20201130_1939.py new file mode 100644 index 00000000..11cf6a3b --- /dev/null +++ b/bookwyrm/migrations/0019_auto_20201130_1939.py @@ -0,0 +1,36 @@ +# Generated by Django 3.0.7 on 2020-11-30 19:39 + +import bookwyrm.models.fields +from django.db import migrations + +def update_notnull(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + users = app_registry.get_model('bookwyrm', 'User') + for user in users.objects.using(db_alias): + if user.name and user.summary: + continue + if not user.summary: + user.summary = '' + if not user.name: + user.name = '' + user.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0018_auto_20201130_1832'), + ] + + operations = [ + migrations.RunPython(update_notnull), + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(default='', max_length=100), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.TextField(default=''), + ), + ] diff --git a/bookwyrm/migrations/0020_auto_20201208_0213.py b/bookwyrm/migrations/0020_auto_20201208_0213.py new file mode 100644 index 00000000..9c5345c7 --- /dev/null +++ b/bookwyrm/migrations/0020_auto_20201208_0213.py @@ -0,0 +1,353 @@ +# Generated by Django 3.0.7 on 2020-12-08 02:13 + +import bookwyrm.models.fields +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0019_auto_20201130_1939'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='aliases', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='author', + name='bio', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='born', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='died', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='author', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='author', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='author', + name='wikipedia_link', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='authors', + field=bookwyrm.models.fields.ManyToManyField(to='bookwyrm.Author'), + ), + migrations.AlterField( + model_name='book', + name='cover', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='covers/'), + ), + migrations.AlterField( + model_name='book', + name='description', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='first_published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='goodreads_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='languages', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='book', + name='librarything_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='openlibrary_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='series', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='series_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='sort_title', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='subject_places', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subjects', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, null=True, size=None), + ), + migrations.AlterField( + model_name='book', + name='subtitle', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='book', + name='title', + field=bookwyrm.models.fields.CharField(max_length=255), + ), + migrations.AlterField( + model_name='boost', + name='boosted_status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='boosters', to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='comment', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='edition', + name='asin', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_10', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='isbn_13', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='oclc_number', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='pages', + field=bookwyrm.models.fields.IntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='edition', + name='parent_work', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='editions', to='bookwyrm.Work'), + ), + migrations.AlterField( + model_name='edition', + name='physical_format', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AlterField( + model_name='edition', + name='publishers', + field=bookwyrm.models.fields.ArrayField(base_field=models.CharField(max_length=255), blank=True, default=list, size=None), + ), + migrations.AlterField( + model_name='favorite', + name='status', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='favorite', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='image', + name='caption', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='image', + name='image', + field=bookwyrm.models.fields.ImageField(blank=True, null=True, upload_to='status/'), + ), + migrations.AlterField( + model_name='quotation', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='quotation', + name='quote', + field=bookwyrm.models.fields.TextField(), + ), + migrations.AlterField( + model_name='review', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='review', + name='name', + field=bookwyrm.models.fields.CharField(max_length=255, null=True), + ), + migrations.AlterField( + model_name='review', + name='rating', + field=bookwyrm.models.fields.IntegerField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(1), django.core.validators.MaxValueValidator(5)]), + ), + migrations.AlterField( + model_name='shelf', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100), + ), + migrations.AlterField( + model_name='shelf', + name='privacy', + field=bookwyrm.models.fields.CharField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255), + ), + migrations.AlterField( + model_name='shelf', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='added_by', + field=bookwyrm.models.fields.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='shelfbook', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='shelfbook', + name='shelf', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Shelf'), + ), + migrations.AlterField( + model_name='status', + name='content', + field=bookwyrm.models.fields.TextField(blank=True, null=True), + ), + migrations.AlterField( + model_name='status', + name='mention_books', + field=bookwyrm.models.fields.TagField(related_name='mention_book', to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='status', + name='mention_users', + field=bookwyrm.models.fields.TagField(related_name='mention_user', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='status', + name='published_date', + field=bookwyrm.models.fields.DateTimeField(default=django.utils.timezone.now), + ), + migrations.AlterField( + model_name='status', + name='reply_parent', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Status'), + ), + migrations.AlterField( + model_name='status', + name='sensitive', + field=bookwyrm.models.fields.BooleanField(default=False), + ), + migrations.AlterField( + model_name='status', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='tag', + name='name', + field=bookwyrm.models.fields.CharField(max_length=100, unique=True), + ), + migrations.AlterField( + model_name='userblocks', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userblocks', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userblocks_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollowrequest', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollowrequest_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_object', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_object', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='userfollows', + name='user_subject', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='userfollows_user_subject', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='usertag', + name='book', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='usertag', + name='tag', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Tag'), + ), + migrations.AlterField( + model_name='usertag', + name='user', + field=bookwyrm.models.fields.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='work', + name='default_edition', + field=bookwyrm.models.fields.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='bookwyrm.Edition'), + ), + migrations.AlterField( + model_name='work', + name='lccn', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/migrations/0021_merge_20201212_1737.py b/bookwyrm/migrations/0021_merge_20201212_1737.py new file mode 100644 index 00000000..4ccf8c8c --- /dev/null +++ b/bookwyrm/migrations/0021_merge_20201212_1737.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0020_auto_20201208_0213'), + ('bookwyrm', '0016_auto_20201211_2026'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0022_auto_20201212_1744.py b/bookwyrm/migrations/0022_auto_20201212_1744.py new file mode 100644 index 00000000..0a98597f --- /dev/null +++ b/bookwyrm/migrations/0022_auto_20201212_1744.py @@ -0,0 +1,30 @@ +# Generated by Django 3.0.7 on 2020-12-12 17:44 + +from django.db import migrations + + +def set_author_name(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + authors = app_registry.get_model('bookwyrm', 'Author') + for author in authors.objects.using(db_alias): + if not author.name: + author.name = '%s %s' % (author.first_name, author.last_name) + author.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0021_merge_20201212_1737'), + ] + + operations = [ + migrations.RunPython(set_author_name), + migrations.RemoveField( + model_name='author', + name='first_name', + ), + migrations.RemoveField( + model_name='author', + name='last_name', + ), + ] diff --git a/bookwyrm/migrations/0023_auto_20201214_0511.py b/bookwyrm/migrations/0023_auto_20201214_0511.py new file mode 100644 index 00000000..e811bded --- /dev/null +++ b/bookwyrm/migrations/0023_auto_20201214_0511.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-14 05:11 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0022_auto_20201212_1744'), + ] + + operations = [ + migrations.AlterField( + model_name='status', + name='privacy', + field=bookwyrm.models.fields.PrivacyField(choices=[('public', 'Public'), ('unlisted', 'Unlisted'), ('followers', 'Followers'), ('direct', 'Direct')], default='public', max_length=255), + ), + ] diff --git a/bookwyrm/migrations/0023_merge_20201216_0112.py b/bookwyrm/migrations/0023_merge_20201216_0112.py new file mode 100644 index 00000000..e3af4849 --- /dev/null +++ b/bookwyrm/migrations/0023_merge_20201216_0112.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-16 01:12 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0017_auto_20201212_0059'), + ('bookwyrm', '0022_auto_20201212_1744'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0024_merge_20201216_1721.py b/bookwyrm/migrations/0024_merge_20201216_1721.py new file mode 100644 index 00000000..41f81335 --- /dev/null +++ b/bookwyrm/migrations/0024_merge_20201216_1721.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.7 on 2020-12-16 17:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0023_auto_20201214_0511'), + ('bookwyrm', '0023_merge_20201216_0112'), + ] + + operations = [ + ] diff --git a/bookwyrm/migrations/0025_auto_20201217_0046.py b/bookwyrm/migrations/0025_auto_20201217_0046.py new file mode 100644 index 00000000..a3ffe8c1 --- /dev/null +++ b/bookwyrm/migrations/0025_auto_20201217_0046.py @@ -0,0 +1,39 @@ +# Generated by Django 3.0.7 on 2020-12-17 00:46 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0024_merge_20201216_1721'), + ] + + operations = [ + migrations.AlterField( + model_name='author', + name='bio', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='book', + name='description', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='quotation', + name='quote', + field=bookwyrm.models.fields.HtmlField(), + ), + migrations.AlterField( + model_name='status', + name='content', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.HtmlField(default=''), + ), + ] diff --git a/bookwyrm/migrations/0026_status_content_warning.py b/bookwyrm/migrations/0026_status_content_warning.py new file mode 100644 index 00000000..f4e494db --- /dev/null +++ b/bookwyrm/migrations/0026_status_content_warning.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-17 03:17 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0025_auto_20201217_0046'), + ] + + operations = [ + migrations.AddField( + model_name='status', + name='content_warning', + field=bookwyrm.models.fields.CharField(blank=True, max_length=500, null=True), + ), + ] diff --git a/bookwyrm/migrations/0027_auto_20201220_2007.py b/bookwyrm/migrations/0027_auto_20201220_2007.py new file mode 100644 index 00000000..a3ad4dda --- /dev/null +++ b/bookwyrm/migrations/0027_auto_20201220_2007.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.7 on 2020-12-20 20:07 + +import bookwyrm.models.fields +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0026_status_content_warning'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='name', + field=bookwyrm.models.fields.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name='user', + name='summary', + field=bookwyrm.models.fields.HtmlField(blank=True, null=True), + ), + ] diff --git a/bookwyrm/migrations/0028_remove_book_author_text.py b/bookwyrm/migrations/0028_remove_book_author_text.py new file mode 100644 index 00000000..8743c910 --- /dev/null +++ b/bookwyrm/migrations/0028_remove_book_author_text.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.7 on 2020-12-21 19:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0027_auto_20201220_2007'), + ] + + operations = [ + migrations.RemoveField( + model_name='book', + name='author_text', + ), + ] diff --git a/bookwyrm/migrations/0029_auto_20201221_2014.py b/bookwyrm/migrations/0029_auto_20201221_2014.py new file mode 100644 index 00000000..ebf27a74 --- /dev/null +++ b/bookwyrm/migrations/0029_auto_20201221_2014.py @@ -0,0 +1,61 @@ +# Generated by Django 3.0.7 on 2020-12-21 20:14 + +import bookwyrm.models.fields +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0028_remove_book_author_text'), + ] + + operations = [ + migrations.RemoveField( + model_name='author', + name='last_sync_date', + ), + migrations.RemoveField( + model_name='author', + name='sync', + ), + migrations.RemoveField( + model_name='book', + name='last_sync_date', + ), + migrations.RemoveField( + model_name='book', + name='sync', + ), + migrations.RemoveField( + model_name='book', + name='sync_cover', + ), + migrations.AddField( + model_name='author', + name='goodreads_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='author', + name='last_edited_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='author', + name='librarything_key', + field=bookwyrm.models.fields.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='last_edited_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='author', + name='origin_id', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/bookwyrm/migrations/0030_auto_20201224_1939.py b/bookwyrm/migrations/0030_auto_20201224_1939.py new file mode 100644 index 00000000..6de5d37f --- /dev/null +++ b/bookwyrm/migrations/0030_auto_20201224_1939.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.7 on 2020-12-24 19:39 + +import bookwyrm.models.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0029_auto_20201221_2014'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='localname', + field=models.CharField(max_length=255, null=True, unique=True, validators=[bookwyrm.models.fields.validate_localname]), + ), + ] diff --git a/bookwyrm/migrations/0031_auto_20210104_2040.py b/bookwyrm/migrations/0031_auto_20210104_2040.py new file mode 100644 index 00000000..604392d4 --- /dev/null +++ b/bookwyrm/migrations/0031_auto_20210104_2040.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.7 on 2021-01-04 20:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0030_auto_20201224_1939'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='favicon', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + migrations.AddField( + model_name='sitesettings', + name='logo', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + migrations.AddField( + model_name='sitesettings', + name='logo_small', + field=models.ImageField(blank=True, null=True, upload_to='logos/'), + ), + ] diff --git a/bookwyrm/migrations/0032_auto_20210104_2055.py b/bookwyrm/migrations/0032_auto_20210104_2055.py new file mode 100644 index 00000000..692cd581 --- /dev/null +++ b/bookwyrm/migrations/0032_auto_20210104_2055.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.7 on 2021-01-04 20:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0031_auto_20210104_2040'), + ] + + operations = [ + migrations.AddField( + model_name='sitesettings', + name='instance_tagline', + field=models.CharField(default='Social Reading and Reviewing', max_length=150), + ), + migrations.AddField( + model_name='sitesettings', + name='registration_closed_text', + field=models.TextField(default='Contact an administrator to get an invite'), + ), + ] diff --git a/bookwyrm/migrations/0033_siteinvite_created_date.py b/bookwyrm/migrations/0033_siteinvite_created_date.py new file mode 100644 index 00000000..9a3f9896 --- /dev/null +++ b/bookwyrm/migrations/0033_siteinvite_created_date.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.7 on 2021-01-05 19:08 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0032_auto_20210104_2055'), + ] + + operations = [ + migrations.AddField( + model_name='siteinvite', + name='created_date', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + ] diff --git a/bookwyrm/migrations/0034_importjob_complete.py b/bookwyrm/migrations/0034_importjob_complete.py new file mode 100644 index 00000000..14170607 --- /dev/null +++ b/bookwyrm/migrations/0034_importjob_complete.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2021-01-07 16:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0033_siteinvite_created_date'), + ] + + operations = [ + migrations.AddField( + model_name='importjob', + name='complete', + field=models.BooleanField(default=False), + ), + ] diff --git a/bookwyrm/migrations/0035_edition_edition_rank.py b/bookwyrm/migrations/0035_edition_edition_rank.py new file mode 100644 index 00000000..6ccb2142 --- /dev/null +++ b/bookwyrm/migrations/0035_edition_edition_rank.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.7 on 2021-01-11 17:18 + +import bookwyrm.models.fields +from django.db import migrations + + +def set_rank(app_registry, schema_editor): + db_alias = schema_editor.connection.alias + books = app_registry.get_model('bookwyrm', 'Edition') + for book in books.objects.using(db_alias): + book.edition_rank = book.get_rank + book.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('bookwyrm', '0034_importjob_complete'), + ] + + operations = [ + migrations.AddField( + model_name='edition', + name='edition_rank', + field=bookwyrm.models.fields.IntegerField(default=0), + ), + migrations.RunPython(set_rank), + ] diff --git a/bookwyrm/models/__init__.py b/bookwyrm/models/__init__.py index 96006135..86211aca 100644 --- a/bookwyrm/models/__init__.py +++ b/bookwyrm/models/__init__.py @@ -2,19 +2,22 @@ import inspect import sys -from .book import Book, Work, Edition +from .book import Book, Work, Edition, BookDataModel from .author import Author from .connector import Connector from .shelf import Shelf, ShelfBook from .status import Status, GeneratedNote, Review, Comment, Quotation -from .status import Favorite, Boost, Notification, ReadThrough, ProgressMode, ProgressUpdate +from .status import Boost from .attachment import Image +from .favorite import Favorite +from .notification import Notification +from .readthrough import ReadThrough, ProgressUpdate, ProgressMode -from .tag import Tag +from .tag import Tag, UserTag -from .user import User +from .user import User, KeyPair from .relationship import UserFollows, UserFollowRequest, UserBlocks from .federated_server import FederatedServer @@ -25,3 +28,6 @@ from .site import SiteSettings, SiteInvite, PasswordReset cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass) activity_models = {c[1].activity_serializer.__name__: c[1] \ for c in cls_members if hasattr(c[1], 'activity_serializer')} + +status_models = [ + c.__name__ for (_, c) in activity_models.items() if issubclass(c, Status)] diff --git a/bookwyrm/models/attachment.py b/bookwyrm/models/attachment.py index 7329e65d..b3337e15 100644 --- a/bookwyrm/models/attachment.py +++ b/bookwyrm/models/attachment.py @@ -3,7 +3,8 @@ from django.db import models from bookwyrm import activitypub from .base_model import ActivitypubMixin -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel +from . import fields class Attachment(ActivitypubMixin, BookWyrmModel): @@ -14,19 +15,16 @@ class Attachment(ActivitypubMixin, BookWyrmModel): related_name='attachments', null=True ) + reverse_unfurl = True class Meta: ''' one day we'll have other types of attachments besides images ''' abstract = True - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('url', 'image'), - ActivityMapping('name', 'caption'), - ] class Image(Attachment): ''' an image attachment ''' - image = models.ImageField(upload_to='status/', null=True, blank=True) - caption = models.TextField(null=True, blank=True) + image = fields.ImageField( + upload_to='status/', null=True, blank=True, activitypub_field='url') + caption = fields.TextField(null=True, blank=True, activitypub_field='name') activity_serializer = activitypub.Image diff --git a/bookwyrm/models/author.py b/bookwyrm/models/author.py index 1d701797..d0cb8d19 100644 --- a/bookwyrm/models/author.py +++ b/bookwyrm/models/author.py @@ -1,50 +1,28 @@ ''' database schema for info about authors ''' from django.db import models -from django.utils import timezone from bookwyrm import activitypub -from bookwyrm.utils.fields import ArrayField +from bookwyrm.settings import DOMAIN -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .book import BookDataModel +from . import fields -class Author(ActivitypubMixin, BookWyrmModel): +class Author(BookDataModel): ''' basic biographic info ''' - origin_id = models.CharField(max_length=255, null=True) - ''' copy of an author from OL ''' - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) - sync = models.BooleanField(default=True) - last_sync_date = models.DateTimeField(default=timezone.now) - wikipedia_link = models.CharField(max_length=255, blank=True, null=True) + wikipedia_link = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # idk probably other keys would be useful here? - born = models.DateTimeField(blank=True, null=True) - died = models.DateTimeField(blank=True, null=True) - name = models.CharField(max_length=255) - last_name = models.CharField(max_length=255, blank=True, null=True) - first_name = models.CharField(max_length=255, blank=True, null=True) - aliases = ArrayField( + born = fields.DateTimeField(blank=True, null=True) + died = fields.DateTimeField(blank=True, null=True) + name = fields.CharField(max_length=255, deduplication_field=True) + aliases = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - bio = models.TextField(null=True, blank=True) + bio = fields.HtmlField(null=True, blank=True) - @property - def display_name(self): - ''' Helper to return a displayable name''' - if self.name: - return self.name - # don't want to return a spurious space if all of these are None - if self.first_name and self.last_name: - return self.first_name + ' ' + self.last_name - return self.last_name or self.first_name + def get_remote_id(self): + ''' editions and works both use "book" instead of model_name ''' + return 'https://%s/author/%s' % (DOMAIN, self.id) - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('name', 'name'), - ActivityMapping('born', 'born'), - ActivityMapping('died', 'died'), - ActivityMapping('aliases', 'aliases'), - ActivityMapping('bio', 'bio'), - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('wikipediaLink', 'wikipedia_link'), - ] activity_serializer = activitypub.Author diff --git a/bookwyrm/models/base_model.py b/bookwyrm/models/base_model.py index 4109a49b..430097c0 100644 --- a/bookwyrm/models/base_model.py +++ b/bookwyrm/models/base_model.py @@ -1,34 +1,27 @@ ''' base model with default fields ''' -from datetime import datetime from base64 import b64encode -from dataclasses import dataclass -from typing import Callable +from functools import reduce +import operator from uuid import uuid4 -from urllib.parse import urlencode from Crypto.PublicKey import RSA from Crypto.Signature import pkcs1_15 from Crypto.Hash import SHA256 +from django.core.paginator import Paginator from django.db import models -from django.db.models.fields.files import ImageFieldFile +from django.db.models import Q from django.dispatch import receiver from bookwyrm import activitypub -from bookwyrm.settings import DOMAIN +from bookwyrm.settings import DOMAIN, PAGE_LENGTH +from .fields import ImageField, ManyToManyField, RemoteIdField -PrivacyLevels = models.TextChoices('Privacy', [ - 'public', - 'unlisted', - 'followers', - 'direct' -]) - class BookWyrmModel(models.Model): ''' shared fields ''' created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) - remote_id = models.CharField(max_length=255, null=True) + remote_id = RemoteIdField(null=True, activitypub_field='id') def get_remote_id(self): ''' generate a url that resolves to the local object ''' @@ -42,8 +35,14 @@ class BookWyrmModel(models.Model): ''' this is just here to provide default fields for other models ''' abstract = True + @property + def local_path(self): + ''' how to link to this object in the local app ''' + return self.get_remote_id().replace('https://%s' % DOMAIN, '') + @receiver(models.signals.post_save) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' set the remote_id after save (when the id is available) ''' if not created or not hasattr(instance, 'get_remote_id'): @@ -53,58 +52,116 @@ def execute_after_save(sender, instance, created, *args, **kwargs): instance.save() +def unfurl_related_field(related_field, sort_field=None): + ''' load reverse lookups (like public key owner or Status attachment ''' + if hasattr(related_field, 'all'): + return [unfurl_related_field(i) for i in related_field.order_by( + sort_field).all()] + if related_field.reverse_unfurl: + return related_field.field_to_activity() + return related_field.remote_id + + class ActivitypubMixin: ''' add this mixin for models that are AP serializable ''' activity_serializer = lambda: {} + reverse_unfurl = False - def to_activity(self, pure=False): - ''' convert from a model to an activity ''' - if pure: - # works around bookwyrm-specific fields for vanilla AP services - mappings = self.pure_activity_mappings - else: - # may include custom fields that bookwyrm instances will understand - mappings = self.activity_mappings - - fields = {} - for mapping in mappings: - if not hasattr(self, mapping.model_key) or not mapping.activity_key: - # this field on the model isn't serialized + def __init__(self, *args, **kwargs): + ''' collect some info on model fields ''' + self.image_fields = [] + self.many_to_many_fields = [] + self.simple_fields = [] # "simple" + for field in self._meta.get_fields(): + if not hasattr(field, 'field_to_activity'): continue - value = getattr(self, mapping.model_key) - if hasattr(value, 'remote_id'): - # this is probably a foreign key field, which we want to - # serialize as just the remote_id url reference - value = value.remote_id - elif isinstance(value, datetime): - value = value.isoformat() - elif isinstance(value, ImageFieldFile): - value = image_formatter(value) - # run the custom formatter function set in the model - formatted_value = mapping.activity_formatter(value) - if mapping.activity_key in fields and \ - isinstance(fields[mapping.activity_key], list): - # there can be two database fields that map to the same AP list - # this happens in status tags, which combines user and book tags - fields[mapping.activity_key] += formatted_value + if isinstance(field, ImageField): + self.image_fields.append(field) + elif isinstance(field, ManyToManyField): + self.many_to_many_fields.append(field) else: - fields[mapping.activity_key] = formatted_value + self.simple_fields.append(field) - if pure: - return self.pure_activity_serializer( - **fields - ).serialize() - return self.activity_serializer( - **fields - ).serialize() + self.activity_fields = self.image_fields + \ + self.many_to_many_fields + self.simple_fields + + self.deserialize_reverse_fields = self.deserialize_reverse_fields \ + if hasattr(self, 'deserialize_reverse_fields') else [] + self.serialize_reverse_fields = self.serialize_reverse_fields \ + if hasattr(self, 'serialize_reverse_fields') else [] + + super().__init__(*args, **kwargs) - def to_create_activity(self, user, pure=False): + @classmethod + def find_existing_by_remote_id(cls, remote_id): + ''' look up a remote id in the db ''' + return cls.find_existing({'id': remote_id}) + + @classmethod + def find_existing(cls, data): + ''' compare data to fields that can be used for deduplation. + This always includes remote_id, but can also be unique identifiers + like an isbn for an edition ''' + filters = [] + for field in cls._meta.get_fields(): + if not hasattr(field, 'deduplication_field') or \ + not field.deduplication_field: + continue + + value = data.get(field.get_activitypub_field()) + if not value: + continue + filters.append({field.name: value}) + + if hasattr(cls, 'origin_id') and 'id' in data: + # kinda janky, but this handles special case for books + filters.append({'origin_id': data['id']}) + + if not filters: + # if there are no deduplication fields, it will match the first + # item no matter what. this shouldn't happen but just in case. + return None + + objects = cls.objects + if hasattr(objects, 'select_subclasses'): + objects = objects.select_subclasses() + + # an OR operation on all the match fields + match = objects.filter( + reduce( + operator.or_, (Q(**f) for f in filters) + ) + ) + # there OUGHT to be only one match + return match.first() + + + def to_activity(self): + ''' convert from a model to an activity ''' + activity = {} + for field in self.activity_fields: + field.set_activity_from_field(activity, self) + + if hasattr(self, 'serialize_reverse_fields'): + # for example, editions of a work + for model_field_name, activity_field_name, sort_field in \ + self.serialize_reverse_fields: + related_field = getattr(self, model_field_name) + activity[activity_field_name] = \ + unfurl_related_field(related_field, sort_field) + + if not activity.get('id'): + activity['id'] = self.get_remote_id() + return self.activity_serializer(**activity).serialize() + + + def to_create_activity(self, user, **kwargs): ''' returns the object wrapped in a Create activity ''' - activity_object = self.to_activity(pure=pure) + activity_object = self.to_activity(**kwargs) - signer = pkcs1_15.new(RSA.import_key(user.private_key)) + signer = pkcs1_15.new(RSA.import_key(user.key_pair.private_key)) content = activity_object['content'] signed_message = signer.sign(SHA256.new(content.encode('utf8'))) create_id = self.remote_id + '/activity' @@ -118,8 +175,8 @@ class ActivitypubMixin: return activitypub.Create( id=create_id, actor=user.remote_id, - to=['%s/followers' % user.remote_id], - cc=['https://www.w3.org/ns/activitystreams#Public'], + to=activity_object['to'], + cc=activity_object['cc'], object=activity_object, signature=signature, ).serialize() @@ -127,21 +184,18 @@ class ActivitypubMixin: def to_delete_activity(self, user): ''' notice of deletion ''' - # this should be a tombstone - activity_object = self.to_activity() - return activitypub.Delete( id=self.remote_id + '/activity', actor=user.remote_id, to=['%s/followers' % user.remote_id], cc=['https://www.w3.org/ns/activitystreams#Public'], - object=activity_object, + object=self.to_activity(), ).serialize() def to_update_activity(self, user): ''' wrapper for Updates to an activity ''' - activity_id = '%s#update/%s' % (user.remote_id, uuid4()) + activity_id = '%s#update/%s' % (self.remote_id, uuid4()) return activitypub.Update( id=activity_id, actor=user.remote_id, @@ -153,10 +207,10 @@ class ActivitypubMixin: def to_undo_activity(self, user): ''' undo an action ''' return activitypub.Undo( - id='%s#undo' % user.remote_id, + id='%s#undo' % self.remote_id, actor=user.remote_id, object=self.to_activity() - ) + ).serialize() class OrderedCollectionPageMixin(ActivitypubMixin): @@ -167,77 +221,55 @@ class OrderedCollectionPageMixin(ActivitypubMixin): ''' this can be overriden if there's a special remote id, ie outbox ''' return self.remote_id - def page(self, min_id=None, max_id=None): - ''' helper function to create the pagination url ''' - params = {'page': 'true'} - if min_id: - params['min_id'] = min_id - if max_id: - params['max_id'] = max_id - return '?%s' % urlencode(params) - - def next_page(self, items): - ''' use the max id of the last item ''' - if not items.count(): - return '' - return self.page(max_id=items[items.count() - 1].id) - - def prev_page(self, items): - ''' use the min id of the first item ''' - if not items.count(): - return '' - return self.page(min_id=items[0].id) - - def to_ordered_collection_page(self, queryset, remote_id, \ - id_only=False, min_id=None, max_id=None): - ''' serialize and pagiante a queryset ''' - # TODO: weird place to define this - limit = 20 - # filters for use in the django queryset min/max - filters = {} - if min_id is not None: - filters['id__gt'] = min_id - if max_id is not None: - filters['id__lte'] = max_id - page_id = self.page(min_id=min_id, max_id=max_id) - - items = queryset.filter( - **filters - ).all()[:limit] - - if id_only: - page = [s.remote_id for s in items] - else: - page = [s.to_activity() for s in items] - return activitypub.OrderedCollectionPage( - id='%s%s' % (remote_id, page_id), - partOf=remote_id, - orderedItems=page, - next='%s%s' % (remote_id, self.next_page(items)), - prev='%s%s' % (remote_id, self.prev_page(items)) - ).serialize() def to_ordered_collection(self, queryset, \ remote_id=None, page=False, **kwargs): ''' an ordered collection of whatevers ''' remote_id = remote_id or self.remote_id if page: - return self.to_ordered_collection_page( + return to_ordered_collection_page( queryset, remote_id, **kwargs) - name = '' - if hasattr(self, 'name'): - name = self.name + name = self.name if hasattr(self, 'name') else None + owner = self.user.remote_id if hasattr(self, 'user') else '' - size = queryset.count() + paginated = Paginator(queryset, PAGE_LENGTH) return activitypub.OrderedCollection( id=remote_id, - totalItems=size, + totalItems=paginated.count, name=name, - first='%s%s' % (remote_id, self.page()), - last='%s%s' % (remote_id, self.page(min_id=0)) + owner=owner, + first='%s?page=1' % remote_id, + last='%s?page=%d' % (remote_id, paginated.num_pages) ).serialize() +# pylint: disable=unused-argument +def to_ordered_collection_page( + queryset, remote_id, id_only=False, page=1, **kwargs): + ''' serialize and pagiante a queryset ''' + paginated = Paginator(queryset, PAGE_LENGTH) + + activity_page = paginated.page(page) + if id_only: + items = [s.remote_id for s in activity_page.object_list] + else: + items = [s.to_activity() for s in activity_page.object_list] + + prev_page = next_page = None + if activity_page.has_next(): + next_page = '%s?page=%d' % (remote_id, activity_page.next_page_number()) + if activity_page.has_previous(): + prev_page = '%s?page=%d' % \ + (remote_id, activity_page.previous_page_number()) + return activitypub.OrderedCollectionPage( + id='%s?page=%s' % (remote_id, page), + partOf=remote_id, + orderedItems=items, + next=next_page, + prev=prev_page + ).serialize() + + class OrderedCollectionMixin(OrderedCollectionPageMixin): ''' extends activitypub models to work as ordered collections ''' @property @@ -250,39 +282,3 @@ class OrderedCollectionMixin(OrderedCollectionPageMixin): def to_activity(self, **kwargs): ''' an ordered collection of the specified model queryset ''' return self.to_ordered_collection(self.collection_queryset, **kwargs) - - -@dataclass(frozen=True) -class ActivityMapping: - ''' translate between an activitypub json field and a model field ''' - activity_key: str - model_key: str - activity_formatter: Callable = lambda x: x - model_formatter: Callable = lambda x: x - - -def tag_formatter(items, name_field, activity_type): - ''' helper function to format lists of foreign keys into Tags ''' - tags = [] - for item in items.all(): - tags.append(activitypub.Link( - href=item.remote_id, - name=getattr(item, name_field), - type=activity_type - )) - return tags - - -def image_formatter(image): - ''' convert images into activitypub json ''' - if image and hasattr(image, 'url'): - url = image.url - else: - return None - url = 'https://%s%s' % (DOMAIN, url) - return activitypub.Image(url=url) - - -def image_attachments_formatter(images): - ''' create a list of image attachments ''' - return [image_formatter(i) for i in images] diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index 236f0449..14aec9cb 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -2,111 +2,104 @@ import re from django.db import models -from django.db.models import Q -from django.utils import timezone from model_utils.managers import InheritanceManager from bookwyrm import activitypub from bookwyrm.settings import DOMAIN -from bookwyrm.utils.fields import ArrayField -from .base_model import ActivityMapping, BookWyrmModel +from .base_model import BookWyrmModel from .base_model import ActivitypubMixin, OrderedCollectionPageMixin +from . import fields -class Book(ActivitypubMixin, BookWyrmModel): - ''' a generic book, which can mean either an edition or a work ''' +class BookDataModel(ActivitypubMixin, BookWyrmModel): + ''' fields shared between editable book data (books, works, authors) ''' origin_id = models.CharField(max_length=255, null=True, blank=True) - # these identifiers apply to both works and editions - openlibrary_key = models.CharField(max_length=255, blank=True, null=True) - librarything_key = models.CharField(max_length=255, blank=True, null=True) - goodreads_key = models.CharField(max_length=255, blank=True, null=True) + openlibrary_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + librarything_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + goodreads_key = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) - # info about where the data comes from and where/if to sync - sync = models.BooleanField(default=True) - sync_cover = models.BooleanField(default=True) - last_sync_date = models.DateTimeField(default=timezone.now) + last_edited_by = models.ForeignKey( + 'User', on_delete=models.PROTECT, null=True) + + class Meta: + ''' can't initialize this model, that wouldn't make sense ''' + abstract = True + + def save(self, *args, **kwargs): + ''' ensure that the remote_id is within this instance ''' + if self.id: + self.remote_id = self.get_remote_id() + else: + self.origin_id = self.remote_id + self.remote_id = None + return super().save(*args, **kwargs) + + +class Book(BookDataModel): + ''' a generic book, which can mean either an edition or a work ''' connector = models.ForeignKey( 'Connector', on_delete=models.PROTECT, null=True) - # TODO: edit history - # book/work metadata - title = models.CharField(max_length=255) - sort_title = models.CharField(max_length=255, blank=True, null=True) - subtitle = models.CharField(max_length=255, blank=True, null=True) - description = models.TextField(blank=True, null=True) - languages = ArrayField( + title = fields.CharField(max_length=255) + sort_title = fields.CharField(max_length=255, blank=True, null=True) + subtitle = fields.CharField(max_length=255, blank=True, null=True) + description = fields.HtmlField(blank=True, null=True) + languages = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) - series = models.CharField(max_length=255, blank=True, null=True) - series_number = models.CharField(max_length=255, blank=True, null=True) - subjects = ArrayField( - models.CharField(max_length=255), blank=True, default=list + series = fields.CharField(max_length=255, blank=True, null=True) + series_number = fields.CharField(max_length=255, blank=True, null=True) + subjects = fields.ArrayField( + models.CharField(max_length=255), blank=True, null=True, default=list ) - subject_places = ArrayField( - models.CharField(max_length=255), blank=True, default=list + subject_places = fields.ArrayField( + models.CharField(max_length=255), blank=True, null=True, default=list ) - # TODO: include an annotation about the type of authorship (ie, translator) - authors = models.ManyToManyField('Author') - # preformatted authorship string for search and easier display - author_text = models.CharField(max_length=255, blank=True, null=True) - cover = models.ImageField(upload_to='covers/', blank=True, null=True) - first_published_date = models.DateTimeField(blank=True, null=True) - published_date = models.DateTimeField(blank=True, null=True) + authors = fields.ManyToManyField('Author') + cover = fields.ImageField( + upload_to='covers/', blank=True, null=True, alt_field='alt_text') + first_published_date = fields.DateTimeField(blank=True, null=True) + published_date = fields.DateTimeField(blank=True, null=True) + objects = InheritanceManager() @property - def ap_authors(self): - ''' the activitypub serialization should be a list of author ids ''' - return [a.remote_id for a in self.authors.all()] + def author_text(self): + ''' format a list of authors ''' + return ', '.join(a.name for a in self.authors.all()) @property def latest_readthrough(self): return self.readthrough_set.order_by('-updated_date').first() - activity_mappings = [ - ActivityMapping('id', 'remote_id'), + @property + def edition_info(self): + ''' properties of this edition, as a string ''' + items = [ + self.physical_format if hasattr(self, 'physical_format') else None, + self.languages[0] + ' language' if self.languages and \ + self.languages[0] != 'English' else None, + str(self.published_date.year) if self.published_date else None, + ] + return ', '.join(i for i in items if i) - ActivityMapping('authors', 'ap_authors'), - ActivityMapping('firstPublishedDate', 'firstpublished_date'), - ActivityMapping('publishedDate', 'published_date'), - - ActivityMapping('title', 'title'), - ActivityMapping('sortTitle', 'sort_title'), - ActivityMapping('subtitle', 'subtitle'), - ActivityMapping('description', 'description'), - ActivityMapping('languages', 'languages'), - ActivityMapping('series', 'series'), - ActivityMapping('seriesNumber', 'series_number'), - ActivityMapping('subjects', 'subjects'), - ActivityMapping('subjectPlaces', 'subject_places'), - - ActivityMapping('openlibraryKey', 'openlibrary_key'), - ActivityMapping('librarythingKey', 'librarything_key'), - ActivityMapping('goodreadsKey', 'goodreads_key'), - - ActivityMapping('work', 'parent_work'), - ActivityMapping('isbn10', 'isbn_10'), - ActivityMapping('isbn13', 'isbn_13'), - ActivityMapping('oclcNumber', 'oclc_number'), - ActivityMapping('asin', 'asin'), - ActivityMapping('pages', 'pages'), - ActivityMapping('physicalFormat', 'physical_format'), - ActivityMapping('publishers', 'publishers'), - - ActivityMapping('lccn', 'lccn'), - ActivityMapping('editions', 'editions_path'), - ActivityMapping('cover', 'cover'), - ] + @property + def alt_text(self): + ''' image alt test ''' + text = '%s cover' % self.title + if self.edition_info: + text += ' (%s)' % self.edition_info + return text def save(self, *args, **kwargs): ''' can't be abstract for query reasons, but you shouldn't USE it ''' if not isinstance(self, Edition) and not isinstance(self, Work): raise ValueError('Books should be added as Editions or Works') - if self.id and not self.remote_id: - self.remote_id = self.get_remote_id() - - super().save(*args, **kwargs) + return super().save(*args, **kwargs) def get_remote_id(self): ''' editions and works both use "book" instead of model_name ''' @@ -123,47 +116,56 @@ class Book(ActivitypubMixin, BookWyrmModel): class Work(OrderedCollectionPageMixin, Book): ''' a work (an abstract concept of a book that manifests in an edition) ''' # library of congress catalog control number - lccn = models.CharField(max_length=255, blank=True, null=True) + lccn = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) # this has to be nullable but should never be null - default_edition = models.ForeignKey( + default_edition = fields.ForeignKey( 'Edition', on_delete=models.PROTECT, - null=True + null=True, + load_remote=False ) - @property - def editions_path(self): - ''' it'd be nice to serialize the edition instead but, recursion ''' - default = self.default_edition - ed_list = [ - e.remote_id for e in self.edition_set.filter(~Q(id=default.id)).all() - ] - return [default.remote_id] + ed_list + def save(self, *args, **kwargs): + ''' set some fields on the edition object ''' + # set rank + for edition in self.editions.all(): + edition.save() + return super().save(*args, **kwargs) + def get_default_edition(self): + ''' in case the default edition is not set ''' + return self.default_edition or self.editions.order_by( + '-edition_rank' + ).first() def to_edition_list(self, **kwargs): - ''' activitypub serialization for this work's editions ''' - remote_id = self.remote_id + '/editions' + ''' an ordered collection of editions ''' return self.to_ordered_collection( - self.edition_set, - remote_id=remote_id, + self.editions.order_by('-edition_rank').all(), + remote_id='%s/editions' % self.remote_id, **kwargs ) - activity_serializer = activitypub.Work + serialize_reverse_fields = [('editions', 'editions', '-edition_rank')] + deserialize_reverse_fields = [('editions', 'editions')] class Edition(Book): ''' an edition of a book ''' # these identifiers only apply to editions, not works - isbn_10 = models.CharField(max_length=255, blank=True, null=True) - isbn_13 = models.CharField(max_length=255, blank=True, null=True) - oclc_number = models.CharField(max_length=255, blank=True, null=True) - asin = models.CharField(max_length=255, blank=True, null=True) - pages = models.IntegerField(blank=True, null=True) - physical_format = models.CharField(max_length=255, blank=True, null=True) - publishers = ArrayField( + isbn_10 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + isbn_13 = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + oclc_number = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + asin = fields.CharField( + max_length=255, blank=True, null=True, deduplication_field=True) + pages = fields.IntegerField(blank=True, null=True) + physical_format = fields.CharField(max_length=255, blank=True, null=True) + publishers = fields.ArrayField( models.CharField(max_length=255), blank=True, default=list ) shelves = models.ManyToManyField( @@ -172,17 +174,42 @@ class Edition(Book): through='ShelfBook', through_fields=('book', 'shelf') ) - parent_work = models.ForeignKey('Work', on_delete=models.PROTECT, null=True) + parent_work = fields.ForeignKey( + 'Work', on_delete=models.PROTECT, null=True, + related_name='editions', activitypub_field='work') + edition_rank = fields.IntegerField(default=0) activity_serializer = activitypub.Edition + name_field = 'title' + + @property + def get_rank(self): + ''' calculate how complete the data is on this edition ''' + if self.parent_work and self.parent_work.default_edition == self: + # default edition has the highest rank + return 20 + rank = 0 + rank += int(bool(self.cover)) * 3 + rank += int(bool(self.isbn_13)) + rank += int(bool(self.isbn_10)) + rank += int(bool(self.oclc_number)) + rank += int(bool(self.pages)) + rank += int(bool(self.physical_format)) + rank += int(bool(self.description)) + # max rank is 9 + return rank def save(self, *args, **kwargs): - ''' calculate isbn 10/13 ''' + ''' set some fields on the edition object ''' + # calculate isbn 10/13 if self.isbn_13 and self.isbn_13[:3] == '978' and not self.isbn_10: self.isbn_10 = isbn_13_to_10(self.isbn_13) if self.isbn_10 and not self.isbn_13: self.isbn_13 = isbn_10_to_13(self.isbn_10) + # set rank + self.edition_rank = self.get_rank + return super().save(*args, **kwargs) diff --git a/bookwyrm/models/favorite.py b/bookwyrm/models/favorite.py new file mode 100644 index 00000000..8373b016 --- /dev/null +++ b/bookwyrm/models/favorite.py @@ -0,0 +1,26 @@ +''' like/fav/star a status ''' +from django.db import models +from django.utils import timezone + +from bookwyrm import activitypub +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields + +class Favorite(ActivitypubMixin, BookWyrmModel): + ''' fav'ing a post ''' + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + status = fields.ForeignKey( + 'Status', on_delete=models.PROTECT, activitypub_field='object') + + activity_serializer = activitypub.Like + + def save(self, *args, **kwargs): + ''' update user active time ''' + self.user.last_active_date = timezone.now() + self.user.save() + super().save(*args, **kwargs) + + class Meta: + ''' can't fav things twice ''' + unique_together = ('user', 'status') diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py new file mode 100644 index 00000000..c6571ff4 --- /dev/null +++ b/bookwyrm/models/fields.py @@ -0,0 +1,427 @@ +''' activitypub-aware django model fields ''' +from dataclasses import MISSING +import re +from uuid import uuid4 + +import dateutil.parser +from dateutil.parser import ParserError +from django.contrib.postgres.fields import ArrayField as DjangoArrayField +from django.core.exceptions import ValidationError +from django.core.files.base import ContentFile +from django.db import models +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from bookwyrm import activitypub +from bookwyrm.connectors import get_image +from bookwyrm.sanitize_html import InputHtmlParser +from bookwyrm.settings import DOMAIN + + +def validate_remote_id(value): + ''' make sure the remote_id looks like a url ''' + if not value or not re.match(r'^http.?:\/\/[^\s]+$', value): + raise ValidationError( + _('%(value)s is not a valid remote_id'), + params={'value': value}, + ) + + +def validate_localname(value): + ''' make sure localnames look okay ''' + if not re.match(r'^[A-Za-z\-_\.0-9]+$', value): + raise ValidationError( + _('%(value)s is not a valid username'), + params={'value': value}, + ) + + +def validate_username(value): + ''' make sure usernames look okay ''' + if not re.match(r'^[A-Za-z\-_\.0-9]+@[A-Za-z\-_\.0-9]+\.[a-z]{2,}$', value): + raise ValidationError( + _('%(value)s is not a valid username'), + params={'value': value}, + ) + + +class ActivitypubFieldMixin: + ''' make a database field serializable ''' + def __init__(self, *args, \ + activitypub_field=None, activitypub_wrapper=None, + deduplication_field=False, **kwargs): + self.deduplication_field = deduplication_field + if activitypub_wrapper: + self.activitypub_wrapper = activitypub_field + self.activitypub_field = activitypub_wrapper + else: + self.activitypub_field = activitypub_field + super().__init__(*args, **kwargs) + + + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + try: + value = getattr(data, self.get_activitypub_field()) + except AttributeError: + # masssively hack-y workaround for boosts + if self.get_activitypub_field() != 'attributedTo': + raise + value = getattr(data, 'actor') + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + setattr(instance, self.name, formatted) + + + def set_activity_from_field(self, activity, instance): + ''' update the json object ''' + value = getattr(instance, self.name) + formatted = self.field_to_activity(value) + if formatted is None: + return + + key = self.get_activitypub_field() + # TODO: surely there's a better way + if instance.__class__.__name__ == 'Boost' and key == 'attributedTo': + key = 'actor' + if isinstance(activity.get(key), list): + activity[key] += formatted + else: + activity[key] = formatted + + + def field_to_activity(self, value): + ''' formatter to convert a model value into activitypub ''' + if hasattr(self, 'activitypub_wrapper'): + return {self.activitypub_wrapper: value} + return value + + def field_from_activity(self, value): + ''' formatter to convert activitypub into a model value ''' + if hasattr(self, 'activitypub_wrapper'): + value = value.get(self.activitypub_wrapper) + return value + + def get_activitypub_field(self): + ''' model_field_name to activitypubFieldName ''' + if self.activitypub_field: + return self.activitypub_field + name = self.name.split('.')[-1] + components = name.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + + +class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin): + ''' default (de)serialization for foreign key and one to one ''' + def __init__(self, *args, load_remote=True, **kwargs): + self.load_remote = load_remote + super().__init__(*args, **kwargs) + + def field_from_activity(self, value): + if not value: + return None + + related_model = self.related_model + if isinstance(value, dict) and value.get('id'): + if not self.load_remote: + # only look in the local database + return related_model.find_existing(value) + # this is an activitypub object, which we can deserialize + activity_serializer = related_model.activity_serializer + return activity_serializer(**value).to_model(related_model) + try: + # make sure the value looks like a remote id + validate_remote_id(value) + except ValidationError: + # we don't know what this is, ignore it + return None + # gets or creates the model field from the remote id + if not self.load_remote: + # only look in the local database + return related_model.find_existing_by_remote_id(value) + return activitypub.resolve_remote_id(related_model, value) + + +class RemoteIdField(ActivitypubFieldMixin, models.CharField): + ''' a url that serves as a unique identifier ''' + def __init__(self, *args, max_length=255, validators=None, **kwargs): + validators = validators or [validate_remote_id] + super().__init__( + *args, max_length=max_length, validators=validators, + **kwargs + ) + # for this field, the default is true. false everywhere else. + self.deduplication_field = kwargs.get('deduplication_field', True) + + +class UsernameField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware username field ''' + def __init__(self, activitypub_field='preferredUsername', **kwargs): + self.activitypub_field = activitypub_field + # I don't totally know why pylint is mad at this, but it makes it work + super( #pylint: disable=bad-super-call + ActivitypubFieldMixin, self + ).__init__( + _('username'), + max_length=150, + unique=True, + validators=[validate_username], + error_messages={ + 'unique': _('A user with that username already exists.'), + }, + ) + + def deconstruct(self): + ''' implementation of models.Field deconstruct ''' + name, path, args, kwargs = super().deconstruct() + del kwargs['verbose_name'] + del kwargs['max_length'] + del kwargs['unique'] + del kwargs['validators'] + del kwargs['error_messages'] + return name, path, args, kwargs + + def field_to_activity(self, value): + return value.split('@')[0] + + +PrivacyLevels = models.TextChoices('Privacy', [ + 'public', + 'unlisted', + 'followers', + 'direct' +]) + +class PrivacyField(ActivitypubFieldMixin, models.CharField): + ''' this maps to two differente activitypub fields ''' + public = 'https://www.w3.org/ns/activitystreams#Public' + def __init__(self, *args, **kwargs): + super().__init__( + *args, max_length=255, + choices=PrivacyLevels.choices, default='public') + + def set_field_from_activity(self, instance, data): + to = data.to + cc = data.cc + if to == [self.public]: + setattr(instance, self.name, 'public') + elif cc == []: + setattr(instance, self.name, 'direct') + elif self.public in cc: + setattr(instance, self.name, 'unlisted') + else: + setattr(instance, self.name, 'followers') + + def set_activity_from_field(self, activity, instance): + mentions = [u.remote_id for u in instance.mention_users.all()] + # this is a link to the followers list + followers = instance.user.__class__._meta.get_field('followers')\ + .field_to_activity(instance.user.followers) + if instance.privacy == 'public': + activity['to'] = [self.public] + activity['cc'] = [followers] + mentions + elif instance.privacy == 'unlisted': + activity['to'] = [followers] + activity['cc'] = [self.public] + mentions + elif instance.privacy == 'followers': + activity['to'] = [followers] + activity['cc'] = mentions + if instance.privacy == 'direct': + activity['to'] = mentions + activity['cc'] = [] + + +class ForeignKey(ActivitypubRelatedFieldMixin, models.ForeignKey): + ''' activitypub-aware foreign key field ''' + def field_to_activity(self, value): + if not value: + return None + return value.remote_id + + +class OneToOneField(ActivitypubRelatedFieldMixin, models.OneToOneField): + ''' activitypub-aware foreign key field ''' + def field_to_activity(self, value): + if not value: + return None + return value.to_activity() + + +class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField): + ''' activitypub-aware many to many field ''' + def __init__(self, *args, link_only=False, **kwargs): + self.link_only = link_only + super().__init__(*args, **kwargs) + + def set_field_from_activity(self, instance, data): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).set(formatted) + + def field_to_activity(self, value): + if self.link_only: + return '%s/%s' % (value.instance.remote_id, self.name) + return [i.remote_id for i in value.all()] + + def field_from_activity(self, value): + items = [] + if value is None or value is MISSING: + return [] + for remote_id in value: + try: + validate_remote_id(remote_id) + except ValidationError: + continue + items.append( + activitypub.resolve_remote_id(self.related_model, remote_id) + ) + return items + + +class TagField(ManyToManyField): + ''' special case of many to many that uses Tags ''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.activitypub_field = 'tag' + + def field_to_activity(self, value): + tags = [] + for item in value.all(): + activity_type = item.__class__.__name__ + if activity_type == 'User': + activity_type = 'Mention' + tags.append(activitypub.Link( + href=item.remote_id, + name=getattr(item, item.name_field), + type=activity_type + )) + return tags + + def field_from_activity(self, value): + if not isinstance(value, list): + return None + items = [] + for link_json in value: + link = activitypub.Link(**link_json) + tag_type = link.type if link.type != 'Mention' else 'Person' + if tag_type == 'Book': + tag_type = 'Edition' + if tag_type != self.related_model.activity_serializer.type: + # tags can contain multiple types + continue + items.append( + activitypub.resolve_remote_id(self.related_model, link.href) + ) + return items + + +def image_serializer(value, alt): + ''' helper for serializing images ''' + if value and hasattr(value, 'url'): + url = value.url + else: + return None + url = 'https://%s%s' % (DOMAIN, url) + return activitypub.Image(url=url, name=alt) + + +class ImageField(ActivitypubFieldMixin, models.ImageField): + ''' activitypub-aware image field ''' + def __init__(self, *args, alt_field=None, **kwargs): + self.alt_field = alt_field + super().__init__(*args, **kwargs) + + # pylint: disable=arguments-differ + def set_field_from_activity(self, instance, data, save=True): + ''' helper function for assinging a value to the field ''' + value = getattr(data, self.get_activitypub_field()) + formatted = self.field_from_activity(value) + if formatted is None or formatted is MISSING: + return + getattr(instance, self.name).save(*formatted, save=save) + + def set_activity_from_field(self, activity, instance): + value = getattr(instance, self.name) + if value is None: + return + alt_text = getattr(instance, self.alt_field) + formatted = self.field_to_activity(value, alt_text) + + key = self.get_activitypub_field() + activity[key] = formatted + + + def field_to_activity(self, value, alt=None): + return image_serializer(value, alt) + + + def field_from_activity(self, value): + image_slug = value + # when it's an inline image (User avatar/icon, Book cover), it's a json + # blob, but when it's an attached image, it's just a url + if isinstance(image_slug, dict): + url = image_slug.get('url') + elif isinstance(image_slug, str): + url = image_slug + else: + return None + + try: + validate_remote_id(url) + except ValidationError: + return None + + response = get_image(url) + if not response: + return None + + image_name = str(uuid4()) + '.' + url.split('.')[-1] + image_content = ContentFile(response.content) + return [image_name, image_content] + + +class DateTimeField(ActivitypubFieldMixin, models.DateTimeField): + ''' activitypub-aware datetime field ''' + def field_to_activity(self, value): + if not value: + return None + return value.isoformat() + + def field_from_activity(self, value): + try: + date_value = dateutil.parser.parse(value) + try: + return timezone.make_aware(date_value) + except ValueError: + return date_value + except (ParserError, TypeError): + return None + +class HtmlField(ActivitypubFieldMixin, models.TextField): + ''' a text field for storing html ''' + def field_from_activity(self, value): + if not value or value == MISSING: + return None + sanitizer = InputHtmlParser() + sanitizer.feed(value) + return sanitizer.get_output() + +class ArrayField(ActivitypubFieldMixin, DjangoArrayField): + ''' activitypub-aware array field ''' + def field_to_activity(self, value): + return [str(i) for i in value] + +class CharField(ActivitypubFieldMixin, models.CharField): + ''' activitypub-aware char field ''' + +class TextField(ActivitypubFieldMixin, models.TextField): + ''' activitypub-aware text field ''' + +class BooleanField(ActivitypubFieldMixin, models.BooleanField): + ''' activitypub-aware boolean field ''' + +class IntegerField(ActivitypubFieldMixin, models.IntegerField): + ''' activitypub-aware boolean field ''' diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index fe39325f..b10651b9 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -2,14 +2,13 @@ import re import dateutil.parser +from django.contrib.postgres.fields import JSONField from django.db import models from django.utils import timezone -from bookwyrm import books_manager -from bookwyrm.connectors import ConnectorException +from bookwyrm.connectors import connector_manager from bookwyrm.models import ReadThrough, User, Book -from bookwyrm.utils.fields import JSONField -from .base_model import PrivacyLevels +from .fields import PrivacyLevels # Mapping goodreads -> bookwyrm shelf titles. @@ -43,6 +42,7 @@ class ImportJob(models.Model): created_date = models.DateTimeField(default=timezone.now) task_id = models.CharField(max_length=100, null=True) include_reviews = models.BooleanField(default=True) + complete = models.BooleanField(default=False) privacy = models.CharField( max_length=255, default='public', @@ -72,12 +72,12 @@ class ImportItem(models.Model): def get_book_from_isbn(self): ''' search by isbn ''' - search_result = books_manager.first_search_result( + search_result = connector_manager.first_search_result( self.isbn, min_confidence=0.999 ) if search_result: # raises ConnectorException - return books_manager.get_or_create_book(search_result.key) + return search_result.connector.get_or_create_book(search_result.key) return None @@ -87,12 +87,12 @@ class ImportItem(models.Model): self.data['Title'], self.data['Author'] ) - search_result = books_manager.first_search_result( + search_result = connector_manager.first_search_result( search_term, min_confidence=0.999 ) if search_result: # raises ConnectorException - return books_manager.get_or_create_book(search_result.key) + return search_result.connector.get_or_create_book(search_result.key) return None diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py new file mode 100644 index 00000000..4ce5dcea --- /dev/null +++ b/bookwyrm/models/notification.py @@ -0,0 +1,33 @@ +''' alert a user to activity ''' +from django.db import models +from .base_model import BookWyrmModel + + +NotificationType = models.TextChoices( + 'NotificationType', + 'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT') + +class Notification(BookWyrmModel): + ''' you've been tagged, liked, followed, etc ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + related_book = models.ForeignKey( + 'Edition', on_delete=models.PROTECT, null=True) + related_user = models.ForeignKey( + 'User', + on_delete=models.PROTECT, null=True, related_name='related_user') + related_status = models.ForeignKey( + 'Status', on_delete=models.PROTECT, null=True) + related_import = models.ForeignKey( + 'ImportJob', on_delete=models.PROTECT, null=True) + read = models.BooleanField(default=False) + notification_type = models.CharField( + max_length=255, choices=NotificationType.choices) + + class Meta: + ''' checks if notifcation is in enum list for valid types ''' + constraints = [ + models.CheckConstraint( + check=models.Q(notification_type__in=NotificationType.values), + name="notification_type_valid", + ) + ] diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py new file mode 100644 index 00000000..3afde306 --- /dev/null +++ b/bookwyrm/models/readthrough.py @@ -0,0 +1,56 @@ +''' progress in a book ''' +from django.db import models +from django.utils import timezone + +from .base_model import BookWyrmModel + +class ProgressMode(models.TextChoices): + PAGE = 'PG', 'page' + PERCENT = 'PCT', 'percent' + +class ReadThrough(BookWyrmModel): + ''' Store a read through a book in the database. ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + book = models.ForeignKey('Edition', on_delete=models.PROTECT) + progress = models.IntegerField( + null=True, + blank=True) + progress_mode = models.CharField( + max_length=3, + choices=ProgressMode.choices, + default=ProgressMode.PAGE) + start_date = models.DateTimeField( + blank=True, + null=True) + finish_date = models.DateTimeField( + blank=True, + null=True) + + def save(self, *args, **kwargs): + ''' update user active time ''' + self.user.last_active_date = timezone.now() + self.user.save() + super().save(*args, **kwargs) + + def create_update(self): + if self.progress: + return self.progressupdate_set.create( + user=self.user, + progress=self.progress, + mode=self.progress_mode) + +class ProgressUpdate(BookWyrmModel): + ''' Store progress through a book in the database. ''' + user = models.ForeignKey('User', on_delete=models.PROTECT) + readthrough = models.ForeignKey('ReadThrough', on_delete=models.PROTECT) + progress = models.IntegerField() + mode = models.CharField( + max_length=3, + choices=ProgressMode.choices, + default=ProgressMode.PAGE) + + def save(self, *args, **kwargs): + ''' update user active time ''' + self.user.last_active_date = timezone.now() + self.user.save() + super().save(*args, **kwargs) diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index dbf99778..0f3c1dab 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -2,20 +2,23 @@ from django.db import models from bookwyrm import activitypub -from .base_model import ActivitypubMixin, ActivityMapping, BookWyrmModel +from .base_model import ActivitypubMixin, BookWyrmModel +from . import fields class UserRelationship(ActivitypubMixin, BookWyrmModel): ''' many-to-many through table for followers ''' - user_subject = models.ForeignKey( + user_subject = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_subject' + related_name='%(class)s_user_subject', + activitypub_field='actor', ) - user_object = models.ForeignKey( + user_object = fields.ForeignKey( 'User', on_delete=models.PROTECT, - related_name='%(class)s_user_object' + related_name='%(class)s_user_object', + activitypub_field='object', ) class Meta: @@ -32,14 +35,9 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): ) ] - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user_subject'), - ActivityMapping('object', 'user_object'), - ] activity_serializer = activitypub.Follow - def get_remote_id(self, status=None): + def get_remote_id(self, status=None):# pylint: disable=arguments-differ ''' use shelf identifier in remote_id ''' status = status or 'follows' base_path = self.user_subject.remote_id @@ -56,7 +54,7 @@ class UserRelationship(ActivitypubMixin, BookWyrmModel): def to_reject_activity(self): - ''' generate an Accept for this follow request ''' + ''' generate a Reject for this follow request ''' return activitypub.Reject( id=self.get_remote_id(status='rejects'), actor=self.user_object.remote_id, diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index e85294ba..a06f78dc 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -3,19 +3,22 @@ import re from django.db import models from bookwyrm import activitypub -from .base_model import BookWyrmModel, OrderedCollectionMixin, PrivacyLevels +from .base_model import ActivitypubMixin, BookWyrmModel +from .base_model import OrderedCollectionMixin +from . import fields class Shelf(OrderedCollectionMixin, BookWyrmModel): ''' a list of books owned by a user ''' - name = models.CharField(max_length=100) + name = fields.CharField(max_length=100) identifier = models.CharField(max_length=100) - user = models.ForeignKey('User', on_delete=models.PROTECT) + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='owner') editable = models.BooleanField(default=True) - privacy = models.CharField( + privacy = fields.CharField( max_length=255, default='public', - choices=PrivacyLevels.choices + choices=fields.PrivacyLevels.choices ) books = models.ManyToManyField( 'Edition', @@ -36,7 +39,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): @property def collection_queryset(self): ''' list of books for this shelf, overrides OrderedCollectionMixin ''' - return self.books + return self.books.all() def get_remote_id(self): ''' shelf identifier instead of id ''' @@ -48,17 +51,22 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): unique_together = ('user', 'identifier') -class ShelfBook(BookWyrmModel): +class ShelfBook(ActivitypubMixin, BookWyrmModel): ''' many to many join table for books and shelves ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - shelf = models.ForeignKey('Shelf', on_delete=models.PROTECT) - added_by = models.ForeignKey( + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + shelf = fields.ForeignKey( + 'Shelf', on_delete=models.PROTECT, activitypub_field='target') + added_by = fields.ForeignKey( 'User', blank=True, null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='actor' ) + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index aa2e2a67..4670bd94 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -12,11 +12,27 @@ from .user import User class SiteSettings(models.Model): ''' customized settings for this instance ''' name = models.CharField(default='BookWyrm', max_length=100) + instance_tagline = models.CharField( + max_length=150, default='Social Reading and Reviewing') instance_description = models.TextField( - default="This instance has no description.") + default='This instance has no description.') + registration_closed_text = models.TextField( + default='Contact an administrator to get an invite') code_of_conduct = models.TextField( - default="Add a code of conduct here.") + default='Add a code of conduct here.') allow_registration = models.BooleanField(default=True) + logo = models.ImageField( + upload_to='logos/', null=True, blank=True + ) + logo_small = models.ImageField( + upload_to='logos/', null=True, blank=True + ) + favicon = models.ImageField( + upload_to='logos/', null=True, blank=True + ) + support_link = models.CharField(max_length=255, null=True, blank=True) + support_title = models.CharField(max_length=100, null=True, blank=True) + admin_email = models.EmailField(max_length=255, null=True, blank=True) @classmethod def get(cls): @@ -34,6 +50,7 @@ def new_access_code(): class SiteInvite(models.Model): ''' gives someone access to create an account on the instance ''' + created_date = models.DateTimeField(auto_now_add=True) code = models.CharField(max_length=32, default=new_access_code) expiry = models.DateTimeField(blank=True, null=True) use_limit = models.IntegerField(blank=True, null=True) @@ -49,7 +66,7 @@ class SiteInvite(models.Model): @property def link(self): ''' formats the invite link ''' - return "https://{}/invite/{}".format(DOMAIN, self.code) + return 'https://{}/invite/{}'.format(DOMAIN, self.code) def get_passowrd_reset_expiry(): @@ -71,4 +88,4 @@ class PasswordReset(models.Model): @property def link(self): ''' formats the invite link ''' - return "https://{}/password-reset/{}".format(DOMAIN, self.code) + return 'https://{}/password-reset/{}'.format(DOMAIN, self.code) diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 5e9ccdac..dad65974 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -1,31 +1,34 @@ ''' models for storing different kinds of Activities ''' -from django.utils import timezone +from dataclasses import MISSING +import re + +from django.apps import apps from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models +from django.utils import timezone from model_utils.managers import InheritanceManager from bookwyrm import activitypub from .base_model import ActivitypubMixin, OrderedCollectionPageMixin -from .base_model import ActivityMapping, BookWyrmModel, PrivacyLevels -from .base_model import tag_formatter, image_attachments_formatter - +from .base_model import BookWyrmModel +from . import fields +from .fields import image_serializer class Status(OrderedCollectionPageMixin, BookWyrmModel): ''' any post, like a reply to a review, etc ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - content = models.TextField(blank=True, null=True) - mention_users = models.ManyToManyField('User', related_name='mention_user') - mention_books = models.ManyToManyField( - 'Edition', related_name='mention_book') + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='attributedTo') + content = fields.HtmlField(blank=True, null=True) + mention_users = fields.TagField('User', related_name='mention_user') + mention_books = fields.TagField('Edition', related_name='mention_book') local = models.BooleanField(default=True) - privacy = models.CharField( - max_length=255, - default='public', - choices=PrivacyLevels.choices - ) - sensitive = models.BooleanField(default=False) - # the created date can't be this, because of receiving federated posts - published_date = models.DateTimeField(default=timezone.now) + content_warning = fields.CharField( + max_length=500, blank=True, null=True, activitypub_field='summary') + privacy = fields.PrivacyField(max_length=255) + sensitive = fields.BooleanField(default=False) + # created date is different than publish date because of federated posts + published_date = fields.DateTimeField( + default=timezone.now, activitypub_field='published') deleted = models.BooleanField(default=False) deleted_date = models.DateTimeField(blank=True, null=True) favorites = models.ManyToManyField( @@ -35,94 +38,57 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): through_fields=('status', 'user'), related_name='user_favorites' ) - reply_parent = models.ForeignKey( + reply_parent = fields.ForeignKey( 'self', null=True, - on_delete=models.PROTECT + on_delete=models.PROTECT, + activitypub_field='inReplyTo', ) objects = InheritanceManager() - # ---- activitypub serialization settings for this model ----- # - @property - def ap_to(self): - ''' should be related to post privacy I think ''' - return ['https://www.w3.org/ns/activitystreams#Public'] - - @property - def ap_cc(self): - ''' should be related to post privacy I think ''' - return [self.user.ap_followers] - - @property - def ap_replies(self): - ''' structured replies block ''' - return self.to_replies() - - @property - def ap_status_image(self): - ''' attach a book cover, if relevent ''' - if hasattr(self, 'book'): - return self.book.ap_cover - if self.mention_books.first(): - return self.mention_books.first().ap_cover - return None - - - shared_mappings = [ - ActivityMapping('url', 'remote_id', lambda x: None), - ActivityMapping('id', 'remote_id'), - ActivityMapping('inReplyTo', 'reply_parent'), - ActivityMapping('published', 'published_date'), - ActivityMapping('attributedTo', 'user'), - ActivityMapping('to', 'ap_to'), - ActivityMapping('cc', 'ap_cc'), - ActivityMapping('replies', 'ap_replies'), - ActivityMapping( - 'tag', 'mention_books', - lambda x: tag_formatter(x, 'title', 'Book'), - lambda x: activitypub.tag_formatter(x, 'Book') - ), - ActivityMapping( - 'tag', 'mention_users', - lambda x: tag_formatter(x, 'username', 'Mention'), - lambda x: activitypub.tag_formatter(x, 'Mention') - ), - ActivityMapping( - 'attachment', 'attachments', - lambda x: image_attachments_formatter(x.all()), - ) - ] - - # serializing to bookwyrm expanded activitypub - activity_mappings = shared_mappings + [ - ActivityMapping('name', 'name'), - ActivityMapping('inReplyToBook', 'book'), - ActivityMapping('rating', 'rating'), - ActivityMapping('quote', 'quote'), - ActivityMapping('content', 'content'), - ] - - # for serializing to standard activitypub without extended types - pure_activity_mappings = shared_mappings + [ - ActivityMapping('name', 'ap_pure_name'), - ActivityMapping('content', 'ap_pure_content'), - ActivityMapping('attachment', 'ap_status_image'), - ] - activity_serializer = activitypub.Note + serialize_reverse_fields = [('attachments', 'attachment', 'id')] + deserialize_reverse_fields = [('attachments', 'attachment')] + + @classmethod + def ignore_activity(cls, activity): + ''' keep notes if they are replies to existing statuses ''' + if activity.type != 'Note': + return False + if cls.objects.filter( + remote_id=activity.inReplyTo).exists(): + return False + + # keep notes if they mention local users + if activity.tag == MISSING or activity.tag is None: + return True + tags = [l['href'] for l in activity.tag if l['type'] == 'Mention'] + for tag in tags: + user_model = apps.get_model('bookwyrm.User', require_ready=True) + if user_model.objects.filter( + remote_id=tag, local=True).exists(): + # we found a mention of a known use boost + return False + return True - #----- replies collection activitypub ----# @classmethod def replies(cls, status): ''' load all replies to a status. idk if there's a better way to write this so it's just a property ''' - return cls.objects.filter(reply_parent=status).select_subclasses() + return cls.objects.filter( + reply_parent=status + ).select_subclasses().order_by('published_date') @property def status_type(self): ''' expose the type of status for the ui using activity type ''' return self.activity_serializer.__name__ + @property + def boostable(self): + ''' you can't boost dms ''' + return self.privacy in ['unlisted', 'public'] + def to_replies(self, **kwargs): ''' helper function for loading AP serialized replies to a status ''' return self.to_ordered_collection( @@ -131,7 +97,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): **kwargs ) - def to_activity(self, pure=False): + def to_activity(self, pure=False):# pylint: disable=arguments-differ ''' return tombstone if the status is deleted ''' if self.deleted: return activitypub.Tombstone( @@ -140,7 +106,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): deleted=self.deleted_date.isoformat(), published=self.deleted_date.isoformat() ).serialize() - return ActivitypubMixin.to_activity(self, pure=pure) + activity = ActivitypubMixin.to_activity(self) + activity['replies'] = self.to_replies() + + # "pure" serialization for non-bookwyrm instances + if pure and hasattr(self, 'pure_content'): + activity['content'] = self.pure_content + if 'name' in activity: + activity['name'] = self.pure_name + activity['type'] = self.pure_type + activity['attachment'] = [ + image_serializer(b.cover, b.alt_text) \ + for b in self.mention_books.all()[:4] if b.cover] + if hasattr(self, 'book') and self.book.cover: + activity['attachment'].append( + image_serializer(self.book.cover, self.book.alt_text) + ) + return activity + def save(self, *args, **kwargs): ''' update user active time ''' @@ -153,57 +136,62 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): class GeneratedNote(Status): ''' these are app-generated messages about user activity ''' @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' message = self.content books = ', '.join( - '"%s"' % (self.book.remote_id, self.book.title) \ + '"%s"' % (book.remote_id, book.title) \ for book in self.mention_books.all() ) - return '%s %s' % (message, books) + return '%s %s %s' % (self.user.display_name, message, books) activity_serializer = activitypub.GeneratedNote - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Comment(Status): ''' like a review but without a rating and transient ''' - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return self.content + '

(comment on "%s")' % \ - (self.book.remote_id, self.book.title) + return '%s

(comment on "%s")

' % \ + (self.content, self.book.remote_id, self.book.title) activity_serializer = activitypub.Comment - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Quotation(Status): ''' like a review but without a rating and transient ''' - quote = models.TextField() - book = models.ForeignKey('Edition', on_delete=models.PROTECT) + quote = fields.HtmlField() + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return '"%s"
-- "%s"

%s' % ( - self.quote, + quote = re.sub(r'^

', '

"', self.quote) + quote = re.sub(r'

$', '"

', quote) + return '%s

-- "%s"

%s' % ( + quote, self.book.remote_id, self.book.title, self.content, ) activity_serializer = activitypub.Quotation - pure_activity_serializer = activitypub.Note + pure_type = 'Note' class Review(Status): ''' a book review ''' - name = models.CharField(max_length=255, null=True) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - rating = models.IntegerField( + name = fields.CharField(max_length=255, null=True) + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='inReplyToBook') + rating = fields.IntegerField( default=None, null=True, blank=True, @@ -211,9 +199,10 @@ class Review(Status): ) @property - def ap_pure_name(self): + def pure_name(self): ''' clarify review names for mastodon serialization ''' if self.rating: + #pylint: disable=bad-string-format-type return 'Review of "%s" (%d stars): %s' % ( self.book.title, self.rating, @@ -225,139 +214,37 @@ class Review(Status): ) @property - def ap_pure_content(self): + def pure_content(self): ''' indicate the book in question for mastodon (or w/e) users ''' - return self.content + '

("%s")' % \ - (self.book.remote_id, self.book.title) + return self.content activity_serializer = activitypub.Review - pure_activity_serializer = activitypub.Article - - -class Favorite(ActivitypubMixin, BookWyrmModel): - ''' fav'ing a post ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - status = models.ForeignKey('Status', on_delete=models.PROTECT) - - # ---- activitypub serialization settings for this model ----- # - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'status'), - ] - - activity_serializer = activitypub.Like - - def save(self, *args, **kwargs): - ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) - - - class Meta: - ''' can't fav things twice ''' - unique_together = ('user', 'status') + pure_type = 'Article' class Boost(Status): ''' boost'ing a post ''' - boosted_status = models.ForeignKey( + boosted_status = fields.ForeignKey( 'Status', on_delete=models.PROTECT, - related_name="boosters") + related_name='boosters', + activitypub_field='object', + ) - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping('actor', 'user'), - ActivityMapping('object', 'boosted_status'), - ] + def __init__(self, *args, **kwargs): + ''' the user field is "actor" here instead of "attributedTo" ''' + super().__init__(*args, **kwargs) + + reserve_fields = ['user', 'boosted_status'] + self.simple_fields = [f for f in self.simple_fields if \ + f.name in reserve_fields] + self.activity_fields = self.simple_fields + self.many_to_many_fields = [] + self.image_fields = [] + self.deserialize_reverse_fields = [] activity_serializer = activitypub.Boost # This constraint can't work as it would cross tables. # class Meta: # unique_together = ('user', 'boosted_status') - - -class ProgressMode(models.TextChoices): - PAGE = 'PG', 'page' - PERCENT = 'PCT', 'percent' - -class ReadThrough(BookWyrmModel): - ''' Store a read through a book in the database. ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Book', on_delete=models.PROTECT) - progress = models.IntegerField( - null=True, - blank=True) - progress_mode = models.CharField( - max_length=3, - choices=ProgressMode.choices, - default=ProgressMode.PAGE) - start_date = models.DateTimeField( - blank=True, - null=True) - finish_date = models.DateTimeField( - blank=True, - null=True) - - def save(self, *args, **kwargs): - ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) - - def create_update(self): - if self.progress: - return self.progressupdate_set.create( - user=self.user, - progress=self.progress, - mode=self.progress_mode) - - -class ProgressUpdate(BookWyrmModel): - ''' Store progress through a book in the database. ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - readthrough = models.ForeignKey('ReadThrough', on_delete=models.PROTECT) - progress = models.IntegerField() - mode = models.CharField( - max_length=3, - choices=ProgressMode.choices, - default=ProgressMode.PAGE) - - def save(self, *args, **kwargs): - ''' update user active time ''' - self.user.last_active_date = timezone.now() - self.user.save() - super().save(*args, **kwargs) - - -NotificationType = models.TextChoices( - 'NotificationType', - 'FAVORITE REPLY MENTION TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT') - -class Notification(BookWyrmModel): - ''' you've been tagged, liked, followed, etc ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - related_book = models.ForeignKey( - 'Edition', on_delete=models.PROTECT, null=True) - related_user = models.ForeignKey( - 'User', - on_delete=models.PROTECT, null=True, related_name='related_user') - related_status = models.ForeignKey( - 'Status', on_delete=models.PROTECT, null=True) - related_import = models.ForeignKey( - 'ImportJob', on_delete=models.PROTECT, null=True) - read = models.BooleanField(default=False) - notification_type = models.CharField( - max_length=255, choices=NotificationType.choices) - - class Meta: - ''' checks if notifcation is in enum list for valid types ''' - constraints = [ - models.CheckConstraint( - check=models.Q(notification_type__in=NotificationType.values), - name="notification_type_valid", - ) - ] diff --git a/bookwyrm/models/tag.py b/bookwyrm/models/tag.py index cd98e2b1..6e0ba8ab 100644 --- a/bookwyrm/models/tag.py +++ b/bookwyrm/models/tag.py @@ -6,19 +6,20 @@ from django.db import models from bookwyrm import activitypub from bookwyrm.settings import DOMAIN from .base_model import OrderedCollectionMixin, BookWyrmModel +from . import fields class Tag(OrderedCollectionMixin, BookWyrmModel): ''' freeform tags for books ''' - user = models.ForeignKey('User', on_delete=models.PROTECT) - book = models.ForeignKey('Edition', on_delete=models.PROTECT) - name = models.CharField(max_length=100) + name = fields.CharField(max_length=100, unique=True) identifier = models.CharField(max_length=100) @classmethod def book_queryset(cls, identifier): ''' county of books associated with this tag ''' - return cls.objects.filter(identifier=identifier) + return cls.objects.filter( + identifier=identifier + ).order_by('-updated_date') @property def collection_queryset(self): @@ -30,6 +31,26 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): base_path = 'https://%s' % DOMAIN return '%s/tag/%s' % (base_path, self.identifier) + + def save(self, *args, **kwargs): + ''' create a url-safe lookup key for the tag ''' + if not self.id: + # add identifiers to new tags + self.identifier = urllib.parse.quote_plus(self.name) + super().save(*args, **kwargs) + + +class UserTag(BookWyrmModel): + ''' an instance of a tag on a book by a user ''' + user = fields.ForeignKey( + 'User', on_delete=models.PROTECT, activitypub_field='actor') + book = fields.ForeignKey( + 'Edition', on_delete=models.PROTECT, activitypub_field='object') + tag = fields.ForeignKey( + 'Tag', on_delete=models.PROTECT, activitypub_field='target') + + activity_serializer = activitypub.AddBook + def to_add_activity(self, user): ''' AP for shelving a book''' return activitypub.Add( @@ -45,16 +66,10 @@ class Tag(OrderedCollectionMixin, BookWyrmModel): id='%s#remove' % self.remote_id, actor=user.remote_id, object=self.book.to_activity(), - target=self.to_activity(), + target=self.remote_id, ).serialize() - def save(self, *args, **kwargs): - ''' create a url-safe lookup key for the tag ''' - if not self.id: - # add identifiers to new tags - self.identifier = urllib.parse.quote_plus(self.name) - super().save(*args, **kwargs) class Meta: ''' unqiueness constraint ''' - unique_together = ('user', 'book', 'name') + unique_together = ('user', 'book', 'tag') diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 4d511d56..ef68f992 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -1,49 +1,71 @@ ''' database schema for user data ''' +import re from urllib.parse import urlparse +from django.apps import apps from django.contrib.auth.models import AbstractUser from django.db import models from django.dispatch import receiver from bookwyrm import activitypub +from bookwyrm.connectors import get_data from bookwyrm.models.shelf import Shelf -from bookwyrm.models.status import Status +from bookwyrm.models.status import Status, Review from bookwyrm.settings import DOMAIN from bookwyrm.signatures import create_key_pair -from .base_model import ActivityMapping, OrderedCollectionPageMixin -from .base_model import image_formatter +from bookwyrm.tasks import app +from bookwyrm.utils import regex +from .base_model import OrderedCollectionPageMixin +from .base_model import ActivitypubMixin, BookWyrmModel +from .federated_server import FederatedServer +from . import fields class User(OrderedCollectionPageMixin, AbstractUser): ''' a user who wants to read books ''' - private_key = models.TextField(blank=True, null=True) - public_key = models.TextField(blank=True, null=True) - inbox = models.CharField(max_length=255, unique=True) - shared_inbox = models.CharField(max_length=255, blank=True, null=True) + username = fields.UsernameField() + + key_pair = fields.OneToOneField( + 'KeyPair', + on_delete=models.CASCADE, + blank=True, null=True, + activitypub_field='publicKey', + related_name='owner' + ) + inbox = fields.RemoteIdField(unique=True) + shared_inbox = fields.RemoteIdField( + activitypub_field='sharedInbox', + activitypub_wrapper='endpoints', + deduplication_field=False, + null=True) federated_server = models.ForeignKey( 'FederatedServer', on_delete=models.PROTECT, null=True, blank=True, ) - outbox = models.CharField(max_length=255, unique=True) - summary = models.TextField(blank=True, null=True) - local = models.BooleanField(default=True) - bookwyrm_user = models.BooleanField(default=True) + outbox = fields.RemoteIdField(unique=True) + summary = fields.HtmlField(null=True, blank=True) + local = models.BooleanField(default=False) + bookwyrm_user = fields.BooleanField(default=True) localname = models.CharField( max_length=255, null=True, - unique=True + unique=True, + validators=[fields.validate_localname], ) # name is your display name, which you can change at will - name = models.CharField(max_length=100, blank=True, null=True) - avatar = models.ImageField(upload_to='avatars/', blank=True, null=True) - following = models.ManyToManyField( + name = fields.CharField(max_length=100, null=True, blank=True) + avatar = fields.ImageField( + upload_to='avatars/', blank=True, null=True, + activitypub_field='icon', alt_field='alt_text') + followers = fields.ManyToManyField( 'self', + link_only=True, symmetrical=False, through='UserFollows', - through_fields=('user_subject', 'user_object'), - related_name='followers' + through_fields=('user_object', 'user_subject'), + related_name='following' ) follow_requests = models.ManyToManyField( 'self', @@ -66,84 +88,69 @@ class User(OrderedCollectionPageMixin, AbstractUser): through_fields=('user', 'status'), related_name='favorite_statuses' ) - remote_id = models.CharField(max_length=255, null=True, unique=True) + remote_id = fields.RemoteIdField( + null=True, unique=True, activitypub_field='id') created_date = models.DateTimeField(auto_now_add=True) updated_date = models.DateTimeField(auto_now=True) last_active_date = models.DateTimeField(auto_now=True) - manually_approves_followers = models.BooleanField(default=False) + manually_approves_followers = fields.BooleanField(default=False) - # ---- activitypub serialization settings for this model ----- # + name_field = 'username' @property - def ap_followers(self): - ''' generates url for activitypub followers page ''' - return '%s/followers' % self.remote_id + def alt_text(self): + ''' alt text with username ''' + return 'avatar for %s' % (self.localname or self.username) @property - def ap_public_key(self): - ''' format the public key block for activitypub ''' - return activitypub.PublicKey(**{ - 'id': '%s/#main-key' % self.remote_id, - 'owner': self.remote_id, - 'publicKeyPem': self.public_key, - }) + def display_name(self): + ''' show the cleanest version of the user's name possible ''' + if self.name and self.name != '': + return self.name + return self.localname or self.username - activity_mappings = [ - ActivityMapping('id', 'remote_id'), - ActivityMapping( - 'preferredUsername', - 'username', - activity_formatter=lambda x: x.split('@')[0] - ), - ActivityMapping('name', 'name'), - ActivityMapping('bookwyrmUser', 'bookwyrm_user'), - ActivityMapping('inbox', 'inbox'), - ActivityMapping('outbox', 'outbox'), - ActivityMapping('followers', 'ap_followers'), - ActivityMapping('summary', 'summary'), - ActivityMapping( - 'publicKey', - 'public_key', - model_formatter=lambda x: x.get('publicKeyPem') - ), - ActivityMapping('publicKey', 'ap_public_key'), - ActivityMapping( - 'endpoints', - 'shared_inbox', - activity_formatter=lambda x: {'sharedInbox': x}, - model_formatter=lambda x: x.get('sharedInbox') - ), - ActivityMapping('icon', 'avatar'), - ActivityMapping( - 'manuallyApprovesFollowers', - 'manually_approves_followers' - ), - # this field isn't in the activity but should always be false - ActivityMapping(None, 'local', model_formatter=lambda x: False), - ] activity_serializer = activitypub.Person - def to_outbox(self, **kwargs): + def to_outbox(self, filter_type=None, **kwargs): ''' an ordered collection of statuses ''' - queryset = Status.objects.filter( + if filter_type: + filter_class = apps.get_model( + 'bookwyrm.%s' % filter_type, require_ready=True) + if not issubclass(filter_class, Status): + raise TypeError( + 'filter_status_class must be a subclass of models.Status') + queryset = filter_class.objects + else: + queryset = Status.objects + + queryset = queryset.filter( user=self, deleted=False, - ).select_subclasses() + privacy__in=['public', 'unlisted'], + ).select_subclasses().order_by('-published_date') return self.to_ordered_collection(queryset, \ remote_id=self.outbox, **kwargs) def to_following_activity(self, **kwargs): ''' activitypub following list ''' remote_id = '%s/following' % self.remote_id - return self.to_ordered_collection(self.following, \ - remote_id=remote_id, id_only=True, **kwargs) + return self.to_ordered_collection( + self.following.order_by('-updated_date').all(), + remote_id=remote_id, + id_only=True, + **kwargs + ) def to_followers_activity(self, **kwargs): ''' activitypub followers list ''' remote_id = '%s/followers' % self.remote_id - return self.to_ordered_collection(self.followers, \ - remote_id=remote_id, id_only=True, **kwargs) + return self.to_ordered_collection( + self.followers.order_by('-updated_date').all(), + remote_id=remote_id, + id_only=True, + **kwargs + ) - def to_activity(self, pure=False): + def to_activity(self): ''' override default AP serializer to add context object idk if this is the best way to go about this ''' activity_object = super().to_activity() @@ -163,35 +170,72 @@ class User(OrderedCollectionPageMixin, AbstractUser): def save(self, *args, **kwargs): ''' populate fields for new local users ''' # this user already exists, no need to populate fields - if self.id: - return super().save(*args, **kwargs) - - if not self.local: + if not self.local and not re.match(regex.full_username, self.username): # generate a username that uses the domain (webfinger format) actor_parts = urlparse(self.remote_id) self.username = '%s@%s' % (self.username, actor_parts.netloc) return super().save(*args, **kwargs) + if self.id or not self.local: + return super().save(*args, **kwargs) + # populate fields for local users - self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.username) - self.localname = self.username - self.username = '%s@%s' % (self.username, DOMAIN) - self.actor = self.remote_id + self.remote_id = 'https://%s/user/%s' % (DOMAIN, self.localname) self.inbox = '%s/inbox' % self.remote_id self.shared_inbox = 'https://%s/inbox' % DOMAIN self.outbox = '%s/outbox' % self.remote_id - if not self.private_key: - self.private_key, self.public_key = create_key_pair() return super().save(*args, **kwargs) + @property + def local_path(self): + ''' this model doesn't inherit bookwyrm model, so here we are ''' + return '/user/%s' % (self.localname or self.username) + + +class KeyPair(ActivitypubMixin, BookWyrmModel): + ''' public and private keys for a user ''' + private_key = models.TextField(blank=True, null=True) + public_key = fields.TextField( + blank=True, null=True, activitypub_field='publicKeyPem') + + activity_serializer = activitypub.PublicKey + serialize_reverse_fields = [('owner', 'owner', 'id')] + + def get_remote_id(self): + # self.owner is set by the OneToOneField on User + return '%s/#main-key' % self.owner.remote_id + + def save(self, *args, **kwargs): + ''' create a key pair ''' + if not self.public_key: + self.private_key, self.public_key = create_key_pair() + return super().save(*args, **kwargs) + + def to_activity(self): + ''' override default AP serializer to add context object + idk if this is the best way to go about this ''' + activity_object = super().to_activity() + del activity_object['@context'] + del activity_object['type'] + return activity_object + @receiver(models.signals.post_save, sender=User) +#pylint: disable=unused-argument def execute_after_save(sender, instance, created, *args, **kwargs): ''' create shelves for new users ''' - if not instance.local or not created: + if not created: return + if not instance.local: + set_remote_server.delay(instance.id) + return + + instance.key_pair = KeyPair.objects.create( + remote_id='%s/#main-key' % instance.remote_id) + instance.save() + shelves = [{ 'name': 'To Read', 'identifier': 'to-read', @@ -210,3 +254,54 @@ def execute_after_save(sender, instance, created, *args, **kwargs): user=instance, editable=False ).save() + + +@app.task +def set_remote_server(user_id): + ''' figure out the user's remote server in the background ''' + user = User.objects.get(id=user_id) + actor_parts = urlparse(user.remote_id) + user.federated_server = \ + get_or_create_remote_server(actor_parts.netloc) + user.save() + if user.bookwyrm_user: + get_remote_reviews.delay(user.outbox) + + +def get_or_create_remote_server(domain): + ''' get info on a remote server ''' + try: + return FederatedServer.objects.get( + server_name=domain + ) + except FederatedServer.DoesNotExist: + pass + + data = get_data('https://%s/.well-known/nodeinfo' % domain) + + try: + nodeinfo_url = data.get('links')[0].get('href') + except (TypeError, KeyError): + return None + + data = get_data(nodeinfo_url) + + server = FederatedServer.objects.create( + server_name=domain, + application_type=data['software']['name'], + application_version=data['software']['version'], + ) + return server + + +@app.task +def get_remote_reviews(outbox): + ''' ingest reviews by a new remote bookwyrm user ''' + outbox_page = outbox + '?page=true&type=Review' + data = get_data(outbox_page) + + # TODO: pagination? + for activity in data['orderedItems']: + if not activity['type'] == 'Review': + continue + activitypub.Review(**activity).to_model(Review) diff --git a/bookwyrm/outgoing.py b/bookwyrm/outgoing.py index a196fcec..2a62fd04 100644 --- a/bookwyrm/outgoing.py +++ b/bookwyrm/outgoing.py @@ -2,35 +2,36 @@ import re from django.db import IntegrityError, transaction -from django.http import HttpResponseNotFound, JsonResponse +from django.http import JsonResponse +from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt -import requests +from django.views.decorators.http import require_GET +from markdown import markdown +from requests import HTTPError from bookwyrm import activitypub from bookwyrm import models +from bookwyrm.connectors import get_data, ConnectorException from bookwyrm.broadcast import broadcast +from bookwyrm.sanitize_html import InputHtmlParser from bookwyrm.status import create_notification from bookwyrm.status import create_generated_note from bookwyrm.status import delete_status -from bookwyrm.remote_user import get_or_create_remote_user from bookwyrm.settings import DOMAIN from bookwyrm.utils import regex @csrf_exempt +@require_GET def outbox(request, username): ''' outbox for the requested user ''' - if request.method != 'GET': - return HttpResponseNotFound() + user = get_object_or_404(models.User, localname=username) + filter_type = request.GET.get('type') + if filter_type not in models.status_models: + filter_type = None - try: - user = models.User.objects.get(localname=username) - except models.User.DoesNotExist: - return HttpResponseNotFound() - - # collection overview return JsonResponse( - user.to_outbox(**request.GET), + user.to_outbox(**request.GET, filter_type=filter_type), encoder=activitypub.ActivityEncoder ) @@ -40,6 +41,9 @@ def handle_remote_webfinger(query): user = None # usernames could be @user@domain or user@domain + if not query: + return None + if query[0] == '@': query = query[1:] @@ -54,16 +58,16 @@ def handle_remote_webfinger(query): url = 'https://%s/.well-known/webfinger?resource=acct:%s' % \ (domain, query) try: - response = requests.get(url) - except requests.exceptions.ConnectionError: + data = get_data(url) + except (ConnectorException, HTTPError): return None - if not response.ok: - return None - data = response.json() - for link in data['links']: - if link['rel'] == 'self': + + for link in data.get('links'): + if link.get('rel') == 'self': try: - user = get_or_create_remote_user(link['href']) + user = activitypub.resolve_remote_id( + models.User, link['href'] + ) except KeyError: return None return user @@ -162,22 +166,30 @@ def handle_imported_book(user, item, include_reviews, privacy): if not item.book: return - if item.shelf: + existing_shelf = models.ShelfBook.objects.filter( + book=item.book, added_by=user).exists() + + # shelve the book if it hasn't been shelved already + if item.shelf and not existing_shelf: desired_shelf = models.Shelf.objects.get( identifier=item.shelf, user=user ) - # shelve the book if it hasn't been shelved already - shelf_book, created = models.ShelfBook.objects.get_or_create( + shelf_book = models.ShelfBook.objects.create( book=item.book, shelf=desired_shelf, added_by=user) - if created: - broadcast(user, shelf_book.to_add_activity(user), privacy=privacy) + broadcast(user, shelf_book.to_add_activity(user), privacy=privacy) - # only add new read-throughs if the item isn't already shelved - for read in item.reads: - read.book = item.book - read.user = user - read.save() + for read in item.reads: + # check for an existing readthrough with the same dates + if models.ReadThrough.objects.filter( + user=user, book=item.book, + start_date=read.start_date, + finish_date=read.finish_date + ).exists(): + continue + read.book = item.book + read.user = user + read.save() if include_reviews and (item.rating or item.review): review_title = 'Review of {!r} on Goodreads'.format( @@ -209,15 +221,72 @@ def handle_delete_status(user, status): def handle_status(user, form): ''' generic handler for statuses ''' - status = form.save() + status = form.save(commit=False) + if not status.sensitive and status.content_warning: + # the cw text field remains populated when you click "remove" + status.content_warning = None + status.save() # inspect the text for user tags - text = status.content - matches = re.finditer( - regex.username, - text - ) - for match in matches: + content = status.content + for (mention_text, mention_user) in find_mentions(content): + # add them to status mentions fk + status.mention_users.add(mention_user) + + # turn the mention into a link + content = re.sub( + r'%s([^@]|$)' % mention_text, + r'%s\g<1>' % \ + (mention_user.remote_id, mention_text), + content) + + # add reply parent to mentions and notify + if status.reply_parent: + status.mention_users.add(status.reply_parent.user) + for mention_user in status.reply_parent.mention_users.all(): + status.mention_users.add(mention_user) + + if status.reply_parent.user.local: + create_notification( + status.reply_parent.user, + 'REPLY', + related_user=user, + related_status=status + ) + + # deduplicate mentions + status.mention_users.set(set(status.mention_users.all())) + # create mention notifications + for mention_user in status.mention_users.all(): + if status.reply_parent and mention_user == status.reply_parent.user: + continue + if mention_user.local: + create_notification( + mention_user, + 'MENTION', + related_user=user, + related_status=status + ) + + # don't apply formatting to generated notes + if not isinstance(status, models.GeneratedNote): + status.content = to_markdown(content) + # do apply formatting to quotes + if hasattr(status, 'quote'): + status.quote = to_markdown(status.quote) + + status.save() + + broadcast(user, status.to_create_activity(user), software='bookwyrm') + + # re-format the activity for non-bookwyrm servers + remote_activity = status.to_create_activity(user, pure=True) + broadcast(user, remote_activity, software='other') + + +def find_mentions(content): + ''' detect @mentions in raw status content ''' + for match in re.finditer(regex.strict_username, content): username = match.group().strip().split('@')[1:] if len(username) == 1: # this looks like a local user (@user), fill in the domain @@ -228,48 +297,25 @@ def handle_status(user, form): if not mention_user: # we can ignore users we don't know about continue - # add them to status mentions fk - status.mention_users.add(mention_user) - # create notification if the mentioned user is local - if mention_user.local: - create_notification( - mention_user, - 'MENTION', - related_user=user, - related_status=status - ) - status.save() - - # notify reply parent or tagged users - if status.reply_parent and status.reply_parent.user.local: - create_notification( - status.reply_parent.user, - 'REPLY', - related_user=user, - related_status=status - ) - - broadcast(user, status.to_create_activity(user), software='bookwyrm') - - # re-format the activity for non-bookwyrm servers - if hasattr(status, 'pure_activity_serializer'): - remote_activity = status.to_create_activity(user, pure=True) - broadcast(user, remote_activity, software='other') + yield (match.group(), mention_user) -def handle_tag(user, tag): - ''' tag a book ''' - broadcast(user, tag.to_add_activity(user)) +def format_links(content): + ''' detect and format links ''' + return re.sub( + r'([^(href=")]|^|\()(https?:\/\/(%s([\w\.\-_\/+&\?=:;,])*))' % \ + regex.domain, + r'\g<1>\g<3>', + content) - -def handle_untag(user, book, name): - ''' tag a book ''' - book = models.Book.objects.get(id=book) - tag = models.Tag.objects.get(name=name, book=book, user=user) - tag_activity = tag.to_remove_activity(user) - tag.delete() - - broadcast(user, tag_activity) +def to_markdown(content): + ''' catch links and convert to markdown ''' + content = format_links(content) + content = markdown(content) + # sanitize resulting html + sanitizer = InputHtmlParser() + sanitizer.feed(content) + return sanitizer.get_output() def handle_favorite(user, status): @@ -286,12 +332,13 @@ def handle_favorite(user, status): fav_activity = favorite.to_activity() broadcast( user, fav_activity, privacy='direct', direct_recipients=[status.user]) - create_notification( - status.user, - 'FAVORITE', - related_user=user, - related_status=status - ) + if status.user.local: + create_notification( + status.user, + 'FAVORITE', + related_user=user, + related_status=status + ) def handle_unfavorite(user, status): @@ -309,28 +356,42 @@ def handle_unfavorite(user, status): favorite.delete() broadcast(user, fav_activity, direct_recipients=[status.user]) + # check for notification + if status.user.local: + notification = models.Notification.objects.filter( + user=status.user, related_user=user, + related_status=status, notification_type='FAVORITE' + ).first() + if notification: + notification.delete() + def handle_boost(user, status): ''' a user wishes to boost a status ''' + # is it boostable? + if not status.boostable: + return + if models.Boost.objects.filter( boosted_status=status, user=user).exists(): # you already boosted that. return boost = models.Boost.objects.create( boosted_status=status, + privacy=status.privacy, user=user, ) - boost.save() boost_activity = boost.to_activity() broadcast(user, boost_activity) - create_notification( - status.user, - 'BOOST', - related_user=user, - related_status=status - ) + if status.user.local: + create_notification( + status.user, + 'BOOST', + related_user=user, + related_status=status + ) def handle_unboost(user, status): @@ -343,12 +404,11 @@ def handle_unboost(user, status): boost.delete() broadcast(user, activity) - -def handle_update_book(user, book): - ''' broadcast the news about our book ''' - broadcast(user, book.to_update_activity(user)) - - -def handle_update_user(user): - ''' broadcast editing a user's profile ''' - broadcast(user, user.to_update_activity(user)) + # delete related notification + if status.user.local: + notification = models.Notification.objects.filter( + user=status.user, related_user=user, + related_status=status, notification_type='BOOST' + ).first() + if notification: + notification.delete() diff --git a/bookwyrm/remote_user.py b/bookwyrm/remote_user.py deleted file mode 100644 index 23a805b3..00000000 --- a/bookwyrm/remote_user.py +++ /dev/null @@ -1,111 +0,0 @@ -''' manage remote users ''' -from urllib.parse import urlparse -import requests - -from django.db import transaction - -from bookwyrm import activitypub, models -from bookwyrm import status as status_builder -from bookwyrm.tasks import app - - -def get_or_create_remote_user(actor): - ''' look up a remote user or add them ''' - try: - return models.User.objects.get(remote_id=actor) - except models.User.DoesNotExist: - pass - - data = fetch_user_data(actor) - - actor_parts = urlparse(actor) - with transaction.atomic(): - user = activitypub.Person(**data).to_model(models.User) - user.federated_server = get_or_create_remote_server(actor_parts.netloc) - user.save() - if user.bookwyrm_user: - get_remote_reviews.delay(user.id) - return user - - -def fetch_user_data(actor): - ''' load the user's info from the actor url ''' - try: - response = requests.get( - actor, - headers={'Accept': 'application/activity+json'} - ) - except ConnectionError: - return None - - if not response.ok: - response.raise_for_status() - data = response.json() - - # make sure our actor is who they say they are - if actor != data['id']: - raise ValueError("Remote actor id must match url.") - return data - - -def refresh_remote_user(user): - ''' get updated user data from its home instance ''' - data = fetch_user_data(user.remote_id) - - activity = activitypub.Person(**data) - activity.to_model(models.User, instance=user) - - -@app.task -def get_remote_reviews(user_id): - ''' ingest reviews by a new remote bookwyrm user ''' - try: - user = models.User.objects.get(id=user_id) - except models.User.DoesNotExist: - return - outbox_page = user.outbox + '?page=true' - response = requests.get( - outbox_page, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - # TODO: pagination? - for activity in data['orderedItems']: - status_builder.create_status(activity) - - -def get_or_create_remote_server(domain): - ''' get info on a remote server ''' - try: - return models.FederatedServer.objects.get( - server_name=domain - ) - except models.FederatedServer.DoesNotExist: - pass - - response = requests.get( - 'https://%s/.well-known/nodeinfo' % domain, - headers={'Accept': 'application/activity+json'} - ) - - if response.status_code != 200: - return None - - data = response.json() - try: - nodeinfo_url = data.get('links')[0].get('href') - except (TypeError, KeyError): - return None - - response = requests.get( - nodeinfo_url, - headers={'Accept': 'application/activity+json'} - ) - data = response.json() - - server = models.FederatedServer.objects.create( - server_name=domain, - application_type=data['software']['name'], - application_version=data['software']['version'], - ) - return server diff --git a/bookwyrm/routine_book_tasks.py b/bookwyrm/routine_book_tasks.py deleted file mode 100644 index eaa28d90..00000000 --- a/bookwyrm/routine_book_tasks.py +++ /dev/null @@ -1,16 +0,0 @@ -''' Routine tasks for keeping your library tidy ''' -from datetime import timedelta -from django.utils import timezone -from bookwyrm import books_manager -from bookwyrm import models - -def sync_book_data(): - ''' update books with any changes to their canonical source ''' - expiry = timezone.now() - timedelta(days=1) - books = models.Edition.objects.filter( - sync=True, - last_sync_date__lte=expiry - ).all() - for book in books: - # TODO: create background tasks - books_manager.update_book(book) diff --git a/bookwyrm/sanitize_html.py b/bookwyrm/sanitize_html.py index 9c5ca73a..de13ede8 100644 --- a/bookwyrm/sanitize_html.py +++ b/bookwyrm/sanitize_html.py @@ -1,12 +1,16 @@ ''' html parser to clean up incoming text from unknown sources ''' from html.parser import HTMLParser -class InputHtmlParser(HTMLParser): +class InputHtmlParser(HTMLParser):#pylint: disable=abstract-method ''' Removes any html that isn't allowed_tagsed from a block ''' def __init__(self): HTMLParser.__init__(self) - self.allowed_tags = ['p', 'b', 'i', 'pre', 'a', 'span'] + self.allowed_tags = [ + 'p', 'br', + 'b', 'i', 'strong', 'em', 'pre', + 'a', 'span', 'ul', 'ol', 'li' + ] self.tag_stack = [] self.output = [] # if the html appears invalid, we just won't allow any at all diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 5adf960f..46c38b5a 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -3,8 +3,11 @@ import os from environs import Env +import requests + env = Env() DOMAIN = env('DOMAIN') +VERSION = '0.0.1' PAGE_LENGTH = env('PAGE_LENGTH', 15) @@ -15,6 +18,13 @@ CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' +# email +EMAIL_HOST = env('EMAIL_HOST') +EMAIL_PORT = env('EMAIL_PORT', 587) +EMAIL_HOST_USER = env('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = env('EMAIL_HOST_PASSWORD') +EMAIL_USE_TLS = env('EMAIL_USE_TLS', True) + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -68,6 +78,7 @@ TEMPLATES = [ 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', + 'bookwyrm.context_processors.site_settings', ], }, }, @@ -91,10 +102,6 @@ BOOKWYRM_DBS = { 'HOST': env('POSTGRES_HOST', ''), 'PORT': 5432 }, - 'sqlite': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'fedireads.db') - } } DATABASES = { @@ -146,3 +153,6 @@ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, env('STATIC_ROOT', 'static')) MEDIA_URL = '/images/' MEDIA_ROOT = os.path.join(BASE_DIR, env('MEDIA_ROOT', 'images')) + +USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % ( + requests.utils.default_user_agent(), VERSION, DOMAIN) diff --git a/bookwyrm/signatures.py b/bookwyrm/signatures.py index 57c181df..ff281664 100644 --- a/bookwyrm/signatures.py +++ b/bookwyrm/signatures.py @@ -31,7 +31,7 @@ def make_signature(sender, destination, date, digest): 'digest: %s' % digest, ] message_to_sign = '\n'.join(signature_headers) - signer = pkcs1_15.new(RSA.import_key(sender.private_key)) + signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key)) signed_message = signer.sign(SHA256.new(message_to_sign.encode('utf8'))) signature = { 'keyId': '%s#main-key' % sender.remote_id, diff --git a/bookwyrm/static/css/format.css b/bookwyrm/static/css/format.css index 9e8a24ba..fab335b1 100644 --- a/bookwyrm/static/css/format.css +++ b/bookwyrm/static/css/format.css @@ -5,6 +5,11 @@ .navbar .logo { max-height: 50px; } + +.card { + overflow: visible; +} + /* --- TOGGLES --- */ input.toggle-control { display: none; @@ -65,6 +70,15 @@ input.toggle-control:checked ~ .modal.toggle-content { .cover-container { height: 250px; width: max-content; + max-width: 250px; +} +.cover-container.is-large { + height: max-content; + max-width: 330px; +} +.cover-container.is-large img { + max-height: 500px; + height: auto; } .cover-container.is-medium { height: 150px; @@ -116,6 +130,9 @@ input.toggle-control:checked ~ .modal.toggle-content { vertical-align: middle; display: inline; } +.navbar .avatar { + max-height: none; +} /* --- QUOTES --- */ @@ -136,8 +153,3 @@ input.toggle-control:checked ~ .modal.toggle-content { content: "\e904"; right: 0; } - -/* --- BLOCKQUOTE --- */ -blockquote { - white-space: pre-line; -} diff --git a/bookwyrm/status.py b/bookwyrm/status.py index 6a86209f..648f2e7d 100644 --- a/bookwyrm/status.py +++ b/bookwyrm/status.py @@ -1,7 +1,7 @@ ''' Handle user activity ''' from django.utils import timezone -from bookwyrm import activitypub, books_manager, models +from bookwyrm import models from bookwyrm.sanitize_html import InputHtmlParser @@ -12,37 +12,6 @@ def delete_status(status): status.save() -def create_status(activity): - ''' unfortunately, it's not QUITE as simple as deserializing it ''' - # render the json into an activity object - serializer = activitypub.activity_objects[activity['type']] - activity = serializer(**activity) - try: - model = models.activity_models[activity.type] - except KeyError: - # not a type of status we are prepared to deserialize - return None - - # ignore notes that aren't replies to known statuses - if activity.type == 'Note': - reply = models.Status.objects.filter( - remote_id=activity.inReplyTo - ).first() - if not reply: - return None - - # look up books - book_urls = [] - if hasattr(activity, 'inReplyToBook'): - book_urls.append(activity.inReplyToBook) - if hasattr(activity, 'tag'): - book_urls += [t['href'] for t in activity.tag if t['type'] == 'Book'] - for remote_id in book_urls: - books_manager.get_or_create_book(remote_id) - - return activity.to_model(model) - - def create_generated_note(user, content, mention_books=None, privacy='public'): ''' a note created by the app about user activity ''' # sanitize input html diff --git a/bookwyrm/templates/about.html b/bookwyrm/templates/about.html index 25caf5e2..aa7426ca 100644 --- a/bookwyrm/templates/about.html +++ b/bookwyrm/templates/about.html @@ -3,13 +3,13 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}

Code of Conduct

- {{ site_settings.code_of_conduct | safe }} + {{ site.code_of_conduct | safe }}
diff --git a/bookwyrm/templates/author.html b/bookwyrm/templates/author.html index fb4970e4..4235b266 100644 --- a/bookwyrm/templates/author.html +++ b/bookwyrm/templates/author.html @@ -1,18 +1,36 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
-

{{ author.display_name }}

+
+
+

{{ author.name }}

+
+ {% if request.user.is_authenticated and perms.bookwyrm.edit_book %} +
+ + + Edit Author + + +
+ {% endif %} +
+
+
{% if author.bio %}

- {{ author.bio }} + {{ author.bio | to_markdown | safe }}

{% endif %} + {% if author.wikipedia_link %} +

Wikipedia

+ {% endif %}
-

Books by {{ author.display_name }}

+

Books by {{ author.name }}

{% include 'snippets/book_tiles.html' with books=books %}
{% endblock %} diff --git a/bookwyrm/templates/book.html b/bookwyrm/templates/book.html index ec48d2e5..35c8dd5f 100644 --- a/bookwyrm/templates/book.html +++ b/bookwyrm/templates/book.html @@ -1,16 +1,27 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
-
-

- {% include 'snippets/book_titleby.html' with book=book %} -

+
+
+

+ {{ book.title }}{% if book.subtitle %}: + {{ book.subtitle }}{% endif %} + {% if book.series %} + ({{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %})
+ {% endif %} +

+ {% if book.authors %} +

+ by {% include 'snippets/authors.html' with book=book %} +

+ {% endif %} +
{% if request.user.is_authenticated and perms.bookwyrm.edit_book %} -
+
Edit Book @@ -41,19 +52,44 @@
{% endif %} -
- {% for field in info_fields %} - {% if field.value %} -
{{ field.name }}:
-
{{ field.value }}
+
+
+ {% if book.isbn_13 %} +
+
ISBN:
+
{{ book.isbn_13 }}
+
+ {% endif %} + + {% if book.oclc_number %} +
+
OCLC Number:
+
{{ book.oclc_number }}
+
+ {% endif %} + + {% if book.asin %} +
+
ASIN:
+
{{ book.asin }}
+
+ {% endif %} +
+ +

+ {% if book.physical_format %}{{ book.physical_format | title }}{% if book.pages %},
{% endif %}{% endif %} + {% if book.pages %}{{ book.pages }} pages{% endif %} +

+ + {% if book.openlibrary_key %} +

View on OpenLibrary

{% endif %} - {% endfor %} -
+
-

{% include 'snippets/stars.html' with rating=rating %} ({{ reviews|length }} review{{ reviews|length|pluralize }})

+

{% include 'snippets/stars.html' with rating=rating %} ({{ review_count }} review{{ reviews|length|pluralize }})

{% include 'snippets/trimmed_text.html' with full=book|book_description %} @@ -86,143 +122,56 @@ {% endif %} - {% if book.parent_work.edition_set.count > 1 %} -

{{ book.parent_work.edition_set.count }} editions

+ {% if book.parent_work.editions.count > 1 %} +

{{ book.parent_work.editions.count }} editions

{% endif %}
- {% for readthrough in readthroughs %} -
- - + {# user's relationship to the book #} +
+ {% for shelf in user_shelves %} +

+ This edition is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/shelf_selector.html' with current=shelf.shelf %} +

+ {% endfor %} + {% for shelf in other_edition_shelves %} +

+ A different edition of this book is on your {{ shelf.shelf.name }} shelf. + {% include 'snippets/switch_edition_button.html' with edition=book %} +

+ {% endfor %}
+ {% if readthroughs.exists %} +
+

Your reading activity

+ {% for readthrough in readthroughs %} + {% include 'snippets/readthrough.html' with readthrough=readthrough %} + {% endfor %} +
+ {% endif %}
- - +
+ {% if book.subjects %} +
+

Subjects

+
    + {% for subject in book.subjects %} +
  • {{ subject }}
    • + {% endfor %} +
+
+ {% endif %} + + {% if book.subject_places %} +
+

Places

+
    + {% for place in book.subject_placess %} +
  • {{ place }}
    • + {% endfor %} +
+
+ {% endif %} +
- -{% if not reviews %} -
-

No reviews yet!

-
-{% endif %} -
{% for review in reviews %}
@@ -265,14 +230,14 @@
{% endfor %} -
+
{% for rating in ratings %} -
+
-
{% include 'snippets/avatar.html' %}
+
{% include 'snippets/avatar.html' with user=rating.user %}
- {% include 'snippets/username.html' %} + {% include 'snippets/username.html' with user=rating.user %}
rated it
@@ -288,6 +253,5 @@
- {% endblock %} diff --git a/bookwyrm/templates/direct_messages.html b/bookwyrm/templates/direct_messages.html new file mode 100644 index 00000000..6a20b111 --- /dev/null +++ b/bookwyrm/templates/direct_messages.html @@ -0,0 +1,37 @@ +{% extends 'layout.html' %} +{% block content %} + +
+

Direct Messages

+ + {% if not activities %} +

You have no messages right now.

+ {% endif %} + {% for activity in activities %} +
+ {% include 'snippets/status.html' with status=activity %} +
+ {% endfor %} + + +
+ +{% endblock %} diff --git a/bookwyrm/templates/discover.html b/bookwyrm/templates/discover.html new file mode 100644 index 00000000..79e31f56 --- /dev/null +++ b/bookwyrm/templates/discover.html @@ -0,0 +1,80 @@ +{% extends 'layout.html' %} +{% block content %} + +{% if not request.user.is_authenticated %} +
+

{{ site.name }}: {{ site.instance_tagline }}

+
+ +
+
+
+ {% include 'snippets/about.html' %} +
+
+
+
+ {% if site.allow_registration %} +

Join {{ site.name }}

+
+ {% include 'snippets/register_form.html' %} +
+ {% else %} +

This instance is closed

+

{{ site.registration_closed_text | safe}}

+ {% endif %} +
+
+
+{% else %} +
+

Discover

+
+{% endif %} + +
+

Recent Books

+
+ +
+
+
+
+ {% include 'snippets/discover/large-book.html' with book=books.0 %} +
+
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.1 %} +
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.2 %} +
+
+
+
+
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.3 %} +
+
+
+
+ {% include 'snippets/discover/small-book.html' with book=books.4 %} +
+
+
+
+
+ {% include 'snippets/discover/large-book.html' with book=books.5 %} +
+
+
+
+ +{% endblock %} diff --git a/bookwyrm/templates/edit_author.html b/bookwyrm/templates/edit_author.html new file mode 100644 index 00000000..b08aa983 --- /dev/null +++ b/bookwyrm/templates/edit_author.html @@ -0,0 +1,89 @@ +{% extends 'layout.html' %} +{% load humanize %} +{% block content %} +
+
+

+ Edit "{{ author.name }}" +

+
+ + + Close + + +
+
+
+

Added: {{ author.created_date | naturaltime }}

+

Updated: {{ author.updated_date | naturaltime }}

+

Last edited by: {{ author.last_edited_by.display_name }}

+
+
+ +{% if form.non_field_errors %} +
+

{{ form.non_field_errors }}

+
+{% endif %} + +
+ {% csrf_token %} + + +
+
+

Metadata

+

{{ form.name }}

+ {% for error in form.name.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.bio }}

+ {% for error in form.bio.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.wikipedia_link }}

+ {% for error in form.wikipedia_link.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.born }}

+ {% for error in form.born.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.died }}

+ {% for error in form.died.errors %} +

{{ error | escape }}

+ {% endfor %} +
+
+

Author Identifiers

+

{{ form.openlibrary_key }}

+ {% for error in form.openlibrary_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.librarything_key }}

+ {% for error in form.librarything_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +

{{ form.goodreads_key }}

+ {% for error in form.goodreads_key.errors %} +

{{ error | escape }}

+ {% endfor %} + +
+
+ +
+ + Cancel +
+
+ +{% endblock %} + diff --git a/bookwyrm/templates/edit_book.html b/bookwyrm/templates/edit_book.html index 54cefb0a..b0e262dd 100644 --- a/bookwyrm/templates/edit_book.html +++ b/bookwyrm/templates/edit_book.html @@ -17,63 +17,47 @@

Added: {{ book.created_date | naturaltime }}

Updated: {{ book.updated_date | naturaltime }}

+

Last edited by: {{ book.last_edited_by.display_name }}

-{% if login_form.non_field_errors %} +{% if form.non_field_errors %}
-

{{ login_form.non_field_errors }}

+

{{ form.non_field_errors }}

{% endif %}
{% csrf_token %} -
-

Data sync -

If sync is enabled, any changes will be over-written

-

-
-
- -
-
- -
-
-
- +

Metadata

-

{{ form.title }}

+

{{ form.title }}

{% for error in form.title.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.sort_title }}

- {% for error in form.sort_title.errors %} -

{{ error | escape }}

- {% endfor %} -

{{ form.subtitle }}

+

{{ form.subtitle }}

{% for error in form.subtitle.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.description }}

+

{{ form.description }}

{% for error in form.description.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.series }}

+

{{ form.series }}

{% for error in form.series.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.series_number }}

+

{{ form.series_number }}

{% for error in form.series_number.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.first_published_date }}

+

{{ form.first_published_date }}

{% for error in form.first_published_date.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.published_date }}

+

{{ form.published_date }}

{% for error in form.published_date.errors %}

{{ error | escape }}

{% endfor %} @@ -97,7 +81,7 @@

Physical Properties

-

{{ form.physical_format }}

+

{{ form.physical_format }}

{% for error in form.physical_format.errors %}

{{ error | escape }}

{% endfor %} @@ -105,7 +89,7 @@

{{ error | escape }}

{% endfor %} -

{{ form.pages }}

+

{{ form.pages }}

{% for error in form.pages.errors %}

{{ error | escape }}

{% endfor %} @@ -113,24 +97,24 @@

Book Identifiers

-

{{ form.isbn_13 }}

+

{{ form.isbn_13 }}

{% for error in form.isbn_13.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.isbn_10 }}

+

{{ form.isbn_10 }}

{% for error in form.isbn_10.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.openlibrary_key }}

+

{{ form.openlibrary_key }}

{% for error in form.openlibrary_key.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.librarything_key }}

- {% for error in form.librarything_key.errors %} +

{{ form.oclc_number }}

+ {% for error in form.oclc_number.errors %}

{{ error | escape }}

{% endfor %} -

{{ form.goodreads_key }}

- {% for error in form.goodreads_key.errors %} +

{{ form.asin }}

+ {% for error in form.ASIN.errors %}

{{ error | escape }}

{% endfor %}
diff --git a/bookwyrm/templates/editions.html b/bookwyrm/templates/editions.html index 273b2cd6..619ceafb 100644 --- a/bookwyrm/templates/editions.html +++ b/bookwyrm/templates/editions.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}

Editions of "{{ work.title }}"

diff --git a/bookwyrm/templates/feed.html b/bookwyrm/templates/feed.html index b4a81063..d15f7d76 100644 --- a/bookwyrm/templates/feed.html +++ b/bookwyrm/templates/feed.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
@@ -44,26 +44,38 @@
-

- Imports are limited in size, and only the first {{ limit }} items will be imported.

diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index f91e2cce..c58635d1 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% load humanize %} {% block content %}
@@ -8,7 +8,7 @@

Import started: {{ job.created_date | naturaltime }}

- {% if task.successful %} + {% if job.complete %}

Import completed: {{ task.date_done | naturaltime }}

@@ -18,7 +18,7 @@
- {% if not task.ready %} + {% if not job.complete %} Import still in progress.

(Hit reload to update!) diff --git a/bookwyrm/templates/invite.html b/bookwyrm/templates/invite.html index 332a5ed2..458ce3df 100644 --- a/bookwyrm/templates/invite.html +++ b/bookwyrm/templates/invite.html @@ -3,7 +3,8 @@

-
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}
diff --git a/bookwyrm/templates/layout.html b/bookwyrm/templates/layout.html index 4ae70a4b..fe448585 100644 --- a/bookwyrm/templates/layout.html +++ b/bookwyrm/templates/layout.html @@ -1,29 +1,30 @@ -{% load fr_display %} +{% load bookwyrm_tags %} - {% if title %}{{ title }} | {% endif %}BookWyrm + {% if title %}{{ title }} | {% endif %}{{ site.name }} - + - - - - - - + + + + + + + + - -
- {% if site_settings.allow_registration %} + {% if site.allow_registration %}

Create an Account

{% include 'snippets/register_form.html' %} @@ -50,7 +50,7 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}

More about this site diff --git a/bookwyrm/templates/notifications.html b/bookwyrm/templates/notifications.html index bc96f307..ae026a9f 100644 --- a/bookwyrm/templates/notifications.html +++ b/bookwyrm/templates/notifications.html @@ -1,5 +1,6 @@ {% extends 'layout.html' %} -{% load humanize %}l +{% load humanize %} +{% load bookwyrm_tags %} {% block content %}

Notifications

@@ -12,50 +13,89 @@
{% for notification in notifications %} -
-
-

- {% if notification.related_user %} - {% include 'snippets/avatar.html' with user=notification.related_user %} - {% include 'snippets/username.html' with user=notification.related_user %} - {% if notification.notification_type == 'FAVORITE' %} - favorited your - status - - {% elif notification.notification_type == 'MENTION' %} - mentioned you in a - status - + {% related_status notification as related_status %} +

+
+
+ {% if notification.notification_type == 'MENTION' %} + {% elif notification.notification_type == 'REPLY' %} - replied - to your - status - - {% elif notification.notification_type == 'FOLLOW' %} - followed you - - {% elif notification.notification_type == 'FOLLOW_REQUEST' %} - sent you a follow request -
- {% include 'snippets/follow_request_buttons.html' with user=notification.related_user %} -
- + + {% elif notification.notification_type == 'FOLLOW' or notification.notification_type == 'FOLLOW_REQUEST' %} + {% elif notification.notification_type == 'BOOST' %} - boosted your status + + {% elif notification.notification_type == 'FAVORITE' %} + + {% elif notification.notification_type == 'IMPORT' %} + {% endif %} - {% else %} - your import completed. +
+
+
+

+ {# DESCRIPTION #} + {% if notification.related_user %} + {% include 'snippets/avatar.html' with user=notification.related_user %} + {% include 'snippets/username.html' with user=notification.related_user %} + {% if notification.notification_type == 'FAVORITE' %} + favorited your + {{ related_status | status_preview_name|safe }} + {% elif notification.notification_type == 'MENTION' %} + mentioned you in a + {{ related_status | status_preview_name|safe }} + + {% elif notification.notification_type == 'REPLY' %} + replied + to your + {{ related_status | status_preview_name|safe }} + {% elif notification.notification_type == 'FOLLOW' %} + followed you + {% include 'snippets/follow_button.html' with user=notification.related_user %} + {% elif notification.notification_type == 'FOLLOW_REQUEST' %} + sent you a follow request +

+ {% include 'snippets/follow_request_buttons.html' with user=notification.related_user %} +
+ + {% elif notification.notification_type == 'BOOST' %} + boosted your {{ related_status | status_preview_name|safe }} + {% endif %} + {% else %} + your import completed. + {% endif %} +

+
+ {% if related_status %} +
+ {# PREVIEW #} +
+
+
+ {% if related_status.content %} + {{ related_status.content | safe | truncatewords_html:10 }} + {% elif related_status.quote %} + {{ related_status.quote | safe | truncatewords_html:10 }} + {% elif related_status.rating %} + {% include 'snippets/stars.html' with rating=related_status.rating %} + {% endif %} +
+
+ {{ related_status.published_date | post_date }} + {% include 'snippets/privacy-icons.html' with item=related_status %} +
+
+
+
{% endif %} -

+
- -

{{ notification.created_date | naturaltime }}

{% endfor %} + {% if not notifications %}

You're all caught up!

{% endif %}
- {% endblock %} diff --git a/bookwyrm/templates/password_reset.html b/bookwyrm/templates/password_reset.html index ac2ba7bd..99d114e4 100644 --- a/bookwyrm/templates/password_reset.html +++ b/bookwyrm/templates/password_reset.html @@ -34,7 +34,7 @@
- {% include 'snippets/about.html' with site_settings=site_settings %} + {% include 'snippets/about.html' %}
diff --git a/bookwyrm/templates/shelf.html b/bookwyrm/templates/shelf.html index d6842d13..99e0c678 100644 --- a/bookwyrm/templates/shelf.html +++ b/bookwyrm/templates/shelf.html @@ -1,5 +1,5 @@ {% extends 'layout.html' %} -{% load fr_display %} +{% load bookwyrm_tags %} {% block content %}
@@ -106,6 +106,8 @@
+ {% else %} + {% endif %}