From 881cc4d64bcf16f282494a381d4068c77e9a9a35 Mon Sep 17 00:00:00 2001 From: Adam Kelly Date: Mon, 20 Apr 2020 17:10:19 +0100 Subject: [PATCH] First draft async imports. --- fedireads/goodreads_import.py | 48 ++++++++++++++++++++----- fedireads/incoming.py | 1 + fedireads/models/status.py | 3 +- fedireads/outgoing.py | 2 +- fedireads/templates/import_results.html | 12 ++----- fedireads/templates/notifications.html | 5 +++ fedireads/view_actions.py | 41 ++++----------------- fr_celery/celery.py | 1 + 8 files changed, 57 insertions(+), 56 deletions(-) diff --git a/fedireads/goodreads_import.py b/fedireads/goodreads_import.py index 884cd76a..163aa90c 100644 --- a/fedireads/goodreads_import.py +++ b/fedireads/goodreads_import.py @@ -1,11 +1,13 @@ ''' handle reading a csv from goodreads ''' import re import csv -import itertools import dateutil.parser +from requests import HTTPError from fedireads import books_manager -from fedireads.models import Edition, ReadThrough +from fedireads import outgoing +from fedireads.models import Edition, ReadThrough, User +from fedireads.tasks import app # Mapping goodreads -> fedireads shelf titles. @@ -36,14 +38,42 @@ def construct_search_term(title, author): return ' '.join([title, author]) -class GoodreadsCsv: - ''' define a goodreads csv ''' - def __init__(self, csv_file): - self.reader = csv.DictReader(csv_file) +def async_import(user, csv_file): + entries = list(csv.DictReader(csv_file))[:MAX_ENTRIES] + return import_data.delay(user.id, entries) + +@app.task +def import_data(user_id, entries): + user = User.objects.get(pk=user_id) + results = [] + reviews = [] + failures = [] + for item in entries: + item = GoodreadsItem(item) + try: + item.resolve() + except HTTPError: + pass + if item.book: + results.append(item) + if item.rating or item.review: + reviews.append(item) + else: + failures.append(item) + + outgoing.handle_import_books(user, results) + for item in reviews: + review_title = "Review of {!r} on Goodreads".format( + item.book.title, + ) if item.review else "" + outgoing.handle_review( + user, + item.book, + review_title, + item.review, + item.rating, + ) - def __iter__(self): - for line in itertools.islice(self.reader, MAX_ENTRIES): - yield GoodreadsItem(line) class GoodreadsItem: ''' a processed line in a goodreads csv ''' diff --git a/fedireads/incoming.py b/fedireads/incoming.py index f14b7d62..49c9e9b0 100644 --- a/fedireads/incoming.py +++ b/fedireads/incoming.py @@ -14,6 +14,7 @@ from fedireads import models, outgoing from fedireads import status as status_builder from fedireads.remote_user import get_or_create_remote_user from fedireads.tasks import app +from fedireads.status import create_notification @csrf_exempt diff --git a/fedireads/models/status.py b/fedireads/models/status.py index 6f19fe28..7136eb53 100644 --- a/fedireads/models/status.py +++ b/fedireads/models/status.py @@ -150,7 +150,8 @@ class ReadThrough(FedireadsModel): NotificationType = models.TextChoices( - 'NotificationType', 'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST') + 'NotificationType', + 'FAVORITE REPLY TAG FOLLOW FOLLOW_REQUEST BOOST IMPORT_RESULT') class Notification(FedireadsModel): ''' you've been tagged, liked, followed, etc ''' diff --git a/fedireads/outgoing.py b/fedireads/outgoing.py index 435843c0..42b659df 100644 --- a/fedireads/outgoing.py +++ b/fedireads/outgoing.py @@ -203,6 +203,7 @@ def handle_import_books(user, items): status.status_type = 'Update' status.save() + create_notification(user, 'IMPORT_RESULT', related_status=status) create_activity = activitypub.get_create( user, activitypub.get_status(status)) broadcast(user, create_activity) @@ -356,4 +357,3 @@ def handle_update_user(user): actor = activitypub.get_actor(user) update_activity = activitypub.get_update(user, actor) broadcast(user, update_activity) - diff --git a/fedireads/templates/import_results.html b/fedireads/templates/import_results.html index 2994b6ff..376f64cb 100644 --- a/fedireads/templates/import_results.html +++ b/fedireads/templates/import_results.html @@ -2,17 +2,9 @@ {% block content %}
-

The following books could not be imported:

+

Import

-
    - {% for item in failures %} -
  • - {{ item }} -
    • - {% endfor %} -
- -

{{ success_count }} books imported successfully

+ Import uploaded successfully. The import is being processed.
{% endblock %} diff --git a/fedireads/templates/notifications.html b/fedireads/templates/notifications.html index dd5f135c..26197b27 100644 --- a/fedireads/templates/notifications.html +++ b/fedireads/templates/notifications.html @@ -14,6 +14,7 @@ {% for notification in notifications %}
{{ notification.created_date | naturaltime }} + {% if notification.related_user %} {% include 'snippets/username.html' with user=notification.related_user %} {% if notification.notification_type == 'FAVORITE' %} favorited your @@ -36,6 +37,10 @@ {% elif notification.notification_type == 'BOOST' %} boosted your status {% endif %} + {% else %} + your import succeeded. + + {% endif %}
{% endfor %} {% if not notifications %} diff --git a/fedireads/view_actions.py b/fedireads/view_actions.py index a5fb1f07..bddbfbf7 100644 --- a/fedireads/view_actions.py +++ b/fedireads/view_actions.py @@ -2,7 +2,6 @@ from io import BytesIO, TextIOWrapper import re from PIL import Image -from requests import HTTPError from django.contrib.auth import authenticate, login, logout from django.contrib.auth.decorators import login_required @@ -12,7 +11,7 @@ from django.shortcuts import redirect from django.template.response import TemplateResponse from fedireads import forms, models, books_manager, outgoing -from fedireads.goodreads_import import GoodreadsCsv +from fedireads import goodreads_import from fedireads.settings import DOMAIN from fedireads.views import get_user_from_username from fedireads.books_manager import get_or_create_book @@ -419,38 +418,10 @@ def import_data(request): ''' ingest a goodreads csv ''' form = forms.ImportForm(request.POST, request.FILES) if form.is_valid(): - results = [] - reviews = [] - failures = [] - for item in GoodreadsCsv(TextIOWrapper( - request.FILES['csv_file'], - encoding=request.encoding)): - try: - item.resolve() - except HTTPError: - pass - if item.book: - results.append(item) - if item.rating or item.review: - reviews.append(item) - else: - failures.append(item) - - outgoing.handle_import_books(request.user, results) - for item in reviews: - review_title = "Review of {!r} on Goodreads".format( - item.book.title, - ) if item.review else "" - outgoing.handle_review( - request.user, - item.book, - review_title, - item.review, - item.rating, - ) - return TemplateResponse(request, 'import_results.html', { - 'success_count': len(results), - 'failures': failures, - }) + goodreads_import.async_import( + request.user, + TextIOWrapper(request.FILES['csv_file'], encoding=request.encoding) + ) + return TemplateResponse(request, 'import_results.html', {}) return HttpResponseBadRequest() diff --git a/fr_celery/celery.py b/fr_celery/celery.py index 435b796c..f7ac2d33 100644 --- a/fr_celery/celery.py +++ b/fr_celery/celery.py @@ -21,4 +21,5 @@ app.autodiscover_tasks() app.autodiscover_tasks(['fedireads'], related_name='incoming') app.autodiscover_tasks(['fedireads'], related_name='broadcast') app.autodiscover_tasks(['fedireads'], related_name='books_manager') +app.autodiscover_tasks(['fedireads'], related_name='goodreads_import')