yul/bookwyrm-mastodon
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #649 from mmai/feature/importLibrarything

Browse Source 
librarything import
This commit is contained in:
Mouse Reeve
2021-02-27 12:17:41 -08:00
committed by GitHub
parent 89b2a63ec0 bfedb300ca
commit 04ed5d62dd
9 changed files with 513 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bookwyrm/tests/data/librarything.tsv Normal file
View File

@ -0,0 +1,4 @@
Book Id Title Sort Character Primary Author Primary Author Role Secondary Author Secondary Author Roles Publication Date Review Rating Comment Private Comment Summary Media Physical Description Weight Height Thickness Length Dimensions Page Count LCCN Acquired Date Started Date Read Barcode BCID Tags Collections Languages Original Languages LC Classification ISBN ISBNs Subjects Dewey Decimal Dewey Wording Other Call Number Copies Source Entry Date From Where OCLC Work id Lending Patron Lending Status Lending Start Lending End
5498194 Marelle 1 Cortázar, Julio Gallimard (1979), Poche 1979 chef d'oeuvre 4.5 Marelle by Julio Cortázar (1979) Broché 590 p.; 7.24 inches 1.28 pounds 7.24 inches 1.26 inches 4.96 inches 7.24 x 4.96 x 1.26 inches 590 [2007-04-16] [2007-05-08] roman, espagnol, expérimental, bohème, philosophie Your library French Spanish PQ7797 .C7145 [2070291340] 2070291340, 9782070291342 Cort<72>azar, Julio. Rayuela 863 Literature > Spanish And Portuguese > Spanish fiction 1 Amazon.fr [2006-08-09] 57814
5015319 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) 1 Roubaud, Jacques Seuil (1989), Unknown Binding 1989 5 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) by Jacques Roubaud (1989) Broché 411 p.; 7.72 inches 0.88 pounds 7.72 inches 1.02 inches 5.43 inches 7.72 x 5.43 x 1.02 inches 411 Your library English PQ2678 .O77 [2020104725] 2020104725, 9782020104722 Autobiographical fiction|Roubaud, Jacques > Fiction 813 American And Canadian > Fiction > Literature 1 Amazon.com [2006-07-25] 478910
5015399 Le Maître et Marguerite 1 Boulgakov, Mikhaïl Pocket (1994), Poche 1994 Le Maître et Marguerite by Mikhaïl Boulgakov (1994) Broché 579 p.; 7.09 inches 0.66 pounds 7.09 inches 1.18 inches 4.33 inches 7.09 x 4.33 x 1.18 inches 579 Your library French PG3476 .B78 [2266062328] 2266062328, 9782266062329 Allegories|Bulgakov|Good and evil > Fiction|Humanities|Jerusalem > Fiction|Jesus Christ > Fiction|Literature|Mental illness > Fiction|Moscow (Russia) > Fiction|Novel|Pilate, Pontius, 1st cent. > Fiction|Political fiction|Russia > Fiction|Russian fiction|Russian publications (Form Entry)|Soviet Union > History > 1925-1953 > Fiction|literature 891.7342 1917-1945 > 1917-1991 (USSR) > Literature > Literature of other Indo-European languages > Other Languages > Russian > Russian Fiction 1 Amazon.fr [2006-07-25] 10151
1 Book Id Title Sort Character Primary Author Primary Author Role Secondary Author Secondary Author Roles Publication Date Review Rating Comment Private Comment Summary Media Physical Description Weight Height Thickness Length Dimensions Page Count LCCN Acquired Date Started Date Read Barcode BCID Tags Collections Languages Original Languages LC Classification ISBN ISBNs Subjects Dewey Decimal Dewey Wording Other Call Number Copies Source Entry Date From Where OCLC Work id Lending Patron Lending Status Lending Start Lending End
2 5498194 Marelle 1 Cortázar, Julio Gallimard (1979), Poche 1979 chef d'oeuvre 4.5 Marelle by Julio Cortázar (1979) Broché 590 p.; 7.24 inches 1.28 pounds 7.24 inches 1.26 inches 4.96 inches 7.24 x 4.96 x 1.26 inches 590 [2007-04-16] [2007-05-08] roman, espagnol, expérimental, bohème, philosophie Your library French Spanish PQ7797 .C7145 [2070291340] 2070291340, 9782070291342 Cortâazar, Julio. Rayuela 863 Literature > Spanish And Portuguese > Spanish fiction 1 Amazon.fr [2006-08-09] 57814
3 5015319 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) 1 Roubaud, Jacques Seuil (1989), Unknown Binding 1989 5 Le grand incendie de Londres: Récit, avec incises et bifurcations, 1985-1987 (Fiction & Cie) by Jacques Roubaud (1989) Broché 411 p.; 7.72 inches 0.88 pounds 7.72 inches 1.02 inches 5.43 inches 7.72 x 5.43 x 1.02 inches 411 Your library English PQ2678 .O77 [2020104725] 2020104725, 9782020104722 Autobiographical fiction|Roubaud, Jacques > Fiction 813 American And Canadian > Fiction > Literature 1 Amazon.com [2006-07-25] 478910
4 5015399 Le Maître et Marguerite 1 Boulgakov, Mikhaïl Pocket (1994), Poche 1994 Le Maître et Marguerite by Mikhaïl Boulgakov (1994) Broché 579 p.; 7.09 inches 0.66 pounds 7.09 inches 1.18 inches 4.33 inches 7.09 x 4.33 x 1.18 inches 579 Your library French PG3476 .B78 [2266062328] 2266062328, 9782266062329 Allegories|Bulgakov|Good and evil > Fiction|Humanities|Jerusalem > Fiction|Jesus Christ > Fiction|Literature|Mental illness > Fiction|Moscow (Russia) > Fiction|Novel|Pilate, Pontius, 1st cent. > Fiction|Political fiction|Russia > Fiction|Russian fiction|Russian publications (Form Entry)|Soviet Union > History > 1925-1953 > Fiction|literature 891.7342 1917-1945 > 1917-1991 (USSR) > Literature > Literature of other Indo-European languages > Other Languages > Russian > Russian Fiction 1 Amazon.fr [2006-07-25] 10151

54
bookwyrm/tests/test_goodreads_import.py
View File

@ -7,16 +7,19 @@ from unittest.mock import patch
from django.test import TestCase
import responses
from bookwyrm import goodreads_import, models
from bookwyrm import models, importer
from bookwyrm.goodreads_import import GoodreadsImporter
from bookwyrm import importer
from bookwyrm.settings import DOMAIN
class GoodreadsImport(TestCase):
''' importing from goodreads csv '''
def setUp(self):
self.importer = GoodreadsImporter()
''' use a test csv '''
datafile = pathlib.Path(__file__).parent.joinpath(
'data/goodreads.csv')
self.csv = open(datafile, 'r')
self.csv = open(datafile, 'r', encoding=self.importer.encoding)
self.user = models.User.objects.create_user(
'mouse', 'mouse@mouse.mouse', 'password', local=True)
@ -41,7 +44,7 @@ class GoodreadsImport(TestCase):
def test_create_job(self):
''' creates the import job entry and checks csv '''
import_job = goodreads_import.create_job(
import_job = self.importer.create_job(
self.user, self.csv, False, 'public')
self.assertEqual(import_job.user, self.user)
self.assertEqual(import_job.include_reviews, False)
@ -59,13 +62,13 @@ class GoodreadsImport(TestCase):
def test_create_retry_job(self):
''' trying again with items that didn't import '''
import_job = goodreads_import.create_job(
import_job = self.importer.create_job(
self.user, self.csv, False, 'unlisted')
import_items = models.ImportItem.objects.filter(
job=import_job
).all()[:2]
retry = goodreads_import.create_retry_job(
retry = self.importer.create_retry_job(
self.user, import_job, import_items)
self.assertNotEqual(import_job, retry)
self.assertEqual(retry.user, self.user)
@ -82,13 +85,13 @@ class GoodreadsImport(TestCase):
def test_start_import(self):
''' begin loading books '''
import_job = goodreads_import.create_job(
import_job = self.importer.create_job(
self.user, self.csv, False, 'unlisted')
MockTask = namedtuple('Task', ('id'))
mock_task = MockTask(7)
with patch('bookwyrm.goodreads_import.import_data.delay') as start:
with patch('bookwyrm.importer.import_data.delay') as start:
start.return_value = mock_task
goodreads_import.start_import(import_job)
self.importer.start_import(import_job)
import_job.refresh_from_db()
self.assertEqual(import_job.task_id, '7')
@ -96,7 +99,7 @@ class GoodreadsImport(TestCase):
@responses.activate
def test_import_data(self):
''' resolve entry '''
import_job = goodreads_import.create_job(
import_job = self.importer.create_job(
self.user, self.csv, False, 'unlisted')
book = models.Edition.objects.create(title='Test Book')
@ -104,8 +107,8 @@ class GoodreadsImport(TestCase):
'bookwyrm.models.import_job.ImportItem.get_book_from_isbn'
) as resolve:
resolve.return_value = book
with patch('bookwyrm.goodreads_import.handle_imported_book'):
goodreads_import.import_data(import_job.id)
with patch('bookwyrm.importer.handle_imported_book'):
importer.import_data(self.importer.service, import_job.id)
import_item = models.ImportItem.objects.get(job=import_job, index=0)
self.assertEqual(import_item.book.id, book.id)
@ -120,13 +123,14 @@ class GoodreadsImport(TestCase):
datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
csv_file = open(datafile, 'r')
for index, entry in enumerate(list(csv.DictReader(csv_file))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
goodreads_import.handle_imported_book(
self.user, import_item, False, 'public')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
@ -153,13 +157,14 @@ class GoodreadsImport(TestCase):
datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
csv_file = open(datafile, 'r')
for index, entry in enumerate(list(csv.DictReader(csv_file))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
goodreads_import.handle_imported_book(
self.user, import_item, False, 'public')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
@ -182,15 +187,16 @@ class GoodreadsImport(TestCase):
datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
csv_file = open(datafile, 'r')
for index, entry in enumerate(list(csv.DictReader(csv_file))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
goodreads_import.handle_imported_book(
self.user, import_item, False, 'public')
goodreads_import.handle_imported_book(
self.user, import_item, False, 'public')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
@ -212,12 +218,13 @@ class GoodreadsImport(TestCase):
datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
csv_file = open(datafile, 'r')
entry = list(csv.DictReader(csv_file))[2]
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=0, data=entry, book=self.book)
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
goodreads_import.handle_imported_book(
self.user, import_item, True, 'unlisted')
importer.handle_imported_book(
self.importer.service, self.user, import_item, True, 'unlisted')
review = models.Review.objects.get(book=self.book, user=self.user)
self.assertEqual(review.content, 'mixed feelings')
self.assertEqual(review.rating, 2)
@ -233,12 +240,13 @@ class GoodreadsImport(TestCase):
datafile = pathlib.Path(__file__).parent.joinpath('data/goodreads.csv')
csv_file = open(datafile, 'r')
entry = list(csv.DictReader(csv_file))[2]
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=0, data=entry, book=self.book)
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
goodreads_import.handle_imported_book(
self.user, import_item, False, 'unlisted')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'unlisted')
self.assertFalse(models.Review.objects.filter(
book=self.book, user=self.user
).exists())

240
bookwyrm/tests/test_librarything_import.py Normal file
View File

@ -0,0 +1,240 @@
''' testing import '''
from collections import namedtuple
import csv
import pathlib
from unittest.mock import patch
from django.test import TestCase
import responses
from bookwyrm import models, importer
from bookwyrm.librarything_import import LibrarythingImporter
from bookwyrm.settings import DOMAIN
class LibrarythingImport(TestCase):
''' importing from librarything tsv '''
def setUp(self):
self.importer = LibrarythingImporter()
''' use a test tsv '''
datafile = pathlib.Path(__file__).parent.joinpath(
'data/librarything.tsv')
# Librarything generates latin encoded exports...
self.csv = open(datafile, 'r', encoding=self.importer.encoding)
self.user = models.User.objects.create_user(
'mmai', 'mmai@mmai.mmai', 'password', local=True)
models.Connector.objects.create(
identifier=DOMAIN,
name='Local',
local=True,
connector_file='self_connector',
base_url='https://%s' % DOMAIN,
books_url='https://%s/book' % DOMAIN,
covers_url='https://%s/images/covers' % DOMAIN,
search_url='https://%s/search?q=' % DOMAIN,
priority=1,
)
work = models.Work.objects.create(title='Test Work')
self.book = models.Edition.objects.create(
title='Example Edition',
remote_id='https://example.com/book/1',
parent_work=work
)
def test_create_job(self):
''' creates the import job entry and checks csv '''
import_job = self.importer.create_job(
self.user, self.csv, False, 'public')
self.assertEqual(import_job.user, self.user)
self.assertEqual(import_job.include_reviews, False)
self.assertEqual(import_job.privacy, 'public')
import_items = models.ImportItem.objects.filter(job=import_job).all()
self.assertEqual(len(import_items), 3)
self.assertEqual(import_items[0].index, 0)
self.assertEqual(import_items[0].data['Book Id'], '5498194')
self.assertEqual(import_items[1].index, 1)
self.assertEqual(import_items[1].data['Book Id'], '5015319')
self.assertEqual(import_items[2].index, 2)
self.assertEqual(import_items[2].data['Book Id'], '5015399')
def test_create_retry_job(self):
''' trying again with items that didn't import '''
import_job = self.importer.create_job(
self.user, self.csv, False, 'unlisted')
import_items = models.ImportItem.objects.filter(
job=import_job
).all()[:2]
retry = self.importer.create_retry_job(
self.user, import_job, import_items)
self.assertNotEqual(import_job, retry)
self.assertEqual(retry.user, self.user)
self.assertEqual(retry.include_reviews, False)
self.assertEqual(retry.privacy, 'unlisted')
retry_items = models.ImportItem.objects.filter(job=retry).all()
self.assertEqual(len(retry_items), 2)
self.assertEqual(retry_items[0].index, 0)
self.assertEqual(import_items[0].data['Book Id'], '5498194')
self.assertEqual(retry_items[1].index, 1)
self.assertEqual(retry_items[1].data['Book Id'], '5015319')
@responses.activate
def test_import_data(self):
''' resolve entry '''
import_job = self.importer.create_job(
self.user, self.csv, False, 'unlisted')
book = models.Edition.objects.create(title='Test Book')
with patch(
'bookwyrm.models.import_job.ImportItem.get_book_from_isbn'
) as resolve:
resolve.return_value = book
with patch('bookwyrm.importer.handle_imported_book'):
importer.import_data(self.importer.service, import_job.id)
import_item = models.ImportItem.objects.get(job=import_job, index=0)
self.assertEqual(import_item.book.id, book.id)
def test_handle_imported_book(self):
''' librarything import added a book, this adds related connections '''
shelf = self.user.shelf_set.filter(identifier='read').first()
self.assertIsNone(shelf.books.first())
import_job = models.ImportJob.objects.create(user=self.user)
datafile = pathlib.Path(__file__).parent.joinpath('data/librarything.tsv')
csv_file = open(datafile, 'r', encoding=self.importer.encoding)
for index, entry in enumerate(list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
readthrough = models.ReadThrough.objects.get(user=self.user)
self.assertEqual(readthrough.book, self.book)
# I can't remember how to create dates and I don't want to look it up.
self.assertEqual(readthrough.start_date.year, 2007)
self.assertEqual(readthrough.start_date.month, 4)
self.assertEqual(readthrough.start_date.day, 16)
self.assertEqual(readthrough.finish_date.year, 2007)
self.assertEqual(readthrough.finish_date.month, 5)
self.assertEqual(readthrough.finish_date.day, 8)
def test_handle_imported_book_already_shelved(self):
''' librarything import added a book, this adds related connections '''
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
shelf = self.user.shelf_set.filter(identifier='to-read').first()
models.ShelfBook.objects.create(
shelf=shelf, user=self.user, book=self.book)
import_job = models.ImportJob.objects.create(user=self.user)
datafile = pathlib.Path(__file__).parent.joinpath('data/librarything.tsv')
csv_file = open(datafile, 'r', encoding=self.importer.encoding)
for index, entry in enumerate(list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
self.assertIsNone(
self.user.shelf_set.get(identifier='read').books.first())
readthrough = models.ReadThrough.objects.get(user=self.user)
self.assertEqual(readthrough.book, self.book)
self.assertEqual(readthrough.start_date.year, 2007)
self.assertEqual(readthrough.start_date.month, 4)
self.assertEqual(readthrough.start_date.day, 16)
self.assertEqual(readthrough.finish_date.year, 2007)
self.assertEqual(readthrough.finish_date.month, 5)
self.assertEqual(readthrough.finish_date.day, 8)
def test_handle_import_twice(self):
''' re-importing books '''
shelf = self.user.shelf_set.filter(identifier='read').first()
import_job = models.ImportJob.objects.create(user=self.user)
datafile = pathlib.Path(__file__).parent.joinpath('data/librarything.tsv')
csv_file = open(datafile, 'r', encoding=self.importer.encoding)
for index, entry in enumerate(list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))):
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=index, data=entry, book=self.book)
break
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'public')
shelf.refresh_from_db()
self.assertEqual(shelf.books.first(), self.book)
readthrough = models.ReadThrough.objects.get(user=self.user)
self.assertEqual(readthrough.book, self.book)
# I can't remember how to create dates and I don't want to look it up.
self.assertEqual(readthrough.start_date.year, 2007)
self.assertEqual(readthrough.start_date.month, 4)
self.assertEqual(readthrough.start_date.day, 16)
self.assertEqual(readthrough.finish_date.year, 2007)
self.assertEqual(readthrough.finish_date.month, 5)
self.assertEqual(readthrough.finish_date.day, 8)
def test_handle_imported_book_review(self):
''' librarything review import '''
import_job = models.ImportJob.objects.create(user=self.user)
datafile = pathlib.Path(__file__).parent.joinpath('data/librarything.tsv')
csv_file = open(datafile, 'r', encoding=self.importer.encoding)
entry = list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))[0]
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=0, data=entry, book=self.book)
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
importer.handle_imported_book(
self.importer.service, self.user, import_item, True, 'unlisted')
review = models.Review.objects.get(book=self.book, user=self.user)
self.assertEqual(review.content, 'chef d\'oeuvre')
self.assertEqual(review.rating, 5)
self.assertEqual(review.published_date.year, 2007)
self.assertEqual(review.published_date.month, 5)
self.assertEqual(review.published_date.day, 8)
self.assertEqual(review.privacy, 'unlisted')
def test_handle_imported_book_reviews_disabled(self):
''' librarything review import '''
import_job = models.ImportJob.objects.create(user=self.user)
datafile = pathlib.Path(__file__).parent.joinpath('data/librarything.tsv')
csv_file = open(datafile, 'r', encoding=self.importer.encoding)
entry = list(csv.DictReader(csv_file, delimiter=self.importer.delimiter))[2]
entry = self.importer.parse_fields(entry)
import_item = models.ImportItem.objects.create(
job_id=import_job.id, index=0, data=entry, book=self.book)
with patch('bookwyrm.models.activitypub_mixin.broadcast_task.delay'):
importer.handle_imported_book(
self.importer.service, self.user, import_item, False, 'unlisted')
self.assertFalse(models.Review.objects.filter(
book=self.book, user=self.user
).exists())