yul
/
bookwyrm-mastodon
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use many small tasks instead of one big task

This commit is contained in:
Mouse Reeve 2021-11-11 14:39:53 -08:00
parent f0ce236ffc
commit 908c9dc689
1 changed files with 25 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
bookwyrm/importers/importer.py
View File

@ -102,39 +102,40 @@ class Importer:
def start_import(self, job): def start_import(self, job):
"""initalizes a csv import job""" """initalizes a csv import job"""
result = import_data.delay(self.service, job.id) result = start_import_task.delay(self.service, job.id)
job.task_id = result.id job.task_id = result.id
job.save() job.save()
@app.task(queue="low_priority") @app.task(queue="low_priority")
def import_data(source, job_id): def start_import_task(source, job_id):
"""does the actual lookup work in a celery task""" """trigger the child tasks for each row"""
job = ImportJob.objects.get(id=job_id) job = ImportJob.objects.get(id=job_id)
# these are sub-tasks so that one big task doesn't use up all the memory in celery
for item in job.items.values("id").all():
import_item_task.delay(source, item.id)
@app.task(queue="low_priority")
def import_item_task(source, item_id):
"""resolve a row into a book"""
item = models.ImportItem.objets.get(id=item_id)
try: try:
for item in job.items.all(): item.resolve()
try: except Exception as err: # pylint: disable=broad-except
item.resolve() logger.exception(err)
except Exception as err: # pylint: disable=broad-except item.fail_reason = _("Error loading book")
logger.exception(err) item.save()
item.fail_reason = _("Error loading book") return
item.save()
continue
if item.book or item.book_guess: if item.book:
item.save() job = item.job
# shelves book and handles reviews
handle_imported_book(source, job.user, item, job.include_reviews, job.privacy)
else:
item.fail_reason = _("Could not find a match for book")
if item.book: item.save()
# shelves book and handles reviews
handle_imported_book(
source, job.user, item, job.include_reviews, job.privacy
)
else:
item.fail_reason = _("Could not find a match for book")
item.save()
finally:
job.complete = True
job.save()
def handle_imported_book(source, user, item, include_reviews, privacy): def handle_imported_book(source, user, item, include_reviews, privacy):