yul/bookwyrm-mastodon
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1163 from joachimesque/storage-s3

Browse Source 
Use S3 as static and media storage
This commit is contained in:
Mouse Reeve
2021-08-02 08:29:09 -06:00
committed by GitHub
parent d76106ded5 d70469d2e0
commit b40b8c2d0e
19 changed files with 196 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.env.dev.example
View File

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG=true
USE_HTTPS=false
DOMAIN=your.domain.here
#EMAIL=your@email.here
@@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true
EMAIL_USE_SSL=false
# S3 configuration
USE_S3=false
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# Commented are example values if you use a non-AWS, S3-compatible service
# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name"
# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud"
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
# Preview image generation can be computing and storage intensive
# ENABLE_PREVIEW_IMAGES=True

16
.env.prod.example
View File

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG=false
USE_HTTPS=true
DOMAIN=your.domain.here
EMAIL=your@email.here
@@ -42,6 +43,21 @@ EMAIL_HOST_PASSWORD=emailpassword123
EMAIL_USE_TLS=true
EMAIL_USE_SSL=false
# S3 configuration
USE_S3=false
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# Commented are example values if you use a non-AWS, S3-compatible service
# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
# AWS_STORAGE_BUCKET_NAME= # "example-bucket-name"
# AWS_S3_CUSTOM_DOMAIN=None # "example-bucket-name.s3.fr-par.scw.cloud"
# AWS_S3_REGION_NAME=None # "fr-par"
# AWS_S3_ENDPOINT_URL=None # "https://s3.fr-par.scw.cloud"
# Preview image generation can be computing and storage intensive
# ENABLE_PREVIEW_IMAGES=True

5
bookwyrm/context_processors.py
View File

@@ -11,10 +11,7 @@ def site_settings(request): # pylint: disable=unused-argument
return {
"site": models.SiteSettings.objects.get(),
"active_announcements": models.Announcement.active_announcements(),
"static_url": settings.STATIC_URL,
"media_url": settings.MEDIA_URL,
"static_path": settings.STATIC_PATH,
"media_path": settings.MEDIA_PATH,
"media_full_url": settings.MEDIA_FULL_URL,
"preview_images_enabled": settings.ENABLE_PREVIEW_IMAGES,
"request_protocol": request_protocol,
}

9
bookwyrm/preview_images.py
View File

@@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageColor
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.storage import default_storage
from django.db.models import Avg
from bookwyrm import models, settings
@@ -319,9 +320,9 @@ def save_and_cleanup(image, instance=None):
try:
try:
old_path = instance.preview_image.path
old_path = instance.preview_image.name
except ValueError:
old_path = ""
old_path = None
# Save
image.save(image_buffer, format="jpeg", quality=75)
@@ -342,8 +343,8 @@ def save_and_cleanup(image, instance=None):
instance.save()
# Clean up old file after saving
if os.path.exists(old_path):
os.remove(old_path)
if old_path and default_storage.exists(old_path):
default_storage.delete(old_path)
finally:
image_buffer.close()

56
bookwyrm/settings.py
View File

@@ -58,6 +58,7 @@ SECRET_KEY = env("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("DEBUG", True)
USE_HTTPS = env.bool("USE_HTTPS", False)
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
@@ -74,6 +75,7 @@ INSTALLED_APPS = [
"django_rename_app",
"bookwyrm",
"celery",
"storages",
]
MIDDLEWARE = [
@@ -179,19 +181,51 @@ USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
STATIC_URL = "/static/"
STATIC_PATH = "%s/%s" % (DOMAIN, env("STATIC_ROOT", "static"))
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_URL = "/images/"
MEDIA_PATH = "%s/%s" % (DOMAIN, env("MEDIA_ROOT", "images"))
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
requests.utils.default_user_agent(),
VERSION,
DOMAIN,
)
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
# Storage
PROTOCOL = "http"
if USE_HTTPS:
PROTOCOL = "https"
USE_S3 = env.bool("USE_S3", False)
if USE_S3:
# AWS settings
AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "")
AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
AWS_DEFAULT_ACL = "public-read"
AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
# S3 Static settings
STATIC_LOCATION = "static"
STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
# S3 Media settings
MEDIA_LOCATION = "images"
MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
MEDIA_FULL_URL = MEDIA_URL
DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
# I don't know if it's used, but the site crashes without it
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
else:
STATIC_URL = "/static/"
STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
MEDIA_URL = "/images/"
MEDIA_FULL_URL = "%s://%s%s" % (PROTOCOL, DOMAIN, MEDIA_URL)
MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))

17
bookwyrm/storage_backends.py Normal file
View File

@@ -0,0 +1,17 @@
"""Handles backends for storages"""
from storages.backends.s3boto3 import S3Boto3Storage
class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for Static contents"""
location = "static"
default_acl = "public-read"
class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method
"""Storage class for Image files"""
location = "images"
default_acl = "public-read"
file_overwrite = False

9
bookwyrm/templates/book/book.html
View File

@@ -1,5 +1,10 @@
{% extends 'layout.html' %}
{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %}{% load layout %}
{% load i18n %}
{% load bookwyrm_tags %}
{% load humanize %}
{% load utilities %}
{% load static %}
{% load layout %}
{% block title %}{{ book|book_title }}{% endblock %}
@@ -321,5 +326,5 @@
{% endblock %}
{% block scripts %}
<script src="/static/js/vendor/tabs.js"></script>
<script src="{% static "js/vendor/tabs.js" %}"></script>
{% endblock %}

3
bookwyrm/templates/feed/layout.html
View File

@@ -1,5 +1,6 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load static %}
{% block title %}{% trans "Updates" %}{% endblock %}
@@ -105,5 +106,5 @@
{% endblock %}
{% block scripts %}
<script src="/static/js/vendor/tabs.js"></script>
<script src="{% static "js/vendor/tabs.js" %}"></script>
{% endblock %}

3
bookwyrm/templates/get_started/layout.html
View File

@@ -1,5 +1,6 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load static %}
{% block title %}{% trans "Welcome" %}{% endblock %}
@@ -9,7 +10,7 @@
<div class="modal-background"></div>
<div class="modal-card is-fullwidth">
<header class="modal-card-head">
<img class="image logo mr-2" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" aria-hidden="true">
<img class="image logo mr-2" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" aria-hidden="true">
<h1 class="modal-card-title" id="get-started-header">
{% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}
<span class="subtitle is-block">

3
bookwyrm/templates/import_status.html
View File

@@ -1,6 +1,7 @@
{% extends 'layout.html' %}
{% load i18n %}
{% load humanize %}
{% load static %}
{% block title %}{% trans "Import Status" %}{% endblock %}
@@ -156,5 +157,5 @@
{% endspaceless %}{% endblock %}
{% block scripts %}
<script src="/static/js/check_all.js"></script>
<script src="{% static "js/check_all.js" %}"></script>
{% endblock %}

18
bookwyrm/templates/layout.html
View File

@@ -1,14 +1,16 @@
{% load layout %}{% load i18n %}
{% load layout %}
{% load i18n %}
{% load static %}
<!DOCTYPE html>
<html lang="{% get_lang %}">
<head>
<title>{% block title %}BookWyrm{% endblock %} | {{ site.name }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="/static/css/vendor/bulma.min.css">
<link rel="stylesheet" href="/static/css/vendor/icons.css">
<link rel="stylesheet" href="/static/css/bookwyrm.css">
<link rel="stylesheet" href="{% static "css/vendor/bulma.min.css" %}">
<link rel="stylesheet" href="{% static "css/vendor/icons.css" %}">
<link rel="stylesheet" href="{% static "css/bookwyrm.css" %}">
<link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{{ media_url }}{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
<link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">
{% if preview_images_enabled is True %}
<meta name="twitter:card" content="summary_large_image">
@@ -30,7 +32,7 @@
<div class="container">
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img class="image logo" src="{% if site.logo_small %}{{ media_url }}{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="Home page">
</a>
<form class="navbar-item column" action="/search/">
<div class="field has-addons">
@@ -242,8 +244,8 @@
<script>
var csrf_token = '{{ csrf_token }}';
</script>
<script src="/static/js/bookwyrm.js"></script>
<script src="/static/js/localstorage.js"></script>
<script src="{% static "js/bookwyrm.js" %}"></script>
<script src="{% static "js/localstorage.js" %}"></script>
{% block scripts %}{% endblock %}
</body>
</html>

4
bookwyrm/templates/snippets/about.html
View File

@@ -1,7 +1,9 @@
{% load static %}
<div class="columns">
<div class="column is-narrow is-hidden-mobile">
<figure class="block is-w-xl">
<img src="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
<img src="{% if site.logo %}/images/{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
</figure>
</div>
<div class="content">

4
bookwyrm/templates/snippets/avatar.html
View File

@@ -1,2 +1,4 @@
<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
{% load static %}
<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">

5
bookwyrm/templates/snippets/book_cover.html
View File

@@ -1,6 +1,7 @@
{% spaceless %}
{% load i18n %}
{% load static %}
<figure
class="
@@ -20,14 +21,14 @@
class="book-cover"
{% if book.cover %}
src="{% if img_path is None %}/images/{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
src="{% if img_path is None %}{% get_media_prefix %}{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
itemprop="thumbnailUrl"
{% if book.alt_text %}
alt="{{ book.alt_text }}"
{% endif %}
{% else %}
src="/static/images/no_cover.jpg"
src="{% static "images/no_cover.jpg" %}"
alt="{% trans "No cover" %}"
{% endif %}
>

14
bookwyrm/templates/snippets/opengraph_images.html
View File

@@ -1,12 +1,14 @@
{% load static %}
{% if preview_images_enabled is True %}
{% if image %}
<meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
<meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
<meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
<meta name="og:image" content="{{ media_full_url }}{{ image }}">
{% else %}
<meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
<meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
<meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
<meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
{% endif %}
{% else %}
<meta name="twitter:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
<meta name="og:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
<meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
<meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
{% endif %}

5
bookwyrm/templates/snippets/status/content_status.html
View File

@@ -1,6 +1,7 @@
{% load bookwyrm_tags %}
{% load markdown %}
{% load i18n %}
{% load static %}
{% with status_type=status.status_type %}
<div
@@ -111,12 +112,12 @@
<div class="column is-narrow">
<figure class="image is-128x128">
<a
href="/images/{{ attachment.image }}"
href="{% get_media_prefix %}{{ attachment.image }}"
target="_blank"
aria-label="{% trans 'Open image in new window' %}"
>
<img
src="/images/{{ attachment.image }}"
src="{% get_media_prefix %}{{ attachment.image }}"
{% if attachment.caption %}
alt="{{ attachment.caption }}"

3
bookwyrm/templates/snippets/status/status_header.html
View File

@@ -2,6 +2,7 @@
{% load status_display %}
{% load i18n %}
{% load humanize %}
{% load static %}
<div class="media">
<figure class="media-left" aria-hidden="true">
@@ -18,7 +19,7 @@
itemtype="https://schema.org/Person"
>
{% if status.user.avatar %}
<meta itemprop="image" content="/images/{{ status.user.avatar }}">
<meta itemprop="image" content="{% get_media_prefix %}{{ status.user.avatar }}">
{% endif %}
<a

50
bw-dev
View File

@@ -38,6 +38,17 @@ function makeitblack {
docker-compose run --rm web black celerywyrm bookwyrm
}
function awscommand {
# expose env vars
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
export AWS_DEFAULT_REGION=${AWS_S3_REGION_NAME}
# first arg is mountpoint, second is the whole aws command
docker run --rm -it -v $1\
-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION\
amazon/aws-cli $2
}
CMD=$1
if [ -n "$CMD" ]; then
shift
@@ -115,6 +126,19 @@ case "$CMD" in
generate_preview_images)
runweb python manage.py generate_preview_images $@
;;
copy_media_to_s3)
awscommand "bookwyrm_media_volume:/images"\
"s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
--endpoint-url ${AWS_S3_ENDPOINT_URL}\
--recursive --acl public-read"
;;
set_cors_to_s3)
awscommand "$(pwd):/bw"\
"s3api put-bucket-cors\
--bucket ${AWS_STORAGE_BUCKET_NAME}\
--endpoint-url ${AWS_S3_ENDPOINT_URL}\
--cors-configuration file:///bw/$@"
;;
runweb)
runweb "$@"
;;
@@ -123,6 +147,30 @@ case "$CMD" in
;;
*)
set +x # No need to echo echo
echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report, black, populate_streams, generate_preview_images"
echo "Unrecognised command. Try:"
echo " up [container]"
echo " run"
echo " initdb"
echo " resetdb"
echo " makemigrations [migration]"
echo " migrate [migration]"
echo " bash"
echo " shell"
echo " dbshell"
echo " restart_celery"
echo " test [path]"
echo " pytest [path]"
echo " collectstatic"
echo " makemessages [locale]"
echo " compilemessages [locale]"
echo " build"
echo " clean"
echo " black"
echo " populate_streams"
echo " generate_preview_images [--all]"
echo " copy_media_to_s3"
echo " set_cors_to_s3 [cors file]"
echo " runweb [command]"
echo " rundb [command]"
;;
esac

2
requirements.txt
View File

@@ -14,6 +14,8 @@ requests==2.22.0
responses==0.10.14
django-rename-app==0.1.2
pytz>=2021.1
boto3==1.17.88
django-storages==1.11.1
# Dev
black==21.4b0