diff --git a/bookwyrm/templates/snippets/shelf_selector.html b/bookwyrm/templates/snippets/shelf_selector.html
index f96f48a8..9a3fd2ce 100644
--- a/bookwyrm/templates/snippets/shelf_selector.html
+++ b/bookwyrm/templates/snippets/shelf_selector.html
@@ -63,16 +63,18 @@
 {% endfor %}
 
 {% if shelf.identifier == 'all' %}
-{% for shelved_in in book.shelves.all %}
+{% for user_shelf in user_shelves %}
+{% if user_shelf in book.shelves.all %}
 <li class="navbar-divider m-0" role="separator" ></li>
 <li role="menuitem" class="dropdown-item p-0">
     <form name="shelve" action="/unshelve/" method="post">
         {% csrf_token %}
         <input type="hidden" name="book" value="{{ book.id }}">
-        <input type="hidden" name="shelf" value="{{ shelved_in.id }}">
-        <button class="button is-fullwidth is-small is-radiusless is-danger is-light" type="submit">{% trans "Remove from" %} {{ shelved_in.name }}</button>
+        <input type="hidden" name="shelf" value="{{ user_shelf.id }}">
+        <button class="button is-fullwidth is-small is-radiusless is-danger is-light" type="submit">{% trans "Remove from" %} {{ user_shelf.name }}</button>
     </form>
 </li>
+{% endif %}
 {% endfor %}
 {% else %}
 <li class="navbar-divider" role="separator" ></li>
diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py
index 34914d97..0662f302 100644
--- a/bookwyrm/views/shelf/shelf.py
+++ b/bookwyrm/views/shelf/shelf.py
@@ -72,9 +72,13 @@ class Shelf(View):
             "start_date"
         )
 
+        if shelf_identifier:
+            books = books.annotate(shelved_date=F("shelfbook__shelved_date"))
+        else:
+            # sorting by shelved date will cause duplicates in the "all books" view
+            books = books.annotate(shelved_date=F("updated_date"))
         books = books.annotate(
             rating=Subquery(reviews.values("rating")[:1]),
-            shelved_date=F("shelfbook__shelved_date"),
             start_date=Subquery(reading.values("start_date")[:1]),
             finish_date=Subquery(reading.values("finish_date")[:1]),
             author=Subquery(