diff --git a/bookwyrm/activitypub/verbs.py b/bookwyrm/activitypub/verbs.py
index 1788dda7..883eb895 100644
--- a/bookwyrm/activitypub/verbs.py
+++ b/bookwyrm/activitypub/verbs.py
@@ -140,7 +140,7 @@ class Add(Verb):
     def action(self):
         """ add obj to collection """
         target = resolve_remote_id(self.target, refresh=False)
-        # we want to related field that isn't the book, this is janky af sorry
+        # we want to get the related field that isn't the book, this is janky af sorry
         model = [t for t in type(target)._meta.related_objects if t.name != "edition"][
             0
         ].related_model
@@ -156,7 +156,11 @@ class Remove(Verb):
 
     def action(self):
         """ find and remove the activity object """
-        obj = self.object.to_model(save=False, allow_create=False)
+        target = resolve_remote_id(self.target, refresh=False)
+        model = [t for t in type(target)._meta.related_objects if t.name != "edition"][
+            0
+        ].related_model
+        obj = self.to_model(model=model, save=False, allow_create=False)
         obj.delete()
 
 
diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py
index 691cad00..0a3c33a1 100644
--- a/bookwyrm/models/activitypub_mixin.py
+++ b/bookwyrm/models/activitypub_mixin.py
@@ -362,14 +362,15 @@ class CollectionItemMixin(ActivitypubMixin):
         """ broadcast a remove activity """
         activity = self.to_remove_activity()
         super().delete(*args, **kwargs)
-        self.broadcast(activity, self.user)
+        if self.user.local:
+            self.broadcast(activity, self.user)
 
     def to_add_activity(self):
         """ AP for shelving a book"""
         object_field = getattr(self, self.object_field)
         collection_field = getattr(self, self.collection_field)
         return activitypub.Add(
-            id="%s#add" % self.remote_id,
+            id=self.remote_id,
             actor=self.user.remote_id,
             object=object_field,
             target=collection_field.remote_id,
@@ -380,7 +381,7 @@ class CollectionItemMixin(ActivitypubMixin):
         object_field = getattr(self, self.object_field)
         collection_field = getattr(self, self.collection_field)
         return activitypub.Remove(
-            id="%s#remove" % self.remote_id,
+            id=self.remote_id,
             actor=self.user.remote_id,
             object=object_field,
             target=collection_field.remote_id,
diff --git a/bookwyrm/tests/views/test_inbox.py b/bookwyrm/tests/views/test_inbox.py
index d43018db..3e52b475 100644
--- a/bookwyrm/tests/views/test_inbox.py
+++ b/bookwyrm/tests/views/test_inbox.py
@@ -608,6 +608,41 @@ class Inbox(TestCase):
         views.inbox.activity_task(activity)
         self.assertEqual(shelf.books.first(), book)
 
+    def test_handle_unshelve_book(self):
+        """ remove a book from a shelf """
+        work = models.Work.objects.create(title="work title")
+        book = models.Edition.objects.create(
+            title="Test",
+            remote_id="https://bookwyrm.social/book/37292",
+            parent_work=work,
+        )
+        shelf = models.Shelf.objects.create(user=self.remote_user, name="Test Shelf")
+        shelf.remote_id = "https://bookwyrm.social/user/mouse/shelf/to-read"
+        shelf.save()
+
+        shelfbook = models.ShelfBook.objects.create(
+            user=self.remote_user, shelf=shelf, book=book
+        )
+
+        self.assertEqual(shelf.books.first(), book)
+        self.assertEqual(shelf.books.count(), 1)
+
+        activity = {
+            "id": shelfbook.remote_id,
+            "type": "Remove",
+            "actor": "https://example.com/users/rat",
+            "object": {
+                "type": "Edition",
+                "title": "Test Title",
+                "work": work.remote_id,
+                "id": "https://bookwyrm.social/book/37292",
+            },
+            "target": "https://bookwyrm.social/user/mouse/shelf/to-read",
+            "@context": "https://www.w3.org/ns/activitystreams",
+        }
+        views.inbox.activity_task(activity)
+        self.assertFalse(shelf.books.exists())
+
     @responses.activate
     def test_handle_add_book_to_list(self):
         """ listing a book """