diff --git a/keyserver/src/fetchers/message-fetchers.js b/keyserver/src/fetchers/message-fetchers.js
--- a/keyserver/src/fetchers/message-fetchers.js
+++ b/keyserver/src/fetchers/message-fetchers.js
@@ -7,6 +7,7 @@
   shimUnsupportedRawMessageInfos,
   isInvalidSidebarSource,
   isUnableToBeRenderedIndependently,
+  isInvalidPinSource,
 } from 'lib/shared/message-utils.js';
 import { messageSpecs } from 'lib/shared/messages/message-specs.js';
 import { getNotifCollapseKey } from 'lib/shared/notif-utils.js';
@@ -745,11 +746,21 @@
 
   for (const message of messages) {
     let { rawMessageInfo } = message;
-    invariant(
-      !isInvalidSidebarSource(rawMessageInfo),
-      'SIDEBAR_SOURCE or TOGGLE_PIN should not point to a ' +
-        'SIDEBAR_SOURCE, REACTION, EDIT_MESSAGE or TOGGLE_PIN',
-    );
+
+    if (rawMessageInfo.type === messageTypes.SIDEBAR_SOURCE) {
+      invariant(
+        !isInvalidSidebarSource(rawMessageInfo),
+        'SIDEBAR_SOURCE should not point to a ' +
+          'SIDEBAR_SOURCE, REACTION, EDIT_MESSAGE or TOGGLE_PIN',
+      );
+    }
+    if (rawMessageInfo.type === messageTypes.TOGGLE_PIN) {
+      invariant(
+        !isInvalidPinSource(rawMessageInfo),
+        'TOGGLE_PIN should not point to a non-composable message type',
+      );
+    }
+
     if (rawMessageInfo.id) {
       const editedContent = edits.get(rawMessageInfo.id);
       if (editedContent && rawMessageInfo.type === messageTypes.TEXT) {
@@ -759,6 +770,11 @@
         };
       }
       invariant(rawMessageInfo.id, 'rawMessageInfo.id should not be null');
+      // Flow doesn't refine the types if we don't explicitly invariant on
+      // or check against all of the unexpected message types, and that list
+      // can both get long and generally defeats the purpose of moving the
+      // logic into message specs to have one 'single source of truth'.
+      // $FlowFixMe
       messagesByID.set(rawMessageInfo.id, rawMessageInfo);
     }
   }