diff --git a/lib/shared/farcaster/farcaster-hooks.js b/lib/shared/farcaster/farcaster-hooks.js
--- a/lib/shared/farcaster/farcaster-hooks.js
+++ b/lib/shared/farcaster/farcaster-hooks.js
@@ -222,6 +222,11 @@
   }
 }
 
+const notRetryableErrors = [
+  'Not authorized to view pending invites',
+  'Not authorized to access this conversation',
+];
+
 async function withRetry<T>(
   operation: () => Promise<T>,
   maxRetries: number = MAX_RETRIES,
@@ -238,12 +243,28 @@
       lastError = error;
 
       if (addLog && operationName) {
+        const message = getMessageForException(error);
+        const errorIsNotRetryable = notRetryableErrors.some(notRetryableError =>
+          message?.includes(notRetryableError),
+        );
+        if (errorIsNotRetryable) {
+          addLog(
+            `Farcaster: Retry attempt ${attempt}/${maxRetries + 1} failed for ${operationName}`,
+            `This isn't a retryable error, so giving up: ${JSON.stringify({
+              attempt,
+              maxRetries,
+              error: message,
+            })}`,
+            new Set([logTypes.FARCASTER]),
+          );
+          throw error;
+        }
         addLog(
           `Farcaster: Retry attempt ${attempt}/${maxRetries + 1} failed for ${operationName}`,
           JSON.stringify({
             attempt,
             maxRetries,
-            error: getMessageForException(error),
+            error: message,
           }),
           new Set([logTypes.FARCASTER]),
         );