diff --git a/lib/handlers/user-infos-handler.react.js b/lib/handlers/user-infos-handler.react.js
--- a/lib/handlers/user-infos-handler.react.js
+++ b/lib/handlers/user-infos-handler.react.js
@@ -18,7 +18,7 @@
 import { IdentityClientContext } from '../shared/identity-client-context.js';
 import { useTunnelbroker } from '../tunnelbroker/tunnelbroker-context.js';
 import { relationshipActions } from '../types/relationship-types.js';
-import { getMessageForException } from '../utils/errors.js';
+import { getMessageForException, FetchTimeout } from '../utils/errors.js';
 import { useDispatchActionPromise } from '../utils/redux-promise-utils.js';
 import { useSelector } from '../utils/redux-utils.js';
 import {
@@ -44,6 +44,7 @@
   const findUserIdentities = useFindUserIdentities();
 
   const requestedIDsRef = React.useRef(new Set<string>());
+  const requestedAvatarsRef = React.useRef(new Set<string>());
 
   const callUpdateRelationships =
     useLegacyAshoatKeyserverCall(updateRelationships);
@@ -94,17 +95,35 @@
       // 2. Fetch avatars from auth keyserver
       if (relyingOnAuthoritativeKeyserver) {
         const userIDsWithoutOwnID = newUserIDs.filter(
-          id => id !== currentUserInfo?.id,
+          id =>
+            id !== currentUserInfo?.id && !requestedAvatarsRef.current.has(id),
         );
+
         if (userIDsWithoutOwnID.length === 0) {
           return;
         }
+
+        userIDsWithoutOwnID.forEach(id => requestedAvatarsRef.current.add(id));
+
+        const updateRelationshipsPromise = (async () => {
+          try {
+            return await callUpdateRelationships({
+              action: relationshipActions.ACKNOWLEDGE,
+              userIDs: userIDsWithoutOwnID,
+            });
+          } catch (e) {
+            if (e instanceof FetchTimeout) {
+              userIDsWithoutOwnID.forEach(id =>
+                requestedAvatarsRef.current.delete(id),
+              );
+            }
+            throw e;
+          }
+        })();
+
         void dispatchActionPromise(
           updateRelationshipsActionTypes,
-          callUpdateRelationships({
-            action: relationshipActions.ACKNOWLEDGE,
-            userIDs: userIDsWithoutOwnID,
-          }),
+          updateRelationshipsPromise,
         );
       }
     })();