diff --git a/lib/handlers/user-infos-handler.react.js b/lib/handlers/user-infos-handler.react.js
--- a/lib/handlers/user-infos-handler.react.js
+++ b/lib/handlers/user-infos-handler.react.js
@@ -44,6 +44,7 @@
   const findUserIdentities = useFindUserIdentities();
 
   const requestedIDsRef = React.useRef(new Set<string>());
+  const requestedAvatarsRef = React.useRef(new Set<string>());
 
   const callUpdateRelationships =
     useLegacyAshoatKeyserverCall(updateRelationships);
@@ -93,12 +94,16 @@
       void dispatchActionPromise(findUserIdentitiesActionTypes, promise);
       // 2. Fetch avatars from auth keyserver
       if (relyingOnAuthoritativeKeyserver) {
-        const userIDsWithoutOwnID = newUserIDs.filter(
-          id => id !== currentUserInfo?.id,
-        );
+        const userIDsWithoutOwnID = newUserIDs
+          .filter(id => id !== currentUserInfo?.id)
+          .filter(id => !requestedAvatarsRef.current.has(id));
+
         if (userIDsWithoutOwnID.length === 0) {
           return;
         }
+
+        userIDsWithoutOwnID.forEach(id => requestedAvatarsRef.current.add(id));
+
         void dispatchActionPromise(
           updateRelationshipsActionTypes,
           callUpdateRelationships({