diff --git a/keyserver/src/creators/farcaster-channel-tag-creator.js b/keyserver/src/creators/farcaster-channel-tag-creator.js --- a/keyserver/src/creators/farcaster-channel-tag-creator.js +++ b/keyserver/src/creators/farcaster-channel-tag-creator.js @@ -8,6 +8,7 @@ CreateOrUpdateFarcasterChannelTagResponse, } from 'lib/types/community-types.js'; import { threadPermissions } from 'lib/types/thread-permission-types.js'; +import type { BlobOperationResult } from 'lib/utils/blob-service.js'; import { ServerError } from 'lib/utils/errors.js'; import { @@ -18,11 +19,10 @@ import { fetchCommunityInfos } from '../fetchers/community-fetchers.js'; import { checkThreadPermission } from '../fetchers/thread-permission-fetchers.js'; import { - uploadBlob, + uploadBlobKeyserverWrapper, assignHolder, download, deleteBlob, - type BlobOperationResult, type BlobDownloadResult, } from '../services/blob.js'; import { Viewer } from '../session/viewer.js'; @@ -162,7 +162,7 @@ const hash = farcasterChannelTagBlobHash(farcasterChannelID); const blob = new Blob([payloadString]); - const uploadResult = await uploadBlob(blob, hash); + const uploadResult = await uploadBlobKeyserverWrapper(blob, hash); if (!uploadResult.success) { return uploadResult; diff --git a/keyserver/src/creators/invite-link-creator.js b/keyserver/src/creators/invite-link-creator.js --- a/keyserver/src/creators/invite-link-creator.js +++ b/keyserver/src/creators/invite-link-creator.js @@ -11,6 +11,7 @@ InviteLink, } from 'lib/types/link-types.js'; import { threadPermissions } from 'lib/types/thread-permission-types.js'; +import type { BlobOperationResult } from 'lib/utils/blob-service.js'; import { ServerError } from 'lib/utils/errors.js'; import { reservedUsernamesSet } from 'lib/utils/reserved-users.js'; @@ -27,9 +28,8 @@ download, type BlobDownloadResult, assignHolder, - uploadBlob, + uploadBlobKeyserverWrapper, deleteBlob, - type BlobOperationResult, } from '../services/blob.js'; import { Viewer } from '../session/viewer.js'; import { thisKeyserverID } from '../user/identity.js'; @@ -272,7 +272,7 @@ const key = inviteLinkBlobHash(linkSecret); const blob = new Blob([payloadString]); - const uploadResult = await uploadBlob(blob, key); + const uploadResult = await uploadBlobKeyserverWrapper(blob, key); if (!uploadResult.success) { return uploadResult; } diff --git a/keyserver/src/push/encrypted-notif-utils-api.js b/keyserver/src/push/encrypted-notif-utils-api.js --- a/keyserver/src/push/encrypted-notif-utils-api.js +++ b/keyserver/src/push/encrypted-notif-utils-api.js @@ -69,6 +69,8 @@ const unencryptedDataBytes = new TextEncoder().encode(unencryptedData); return await encrypt(encryptionKeyBytes, unencryptedDataBytes); }, + normalizeUint8ArrayForBlobUpload: (uint8Array: Uint8Array) => + new Blob([uint8Array]), }; export default encryptedNotifUtilsAPI; diff --git a/keyserver/src/services/blob.js b/keyserver/src/services/blob.js --- a/keyserver/src/services/blob.js +++ b/keyserver/src/services/blob.js @@ -5,6 +5,10 @@ getBlobFetchableURL, makeBlobServiceEndpointURL, } from 'lib/utils/blob-service.js'; +import { + uploadBlob, + type BlobOperationResult, +} from 'lib/utils/blob-service.js'; import { createHTTPAuthorizationHeader } from 'lib/utils/services-utils.js'; import { verifyUserLoggedIn } from '../user/login.js'; @@ -35,49 +39,6 @@ +holder: string, }; -export type BlobOperationResult = - | { - +success: true, - } - | { - +success: false, - +reason: 'HASH_IN_USE' | 'OTHER', - +status: number, - +statusText: string, - }; - -async function uploadBlob( - blob: Blob, - hash: string, -): Promise { - const formData = new FormData(); - formData.append('blob_hash', hash); - formData.append('blob_data', blob); - - const headers = await createRequestHeaders(false); - const uploadBlobResponse = await fetch( - makeBlobServiceEndpointURL(blobService.httpEndpoints.UPLOAD_BLOB), - { - method: blobService.httpEndpoints.UPLOAD_BLOB.method, - body: formData, - headers, - }, - ); - - if (!uploadBlobResponse.ok) { - const { status, statusText } = uploadBlobResponse; - const reason = status === 409 ? 'HASH_IN_USE' : 'OTHER'; - return { - success: false, - reason, - status, - statusText, - }; - } - - return { success: true }; -} - async function assignHolder( params: BlobDescriptor, ): Promise { @@ -103,6 +64,14 @@ return { success: true }; } +async function uploadBlobKeyserverWrapper( + blob: Blob, + hash: string, +): Promise { + const authHeaders = await createRequestHeaders(false); + return uploadBlob(blob, hash, authHeaders); +} + async function upload( blob: Blob, params: BlobDescriptor, @@ -117,10 +86,9 @@ }, > { const { hash, holder } = params; - const [holderResult, uploadResult] = await Promise.all([ assignHolder({ hash, holder }), - uploadBlob(blob, hash), + uploadBlobKeyserverWrapper(blob, hash), ]); if (holderResult.success && uploadResult.success) { return { success: true }; @@ -171,4 +139,11 @@ }); } -export { upload, uploadBlob, assignHolder, download, deleteBlob }; +export { + upload, + uploadBlob, + assignHolder, + download, + deleteBlob, + uploadBlobKeyserverWrapper, +}; diff --git a/lib/facts/blob-service.js b/lib/facts/blob-service.js --- a/lib/facts/blob-service.js +++ b/lib/facts/blob-service.js @@ -2,7 +2,7 @@ import { isDev } from '../utils/dev-utils.js'; -type BlobServicePath = '/blob/:blobHash' | '/blob'; +type BlobServicePath = '/blob/:blobHash' | '/blob' | '/holders'; export type BlobServiceHTTPEndpoint = { +path: BlobServicePath, @@ -23,6 +23,10 @@ path: '/blob', method: 'POST', }, + ASSIGN_MULTIPLE_HOLDERS: { + path: '/holders', + method: 'POST', + }, UPLOAD_BLOB: { path: '/blob', method: 'PUT', diff --git a/lib/push/android-notif-creators.js b/lib/push/android-notif-creators.js --- a/lib/push/android-notif-creators.js +++ b/lib/push/android-notif-creators.js @@ -7,7 +7,9 @@ prepareEncryptedAndroidVisualNotifications, prepareEncryptedAndroidSilentNotifications, prepareLargeNotifData, + type LargeNotifEncryptionResult, type LargeNotifData, + generateBlobHolders, } from './crypto.js'; import { hasMinCodeVersion } from '../shared/version-utils.js'; import type { PlatformDetails } from '../types/device-types.js'; @@ -70,7 +72,7 @@ inputData: AndroidNotifInputData, devices: $ReadOnlyArray, largeNotifToEncryptionResultPromises?: { - [string]: Promise, + [string]: Promise, }, ): Promise<{ +targetedNotifications: $ReadOnlyArray, @@ -210,6 +212,7 @@ const copyWithMessageInfosDataBlob = JSON.stringify( copyWithMessageInfos.data, ); + if ( canQueryBlobService && largeNotifToEncryptionResultPromises && @@ -219,14 +222,13 @@ await largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob]; blobHash = largeNotifData.blobHash; encryptionKey = largeNotifData.encryptionKey; - blobHolders = largeNotifData.blobHolders; + blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length); encryptedCopyWithMessageInfos = largeNotifData.encryptedCopyWithMessageInfos; } else if (canQueryBlobService && largeNotifToEncryptionResultPromises) { largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob] = prepareLargeNotifData( copyWithMessageInfosDataBlob, - devicesWithExcessiveSizeNoHolders.length, encryptedNotifUtilsAPI, ); @@ -234,7 +236,7 @@ await largeNotifToEncryptionResultPromises[copyWithMessageInfosDataBlob]; blobHash = largeNotifData.blobHash; encryptionKey = largeNotifData.encryptionKey; - blobHolders = largeNotifData.blobHolders; + blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length); encryptedCopyWithMessageInfos = largeNotifData.encryptedCopyWithMessageInfos; } else if (canQueryBlobService) { diff --git a/lib/push/apns-notif-creators.js b/lib/push/apns-notif-creators.js --- a/lib/push/apns-notif-creators.js +++ b/lib/push/apns-notif-creators.js @@ -12,6 +12,8 @@ prepareEncryptedAPNsSilentNotifications, prepareLargeNotifData, type LargeNotifData, + type LargeNotifEncryptionResult, + generateBlobHolders, } from './crypto.js'; import { getAPNsNotificationTopic } from '../shared/notif-utils.js'; import { hasMinCodeVersion } from '../shared/version-utils.js'; @@ -45,7 +47,7 @@ inputData: APNsNotifInputData, devices: $ReadOnlyArray, largeNotifToEncryptionResultPromises?: { - [string]: Promise, + [string]: Promise, }, ): Promise<{ +targetedNotifications: $ReadOnlyArray, @@ -260,14 +262,14 @@ await largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob]; blobHash = largeNotifData.blobHash; encryptionKey = largeNotifData.encryptionKey; - blobHolders = largeNotifData.blobHolders; + blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length); encryptedCopyWithMessageInfos = largeNotifData.encryptedCopyWithMessageInfos; } else if (canQueryBlobService && largeNotifToEncryptionResultPromises) { largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob] = prepareLargeNotifData( copyWithMessageInfosBlob, - devicesWithExcessiveSizeNoHolders.length, + encryptedNotifUtilsAPI, ); @@ -275,7 +277,7 @@ await largeNotifToEncryptionResultPromises[copyWithMessageInfosBlob]; blobHash = largeNotifData.blobHash; encryptionKey = largeNotifData.encryptionKey; - blobHolders = largeNotifData.blobHolders; + blobHolders = generateBlobHolders(devicesWithExcessiveSizeNoHolders.length); encryptedCopyWithMessageInfos = largeNotifData.encryptedCopyWithMessageInfos; } else if (canQueryBlobService) { diff --git a/lib/push/crypto.js b/lib/push/crypto.js --- a/lib/push/crypto.js +++ b/lib/push/crypto.js @@ -20,6 +20,7 @@ APNsNotificationRescind, APNsBadgeOnlyNotification, } from '../types/notif-types.js'; +import { toBase64URL } from '../utils/base64.js'; async function encryptAndroidNotificationPayload( encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI, @@ -611,21 +612,26 @@ return Promise.all(notificationPromises); } -export type LargeNotifData = { - +blobHolders: $ReadOnlyArray, +export type LargeNotifEncryptionResult = { +blobHash: string, +encryptionKey: string, +encryptedCopyWithMessageInfos: Uint8Array, }; +export type LargeNotifData = $ReadOnly<{ + ...LargeNotifEncryptionResult, + +blobHolders: $ReadOnlyArray, +}>; + +function generateBlobHolders(numberOfDevices: number): $ReadOnlyArray { + return Array.from({ length: numberOfDevices }, () => uuid.v4()); +} + async function prepareLargeNotifData( copyWithMessageInfos: string, - numberOfDevices: number, encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI, -): Promise { +): Promise { const encryptionKey = await encryptedNotifUtilsAPI.generateAESKey(); - - const blobHolders = Array.from({ length: numberOfDevices }, () => uuid.v4()); const encryptedCopyWithMessageInfos = await encryptedNotifUtilsAPI.encryptWithAESKey( encryptionKey, @@ -634,9 +640,9 @@ const blobHash = await encryptedNotifUtilsAPI.getBlobHash( encryptedCopyWithMessageInfos, ); + const blobHashBase64url = toBase64URL(blobHash); return { - blobHolders, - blobHash, + blobHash: blobHashBase64url, encryptedCopyWithMessageInfos, encryptionKey, }; @@ -650,4 +656,5 @@ prepareEncryptedWebNotifications, prepareEncryptedWNSNotifications, prepareLargeNotifData, + generateBlobHolders, }; diff --git a/lib/push/send-hooks.react.js b/lib/push/send-hooks.react.js --- a/lib/push/send-hooks.react.js +++ b/lib/push/send-hooks.react.js @@ -1,9 +1,11 @@ // @flow import invariant from 'invariant'; +import _groupBy from 'lodash/fp/groupBy.js'; import * as React from 'react'; import uuid from 'uuid'; +import type { LargeNotifData } from './crypto.js'; import { preparePushNotifs, prepareOwnDevicesPushNotifs, @@ -14,6 +16,7 @@ import { usePeerOlmSessionsCreatorContext } from '../components/peer-olm-session-creator-provider.react.js'; import { thickRawThreadInfosSelector } from '../selectors/thread-selectors.js'; import { IdentityClientContext } from '../shared/identity-client-context.js'; +import type { AuthMetadata } from '../shared/identity-client-context.js'; import { useTunnelbroker } from '../tunnelbroker/tunnelbroker-context.js'; import type { TargetedAPNsNotification, @@ -21,6 +24,7 @@ TargetedWebNotification, TargetedWNSNotification, NotificationsCreationData, + EncryptedNotifUtilsAPI, } from '../types/notif-types.js'; import { deviceToTunnelbrokerMessageTypes } from '../types/tunnelbroker/messages.js'; import type { @@ -29,9 +33,12 @@ TunnelbrokerWebPushNotif, TunnelbrokerWNSNotif, } from '../types/tunnelbroker/notif-types.js'; +import { uploadBlob, assignMultipleHolders } from '../utils/blob-service.js'; import { getConfig } from '../utils/config.js'; import { getMessageForException } from '../utils/errors.js'; +import { values } from '../utils/objects.js'; import { useSelector } from '../utils/redux-utils.js'; +import { createDefaultHTTPRequestHeaders } from '../utils/services-utils.js'; function apnsNotifToTunnelbrokerAPNsNotif( targetedNotification: TargetedAPNsNotification, @@ -119,7 +126,9 @@ if (!notifCreationData) { return; } - const { deviceID, userID: senderUserID } = await getAuthMetadata(); + const authMetadata = await getAuthMetadata(); + const { deviceID, userID: senderUserID } = authMetadata; + if (!deviceID || !senderUserID) { return; } @@ -176,6 +185,18 @@ }; } + if (preparedPushNotifs) { + try { + await uploadLargeNotifBlobs( + preparedPushNotifs, + authMetadata, + encryptedNotifUtilsAPI, + ); + } catch (e) { + console.log('Failed to upload blobs', e); + } + } + const sendPromises = []; for (const userID in allPreparedPushNotifs) { for (const notif of allPreparedPushNotifs[userID] @@ -238,4 +259,79 @@ ); } +async function uploadLargeNotifBlobs( + pushNotifs: PerUserTargetedNotifications, + authMetadata: AuthMetadata, + encryptedNotifUtilsAPI: EncryptedNotifUtilsAPI, +): Promise { + const largeNotifArray = values(pushNotifs) + .map(({ largeNotifDataArray }) => largeNotifDataArray) + .flat(); + + if (largeNotifArray.length === 0) { + return; + } + + const largeNotifsByHash: { + +[blobHash: string]: $ReadOnlyArray, + } = _groupBy(largeNotifData => largeNotifData.blobHash)(largeNotifArray); + + const uploads = Object.entries(largeNotifsByHash).map( + ([blobHash, [{ encryptedCopyWithMessageInfos }]]) => ({ + blobHash, + encryptedCopyWithMessageInfos, + }), + ); + + const assignments = Object.entries(largeNotifsByHash) + .map(([blobHash, largeNotifs]) => + largeNotifs + .map(({ blobHolders }) => blobHolders) + .flat() + .map(holder => ({ blobHash, holder })), + ) + .flat(); + + const authHeaders = createDefaultHTTPRequestHeaders(authMetadata); + const uploadPromises = uploads.map( + ({ blobHash, encryptedCopyWithMessageInfos }) => + uploadBlob( + encryptedNotifUtilsAPI.normalizeUint8ArrayForBlobUpload( + encryptedCopyWithMessageInfos, + ), + blobHash, + authHeaders, + ), + ); + const assignmentPromise = assignMultipleHolders(assignments, authHeaders); + + const [uploadResults, assignmentResult] = await Promise.all([ + Promise.all(uploadPromises), + assignmentPromise, + ]); + + for (const uploadResult of uploadResults) { + if (uploadResult.success) { + continue; + } + const { reason, statusText } = uploadResult; + console.log( + `Failed to upload. Reason: ${reason}, status text: ${statusText}`, + ); + } + + if (assignmentResult.success) { + return; + } + + if (assignmentResult.error) { + const { statusText } = assignmentResult; + console.log(`Failed to assign all holders. Status text: ${statusText}`); + return; + } + + for (const [blobHash, holder] of assignmentResult.failedAssignments) { + console.log(`Assingnemt failed for holder: ${holder} and hash ${blobHash}`); + } +} export { useSendPushNotifs }; diff --git a/lib/push/send-utils.js b/lib/push/send-utils.js --- a/lib/push/send-utils.js +++ b/lib/push/send-utils.js @@ -13,7 +13,7 @@ createAPNsBadgeOnlyNotification, createAPNsNotificationRescind, } from './apns-notif-creators.js'; -import type { LargeNotifData } from './crypto.js'; +import type { LargeNotifEncryptionResult, LargeNotifData } from './crypto'; import { stringToVersionKey, getDevicesByPlatform, @@ -470,7 +470,7 @@ input: NotifCreatorInput, devices: $ReadOnlyArray, largeNotifToEncryptionResultPromises?: { - [string]: Promise, + [string]: Promise, }, ) => Promise<{ +targetedNotifications: $ReadOnlyArray, @@ -500,7 +500,7 @@ NotifCreatorInput, >, largeNotifToEncryptionResultPromises?: { - [string]: Promise, + [string]: Promise, }, ): Promise<{ +targetedNotificationsWithPlatform: $ReadOnlyArray<{ @@ -601,7 +601,7 @@ async function buildNotifsForUserDevices( inputData: BuildNotifsForUserDevicesInputData, largeNotifToEncryptionResultPromises: { - [string]: Promise, + [string]: Promise, }, ): Promise, @@ -1011,7 +1011,7 @@ ); const largeNotifToEncryptionResultPromises: { - [string]: Promise, + [string]: Promise, } = {}; for (const userID in mergedUsersToCollapsableInfo) { @@ -1067,7 +1067,7 @@ .map(({ largeNotifDataArray: array }) => array) .filter(Boolean) .flat(); - console.log(largeNotifDataArray); + return { targetedNotifications: targetedNotifsWithPlatform, largeNotifDataArray, diff --git a/lib/types/notif-types.js b/lib/types/notif-types.js --- a/lib/types/notif-types.js +++ b/lib/types/notif-types.js @@ -426,4 +426,5 @@ encryptionKey: string, unencrypotedData: string, ) => Promise, + +normalizeUint8ArrayForBlobUpload: (array: Uint8Array) => string | Blob, }; diff --git a/lib/utils/__mocks__/config.js b/lib/utils/__mocks__/config.js --- a/lib/utils/__mocks__/config.js +++ b/lib/utils/__mocks__/config.js @@ -54,6 +54,7 @@ getEncryptedNotifHash: jest.fn(), getBlobHash: jest.fn(), getNotifByteSize: jest.fn(), + normalizeUint8ArrayForBlobUpload: jest.fn(), }, }); diff --git a/lib/utils/blob-service.js b/lib/utils/blob-service.js --- a/lib/utils/blob-service.js +++ b/lib/utils/blob-service.js @@ -64,6 +64,96 @@ return `${urlSafeDeviceID}:${uuid.v4()}`; } +export type BlobOperationResult = + | { + +success: true, + } + | { + +success: false, + +reason: 'HASH_IN_USE' | 'OTHER', + +status: number, + +statusText: string, + }; + +async function uploadBlob( + blob: Blob | string, + hash: string, + headers: { [string]: string }, +): Promise { + const formData = new FormData(); + formData.append('blob_hash', hash); + if (typeof blob === 'string') { + formData.append('base64_data', blob); + } else { + formData.append('blob_data', blob); + } + + const uploadBlobResponse = await fetch( + makeBlobServiceEndpointURL(blobServiceConfig.httpEndpoints.UPLOAD_BLOB), + { + method: blobServiceConfig.httpEndpoints.UPLOAD_BLOB.method, + body: formData, + headers, + }, + ); + + if (!uploadBlobResponse.ok) { + const { status, statusText } = uploadBlobResponse; + const reason = status === 409 ? 'HASH_IN_USE' : 'OTHER'; + return { + success: false, + reason, + status, + statusText, + }; + } + + return { success: true }; +} + +async function assignMultipleHolders( + holders: $ReadOnlyArray<{ +blobHash: string, +holder: string }>, + headers: { [string]: string }, +): Promise< + | { +success: true } + | { +error: true, status: number, statusText: string } + | { + +failedAssignments: $ReadOnlyArray<{ + +blobHash: string, + +holder: string, + }>, + }, +> { + const assignMultipleHoldersResponse = await fetch( + makeBlobServiceEndpointURL( + blobServiceConfig.httpEndpoints.ASSIGN_MULTIPLE_HOLDERS, + ), + { + method: blobServiceConfig.httpEndpoints.ASSIGN_MULTIPLE_HOLDERS.method, + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + requests: holders, + }), + }, + ); + + if (!assignMultipleHoldersResponse.ok) { + const { status, statusText } = assignMultipleHoldersResponse; + return { error: true, status, statusText }; + } + + const { results } = await assignMultipleHoldersResponse.json(); + const failedRequests = results + .filter(result => !result.success) + .map(({ blobHash, holder }) => ({ blobHash, holder })); + + if (failedRequests.length !== 0) { + return { failedAssignments: failedRequests }; + } + + return { success: true }; +} + export { makeBlobServiceURI, isBlobServiceURI, @@ -72,4 +162,6 @@ generateBlobHolder, getBlobFetchableURL, makeBlobServiceEndpointURL, + uploadBlob, + assignMultipleHolders, }; diff --git a/native/push/encrypted-notif-utils-api.js b/native/push/encrypted-notif-utils-api.js --- a/native/push/encrypted-notif-utils-api.js +++ b/native/push/encrypted-notif-utils-api.js @@ -56,6 +56,8 @@ new Uint8Array(unencryptedDataBytes), ); }, + normalizeUint8ArrayForBlobUpload: (array: Uint8Array) => + commUtilsModule.base64EncodeBuffer(array.buffer), }; export default encryptedNotifUtilsAPI; diff --git a/web/push-notif/encrypted-notif-utils-api.js b/web/push-notif/encrypted-notif-utils-api.js --- a/web/push-notif/encrypted-notif-utils-api.js +++ b/web/push-notif/encrypted-notif-utils-api.js @@ -56,6 +56,8 @@ unencryptedDataBytes, ); }, + normalizeUint8ArrayForBlobUpload: (uint8Array: Uint8Array) => + new Blob([uint8Array]), }; export default encryptedNotifUtilsAPI;