diff --git a/keyserver/src/fetchers/upload-fetchers.js b/keyserver/src/fetchers/upload-fetchers.js --- a/keyserver/src/fetchers/upload-fetchers.js +++ b/keyserver/src/fetchers/upload-fetchers.js @@ -123,7 +123,7 @@ function imagesFromRow(row: Object): Image | EncryptedImage { const uploadExtra = JSON.parse(row.uploadExtra); - const { width, height, blobHolder } = uploadExtra; + const { width, height, blobHolder, thumbHash } = uploadExtra; const { uploadType: type, uploadSecret: secret } = row; const id = row.uploadID.toString(); @@ -134,13 +134,14 @@ throw new ServerError('invalid_parameters'); } if (!isEncrypted) { - return { id, type: 'photo', uri, dimensions }; + return { id, type: 'photo', uri, dimensions, thumbHash }; } return { id, type: 'encrypted_photo', holder: uri, dimensions, + thumbHash, encryptionKey: uploadExtra.encryptionKey, }; } @@ -212,7 +213,7 @@ const media = uploads.map(upload => { const { uploadID, uploadType, uploadSecret, uploadExtra } = upload; - const { width, height, encryptionKey, blobHolder } = + const { width, height, encryptionKey, blobHolder, thumbHash } = JSON.parse(uploadExtra); const dimensions = { width, height }; const uri = makeUploadURI(blobHolder, uploadID, uploadSecret); @@ -225,6 +226,7 @@ holder: uri, encryptionKey, dimensions, + thumbHash, }; } return { @@ -232,6 +234,7 @@ id: uploadID.toString(), uri, dimensions, + thumbHash, }; } @@ -239,6 +242,7 @@ const { encryptionKey: thumbnailEncryptionKey, blobHolder: thumbnailBlobHolder, + thumbHash: thumbnailThumbHash, } = JSON.parse(thumbnailUploadExtra); const thumbnailURI = makeUploadURI( thumbnailBlobHolder, @@ -256,6 +260,7 @@ thumbnailID, thumbnailHolder: thumbnailURI, thumbnailEncryptionKey, + thumbnailThumbHash, }; } @@ -266,6 +271,7 @@ dimensions, thumbnailID, thumbnailURI, + thumbnailThumbHash, }; }); @@ -313,7 +319,8 @@ const primaryUpload = uploadMap[primaryUploadID]; const uploadExtra = JSON.parse(primaryUpload.uploadExtra); - const { width, height, loop, blobHolder, encryptionKey } = uploadExtra; + const { width, height, loop, blobHolder, encryptionKey, thumbHash } = + uploadExtra; const dimensions = { width, height }; const primaryUploadURI = makeUploadURI( @@ -330,6 +337,7 @@ holder: primaryUploadURI, encryptionKey, dimensions, + thumbHash, }); } else { media.push({ @@ -337,6 +345,7 @@ id: primaryUploadID, uri: primaryUploadURI, dimensions, + thumbHash, }); } continue; @@ -346,7 +355,8 @@ const thumbnailUpload = uploadMap[thumbnailUploadID]; const thumbnailUploadExtra = JSON.parse(thumbnailUpload.uploadExtra); - const { blobHolder: thumbnailBlobHolder } = thumbnailUploadExtra; + const { blobHolder: thumbnailBlobHolder, thumbHash: thumbnailThumbHash } = + thumbnailUploadExtra; const thumbnailUploadURI = makeUploadURI( thumbnailBlobHolder, thumbnailUploadID, @@ -363,6 +373,7 @@ thumbnailID: thumbnailUploadID, thumbnailHolder: thumbnailUploadURI, thumbnailEncryptionKey: thumbnailUploadExtra.encryptionKey, + thumbnailThumbHash, }; media.push(loop ? { ...video, loop } : video); } else { @@ -373,6 +384,7 @@ dimensions, thumbnailID: thumbnailUploadID, thumbnailURI: thumbnailUploadURI, + thumbnailThumbHash, }; media.push(loop ? { ...video, loop } : video); } diff --git a/lib/reducers/message-reducer.js b/lib/reducers/message-reducer.js --- a/lib/reducers/message-reducer.js +++ b/lib/reducers/message-reducer.js @@ -1395,6 +1395,7 @@ type: 'photo', uri: mediaUpdate.uri ?? singleMedia.uri, dimensions: mediaUpdate.dimensions ?? singleMedia.dimensions, + thumbHash: mediaUpdate.thumbHash ?? singleMedia.thumbHash, }; if ( diff --git a/lib/reducers/message-reducer.test.js b/lib/reducers/message-reducer.test.js --- a/lib/reducers/message-reducer.test.js +++ b/lib/reducers/message-reducer.test.js @@ -21,6 +21,7 @@ uri: 'assets-library://asset/asset.HEIC?id=CC95F08C-88C3-4012-9D6D-64A413D254B3&ext=HEIC', type: 'photo', dimensions: { height: 3024, width: 4032 }, + thumbHash: 'some_thumb_hash', localMediaSelection: { step: 'photo_library', dimensions: { height: 3024, width: 4032 }, @@ -85,6 +86,7 @@ type: 'photo', uri: 'http://localhost/comm/upload/91172/dfa9b9fe7eb03fde', dimensions: { height: 1440, width: 1920 }, + thumbHash: 'some_thumb_hash', }, ], localID: 'local1', diff --git a/lib/types/media-types.js b/lib/types/media-types.js --- a/lib/types/media-types.js +++ b/lib/types/media-types.js @@ -639,6 +639,7 @@ +uri: string, +type: 'photo', +dimensions: Dimensions, + +thumbHash: ?string, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, }; @@ -648,6 +649,7 @@ uri: t.String, type: tString('photo'), dimensions: dimensionsValidator, + thumbHash: t.maybe(t.String), localMediaSelection: t.maybe(nativeMediaSelectionValidator), }); @@ -658,6 +660,7 @@ +encryptionKey: string, +type: 'encrypted_photo', +dimensions: Dimensions, + +thumbHash: ?string, }; export const encryptedImageValidator: TInterface = @@ -667,6 +670,7 @@ encryptionKey: t.String, type: tString('encrypted_photo'), dimensions: dimensionsValidator, + thumbHash: t.maybe(t.String), }); export type Video = { @@ -677,6 +681,7 @@ +loop?: boolean, +thumbnailID: string, +thumbnailURI: string, + +thumbnailThumbHash: ?string, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, }; @@ -689,6 +694,7 @@ loop: t.maybe(t.Boolean), thumbnailID: tID, thumbnailURI: t.String, + thumbnailThumbHash: t.maybe(t.String), localMediaSelection: t.maybe(nativeMediaSelectionValidator), }); @@ -703,6 +709,7 @@ +thumbnailID: string, +thumbnailHolder: string, +thumbnailEncryptionKey: string, + +thumbnailThumbHash: ?string, }; export const encryptedVideoValidator: TInterface = @@ -716,6 +723,7 @@ thumbnailID: tID, thumbnailHolder: t.String, thumbnailEncryptionKey: t.String, + thumbnailThumbHash: t.maybe(t.String), }); export type Media = Image | Video | EncryptedImage | EncryptedVideo; diff --git a/lib/utils/message-ops-utils.js b/lib/utils/message-ops-utils.js --- a/lib/utils/message-ops-utils.js +++ b/lib/utils/message-ops-utils.js @@ -52,6 +52,7 @@ loop: type === 'video' ? m.loop : false, local_media_selection: m.localMediaSelection, encryption_key: m.encryptionKey, + thumb_hash: m.thumbHash ?? undefined, }), }); if (m.type === 'video' || m.type === 'encrypted_video') { @@ -65,6 +66,7 @@ dimensions: m.dimensions, loop: false, encryption_key: m.thumbnailEncryptionKey, + thumb_hash: m.thumbnailThumbHash ?? undefined, }), }); } @@ -75,7 +77,7 @@ function translateClientDBMediaInfoToImage( clientDBMediaInfo: ClientDBMediaInfo, ): Image { - const { dimensions, local_media_selection } = JSON.parse( + const { dimensions, local_media_selection, thumb_hash } = JSON.parse( clientDBMediaInfo.extras, ); @@ -85,6 +87,7 @@ uri: clientDBMediaInfo.uri, type: 'photo', dimensions: dimensions, + thumbHash: thumb_hash, }; } return { @@ -93,6 +96,7 @@ type: 'photo', dimensions: dimensions, localMediaSelection: local_media_selection, + thumbHash: thumb_hash, }; } @@ -124,7 +128,11 @@ for (const media of messageContent) { if (media.type === 'photo') { const extras = JSON.parse(mediaMap[media.uploadID].extras); - const { dimensions, encryption_key: encryptionKey } = extras; + const { + dimensions, + encryption_key: encryptionKey, + thumb_hash: thumbHash, + } = extras; let image; if (encryptionKey) { @@ -134,6 +142,7 @@ holder: mediaMap[media.uploadID].uri, dimensions, encryptionKey, + thumbHash, }; } else { image = { @@ -141,6 +150,7 @@ type: 'photo', uri: mediaMap[media.uploadID].uri, dimensions, + thumbHash, }; } translatedMedia.push(image); @@ -153,10 +163,11 @@ encryption_key: encryptionKey, } = extras; + const { + encryption_key: thumbnailEncryptionKey, + thumb_hash: thumbnailThumbHash, + } = JSON.parse(mediaMap[media.thumbnailUploadID].extras); if (encryptionKey) { - const thumbnailEncryptionKey = JSON.parse( - mediaMap[media.thumbnailUploadID].extras, - ).encryption_key; const video: EncryptedVideo = { id: media.uploadID, type: 'encrypted_video', @@ -167,6 +178,7 @@ thumbnailID: media.thumbnailUploadID, thumbnailHolder: mediaMap[media.thumbnailUploadID].uri, thumbnailEncryptionKey, + thumbnailThumbHash, }; translatedMedia.push(video); } else { @@ -178,6 +190,7 @@ loop, thumbnailID: media.thumbnailUploadID, thumbnailURI: mediaMap[media.thumbnailUploadID].uri, + thumbnailThumbHash, }; translatedMedia.push( localMediaSelection ? { ...video, localMediaSelection } : video, diff --git a/lib/utils/message-ops-utils.test.js b/lib/utils/message-ops-utils.test.js --- a/lib/utils/message-ops-utils.test.js +++ b/lib/utils/message-ops-utils.test.js @@ -279,6 +279,7 @@ width: 1920, height: 1281, }, + thumbHash: 'some_thumb_hash', }, ], id: '85505', @@ -305,6 +306,7 @@ width: 1920, height: 1281, }, + thumbHash: 'some_thumb_hash', }, ], localID: 'local123', @@ -433,6 +435,7 @@ retries: 0, }, loop: false, + thumbnailThumbHash: undefined, thumbnailID: 'localUpload1', thumbnailURI: 'assets-library://asset/asset.mov?id=6F1BEA56-3875-474C-B3AF-B11DEDCBAFF2&ext=mov', @@ -461,7 +464,7 @@ uri: 'assets-library://asset/asset.jpeg?id=6F1BEA56-3875-474C-B3AF-B11DEDCBAFF2&ext=jpeg', type: 'photo', extras: - '{"dimensions":{"height":1010,"width":576},"loop":false,"encryption_key":"someKey"}', + '{"dimensions":{"height":1010,"width":576},"loop":false,"encryption_key":"someKey","thumb_hash":"thumb"}', }, ], }; @@ -478,6 +481,7 @@ 'assets-library://asset/asset.jpeg?id=6F1BEA56-3875-474C-B3AF-B11DEDCBAFF2&ext=jpeg', encryptionKey: 'someKey', dimensions: { height: 1010, width: 576 }, + thumbHash: 'thumb', }, ], localID: 'local0', @@ -511,7 +515,7 @@ uri: 'assets-library://asset/asset.jpeg?id=6F1BEA56-3875-474C-B3AF-B11DEDCBAFF2&ext=jpeg', type: 'photo', extras: - '{"dimensions":{"height":1010,"width":576},"loop":false,"encryption_key":"someThumbKey"}', + '{"dimensions":{"height":1010,"width":576},"loop":false,"encryption_key":"someThumbKey","thumb_hash":"thumb"}', }, ], }; @@ -533,6 +537,7 @@ thumbnailHolder: 'assets-library://asset/asset.jpeg?id=6F1BEA56-3875-474C-B3AF-B11DEDCBAFF2&ext=jpeg', thumbnailEncryptionKey: 'someThumbKey', + thumbnailThumbHash: 'thumb', }, ], localID: 'local0', diff --git a/native/input/input-state-container.react.js b/native/input/input-state-container.react.js --- a/native/input/input-state-container.react.js +++ b/native/input/input-state-container.react.js @@ -635,6 +635,7 @@ type: 'photo', dimensions: selection.dimensions, localMediaSelection: selection, + thumbHash: null, }); ids = { type: 'photo', localMediaID }; } @@ -649,6 +650,7 @@ loop: false, thumbnailID: localThumbnailID, thumbnailURI: selection.uri, + thumbnailThumbHash: null, }); ids = { type: 'video', localMediaID, localThumbnailID }; } @@ -802,6 +804,7 @@ await encryptMedia(processedMedia); encryptionSteps = encryptionReturn.steps; if (!encryptionResult.success) { + steps.push(...encryptionSteps); onUploadFailed(localMediaID, encryptionResult.reason); return await onUploadFinished(encryptionResult); } @@ -845,6 +848,10 @@ blobHash: processedMedia.blobHash, encryptionKey: processedMedia.encryptionKey, dimensions: processedMedia.dimensions, + thumbHash: + processedMedia.mediaType === 'encrypted_photo' + ? processedMedia.thumbHash + : null, }, { onProgress: (percent: number) => { @@ -869,6 +876,7 @@ encryptionKey: processedMedia.thumbnailEncryptionKey, loop: false, dimensions: processedMedia.dimensions, + thumbHash: processedMedia.thumbHash, }), ); } @@ -887,6 +895,11 @@ ? processedMedia.loop : undefined, encryptionKey: processedMedia.encryptionKey, + thumbHash: + processedMedia.mediaType === 'photo' || + processedMedia.mediaType === 'encrypted_photo' + ? processedMedia.thumbHash + : null, }, { onProgress: (percent: number) => @@ -916,6 +929,7 @@ ...processedMedia.dimensions, loop: false, encryptionKey: processedMedia.thumbnailEncryptionKey, + thumbHash: processedMedia.thumbHash, }, { uploadBlob: this.uploadBlob, @@ -981,7 +995,8 @@ ) { invariant(uploadThumbnailResult, 'uploadThumbnailResult exists'); const { uri: thumbnailURI, id: thumbnailID } = uploadThumbnailResult; - const { thumbnailEncryptionKey } = processedMedia; + const { thumbnailEncryptionKey, thumbHash: thumbnailThumbHash } = + processedMedia; if (processedMedia.mediaType === 'encrypted_video') { updateMediaPayload = { @@ -991,6 +1006,7 @@ thumbnailID, thumbnailHolder: thumbnailURI, thumbnailEncryptionKey, + thumbnailThumbHash, }, }; } else { @@ -1000,9 +1016,18 @@ ...updateMediaPayload.mediaUpdate, thumbnailID, thumbnailURI, + thumbnailThumbHash, }, }; } + } else { + updateMediaPayload = { + ...updateMediaPayload, + mediaUpdate: { + ...updateMediaPayload.mediaUpdate, + thumbHash: processedMedia.thumbHash, + }, + }; } // When we dispatch this action, it updates Redux and triggers the @@ -1142,6 +1167,7 @@ encryptionKey: string, dimensions: Dimensions, loop?: boolean, + thumbHash: ?string, }, options?: ?CallServerEndpointOptions, ): Promise { @@ -1227,7 +1253,8 @@ } // 3. Send upload metadata to the keyserver, return response - const { filename, mimeType, loop, dimensions, encryptionKey } = input; + const { filename, mimeType, loop, dimensions, encryptionKey, thumbHash } = + input; return await this.props.uploadMediaMetadata({ ...dimensions, filename, @@ -1235,6 +1262,7 @@ blobHolder: newHolder, encryptionKey, loop: loop ?? false, + ...(thumbHash ? { thumbHash } : null), }); } diff --git a/native/media/multimedia.react.js b/native/media/multimedia.react.js --- a/native/media/multimedia.react.js +++ b/native/media/multimedia.react.js @@ -170,20 +170,30 @@ // eslint-disable-next-line consistent-return static sourceFromMediaInfo(mediaInfo: MediaInfo | AvatarMediaInfo): Source { if (mediaInfo.type === 'photo') { - return { kind: 'uri', uri: mediaInfo.uri }; + return { + kind: 'uri', + uri: mediaInfo.uri, + thumbHash: mediaInfo.thumbHash, + }; } else if (mediaInfo.type === 'video') { - return { kind: 'uri', uri: mediaInfo.thumbnailURI }; + return { + kind: 'uri', + uri: mediaInfo.thumbnailURI, + thumbHash: mediaInfo.thumbnailThumbHash, + }; } else if (mediaInfo.type === 'encrypted_photo') { return { kind: 'encrypted', holder: mediaInfo.holder, encryptionKey: mediaInfo.encryptionKey, + thumbHash: mediaInfo.thumbHash, }; } else if (mediaInfo.type === 'encrypted_video') { return { kind: 'encrypted', holder: mediaInfo.thumbnailHolder, encryptionKey: mediaInfo.thumbnailEncryptionKey, + thumbHash: mediaInfo.thumbnailThumbHash, }; } else { invariant(false, 'Invalid mediaInfo type'); diff --git a/web/input/input-state-container.react.js b/web/input/input-state-container.react.js --- a/web/input/input-state-container.react.js +++ b/web/input/input-state-container.react.js @@ -329,6 +329,7 @@ uri, type: 'photo', dimensions: shimmedDimensions, + thumbHash: null, }; } invariant( @@ -341,6 +342,7 @@ type: 'encrypted_photo', encryptionKey, dimensions: shimmedDimensions, + thumbHash: null, }; }, );