diff --git a/lib/types/media-types.js b/lib/types/media-types.js --- a/lib/types/media-types.js +++ b/lib/types/media-types.js @@ -623,7 +623,9 @@ +success: false, +reason: 'encryption_failed', } - | { +success: false, +reason: 'digest_failed' }; + | { +success: false, +reason: 'digest_failed' } + | { +success: false, +reason: 'thumbhash_failed' } + | { +success: false, +reason: 'preload_image_failed' }; export type MediaMissionResult = MediaMissionFailure | { +success: true }; diff --git a/web/input/input-state-container.react.js b/web/input/input-state-container.react.js --- a/web/input/input-state-container.react.js +++ b/web/input/input-state-container.react.js @@ -103,6 +103,7 @@ InputStateContext, } from './input-state.js'; import { encryptFile } from '../media/encryption-utils.js'; +import { generateThumbHash } from '../media/image-utils.js'; import { validateFile, preloadImage } from '../media/media-utils.js'; import InvalidUploadModal from '../modals/chat/invalid-upload.react.js'; import { updateNavInfoActionType } from '../redux/action-types.js'; @@ -778,6 +779,13 @@ return { steps, result: encryptionResult }; } + const { steps: thumbHashSteps, result: thumbHashResult } = + await generateThumbHash(fixedFile, encryptionResult?.encryptionKey); + const thumbHash = thumbHashResult.success + ? thumbHashResult.thumbHash + : null; + steps.push(...thumbHashSteps); + return { steps, result: { @@ -795,6 +803,7 @@ uriIsReal: false, blobHash: encryptionResult?.sha256Hash, encryptionKey: encryptionResult?.encryptionKey, + thumbHash, progressPercent: 0, abort: null, steps, @@ -858,7 +867,7 @@ (upload.mediaType === 'encrypted_photo' || upload.mediaType === 'encrypted_video') ) { - const { blobHash, dimensions } = upload; + const { blobHash, dimensions, thumbHash } = upload; invariant( encryptionKey && blobHash && dimensions, 'incomplete encrypted upload', @@ -870,11 +879,16 @@ encryptionKey, dimensions, loop: false, + ...(thumbHash ? { thumbHash } : undefined), }, { ...callbacks }, ); } else { - let uploadExtras = { ...upload.dimensions, loop: false }; + let uploadExtras = { + ...upload.dimensions, + loop: false, + thumbHash: upload.thumbHash, + }; if (encryptionKey) { uploadExtras = { ...uploadExtras, encryptionKey }; } @@ -973,16 +987,24 @@ ); if (uploadAfterPreload.messageID) { const { mediaType, uri, dimensions, loop } = result; - let mediaUpdate; + const { thumbHash } = upload; + let mediaUpdate = { + loop, + dimensions, + ...(thumbHash ? { thumbHash } : undefined), + }; if (!isEncrypted) { - mediaUpdate = { type: mediaType, uri, dimensions, loop }; + mediaUpdate = { + ...mediaUpdate, + type: mediaType, + uri, + }; } else { mediaUpdate = { + ...mediaUpdate, type: outputMediaType, holder: uri, encryptionKey, - dimensions, - loop, }; } this.props.dispatch({ @@ -1037,11 +1059,12 @@ async blobServiceUpload( input: { - file: File, - blobHash: string, - encryptionKey: string, - dimensions: Dimensions, - loop?: boolean, + +file: File, + +blobHash: string, + +encryptionKey: string, + +dimensions: Dimensions, + +loop?: boolean, + +thumbHash?: string, }, options?: ?CallServerEndpointOptions, ): Promise { @@ -1149,6 +1172,7 @@ encryptionKey: input.encryptionKey, mimeType: input.file.type, filename: input.file.name, + thumbHash: input.thumbHash, }); } diff --git a/web/input/input-state.js b/web/input/input-state.js --- a/web/input/input-state.js +++ b/web/input/input-state.js @@ -12,29 +12,30 @@ import type { ThreadInfo, RelativeMemberInfo } from 'lib/types/thread-types.js'; export type PendingMultimediaUpload = { - localID: string, + +localID: string, // Pending uploads are assigned a serverID once they are complete - serverID: ?string, + +serverID: ?string, // Pending uploads are assigned a messageID once they are sent - messageID: ?string, + +messageID: ?string, // This is set to true if the upload fails for whatever reason - failed: boolean, - file: File, - mediaType: MediaType | EncryptedMediaType, - dimensions: ?Dimensions, - uri: string, - blobHash: ?string, - encryptionKey: ?string, - loop: boolean, + +failed: boolean, + +file: File, + +mediaType: MediaType | EncryptedMediaType, + +dimensions: ?Dimensions, + +uri: string, + +blobHash: ?string, + +encryptionKey: ?string, + +thumbHash: ?string, + +loop: boolean, // URLs created with createObjectURL aren't considered "real". The distinction // is required because those "fake" URLs must be disposed properly - uriIsReal: boolean, - progressPercent: number, + +uriIsReal: boolean, + +progressPercent: number, // This is set once the network request begins and used if the upload is // cancelled - abort: ?() => void, - steps: MediaMissionStep[], - selectTime: number, + +abort: ?() => void, + +steps: MediaMissionStep[], + +selectTime: number, }; export type TypeaheadState = { diff --git a/web/media/image-utils.js b/web/media/image-utils.js --- a/web/media/image-utils.js +++ b/web/media/image-utils.js @@ -1,10 +1,20 @@ // @flow import EXIF from 'exif-js'; +import { rgbaToThumbHash } from 'thumbhash'; -import type { GetOrientationMediaMissionStep } from 'lib/types/media-types.js'; +import { hexToUintArray } from 'lib/media/data-utils.js'; +import type { + GetOrientationMediaMissionStep, + MediaMissionFailure, + MediaMissionStep, +} from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; +import * as AES from './aes-crypto-utils.js'; +import { preloadImage } from './media-utils.js'; +import { base64EncodeBuffer } from '../utils/base64-utils.js'; + function getEXIFOrientation(file: File): Promise { return new Promise(resolve => { EXIF.getData(file, function () { @@ -35,4 +45,75 @@ }; } -export { getOrientation }; +type GenerateThumbhashResult = { + +success: true, + +thumbHash: string, +}; + +/** + * Generate a thumbhash for a given image file. If `encryptionKey` is provided, + * the thumbhash string will be encrypted with it. + */ +async function generateThumbHash( + file: File, + encryptionKey: ?string = null, +): Promise<{ + +steps: $ReadOnlyArray, + +result: GenerateThumbhashResult | MediaMissionFailure, +}> { + const steps = []; + const initialURI = URL.createObjectURL(file); + const { steps: preloadSteps, result: image } = await preloadImage(initialURI); + steps.push(...preloadSteps); + if (!image) { + return { + steps, + result: { success: false, reason: 'preload_image_failed' }, + }; + } + + let binaryThumbHash, thumbHashString, exceptionMessage; + try { + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d'); + + // rescale to 100px max as thumbhash doesn't need more + const scale = 100 / Math.max(image.width, image.height); + canvas.width = Math.round(image.width * scale); + canvas.height = Math.round(image.height * scale); + + context.drawImage(image, 0, 0, canvas.width, canvas.height); + const pixels = context.getImageData(0, 0, canvas.width, canvas.height); + binaryThumbHash = rgbaToThumbHash(pixels.width, pixels.height, pixels.data); + thumbHashString = base64EncodeBuffer(binaryThumbHash); + } catch (e) { + exceptionMessage = getMessageForException(e); + } finally { + URL.revokeObjectURL(initialURI); + } + steps.push({ + step: 'generate_thumbhash', + success: !!thumbHashString && !exceptionMessage, + exceptionMessage, + thumbHash: thumbHashString, + }); + if (!binaryThumbHash || !thumbHashString || exceptionMessage) { + return { steps, result: { success: false, reason: 'thumbhash_failed' } }; + } + + if (encryptionKey) { + try { + const encryptedThumbHash = await AES.encrypt( + hexToUintArray(encryptionKey), + binaryThumbHash, + ); + thumbHashString = base64EncodeBuffer(encryptedThumbHash); + } catch { + return { steps, result: { success: false, reason: 'encryption_failed' } }; + } + } + + return { steps, result: { success: true, thumbHash: thumbHashString } }; +} + +export { getOrientation, generateThumbHash }; diff --git a/web/package.json b/web/package.json --- a/web/package.json +++ b/web/package.json @@ -84,6 +84,7 @@ "simple-markdown": "^0.7.2", "siwe": "^1.1.6", "sql.js": "^1.8.0", + "thumbhash": "^0.1.1", "tinycolor2": "^1.4.1", "uuid": "^3.4.0", "visibilityjs": "^2.0.2", diff --git a/web/utils/base64-utils.js b/web/utils/base64-utils.js new file mode 100644 --- /dev/null +++ b/web/utils/base64-utils.js @@ -0,0 +1,14 @@ +// @flow + +function base64EncodeBuffer(data: Uint8Array): string { + return btoa(String.fromCharCode(...data)); +} + +function base64DecodeBuffer(base64String: string): Uint8Array { + const binaryString = atob(base64String); + return new Uint8Array(binaryString.length).map((_, i) => + binaryString.charCodeAt(i), + ); +} + +export { base64EncodeBuffer, base64DecodeBuffer }; diff --git a/yarn.lock b/yarn.lock --- a/yarn.lock +++ b/yarn.lock @@ -22468,6 +22468,11 @@ resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= +thumbhash@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/thumbhash/-/thumbhash-0.1.1.tgz#bd2b8616fc043f2b17151dfce0cce1408e0ebbeb" + integrity sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg== + thunky@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.0.3.tgz#f5df732453407b09191dae73e2a8cc73f381a826"