diff --git a/lib/types/media-types.js b/lib/types/media-types.js --- a/lib/types/media-types.js +++ b/lib/types/media-types.js @@ -623,7 +623,9 @@ +success: false, +reason: 'encryption_failed', } - | { +success: false, +reason: 'digest_failed' }; + | { +success: false, +reason: 'digest_failed' } + | { +success: false, +reason: 'thumbhash_failed' } + | { +success: false, +reason: 'preload_image_failed' }; export type MediaMissionResult = MediaMissionFailure | { +success: true }; diff --git a/web/input/input-state-container.react.js b/web/input/input-state-container.react.js --- a/web/input/input-state-container.react.js +++ b/web/input/input-state-container.react.js @@ -103,6 +103,7 @@ InputStateContext, } from './input-state.js'; import { encryptFile } from '../media/encryption-utils.js'; +import { generateThumbHash } from '../media/image-utils.js'; import { validateFile, preloadImage } from '../media/media-utils.js'; import InvalidUploadModal from '../modals/chat/invalid-upload.react.js'; import { updateNavInfoActionType } from '../redux/action-types.js'; @@ -778,6 +779,13 @@ return { steps, result: encryptionResult }; } + const { steps: thumbhashSteps, result: thumbhashResult } = + await generateThumbHash(fixedFile, encryptionResult?.encryptionKey); + const thumbHash = thumbhashResult.success + ? thumbhashResult.thumbHash + : null; + steps.push(...thumbhashSteps); + return { steps, result: { @@ -795,6 +803,7 @@ uriIsReal: false, blobHash: encryptionResult?.sha256Hash, encryptionKey: encryptionResult?.encryptionKey, + thumbHash, progressPercent: 0, abort: null, steps, @@ -858,9 +867,9 @@ (upload.mediaType === 'encrypted_photo' || upload.mediaType === 'encrypted_video') ) { - const { blobHash, dimensions } = upload; + const { blobHash, dimensions, thumbHash } = upload; invariant( - encryptionKey && blobHash && dimensions, + encryptionKey && blobHash && dimensions && thumbHash, 'incomplete encrypted upload', ); uploadResult = await this.blobServiceUpload( @@ -870,11 +879,16 @@ encryptionKey, dimensions, loop: false, + ...(thumbHash && { thumbHash }), }, { ...callbacks }, ); } else { - let uploadExtras = { ...upload.dimensions, loop: false }; + let uploadExtras = { + ...upload.dimensions, + loop: false, + thumbHash: upload.thumbHash, + }; if (encryptionKey) { uploadExtras = { ...uploadExtras, encryptionKey }; } @@ -1042,6 +1056,7 @@ encryptionKey: string, dimensions: Dimensions, loop?: boolean, + thumbHash?: string, }, options?: ?CallServerEndpointOptions, ): Promise { @@ -1149,6 +1164,7 @@ encryptionKey: input.encryptionKey, mimeType: input.file.type, filename: input.file.name, + thumbHash: input.thumbHash, }); } diff --git a/web/input/input-state.js b/web/input/input-state.js --- a/web/input/input-state.js +++ b/web/input/input-state.js @@ -25,6 +25,7 @@ uri: string, blobHash: ?string, encryptionKey: ?string, + thumbHash: ?string, loop: boolean, // URLs created with createObjectURL aren't considered "real". The distinction // is required because those "fake" URLs must be disposed properly diff --git a/web/media/image-utils.js b/web/media/image-utils.js --- a/web/media/image-utils.js +++ b/web/media/image-utils.js @@ -1,10 +1,20 @@ // @flow import EXIF from 'exif-js'; +import { rgbaToThumbHash } from 'thumbhash'; -import type { GetOrientationMediaMissionStep } from 'lib/types/media-types.js'; +import { hexToUintArray } from 'lib/media/data-utils.js'; +import type { + GetOrientationMediaMissionStep, + MediaMissionFailure, + MediaMissionStep, +} from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; +import * as AES from './aes-crypto-utils.js'; +import { preloadImage } from './media-utils.js'; +import { base64EncodeBuffer } from '../utils/base64-utils.js'; + function getEXIFOrientation(file: File): Promise { return new Promise(resolve => { EXIF.getData(file, function () { @@ -35,4 +45,75 @@ }; } -export { getOrientation }; +type GenerateThumbhashResult = { + +success: true, + +thumbHash: string, +}; + +/** + * Generate a thumbhash for a given image file. If `encryptionKey` is provided, + * the thumbhash string will be encrypted with it. + */ +async function generateThumbHash( + file: File, + encryptionKey: ?string = null, +): Promise<{ + +steps: $ReadOnlyArray, + +result: GenerateThumbhashResult | MediaMissionFailure, +}> { + const steps = []; + const initialURI = URL.createObjectURL(file); + const { steps: preloadSteps, result: image } = await preloadImage(initialURI); + steps.push(...preloadSteps); + if (!image) { + return { + steps, + result: { success: false, reason: 'preload_image_failed' }, + }; + } + + let binaryThumbHash, thumbHashString, exceptionMessage; + try { + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d'); + + // rescale to 100px max as thumbhash doesn't need more + const scale = 100 / Math.max(image.width, image.height); + canvas.width = Math.round(image.width * scale); + canvas.height = Math.round(image.height * scale); + + context.drawImage(image, 0, 0, canvas.width, canvas.height); + const pixels = context.getImageData(0, 0, canvas.width, canvas.height); + binaryThumbHash = rgbaToThumbHash(pixels.width, pixels.height, pixels.data); + thumbHashString = base64EncodeBuffer(binaryThumbHash); + } catch (e) { + exceptionMessage = getMessageForException(e); + } finally { + URL.revokeObjectURL(initialURI); + } + steps.push({ + step: 'generate_thumbhash', + success: !!binaryThumbHash && !exceptionMessage, + exceptionMessage, + thumbHash: thumbHashString, + }); + if (!binaryThumbHash || !thumbHashString || exceptionMessage) { + return { steps, result: { success: false, reason: 'thumbhash_failed' } }; + } + + if (encryptionKey) { + try { + const encryptedThumbHash = await AES.encrypt( + hexToUintArray(encryptionKey), + binaryThumbHash, + ); + thumbHashString = base64EncodeBuffer(encryptedThumbHash); + } catch { + return { steps, result: { success: false, reason: 'encryption_failed' } }; + } + } + + return { steps, result: { success: true, thumbHash: thumbHashString } }; +} + +export { getOrientation, generateThumbHash }; diff --git a/web/package.json b/web/package.json --- a/web/package.json +++ b/web/package.json @@ -84,6 +84,7 @@ "simple-markdown": "^0.7.2", "siwe": "^1.1.6", "sql.js": "^1.8.0", + "thumbhash": "^0.1.1", "tinycolor2": "^1.4.1", "uuid": "^3.4.0", "visibilityjs": "^2.0.2", diff --git a/web/utils/base64-utils.js b/web/utils/base64-utils.js new file mode 100644 --- /dev/null +++ b/web/utils/base64-utils.js @@ -0,0 +1,14 @@ +// @flow + +function base64EncodeBuffer(data: Uint8Array): string { + return btoa(String.fromCharCode(...data)); +} + +function base64DecodeBuffer(base64String: string): Uint8Array { + const binaryString = atob(base64String); + return new Uint8Array(binaryString.length).map((_, i) => + binaryString.charCodeAt(i), + ); +} + +export { base64EncodeBuffer, base64DecodeBuffer }; diff --git a/yarn.lock b/yarn.lock --- a/yarn.lock +++ b/yarn.lock @@ -22463,6 +22463,11 @@ resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= +thumbhash@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/thumbhash/-/thumbhash-0.1.1.tgz#bd2b8616fc043f2b17151dfce0cce1408e0ebbeb" + integrity sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg== + thunky@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.0.3.tgz#f5df732453407b09191dae73e2a8cc73f381a826"