diff --git a/lib/types/media-types.js b/lib/types/media-types.js
--- a/lib/types/media-types.js
+++ b/lib/types/media-types.js
@@ -623,7 +623,9 @@
+success: false,
+reason: 'encryption_failed',
}
- | { +success: false, +reason: 'digest_failed' };
+ | { +success: false, +reason: 'digest_failed' }
+ | { +success: false, +reason: 'thumbhash_failed' }
+ | { +success: false, +reason: 'preload_image_failed' };
export type MediaMissionResult = MediaMissionFailure | { +success: true };
diff --git a/web/input/input-state-container.react.js b/web/input/input-state-container.react.js
--- a/web/input/input-state-container.react.js
+++ b/web/input/input-state-container.react.js
@@ -103,6 +103,7 @@
InputStateContext,
} from './input-state.js';
import { encryptFile } from '../media/encryption-utils.js';
+import { generateThumbHash } from '../media/image-utils.js';
import { validateFile, preloadImage } from '../media/media-utils.js';
import InvalidUploadModal from '../modals/chat/invalid-upload.react.js';
import { updateNavInfoActionType } from '../redux/action-types.js';
@@ -778,6 +779,13 @@
return { steps, result: encryptionResult };
}
+ const { steps: thumbHashSteps, result: thumbHashResult } =
+ await generateThumbHash(fixedFile, encryptionResult?.encryptionKey);
+ const thumbHash = thumbHashResult.success
+ ? thumbHashResult.thumbHash
+ : null;
+ steps.push(...thumbHashSteps);
+
return {
steps,
result: {
@@ -795,6 +803,7 @@
uriIsReal: false,
blobHash: encryptionResult?.sha256Hash,
encryptionKey: encryptionResult?.encryptionKey,
+ thumbHash,
progressPercent: 0,
abort: null,
steps,
@@ -858,7 +867,7 @@
(upload.mediaType === 'encrypted_photo' ||
upload.mediaType === 'encrypted_video')
) {
- const { blobHash, dimensions } = upload;
+ const { blobHash, dimensions, thumbHash } = upload;
invariant(
encryptionKey && blobHash && dimensions,
'incomplete encrypted upload',
@@ -870,11 +879,16 @@
encryptionKey,
dimensions,
loop: false,
+ ...(thumbHash ? { thumbHash } : undefined),
},
{ ...callbacks },
);
} else {
- let uploadExtras = { ...upload.dimensions, loop: false };
+ let uploadExtras = {
+ ...upload.dimensions,
+ loop: false,
+ thumbHash: upload.thumbHash,
+ };
if (encryptionKey) {
uploadExtras = { ...uploadExtras, encryptionKey };
}
@@ -973,16 +987,24 @@
);
if (uploadAfterPreload.messageID) {
const { mediaType, uri, dimensions, loop } = result;
- let mediaUpdate;
+ const { thumbHash } = upload;
+ let mediaUpdate = {
+ loop,
+ dimensions,
+ ...(thumbHash ? { thumbHash } : undefined),
+ };
if (!isEncrypted) {
- mediaUpdate = { type: mediaType, uri, dimensions, loop };
+ mediaUpdate = {
+ ...mediaUpdate,
+ type: mediaType,
+ uri,
+ };
} else {
mediaUpdate = {
+ ...mediaUpdate,
type: outputMediaType,
holder: uri,
encryptionKey,
- dimensions,
- loop,
};
}
this.props.dispatch({
@@ -1037,11 +1059,12 @@
async blobServiceUpload(
input: {
- file: File,
- blobHash: string,
- encryptionKey: string,
- dimensions: Dimensions,
- loop?: boolean,
+ +file: File,
+ +blobHash: string,
+ +encryptionKey: string,
+ +dimensions: Dimensions,
+ +loop?: boolean,
+ +thumbHash?: string,
},
options?: ?CallServerEndpointOptions,
): Promise<void> {
@@ -1149,6 +1172,7 @@
encryptionKey: input.encryptionKey,
mimeType: input.file.type,
filename: input.file.name,
+ thumbHash: input.thumbHash,
});
}
diff --git a/web/input/input-state.js b/web/input/input-state.js
--- a/web/input/input-state.js
+++ b/web/input/input-state.js
@@ -12,29 +12,30 @@
import type { ThreadInfo, RelativeMemberInfo } from 'lib/types/thread-types.js';
export type PendingMultimediaUpload = {
- localID: string,
+ +localID: string,
// Pending uploads are assigned a serverID once they are complete
- serverID: ?string,
+ +serverID: ?string,
// Pending uploads are assigned a messageID once they are sent
- messageID: ?string,
+ +messageID: ?string,
// This is set to true if the upload fails for whatever reason
- failed: boolean,
- file: File,
- mediaType: MediaType | EncryptedMediaType,
- dimensions: ?Dimensions,
- uri: string,
- blobHash: ?string,
- encryptionKey: ?string,
- loop: boolean,
+ +failed: boolean,
+ +file: File,
+ +mediaType: MediaType | EncryptedMediaType,
+ +dimensions: ?Dimensions,
+ +uri: string,
+ +blobHash: ?string,
+ +encryptionKey: ?string,
+ +thumbHash: ?string,
+ +loop: boolean,
// URLs created with createObjectURL aren't considered "real". The distinction
// is required because those "fake" URLs must be disposed properly
- uriIsReal: boolean,
- progressPercent: number,
+ +uriIsReal: boolean,
+ +progressPercent: number,
// This is set once the network request begins and used if the upload is
// cancelled
- abort: ?() => void,
- steps: MediaMissionStep[],
- selectTime: number,
+ +abort: ?() => void,
+ +steps: MediaMissionStep[],
+ +selectTime: number,
};
export type TypeaheadState = {
diff --git a/web/media/image-utils.js b/web/media/image-utils.js
--- a/web/media/image-utils.js
+++ b/web/media/image-utils.js
@@ -1,10 +1,20 @@
// @flow
import EXIF from 'exif-js';
+import { rgbaToThumbHash } from 'thumbhash';
-import type { GetOrientationMediaMissionStep } from 'lib/types/media-types.js';
+import { hexToUintArray } from 'lib/media/data-utils.js';
+import type {
+ GetOrientationMediaMissionStep,
+ MediaMissionFailure,
+ MediaMissionStep,
+} from 'lib/types/media-types.js';
import { getMessageForException } from 'lib/utils/errors.js';
+import * as AES from './aes-crypto-utils.js';
+import { preloadImage } from './media-utils.js';
+import { base64EncodeBuffer } from '../utils/base64-utils.js';
+
function getEXIFOrientation(file: File): Promise<?number> {
return new Promise(resolve => {
EXIF.getData(file, function () {
@@ -35,4 +45,75 @@
};
}
-export { getOrientation };
+type GenerateThumbhashResult = {
+ +success: true,
+ +thumbHash: string,
+};
+
+/**
+ * Generate a thumbhash for a given image file. If `encryptionKey` is provided,
+ * the thumbhash string will be encrypted with it.
+ */
+async function generateThumbHash(
+ file: File,
+ encryptionKey: ?string = null,
+): Promise<{
+ +steps: $ReadOnlyArray<MediaMissionStep>,
+ +result: GenerateThumbhashResult | MediaMissionFailure,
+}> {
+ const steps = [];
+ const initialURI = URL.createObjectURL(file);
+ const { steps: preloadSteps, result: image } = await preloadImage(initialURI);
+ steps.push(...preloadSteps);
+ if (!image) {
+ return {
+ steps,
+ result: { success: false, reason: 'preload_image_failed' },
+ };
+ }
+
+ let binaryThumbHash, thumbHashString, exceptionMessage;
+ try {
+ const canvas = document.createElement('canvas');
+ const context = canvas.getContext('2d');
+
+ // rescale to 100px max as thumbhash doesn't need more
+ const scale = 100 / Math.max(image.width, image.height);
+ canvas.width = Math.round(image.width * scale);
+ canvas.height = Math.round(image.height * scale);
+
+ context.drawImage(image, 0, 0, canvas.width, canvas.height);
+ const pixels = context.getImageData(0, 0, canvas.width, canvas.height);
+ binaryThumbHash = rgbaToThumbHash(pixels.width, pixels.height, pixels.data);
+ thumbHashString = base64EncodeBuffer(binaryThumbHash);
+ } catch (e) {
+ exceptionMessage = getMessageForException(e);
+ } finally {
+ URL.revokeObjectURL(initialURI);
+ }
+ steps.push({
+ step: 'generate_thumbhash',
+ success: !!thumbHashString && !exceptionMessage,
+ exceptionMessage,
+ thumbHash: thumbHashString,
+ });
+ if (!binaryThumbHash || !thumbHashString || exceptionMessage) {
+ return { steps, result: { success: false, reason: 'thumbhash_failed' } };
+ }
+
+ if (encryptionKey) {
+ try {
+ const encryptedThumbHash = await AES.encrypt(
+ hexToUintArray(encryptionKey),
+ binaryThumbHash,
+ );
+ thumbHashString = base64EncodeBuffer(encryptedThumbHash);
+ } catch {
+ return { steps, result: { success: false, reason: 'encryption_failed' } };
+ }
+ }
+
+ return { steps, result: { success: true, thumbHash: thumbHashString } };
+}
+
+export { getOrientation, generateThumbHash };
diff --git a/web/package.json b/web/package.json
--- a/web/package.json
+++ b/web/package.json
@@ -84,6 +84,7 @@
"simple-markdown": "^0.7.2",
"siwe": "^1.1.6",
"sql.js": "^1.8.0",
+ "thumbhash": "^0.1.1",
"tinycolor2": "^1.4.1",
"uuid": "^3.4.0",
"visibilityjs": "^2.0.2",
diff --git a/web/utils/base64-utils.js b/web/utils/base64-utils.js
new file mode 100644
--- /dev/null
+++ b/web/utils/base64-utils.js
@@ -0,0 +1,14 @@
+// @flow
+
+function base64EncodeBuffer(data: Uint8Array): string {
+ return btoa(String.fromCharCode(...data));
+}
+
+function base64DecodeBuffer(base64String: string): Uint8Array {
+ const binaryString = atob(base64String);
+ return new Uint8Array(binaryString.length).map((_, i) =>
+ binaryString.charCodeAt(i),
+ );
+}
+
+export { base64EncodeBuffer, base64DecodeBuffer };
diff --git a/yarn.lock b/yarn.lock
--- a/yarn.lock
+++ b/yarn.lock
@@ -22468,6 +22468,11 @@
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
+thumbhash@^0.1.1:
+ version "0.1.1"
+ resolved "https://registry.yarnpkg.com/thumbhash/-/thumbhash-0.1.1.tgz#bd2b8616fc043f2b17151dfce0cce1408e0ebbeb"
+ integrity sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg==
+
thunky@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.0.3.tgz#f5df732453407b09191dae73e2a8cc73f381a826"