diff --git a/native/input/input-state-container.react.js b/native/input/input-state-container.react.js index 7dc35e6a5..d4546388d 100644 --- a/native/input/input-state-container.react.js +++ b/native/input/input-state-container.react.js @@ -1,1745 +1,1746 @@ // @flow import * as FileSystem from 'expo-file-system'; import invariant from 'invariant'; import * as React from 'react'; import { Platform } from 'react-native'; import { useDispatch } from 'react-redux'; import { createSelector } from 'reselect'; import { createLocalMessageActionType, sendMultimediaMessageActionTypes, useSendMultimediaMessage, sendTextMessageActionTypes, useSendTextMessage, } from 'lib/actions/message-actions.js'; import type { SendMultimediaMessageInput, SendTextMessageInput, } from 'lib/actions/message-actions.js'; import { queueReportsActionType } from 'lib/actions/report-actions.js'; import { useNewThread } from 'lib/actions/thread-actions.js'; import { uploadMultimedia, updateMultimediaMessageMediaActionType, useBlobServiceUpload, type MultimediaUploadCallbacks, type MultimediaUploadExtras, type BlobServiceUploadAction, } from 'lib/actions/upload-actions.js'; import commStaffCommunity from 'lib/facts/comm-staff-community.js'; import { pathFromURI, replaceExtension } from 'lib/media/file-utils.js'; import { isLocalUploadID, getNextLocalUploadID, } from 'lib/media/media-utils.js'; import { videoDurationLimit } from 'lib/media/video-utils.js'; import { createLoadingStatusSelector, combineLoadingStatuses, } from 'lib/selectors/loading-selectors.js'; import { createMediaMessageInfo, useNextLocalID, useMessageCreationSideEffectsFunc, } from 'lib/shared/message-utils.js'; import type { CreationSideEffectsFunc } from 'lib/shared/messages/message-spec.js'; import { createRealThreadFromPendingThread, threadIsPending, threadIsPendingSidebar, patchThreadInfoToIncludeMentionedMembersOfParent, threadInfoInsideCommunity, } from 'lib/shared/thread-utils.js'; import type { CalendarQuery } from 'lib/types/entry-types.js'; import type { UploadMultimediaResult, Media, NativeMediaSelection, MediaMissionResult, MediaMission, + MediaMissionStep, } from 'lib/types/media-types.js'; import { messageTypes } from 'lib/types/message-types-enum.js'; import { type RawMessageInfo, type RawMultimediaMessageInfo, type SendMessageResult, type SendMessagePayload, } from 'lib/types/message-types.js'; import type { RawImagesMessageInfo } from 'lib/types/messages/images.js'; import type { RawMediaMessageInfo } from 'lib/types/messages/media.js'; import { getMediaMessageServerDBContentsFromMedia } from 'lib/types/messages/media.js'; import type { RawTextMessageInfo } from 'lib/types/messages/text.js'; import type { MinimallyEncodedThreadInfo } from 'lib/types/minimally-encoded-thread-permissions-types.js'; import type { Dispatch } from 'lib/types/redux-types.js'; import { type ClientMediaMissionReportCreationRequest, reportTypes, } from 'lib/types/report-types.js'; import { threadTypes } from 'lib/types/thread-types-enum.js'; import { type ClientNewThreadRequest, type NewThreadResult, type ThreadInfo, } from 'lib/types/thread-types.js'; import { type DispatchActionPromise, useServerCall, useDispatchActionPromise, } from 'lib/utils/action-utils.js'; import type { CallServerEndpointOptions, CallServerEndpointResponse, } from 'lib/utils/call-server-endpoint.js'; import { getConfig } from 'lib/utils/config.js'; import { getMessageForException, cloneError } from 'lib/utils/errors.js'; import { values } from 'lib/utils/objects.js'; import { generateReportID, useIsReportEnabled, } from 'lib/utils/report-utils.js'; import { type EditInputBarMessageParameters, InputStateContext, type PendingMultimediaUploads, type MultimediaProcessingStep, } from './input-state.js'; import { encryptMedia } from '../media/encryption-utils.js'; import { disposeTempFile } from '../media/file-utils.js'; import { processMedia } from '../media/media-utils.js'; import { displayActionResultModal } from '../navigation/action-result-modal.js'; import { useCalendarQuery } from '../navigation/nav-selectors.js'; import { useSelector } from '../redux/redux-utils.js'; import blobServiceUploadHandler from '../utils/blob-service-upload.js'; import { useStaffCanSee } from '../utils/staff-utils.js'; type MediaIDs = | { +type: 'photo', +localMediaID: string } | { +type: 'video', +localMediaID: string, +localThumbnailID: string }; type UploadFileInput = { +selection: NativeMediaSelection, +ids: MediaIDs, }; type CompletedUploads = { +[localMessageID: string]: ?Set }; type BaseProps = { +children: React.Node, }; type Props = { ...BaseProps, +viewerID: ?string, +nextLocalID: string, +messageStoreMessages: { +[id: string]: RawMessageInfo }, +ongoingMessageCreation: boolean, +hasWiFi: boolean, +mediaReportsEnabled: boolean, +calendarQuery: () => CalendarQuery, +dispatch: Dispatch, +staffCanSee: boolean, +dispatchActionPromise: DispatchActionPromise, +uploadMultimedia: ( multimedia: Object, extras: MultimediaUploadExtras, callbacks: MultimediaUploadCallbacks, ) => Promise, +blobServiceUpload: BlobServiceUploadAction, +sendMultimediaMessage: ( input: SendMultimediaMessageInput, ) => Promise, +sendTextMessage: (input: SendTextMessageInput) => Promise, +newThread: (request: ClientNewThreadRequest) => Promise, +textMessageCreationSideEffectsFunc: CreationSideEffectsFunc, }; type State = { +pendingUploads: PendingMultimediaUploads, }; class InputStateContainer extends React.PureComponent { state: State = { pendingUploads: {}, }; sendCallbacks: Array<() => void> = []; activeURIs = new Map(); editInputBarCallbacks: Array< (params: EditInputBarMessageParameters) => void, > = []; scrollToMessageCallbacks: Array<(messageID: string) => void> = []; pendingThreadCreations = new Map>(); pendingThreadUpdateHandlers = new Map< string, (ThreadInfo | MinimallyEncodedThreadInfo) => mixed, >(); // TODO: flip the switch // Note that this enables Blob service for encrypted media only useBlobServiceUploads = false; // When the user sends a multimedia message that triggers the creation of a // sidebar, the sidebar gets created right away, but the message needs to wait // for the uploads to complete before sending. We use this Set to track the // message localIDs that need sidebarCreation: true. pendingSidebarCreationMessageLocalIDs = new Set(); static getCompletedUploads(props: Props, state: State): CompletedUploads { const completedUploads = {}; for (const localMessageID in state.pendingUploads) { const messagePendingUploads = state.pendingUploads[localMessageID]; const rawMessageInfo = props.messageStoreMessages[localMessageID]; if (!rawMessageInfo) { continue; } invariant( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA, `rawMessageInfo ${localMessageID} should be multimedia`, ); let allUploadsComplete = true; const completedUploadIDs = new Set(Object.keys(messagePendingUploads)); for (const singleMedia of rawMessageInfo.media) { if (isLocalUploadID(singleMedia.id)) { allUploadsComplete = false; completedUploadIDs.delete(singleMedia.id); } } if (allUploadsComplete) { completedUploads[localMessageID] = null; } else if (completedUploadIDs.size > 0) { completedUploads[localMessageID] = completedUploadIDs; } } return completedUploads; } componentDidUpdate(prevProps: Props, prevState: State) { if (this.props.viewerID !== prevProps.viewerID) { this.setState({ pendingUploads: {} }); return; } const currentlyComplete = InputStateContainer.getCompletedUploads( this.props, this.state, ); const previouslyComplete = InputStateContainer.getCompletedUploads( prevProps, prevState, ); const newPendingUploads = {}; let pendingUploadsChanged = false; const readyMessageIDs = []; for (const localMessageID in this.state.pendingUploads) { const messagePendingUploads = this.state.pendingUploads[localMessageID]; const prevRawMessageInfo = prevProps.messageStoreMessages[localMessageID]; const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; const completedUploadIDs = currentlyComplete[localMessageID]; const previouslyCompletedUploadIDs = previouslyComplete[localMessageID]; if (!rawMessageInfo && prevRawMessageInfo) { pendingUploadsChanged = true; continue; } else if (completedUploadIDs === null) { // All of this message's uploads have been completed newPendingUploads[localMessageID] = {}; if (previouslyCompletedUploadIDs !== null) { readyMessageIDs.push(localMessageID); pendingUploadsChanged = true; } continue; } else if (!completedUploadIDs) { // Nothing has been completed newPendingUploads[localMessageID] = messagePendingUploads; continue; } const newUploads = {}; let uploadsChanged = false; for (const localUploadID in messagePendingUploads) { if (!completedUploadIDs.has(localUploadID)) { newUploads[localUploadID] = messagePendingUploads[localUploadID]; } else if ( !previouslyCompletedUploadIDs || !previouslyCompletedUploadIDs.has(localUploadID) ) { uploadsChanged = true; } } if (uploadsChanged) { pendingUploadsChanged = true; newPendingUploads[localMessageID] = newUploads; } else { newPendingUploads[localMessageID] = messagePendingUploads; } } if (pendingUploadsChanged) { this.setState({ pendingUploads: newPendingUploads }); } for (const localMessageID of readyMessageIDs) { const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; if (!rawMessageInfo) { continue; } invariant( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA, `rawMessageInfo ${localMessageID} should be multimedia`, ); this.dispatchMultimediaMessageAction(rawMessageInfo); } } async dispatchMultimediaMessageAction(messageInfo: RawMultimediaMessageInfo) { if (!threadIsPending(messageInfo.threadID)) { this.props.dispatchActionPromise( sendMultimediaMessageActionTypes, this.sendMultimediaMessageAction(messageInfo), undefined, messageInfo, ); return; } this.props.dispatch({ type: sendMultimediaMessageActionTypes.started, payload: messageInfo, }); let newThreadID = null; try { const threadCreationPromise = this.pendingThreadCreations.get( messageInfo.threadID, ); if (!threadCreationPromise) { // When we create or retry multimedia message, we add a promise to // pendingThreadCreations map. This promise can be removed in // sendMultimediaMessage and sendTextMessage methods. When any of these // method remove the promise, it has to be settled. If the promise was // fulfilled, this method would be called with realized thread, so we // can conclude that the promise was rejected. We don't have enough info // here to retry the thread creation, but we can mark the message as // failed. Then the retry will be possible and promise will be created // again. throw new Error('Thread creation failed'); } newThreadID = await threadCreationPromise; } catch (e) { const copy = cloneError(e); copy.localID = messageInfo.localID; copy.threadID = messageInfo.threadID; this.props.dispatch({ type: sendMultimediaMessageActionTypes.failed, payload: copy, error: true, }); return; } finally { this.pendingThreadCreations.delete(messageInfo.threadID); } const newMessageInfo = { ...messageInfo, threadID: newThreadID, time: Date.now(), }; this.props.dispatchActionPromise( sendMultimediaMessageActionTypes, this.sendMultimediaMessageAction(newMessageInfo), undefined, newMessageInfo, ); } async sendMultimediaMessageAction( messageInfo: RawMultimediaMessageInfo, ): Promise { const { localID, threadID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const sidebarCreation = this.pendingSidebarCreationMessageLocalIDs.has(localID); const mediaMessageContents = getMediaMessageServerDBContentsFromMedia( messageInfo.media, ); try { const result = await this.props.sendMultimediaMessage({ threadID, localID, mediaMessageContents, sidebarCreation, }); this.pendingSidebarCreationMessageLocalIDs.delete(localID); return { localID, serverID: result.id, threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = localID; copy.threadID = threadID; throw copy; } } inputStateSelector = createSelector( (state: State) => state.pendingUploads, (pendingUploads: PendingMultimediaUploads) => ({ pendingUploads, sendTextMessage: this.sendTextMessage, sendMultimediaMessage: this.sendMultimediaMessage, editInputMessage: this.editInputMessage, addEditInputMessageListener: this.addEditInputMessageListener, removeEditInputMessageListener: this.removeEditInputMessageListener, messageHasUploadFailure: this.messageHasUploadFailure, retryMessage: this.retryMessage, registerSendCallback: this.registerSendCallback, unregisterSendCallback: this.unregisterSendCallback, uploadInProgress: this.uploadInProgress, reportURIDisplayed: this.reportURIDisplayed, setPendingThreadUpdateHandler: this.setPendingThreadUpdateHandler, scrollToMessage: this.scrollToMessage, addScrollToMessageListener: this.addScrollToMessageListener, removeScrollToMessageListener: this.removeScrollToMessageListener, }), ); scrollToMessage = (messageID: string) => { this.scrollToMessageCallbacks.forEach(callback => callback(messageID)); }; addScrollToMessageListener = (callback: (messageID: string) => void) => { this.scrollToMessageCallbacks.push(callback); }; removeScrollToMessageListener = ( callbackScrollToMessage: (messageID: string) => void, ) => { this.scrollToMessageCallbacks = this.scrollToMessageCallbacks.filter( candidate => candidate !== callbackScrollToMessage, ); }; uploadInProgress = () => { if (this.props.ongoingMessageCreation) { return true; } const { pendingUploads } = this.state; return values(pendingUploads).some(messagePendingUploads => values(messagePendingUploads).some(upload => !upload.failed), ); }; sendTextMessage = async ( messageInfo: RawTextMessageInfo, inputThreadInfo: ThreadInfo | MinimallyEncodedThreadInfo, parentThreadInfo: ?ThreadInfo | ?MinimallyEncodedThreadInfo, ) => { this.sendCallbacks.forEach(callback => callback()); const { localID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); if (threadIsPendingSidebar(inputThreadInfo.id)) { this.pendingSidebarCreationMessageLocalIDs.add(localID); } if (!threadIsPending(inputThreadInfo.id)) { this.props.dispatchActionPromise( sendTextMessageActionTypes, this.sendTextMessageAction( messageInfo, inputThreadInfo, parentThreadInfo, ), undefined, messageInfo, ); return; } this.props.dispatch({ type: sendTextMessageActionTypes.started, payload: messageInfo, }); let threadInfo = inputThreadInfo; const { viewerID } = this.props; if (viewerID && inputThreadInfo.type === threadTypes.SIDEBAR) { invariant(parentThreadInfo, 'sidebar should have parent'); threadInfo = patchThreadInfoToIncludeMentionedMembersOfParent( inputThreadInfo, parentThreadInfo, messageInfo.text, viewerID, ); if (threadInfo !== inputThreadInfo) { const pendingThreadUpdateHandler = this.pendingThreadUpdateHandlers.get( threadInfo.id, ); pendingThreadUpdateHandler?.(threadInfo); } } let newThreadID = null; try { newThreadID = await this.startThreadCreation(threadInfo); } catch (e) { const copy = cloneError(e); copy.localID = messageInfo.localID; copy.threadID = messageInfo.threadID; this.props.dispatch({ type: sendTextMessageActionTypes.failed, payload: copy, error: true, }); return; } finally { this.pendingThreadCreations.delete(threadInfo.id); } const newMessageInfo = { ...messageInfo, threadID: newThreadID, time: Date.now(), }; // Branching to appease `flow`. const newThreadInfo = threadInfo.minimallyEncoded ? { ...threadInfo, id: newThreadID, } : { ...threadInfo, id: newThreadID, }; this.props.dispatchActionPromise( sendTextMessageActionTypes, this.sendTextMessageAction( newMessageInfo, newThreadInfo, parentThreadInfo, ), undefined, newMessageInfo, ); }; startThreadCreation( threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ): Promise { if (!threadIsPending(threadInfo.id)) { return Promise.resolve(threadInfo.id); } let threadCreationPromise = this.pendingThreadCreations.get(threadInfo.id); if (!threadCreationPromise) { const calendarQuery = this.props.calendarQuery(); threadCreationPromise = createRealThreadFromPendingThread({ threadInfo, dispatchActionPromise: this.props.dispatchActionPromise, createNewThread: this.props.newThread, sourceMessageID: threadInfo.sourceMessageID, viewerID: this.props.viewerID, calendarQuery, }); this.pendingThreadCreations.set(threadInfo.id, threadCreationPromise); } return threadCreationPromise; } async sendTextMessageAction( messageInfo: RawTextMessageInfo, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, parentThreadInfo: ?ThreadInfo | ?MinimallyEncodedThreadInfo, ): Promise { try { await this.props.textMessageCreationSideEffectsFunc( messageInfo, threadInfo, parentThreadInfo, ); const { localID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const sidebarCreation = this.pendingSidebarCreationMessageLocalIDs.has(localID); const result = await this.props.sendTextMessage({ threadID: messageInfo.threadID, localID, text: messageInfo.text, sidebarCreation, }); this.pendingSidebarCreationMessageLocalIDs.delete(localID); return { localID, serverID: result.id, threadID: messageInfo.threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = messageInfo.localID; copy.threadID = messageInfo.threadID; throw copy; } } shouldEncryptMedia( threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ): boolean { return threadInfoInsideCommunity(threadInfo, commStaffCommunity.id); } sendMultimediaMessage = async ( selections: $ReadOnlyArray, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ) => { this.sendCallbacks.forEach(callback => callback()); const localMessageID = this.props.nextLocalID; this.startThreadCreation(threadInfo); if (threadIsPendingSidebar(threadInfo.id)) { this.pendingSidebarCreationMessageLocalIDs.add(localMessageID); } const uploadFileInputs = [], media = []; for (const selection of selections) { const localMediaID = getNextLocalUploadID(); let ids; if ( selection.step === 'photo_library' || selection.step === 'photo_capture' || selection.step === 'photo_paste' ) { media.push({ id: localMediaID, uri: selection.uri, type: 'photo', dimensions: selection.dimensions, localMediaSelection: selection, thumbHash: null, }); ids = { type: 'photo', localMediaID }; } const localThumbnailID = getNextLocalUploadID(); if (selection.step === 'video_library') { media.push({ id: localMediaID, uri: selection.uri, type: 'video', dimensions: selection.dimensions, localMediaSelection: selection, loop: false, thumbnailID: localThumbnailID, thumbnailURI: selection.uri, thumbnailThumbHash: null, }); ids = { type: 'video', localMediaID, localThumbnailID }; } invariant(ids, `unexpected MediaSelection ${selection.step}`); uploadFileInputs.push({ selection, ids }); } const pendingUploads = {}; for (const uploadFileInput of uploadFileInputs) { const { localMediaID } = uploadFileInput.ids; pendingUploads[localMediaID] = { failed: false, progressPercent: 0, processingStep: null, }; if (uploadFileInput.ids.type === 'video') { const { localThumbnailID } = uploadFileInput.ids; pendingUploads[localThumbnailID] = { failed: false, progressPercent: 0, processingStep: null, }; } } this.setState( prevState => { return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: pendingUploads, }, }; }, () => { const creatorID = this.props.viewerID; invariant(creatorID, 'need viewer ID in order to send a message'); const messageInfo = createMediaMessageInfo( { localID: localMessageID, threadID: threadInfo.id, creatorID, media, }, { forceMultimediaMessageType: this.shouldEncryptMedia(threadInfo) }, ); this.props.dispatch({ type: createLocalMessageActionType, payload: messageInfo, }); }, ); await this.uploadFiles(localMessageID, uploadFileInputs, threadInfo); }; async uploadFiles( localMessageID: string, uploadFileInputs: $ReadOnlyArray, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ) { const results = await Promise.all( uploadFileInputs.map(uploadFileInput => this.uploadFile(localMessageID, uploadFileInput, threadInfo), ), ); const errors = [...new Set(results.filter(Boolean))]; if (errors.length > 0) { displayActionResultModal(errors.join(', ') + ' :('); } } async uploadFile( localMessageID: string, uploadFileInput: UploadFileInput, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ): Promise { const { ids, selection } = uploadFileInput; const { localMediaID } = ids; const start = selection.sendTime; - const steps = [selection]; + const steps: Array = [selection]; let encryptionSteps = []; let serverID; let userTime; let errorMessage; let reportPromise; const filesToDispose = []; const onUploadFinished = async (result: MediaMissionResult) => { if (!this.props.mediaReportsEnabled) { return errorMessage; } if (reportPromise) { const finalSteps = await reportPromise; steps.push(...finalSteps); steps.push(...encryptionSteps); } const totalTime = Date.now() - start; userTime = userTime ? userTime : totalTime; this.queueMediaMissionReport( { localID: localMediaID, localMessageID, serverID }, { steps, result, totalTime, userTime }, ); return errorMessage; }; const onUploadFailed = (mediaID: string, message: string) => { errorMessage = message; this.handleUploadFailure(localMessageID, mediaID); userTime = Date.now() - start; }; const onTranscodingProgress = (percent: number) => { this.setProgress(localMessageID, localMediaID, 'transcoding', percent); }; let processedMedia; const processingStart = Date.now(); try { const processMediaReturn = processMedia(selection, { hasWiFi: this.props.hasWiFi, finalFileHeaderCheck: this.props.staffCanSee, onTranscodingProgress, }); reportPromise = processMediaReturn.reportPromise; const processResult = await processMediaReturn.resultPromise; if (!processResult.success) { const message = processResult.reason === 'video_too_long' ? `can't do vids longer than ${videoDurationLimit}min` : 'processing failed'; onUploadFailed(localMediaID, message); return await onUploadFinished(processResult); } if (processResult.shouldDisposePath) { filesToDispose.push(processResult.shouldDisposePath); } processedMedia = processResult; } catch (e) { onUploadFailed(localMediaID, 'processing failed'); return await onUploadFinished({ success: false, reason: 'processing_exception', time: Date.now() - processingStart, exceptionMessage: getMessageForException(e), }); } if (this.shouldEncryptMedia(threadInfo)) { const encryptionStart = Date.now(); try { const { result: encryptionResult, ...encryptionReturn } = await encryptMedia(processedMedia); encryptionSteps = encryptionReturn.steps; if (!encryptionResult.success) { onUploadFailed(localMediaID, encryptionResult.reason); return await onUploadFinished(encryptionResult); } if (encryptionResult.shouldDisposePath) { filesToDispose.push(encryptionResult.shouldDisposePath); } processedMedia = encryptionResult; } catch (e) { onUploadFailed(localMediaID, 'encryption failed'); return await onUploadFinished({ success: false, reason: 'encryption_exception', time: Date.now() - encryptionStart, exceptionMessage: getMessageForException(e), }); } } const { uploadURI, filename, mime } = processedMedia; const { hasWiFi } = this.props; const uploadStart = Date.now(); let uploadExceptionMessage, uploadResult, uploadThumbnailResult, mediaMissionResult; try { const uploadPromises = []; if ( this.useBlobServiceUploads && (processedMedia.mediaType === 'encrypted_photo' || processedMedia.mediaType === 'encrypted_video') ) { uploadPromises.push( this.props.blobServiceUpload({ uploadInput: { blobInput: { type: 'uri', uri: uploadURI, filename: filename, mimeType: mime, }, blobHash: processedMedia.blobHash, encryptionKey: processedMedia.encryptionKey, dimensions: processedMedia.dimensions, thumbHash: processedMedia.mediaType === 'encrypted_photo' ? processedMedia.thumbHash : null, }, keyserverOrThreadID: threadInfo.id, callbacks: { blobServiceUploadHandler, onProgress: (percent: number) => { this.setProgress( localMessageID, localMediaID, 'uploading', percent, ); }, }, }), ); if (processedMedia.mediaType === 'encrypted_video') { uploadPromises.push( this.props.blobServiceUpload({ uploadInput: { blobInput: { type: 'uri', uri: processedMedia.uploadThumbnailURI, filename: replaceExtension(`thumb${filename}`, 'jpg'), mimeType: 'image/jpeg', }, blobHash: processedMedia.thumbnailBlobHash, encryptionKey: processedMedia.thumbnailEncryptionKey, loop: false, dimensions: processedMedia.dimensions, thumbHash: processedMedia.thumbHash, }, keyserverOrThreadID: threadInfo.id, callbacks: { blobServiceUploadHandler, }, }), ); } [uploadResult, uploadThumbnailResult] = await Promise.all( uploadPromises, ); } else { uploadPromises.push( this.props.uploadMultimedia( { uri: uploadURI, name: filename, type: mime }, { ...processedMedia.dimensions, loop: processedMedia.mediaType === 'video' || processedMedia.mediaType === 'encrypted_video' ? processedMedia.loop : undefined, encryptionKey: processedMedia.encryptionKey, thumbHash: processedMedia.mediaType === 'photo' || processedMedia.mediaType === 'encrypted_photo' ? processedMedia.thumbHash : null, }, { onProgress: (percent: number) => this.setProgress( localMessageID, localMediaID, 'uploading', percent, ), uploadBlob: this.uploadBlob, }, ), ); if ( processedMedia.mediaType === 'video' || processedMedia.mediaType === 'encrypted_video' ) { uploadPromises.push( this.props.uploadMultimedia( { uri: processedMedia.uploadThumbnailURI, name: replaceExtension(`thumb${filename}`, 'jpg'), type: 'image/jpeg', }, { ...processedMedia.dimensions, loop: false, encryptionKey: processedMedia.thumbnailEncryptionKey, thumbHash: processedMedia.thumbHash, }, { uploadBlob: this.uploadBlob, }, ), ); } [uploadResult, uploadThumbnailResult] = await Promise.all( uploadPromises, ); } mediaMissionResult = { success: true }; } catch (e) { uploadExceptionMessage = getMessageForException(e); onUploadFailed(localMediaID, 'upload failed'); mediaMissionResult = { success: false, reason: 'http_upload_failed', exceptionMessage: uploadExceptionMessage, }; } if ( ((processedMedia.mediaType === 'photo' || processedMedia.mediaType === 'encrypted_photo') && uploadResult) || ((processedMedia.mediaType === 'video' || processedMedia.mediaType === 'encrypted_video') && uploadResult && uploadThumbnailResult) ) { const { encryptionKey } = processedMedia; const { id, uri, dimensions, loop } = uploadResult; serverID = id; const mediaSourcePayload = processedMedia.mediaType === 'encrypted_photo' || processedMedia.mediaType === 'encrypted_video' ? { type: processedMedia.mediaType, blobURI: uri, encryptionKey, } : { type: uploadResult.mediaType, uri, }; let updateMediaPayload = { messageID: localMessageID, currentMediaID: localMediaID, mediaUpdate: { id, ...mediaSourcePayload, dimensions, localMediaSelection: undefined, loop: uploadResult.mediaType === 'video' ? loop : undefined, }, }; if ( processedMedia.mediaType === 'video' || processedMedia.mediaType === 'encrypted_video' ) { invariant(uploadThumbnailResult, 'uploadThumbnailResult exists'); const { uri: thumbnailURI, id: thumbnailID } = uploadThumbnailResult; const { thumbnailEncryptionKey, thumbHash: thumbnailThumbHash } = processedMedia; if (processedMedia.mediaType === 'encrypted_video') { updateMediaPayload = { ...updateMediaPayload, mediaUpdate: { ...updateMediaPayload.mediaUpdate, thumbnailID, thumbnailBlobURI: thumbnailURI, thumbnailEncryptionKey, thumbnailThumbHash, }, }; } else { updateMediaPayload = { ...updateMediaPayload, mediaUpdate: { ...updateMediaPayload.mediaUpdate, thumbnailID, thumbnailURI, thumbnailThumbHash, }, }; } } else { updateMediaPayload = { ...updateMediaPayload, mediaUpdate: { ...updateMediaPayload.mediaUpdate, thumbHash: processedMedia.thumbHash, }, }; } // When we dispatch this action, it updates Redux and triggers the // componentDidUpdate in this class. componentDidUpdate will handle // calling dispatchMultimediaMessageAction once all the uploads are // complete, and does not wait until this function concludes. this.props.dispatch({ type: updateMultimediaMessageMediaActionType, payload: updateMediaPayload, }); userTime = Date.now() - start; } const processSteps = await reportPromise; reportPromise = null; steps.push(...processSteps); steps.push(...encryptionSteps); steps.push({ step: 'upload', success: !!uploadResult, exceptionMessage: uploadExceptionMessage, time: Date.now() - uploadStart, inputFilename: filename, outputMediaType: uploadResult && uploadResult.mediaType, outputURI: uploadResult && uploadResult.uri, outputDimensions: uploadResult && uploadResult.dimensions, outputLoop: uploadResult && uploadResult.loop, hasWiFi, }); const cleanupPromises = []; if (filesToDispose.length > 0) { // If processMedia needed to do any transcoding before upload, we dispose // of the resultant temporary file here. Since the transcoded temporary // file is only used for upload, we can dispose of it after processMedia // (reportPromise) and the upload are complete filesToDispose.forEach(shouldDisposePath => { cleanupPromises.push( (async () => { const disposeStep = await disposeTempFile(shouldDisposePath); steps.push(disposeStep); })(), ); }); } // if there's a thumbnail we'll temporarily unlink it here // instead of in media-utils, will be changed in later diffs if (processedMedia.mediaType === 'video') { const { uploadThumbnailURI } = processedMedia; cleanupPromises.push( (async () => { const { steps: clearSteps, result: thumbnailPath } = await this.waitForCaptureURIUnload(uploadThumbnailURI); steps.push(...clearSteps); if (!thumbnailPath) { return; } const disposeStep = await disposeTempFile(thumbnailPath); steps.push(disposeStep); })(), ); } if (selection.captureTime || selection.step === 'photo_paste') { // If we are uploading a newly captured photo, we dispose of the original // file here. Note that we try to save photo captures to the camera roll // if we have permission. Even if we fail, this temporary file isn't // visible to the user, so there's no point in keeping it around. Since // the initial URI is used in rendering paths, we have to wait for it to // be replaced with the remote URI before we can dispose. Check out the // Multimedia component to see how the URIs get switched out. const captureURI = selection.uri; cleanupPromises.push( (async () => { const { steps: clearSteps, result: capturePath } = await this.waitForCaptureURIUnload(captureURI); steps.push(...clearSteps); if (!capturePath) { return; } const disposeStep = await disposeTempFile(capturePath); steps.push(disposeStep); })(), ); } await Promise.all(cleanupPromises); return await onUploadFinished(mediaMissionResult); } setProgress( localMessageID: string, localUploadID: string, processingStep: MultimediaProcessingStep, progressPercent: number, ) { this.setState(prevState => { const pendingUploads = prevState.pendingUploads[localMessageID]; if (!pendingUploads) { return {}; } const pendingUpload = pendingUploads[localUploadID]; if (!pendingUpload) { return {}; } const newOutOfHundred = Math.floor(progressPercent * 100); const oldOutOfHundred = Math.floor(pendingUpload.progressPercent * 100); if (newOutOfHundred === oldOutOfHundred) { return {}; } const newPendingUploads = { ...pendingUploads, [localUploadID]: { ...pendingUpload, progressPercent, processingStep, }, }; return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: newPendingUploads, }, }; }); } uploadBlob = async ( url: string, cookie: ?string, sessionID: ?string, input: { +[key: string]: mixed }, options?: ?CallServerEndpointOptions, ): Promise => { invariant( cookie && input.multimedia && Array.isArray(input.multimedia) && input.multimedia.length === 1 && input.multimedia[0] && typeof input.multimedia[0] === 'object', 'InputStateContainer.uploadBlob sent incorrect input', ); const { uri, name, type } = input.multimedia[0]; invariant( typeof uri === 'string' && typeof name === 'string' && typeof type === 'string', 'InputStateContainer.uploadBlob sent incorrect input', ); const parameters = {}; parameters.cookie = cookie; parameters.filename = name; for (const key in input) { if ( key === 'multimedia' || key === 'cookie' || key === 'sessionID' || key === 'filename' ) { continue; } const value = input[key]; invariant( typeof value === 'string', 'blobUpload calls can only handle string values for non-multimedia keys', ); parameters[key] = value; } let path = uri; if (Platform.OS === 'android') { const resolvedPath = pathFromURI(uri); if (resolvedPath) { path = resolvedPath; } } const uploadTask = FileSystem.createUploadTask( url, path, { uploadType: FileSystem.FileSystemUploadType.MULTIPART, fieldName: 'multimedia', headers: { Accept: 'application/json', }, parameters, }, uploadProgress => { if (options && options.onProgress) { const { totalByteSent, totalBytesExpectedToSend } = uploadProgress; options.onProgress(totalByteSent / totalBytesExpectedToSend); } }, ); if (options && options.abortHandler) { options.abortHandler(() => uploadTask.cancelAsync()); } try { const response = await uploadTask.uploadAsync(); return JSON.parse(response.body); } catch (e) { throw new Error( `Failed to upload blob: ${ getMessageForException(e) ?? 'unknown error' }`, ); } }; handleUploadFailure(localMessageID: string, localUploadID: string) { this.setState(prevState => { const uploads = prevState.pendingUploads[localMessageID]; const upload = uploads[localUploadID]; if (!upload) { // The upload has been completed before it failed return {}; } return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: { ...uploads, [localUploadID]: { ...upload, failed: true, progressPercent: 0, }, }, }, }; }); } queueMediaMissionReport( ids: { localID: string, localMessageID: string, serverID: ?string }, mediaMission: MediaMission, ) { const report: ClientMediaMissionReportCreationRequest = { type: reportTypes.MEDIA_MISSION, time: Date.now(), platformDetails: getConfig().platformDetails, mediaMission, uploadServerID: ids.serverID, uploadLocalID: ids.localID, messageLocalID: ids.localMessageID, id: generateReportID(), }; this.props.dispatch({ type: queueReportsActionType, payload: { reports: [report], }, }); } messageHasUploadFailure = (localMessageID: string) => { const pendingUploads = this.state.pendingUploads[localMessageID]; if (!pendingUploads) { return false; } return values(pendingUploads).some(upload => upload.failed); }; editInputMessage = (params: EditInputBarMessageParameters) => { this.editInputBarCallbacks.forEach(addEditInputBarCallback => addEditInputBarCallback(params), ); }; addEditInputMessageListener = ( callbackEditInputBar: (params: EditInputBarMessageParameters) => void, ) => { this.editInputBarCallbacks.push(callbackEditInputBar); }; removeEditInputMessageListener = ( callbackEditInputBar: (params: EditInputBarMessageParameters) => void, ) => { this.editInputBarCallbacks = this.editInputBarCallbacks.filter( candidate => candidate !== callbackEditInputBar, ); }; retryTextMessage = async ( rawMessageInfo: RawTextMessageInfo, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, parentThreadInfo: ?ThreadInfo | ?MinimallyEncodedThreadInfo, ) => { await this.sendTextMessage( { ...rawMessageInfo, time: Date.now(), }, threadInfo, parentThreadInfo, ); }; retryMultimediaMessage = async ( rawMessageInfo: RawMultimediaMessageInfo, localMessageID: string, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, ) => { const pendingUploads = this.state.pendingUploads[localMessageID] ?? {}; const now = Date.now(); this.startThreadCreation(threadInfo); if (threadIsPendingSidebar(threadInfo.id)) { this.pendingSidebarCreationMessageLocalIDs.add(localMessageID); } const updateMedia = (media: $ReadOnlyArray): T[] => media.map(singleMedia => { invariant( singleMedia.type === 'photo' || singleMedia.type === 'video', 'Retry selection must be unencrypted', ); let updatedMedia = singleMedia; const oldMediaID = updatedMedia.id; if ( // not complete isLocalUploadID(oldMediaID) && // not still ongoing (!pendingUploads[oldMediaID] || pendingUploads[oldMediaID].failed) ) { // If we have an incomplete upload that isn't in pendingUploads, that // indicates the app has restarted. We'll reassign a new localID to // avoid collisions. Note that this isn't necessary for the message ID // since the localID reducer prevents collisions there const mediaID = pendingUploads[oldMediaID] ? oldMediaID : getNextLocalUploadID(); if (updatedMedia.type === 'photo') { updatedMedia = { type: 'photo', ...updatedMedia, id: mediaID, }; } else { updatedMedia = { type: 'video', ...updatedMedia, id: mediaID, }; } } if (updatedMedia.type === 'video') { const oldThumbnailID = updatedMedia.thumbnailID; invariant(oldThumbnailID, 'oldThumbnailID not null or undefined'); if ( // not complete isLocalUploadID(oldThumbnailID) && // not still ongoing (!pendingUploads[oldThumbnailID] || pendingUploads[oldThumbnailID].failed) ) { const thumbnailID = pendingUploads[oldThumbnailID] ? oldThumbnailID : getNextLocalUploadID(); updatedMedia = { ...updatedMedia, thumbnailID, }; } } if (updatedMedia === singleMedia) { return singleMedia; } const oldSelection = updatedMedia.localMediaSelection; invariant( oldSelection, 'localMediaSelection should be set on locally created Media', ); const retries = oldSelection.retries ? oldSelection.retries + 1 : 1; // We switch for Flow let selection; if (oldSelection.step === 'photo_capture') { selection = { ...oldSelection, sendTime: now, retries }; } else if (oldSelection.step === 'photo_library') { selection = { ...oldSelection, sendTime: now, retries }; } else if (oldSelection.step === 'photo_paste') { selection = { ...oldSelection, sendTime: now, retries }; } else { selection = { ...oldSelection, sendTime: now, retries }; } if (updatedMedia.type === 'photo') { return { type: 'photo', ...updatedMedia, localMediaSelection: selection, }; } return { type: 'video', ...updatedMedia, localMediaSelection: selection, }; }); let newRawMessageInfo; // This conditional is for Flow if (rawMessageInfo.type === messageTypes.MULTIMEDIA) { newRawMessageInfo = ({ ...rawMessageInfo, time: now, media: updateMedia(rawMessageInfo.media), }: RawMediaMessageInfo); } else if (rawMessageInfo.type === messageTypes.IMAGES) { newRawMessageInfo = ({ ...rawMessageInfo, time: now, media: updateMedia(rawMessageInfo.media), }: RawImagesMessageInfo); } else { invariant(false, `rawMessageInfo ${localMessageID} should be multimedia`); } const incompleteMedia: Media[] = []; for (const singleMedia of newRawMessageInfo.media) { if (isLocalUploadID(singleMedia.id)) { incompleteMedia.push(singleMedia); } } if (incompleteMedia.length === 0) { this.dispatchMultimediaMessageAction(newRawMessageInfo); this.setState(prevState => ({ pendingUploads: { ...prevState.pendingUploads, [localMessageID]: {}, }, })); return; } const retryMedia = incompleteMedia.filter( ({ id }) => !pendingUploads[id] || pendingUploads[id].failed, ); if (retryMedia.length === 0) { // All media are already in the process of being uploaded return; } // We're not actually starting the send here, // we just use this action to update the message in Redux this.props.dispatch({ type: sendMultimediaMessageActionTypes.started, payload: newRawMessageInfo, }); // We clear out the failed status on individual media here, // which makes the UI show pending status instead of error messages for (const singleMedia of retryMedia) { pendingUploads[singleMedia.id] = { failed: false, progressPercent: 0, processingStep: null, }; if (singleMedia.type === 'video') { const { thumbnailID } = singleMedia; invariant(thumbnailID, 'thumbnailID not null or undefined'); pendingUploads[thumbnailID] = { failed: false, progressPercent: 0, processingStep: null, }; } } this.setState(prevState => ({ pendingUploads: { ...prevState.pendingUploads, [localMessageID]: pendingUploads, }, })); const uploadFileInputs = retryMedia.map(singleMedia => { invariant( singleMedia.localMediaSelection, 'localMediaSelection should be set on locally created Media', ); let ids; if (singleMedia.type === 'photo') { ids = { type: 'photo', localMediaID: singleMedia.id }; } else { invariant( singleMedia.thumbnailID, 'singleMedia.thumbnailID should be set for videos', ); ids = { type: 'video', localMediaID: singleMedia.id, localThumbnailID: singleMedia.thumbnailID, }; } return { selection: singleMedia.localMediaSelection, ids, }; }); await this.uploadFiles(localMessageID, uploadFileInputs, threadInfo); }; retryMessage = async ( localMessageID: string, threadInfo: ThreadInfo | MinimallyEncodedThreadInfo, parentThreadInfo: ?ThreadInfo | ?MinimallyEncodedThreadInfo, ) => { this.sendCallbacks.forEach(callback => callback()); const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; invariant(rawMessageInfo, `rawMessageInfo ${localMessageID} should exist`); if (rawMessageInfo.type === messageTypes.TEXT) { await this.retryTextMessage(rawMessageInfo, threadInfo, parentThreadInfo); } else if ( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA ) { await this.retryMultimediaMessage( rawMessageInfo, localMessageID, threadInfo, ); } }; registerSendCallback = (callback: () => void) => { this.sendCallbacks.push(callback); }; unregisterSendCallback = (callback: () => void) => { this.sendCallbacks = this.sendCallbacks.filter( candidate => candidate !== callback, ); }; reportURIDisplayed = (uri: string, loaded: boolean) => { const prevActiveURI = this.activeURIs.get(uri); const curCount = prevActiveURI && prevActiveURI.count; const prevCount = curCount ? curCount : 0; const count = loaded ? prevCount + 1 : prevCount - 1; const prevOnClear = prevActiveURI && prevActiveURI.onClear; const onClear = prevOnClear ? prevOnClear : []; const activeURI = { count, onClear }; if (count) { this.activeURIs.set(uri, activeURI); return; } this.activeURIs.delete(uri); for (const callback of onClear) { callback(); } }; waitForCaptureURIUnload(uri: string) { const start = Date.now(); const path = pathFromURI(uri); if (!path) { return Promise.resolve({ result: null, steps: [ { step: 'wait_for_capture_uri_unload', success: false, time: Date.now() - start, uri, }, ], }); } const getResult = () => ({ result: path, steps: [ { step: 'wait_for_capture_uri_unload', success: true, time: Date.now() - start, uri, }, ], }); const activeURI = this.activeURIs.get(uri); if (!activeURI) { return Promise.resolve(getResult()); } return new Promise(resolve => { const finish = () => resolve(getResult()); const newActiveURI = { ...activeURI, onClear: [...activeURI.onClear, finish], }; this.activeURIs.set(uri, newActiveURI); }); } setPendingThreadUpdateHandler = ( threadID: string, pendingThreadUpdateHandler: ?( ThreadInfo | MinimallyEncodedThreadInfo, ) => mixed, ) => { if (!pendingThreadUpdateHandler) { this.pendingThreadUpdateHandlers.delete(threadID); } else { this.pendingThreadUpdateHandlers.set( threadID, pendingThreadUpdateHandler, ); } }; render() { const inputState = this.inputStateSelector(this.state); return ( {this.props.children} ); } } const mediaCreationLoadingStatusSelector = createLoadingStatusSelector( sendMultimediaMessageActionTypes, ); const textCreationLoadingStatusSelector = createLoadingStatusSelector( sendTextMessageActionTypes, ); const ConnectedInputStateContainer: React.ComponentType = React.memo(function ConnectedInputStateContainer( props: BaseProps, ) { const viewerID = useSelector( state => state.currentUserInfo && state.currentUserInfo.id, ); const nextLocalID = useNextLocalID(); const messageStoreMessages = useSelector( state => state.messageStore.messages, ); const ongoingMessageCreation = useSelector( state => combineLoadingStatuses( mediaCreationLoadingStatusSelector(state), textCreationLoadingStatusSelector(state), ) === 'loading', ); const hasWiFi = useSelector(state => state.connectivity.hasWiFi); const calendarQuery = useCalendarQuery(); const callUploadMultimedia = useServerCall(uploadMultimedia); const callBlobServiceUpload = useBlobServiceUpload(); const callSendMultimediaMessage = useSendMultimediaMessage(); const callSendTextMessage = useSendTextMessage(); const callNewThread = useNewThread(); const dispatchActionPromise = useDispatchActionPromise(); const dispatch = useDispatch(); const mediaReportsEnabled = useIsReportEnabled('mediaReports'); const staffCanSee = useStaffCanSee(); const textMessageCreationSideEffectsFunc = useMessageCreationSideEffectsFunc(messageTypes.TEXT); return ( ); }); export default ConnectedInputStateContainer; diff --git a/native/media/blob-utils.js b/native/media/blob-utils.js index dafc06bc3..20f7cd50b 100644 --- a/native/media/blob-utils.js +++ b/native/media/blob-utils.js @@ -1,149 +1,149 @@ // @flow import base64 from 'base-64'; import invariant from 'invariant'; import { fileInfoFromData, bytesNeededForFileTypeCheck, } from 'lib/media/file-utils.js'; import type { MediaMissionStep, MediaMissionFailure, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { getFetchableURI } from './identifier-utils.js'; function blobToDataURI(blob: Blob): Promise { const fileReader = new FileReader(); return new Promise((resolve, reject) => { fileReader.onerror = error => { fileReader.abort(); reject(error); }; fileReader.onload = () => { invariant( typeof fileReader.result === 'string', 'FileReader.readAsDataURL should result in string', ); resolve(fileReader.result); }; fileReader.readAsDataURL(blob); }); } const base64CharsNeeded = 4 * Math.ceil(bytesNeededForFileTypeCheck / 3); function dataURIToIntArray(dataURI: string): Uint8Array { const uri = dataURI.replace(/\r?\n/g, ''); const firstComma = uri.indexOf(','); if (firstComma <= 4) { throw new TypeError('malformed data-URI'); } const meta = uri.substring(5, firstComma).split(';'); const base64Encoded = meta.some(metum => metum === 'base64'); let data = unescape(uri.substr(firstComma + 1, base64CharsNeeded)); if (base64Encoded) { data = base64.decode(data); } return stringToIntArray(data); } function stringToIntArray(str: string): Uint8Array { const array = new Uint8Array(str.length); for (let i = 0; i < str.length; i++) { array[i] = str.charCodeAt(i); } return array; } type FetchBlobResult = { success: true, base64: string, mime: string, }; async function fetchBlob(inputURI: string): Promise<{ steps: $ReadOnlyArray, result: MediaMissionFailure | FetchBlobResult, }> { const uri = getFetchableURI(inputURI); - const steps = []; + const steps: Array = []; let blob, fetchExceptionMessage; const fetchStart = Date.now(); try { const response = await fetch(uri); blob = await response.blob(); } catch (e) { fetchExceptionMessage = getMessageForException(e); } steps.push({ step: 'fetch_blob', success: !!blob, exceptionMessage: fetchExceptionMessage, time: Date.now() - fetchStart, inputURI, uri, size: blob && blob.size, mime: blob && blob.type, }); if (!blob) { return { result: { success: false, reason: 'fetch_failed' }, steps }; } let dataURI, dataURIExceptionMessage; const dataURIStart = Date.now(); try { dataURI = await blobToDataURI(blob); } catch (e) { dataURIExceptionMessage = getMessageForException(e); } steps.push({ step: 'data_uri_from_blob', success: !!dataURI, exceptionMessage: dataURIExceptionMessage, time: Date.now() - dataURIStart, first255Chars: dataURI && dataURI.substring(0, 255), }); if (!dataURI) { return { result: { success: false, reason: 'data_uri_failed' }, steps }; } const firstComma = dataURI.indexOf(','); invariant(firstComma > 4, 'malformed data-URI'); const base64String = dataURI.substring(firstComma + 1); let mime = blob.type; if (!mime) { let mimeCheckExceptionMessage; const mimeCheckStart = Date.now(); try { const intArray = dataURIToIntArray(dataURI); ({ mime } = fileInfoFromData(intArray)); } catch (e) { mimeCheckExceptionMessage = getMessageForException(e); } steps.push({ step: 'mime_check', success: !!mime, exceptionMessage: mimeCheckExceptionMessage, time: Date.now() - mimeCheckStart, mime, }); } if (!mime) { return { result: { success: false, reason: 'mime_check_failed', mime }, steps, }; } return { result: { success: true, base64: base64String, mime }, steps }; } export { stringToIntArray, fetchBlob }; diff --git a/native/media/encryption-utils.js b/native/media/encryption-utils.js index cf4d0eb49..960cf1a0c 100644 --- a/native/media/encryption-utils.js +++ b/native/media/encryption-utils.js @@ -1,447 +1,447 @@ // @flow import invariant from 'invariant'; import { uintArrayToHexString, hexToUintArray } from 'lib/media/data-utils.js'; import { replaceExtension, fileInfoFromData, filenameFromPathOrURI, readableFilename, pathFromURI, } from 'lib/media/file-utils.js'; import type { MediaMissionFailure, MediaMissionStep, EncryptFileMediaMissionStep, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { pad, unpad, calculatePaddedLength } from 'lib/utils/pkcs7-padding.js'; import { temporaryDirectoryPath } from './file-utils.js'; import { getFetchableURI } from './identifier-utils.js'; import type { MediaResult } from './media-utils.js'; import { commUtilsModule } from '../native-modules.js'; import * as AES from '../utils/aes-crypto-module.js'; import { arrayBufferFromBlob } from '../utils/blob-utils-module.js'; const PADDING_THRESHOLD = 5000000; // we don't pad files larger than this type EncryptedFileResult = { +success: true, +uri: string, +sha256Hash: string, +encryptionKey: string, }; /** * Encrypts a single file and returns the encrypted file URI * and the encryption key. The encryption key is returned as a hex string. * The encrypted file is written to the same directory as the original file, * with the same name, but with the extension ".dat". * * @param uri uri to the file to encrypt * @returns encryption result along with mission steps */ async function encryptFile(uri: string): Promise<{ steps: $ReadOnlyArray, result: MediaMissionFailure | EncryptedFileResult, }> { let success = true, exceptionMessage; const steps: EncryptFileMediaMissionStep[] = []; // prepare destination path for temporary encrypted file const originalFilename = filenameFromPathOrURI(uri); invariant(originalFilename, 'encryptFile: Invalid URI - filename is null'); const targetFilename = replaceExtension(originalFilename, 'dat'); const destinationPath = `${temporaryDirectoryPath}${targetFilename}`; const destinationURI = `file://${destinationPath}`; // Step 1. Read the file const startOpenFile = Date.now(); let data; try { const path = pathFromURI(uri); // for local paths (file:// URI) we can use native module which is faster if (path) { const buffer = await commUtilsModule.readBufferFromFile(path); data = new Uint8Array(buffer); } else { const response = await fetch(getFetchableURI(uri)); const blob = await response.blob(); const buffer = arrayBufferFromBlob(blob); data = new Uint8Array(buffer); } } catch (e) { success = false; exceptionMessage = getMessageForException(e); } steps.push({ step: 'read_plaintext_file', file: uri, time: Date.now() - startOpenFile, success, exceptionMessage, }); if (!success || !data) { return { steps, result: { success: false, reason: 'fetch_failed' }, }; } // Step 2. Encrypt the file const startEncrypt = Date.now(); const paddedLength = calculatePaddedLength(data.byteLength); const shouldPad = paddedLength <= PADDING_THRESHOLD; let key, encryptedData, sha256Hash; try { const plaintextData = shouldPad ? pad(data) : data; key = AES.generateKey(); encryptedData = AES.encrypt(key, plaintextData); sha256Hash = commUtilsModule.sha256(encryptedData.buffer); } catch (e) { success = false; exceptionMessage = getMessageForException(e); } steps.push({ step: 'encrypt_data', dataSize: encryptedData?.byteLength ?? -1, isPadded: shouldPad, time: Date.now() - startEncrypt, sha256: sha256Hash, success, exceptionMessage, }); if (encryptedData && !sha256Hash) { return { steps, result: { success: false, reason: 'digest_failed' } }; } if (!success || !encryptedData || !key || !sha256Hash) { return { steps, result: { success: false, reason: 'encryption_failed' }, }; } // Step 3. Write the encrypted file const startWriteFile = Date.now(); try { await commUtilsModule.writeBufferToFile( destinationPath, encryptedData.buffer, ); } catch (e) { success = false; exceptionMessage = getMessageForException(e); } steps.push({ step: 'write_encrypted_file', file: destinationPath, time: Date.now() - startWriteFile, success, exceptionMessage, }); if (!success) { return { steps, result: { success: false, reason: 'write_file_failed' }, }; } return { steps, result: { success: true, uri: destinationURI, encryptionKey: uintArrayToHexString(key), sha256Hash, }, }; } /** * Encrypts a single photo or video. Replaces the uploadURI with the encrypted * file URI. Attaches `encryptionKey` to the result. Changes the mediaType to * `encrypted_photo` or `encrypted_video`. * * @param preprocessedMedia - Result of `processMedia()` call * @returns a `preprocessedMedia` param, but with encryption applied */ async function encryptMedia(preprocessedMedia: MediaResult): Promise<{ result: MediaResult | MediaMissionFailure, steps: $ReadOnlyArray, }> { invariant(preprocessedMedia.success, 'encryptMedia called on failure result'); invariant( preprocessedMedia.mediaType === 'photo' || preprocessedMedia.mediaType === 'video', 'encryptMedia should only be called on unencrypted photos and videos', ); const { uploadURI } = preprocessedMedia; - const steps = []; + const steps: Array = []; // Encrypt the media file const { steps: encryptionSteps, result: encryptionResult } = await encryptFile(uploadURI); steps.push(...encryptionSteps); if (!encryptionResult.success) { return { steps, result: encryptionResult }; } if (preprocessedMedia.mediaType === 'photo') { const thumbHashResult = preprocessedMedia.thumbHash ? encryptBase64( preprocessedMedia.thumbHash, hexToUintArray(encryptionResult.encryptionKey), ) : null; return { steps, result: { ...preprocessedMedia, mediaType: 'encrypted_photo', uploadURI: encryptionResult.uri, blobHash: encryptionResult.sha256Hash, thumbHash: thumbHashResult?.base64, encryptionKey: encryptionResult.encryptionKey, shouldDisposePath: pathFromURI(encryptionResult.uri), }, }; } // For videos, we also need to encrypt the thumbnail const { steps: thumbnailEncryptionSteps, result: thumbnailEncryptionResult } = await encryptFile(preprocessedMedia.uploadThumbnailURI); steps.push(...thumbnailEncryptionSteps); if (!thumbnailEncryptionResult.success) { return { steps, result: thumbnailEncryptionResult }; } const thumbHashResult = preprocessedMedia.thumbHash ? encryptBase64( preprocessedMedia.thumbHash, hexToUintArray(thumbnailEncryptionResult.encryptionKey), ) : null; return { steps, result: { ...preprocessedMedia, mediaType: 'encrypted_video', uploadURI: encryptionResult.uri, blobHash: encryptionResult.sha256Hash, thumbHash: thumbHashResult?.base64, encryptionKey: encryptionResult.encryptionKey, uploadThumbnailURI: thumbnailEncryptionResult.uri, thumbnailBlobHash: thumbnailEncryptionResult.sha256Hash, thumbnailEncryptionKey: thumbnailEncryptionResult.encryptionKey, shouldDisposePath: pathFromURI(encryptionResult.uri), }, }; } type DecryptFileStep = | { +step: 'fetch_file', +file: string, +time: number, +success: boolean, +exceptionMessage: ?string, } | { +step: 'decrypt_data', +dataSize: number, +time: number, +isPadded: boolean, +success: boolean, +exceptionMessage: ?string, } | { +step: 'write_file', +file: string, +mimeType: string, +time: number, +success: boolean, +exceptionMessage: ?string, } | { +step: 'create_data_uri', +mimeType: string, +time: number, +success: boolean, +exceptionMessage: ?string, }; type DecryptionFailure = | MediaMissionFailure | { +success: false, +reason: | 'fetch_file_failed' | 'decrypt_data_failed' | 'write_file_failed', +exceptionMessage: ?string, }; async function decryptMedia( blobURI: string, encryptionKey: string, options: { +destination: 'file' | 'data_uri' }, ): Promise<{ steps: $ReadOnlyArray, result: DecryptionFailure | { success: true, uri: string }, }> { let success = true, exceptionMessage; const steps: DecryptFileStep[] = []; // Step 1. Fetch the file and convert it to a Uint8Array const fetchStartTime = Date.now(); let data; try { const response = await fetch(getFetchableURI(blobURI)); if (!response.ok) { throw new Error(`HTTP error ${response.status}: ${response.statusText}`); } const blob = await response.blob(); const buffer = arrayBufferFromBlob(blob); data = new Uint8Array(buffer); } catch (e) { success = false; exceptionMessage = getMessageForException(e); } steps.push({ step: 'fetch_file', file: blobURI, time: Date.now() - fetchStartTime, success, exceptionMessage, }); if (!success || !data) { return { steps, result: { success: false, reason: 'fetch_file_failed', exceptionMessage }, }; } // Step 2. Decrypt the data const decryptionStartTime = Date.now(); let plaintextData, decryptedData, isPadded; try { const key = hexToUintArray(encryptionKey); plaintextData = AES.decrypt(key, data); isPadded = plaintextData.byteLength <= PADDING_THRESHOLD; decryptedData = isPadded ? unpad(plaintextData) : plaintextData; } catch (e) { success = false; exceptionMessage = getMessageForException(e); } steps.push({ step: 'decrypt_data', dataSize: decryptedData?.byteLength ?? -1, isPadded: !!isPadded, time: Date.now() - decryptionStartTime, success, exceptionMessage, }); if (!success || !decryptedData) { return { steps, result: { success: false, reason: 'decrypt_data_failed', exceptionMessage, }, }; } // Step 3. Write the file to disk or create a data URI let uri; const writeStartTime = Date.now(); // we need extension for react-native-video to work const { mime } = fileInfoFromData(decryptedData); if (!mime) { return { steps, result: { success: false, reason: 'mime_check_failed', mime, }, }; } if (options.destination === 'file') { // blobURI is a URL, we use the last part of the path as the filename const uriSuffix = blobURI.substring(blobURI.lastIndexOf('/') + 1); const filename = readableFilename(uriSuffix, mime) || uriSuffix; const targetPath = `${temporaryDirectoryPath}${Date.now()}-${filename}`; try { await commUtilsModule.writeBufferToFile(targetPath, decryptedData.buffer); } catch (e) { success = false; exceptionMessage = getMessageForException(e); } uri = `file://${targetPath}`; steps.push({ step: 'write_file', file: uri, mimeType: mime, time: Date.now() - writeStartTime, success, exceptionMessage, }); if (!success) { return { steps, result: { success: false, reason: 'write_file_failed', exceptionMessage, }, }; } } else { const base64 = commUtilsModule.base64EncodeBuffer(decryptedData.buffer); uri = `data:${mime};base64,${base64}`; steps.push({ step: 'create_data_uri', mimeType: mime, time: Date.now() - writeStartTime, success, exceptionMessage, }); } return { steps, result: { success: true, uri }, }; } function encryptBase64( base64: string, keyBytes?: Uint8Array, ): { +base64: string, +keyHex: string } { const rawData = commUtilsModule.base64DecodeBuffer(base64); const aesKey = keyBytes ?? AES.generateKey(); const encrypted = AES.encrypt(aesKey, new Uint8Array(rawData)); return { base64: commUtilsModule.base64EncodeBuffer(encrypted.buffer), keyHex: uintArrayToHexString(aesKey), }; } function decryptBase64(encrypted: string, keyHex: string): string { const encryptedData = commUtilsModule.base64DecodeBuffer(encrypted); const decryptedData = AES.decrypt( hexToUintArray(keyHex), new Uint8Array(encryptedData), ); return commUtilsModule.base64EncodeBuffer(decryptedData.buffer); } export { encryptMedia, decryptMedia, encryptBase64, decryptBase64 }; diff --git a/native/media/file-utils.js b/native/media/file-utils.js index 681604eb9..fd120fe37 100644 --- a/native/media/file-utils.js +++ b/native/media/file-utils.js @@ -1,442 +1,442 @@ // @flow import base64 from 'base-64'; import * as ExpoFileSystem from 'expo-file-system'; import * as MediaLibrary from 'expo-media-library'; import invariant from 'invariant'; import { Platform } from 'react-native'; import filesystem from 'react-native-fs'; import { mediaConfig, pathFromURI, fileInfoFromData, bytesNeededForFileTypeCheck, } from 'lib/media/file-utils.js'; import type { Shape } from 'lib/types/core.js'; import type { MediaMissionStep, MediaMissionFailure, MediaType, ReadFileHeaderMediaMissionStep, DisposeTemporaryFileMediaMissionStep, MakeDirectoryMediaMissionStep, AndroidScanFileMediaMissionStep, FetchFileHashMediaMissionStep, CopyFileMediaMissionStep, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { stringToIntArray } from './blob-utils.js'; import { ffmpeg } from './ffmpeg.js'; const defaultInputs = Object.freeze({}); const defaultFields = Object.freeze({}); type FetchFileInfoResult = { +success: true, +uri: string, +orientation: ?number, +fileSize: number, +mime: ?string, +mediaType: ?MediaType, }; type OptionalInputs = Shape<{ +mediaNativeID: ?string }>; type OptionalFields = Shape<{ +orientation: boolean, +mediaType: boolean, +mime: boolean, }>; async function fetchFileInfo( inputURI: string, optionalInputs?: OptionalInputs = defaultInputs, optionalFields?: OptionalFields = defaultFields, ): Promise<{ steps: $ReadOnlyArray, result: MediaMissionFailure | FetchFileInfoResult, }> { const { mediaNativeID } = optionalInputs; - const steps = []; + const steps: Array = []; let assetInfoPromise, assetURI; const inputPath = pathFromURI(inputURI); if (mediaNativeID && (!inputPath || optionalFields.orientation)) { assetInfoPromise = (async () => { const { steps: assetInfoSteps, result: assetInfoResult } = await fetchAssetInfo(mediaNativeID); steps.push(...assetInfoSteps); assetURI = assetInfoResult.localURI; return assetInfoResult; })(); } const getLocalURIPromise = (async () => { if (inputPath) { return { localURI: inputURI, path: inputPath }; } if (!assetInfoPromise) { return null; } const { localURI } = await assetInfoPromise; if (!localURI) { return null; } const path = pathFromURI(localURI); if (!path) { return null; } return { localURI, path }; })(); const getOrientationPromise = (async () => { if (!optionalFields.orientation || !assetInfoPromise) { return null; } const { orientation } = await assetInfoPromise; return orientation; })(); const getFileSizePromise = (async () => { const localURIResult = await getLocalURIPromise; if (!localURIResult) { return null; } const { localURI } = localURIResult; const { steps: fileSizeSteps, result: fileSize } = await fetchFileSize( localURI, ); steps.push(...fileSizeSteps); return fileSize; })(); const getTypesPromise = (async () => { if (!optionalFields.mime && !optionalFields.mediaType) { return { mime: null, mediaType: null }; } const [localURIResult, fileSize] = await Promise.all([ getLocalURIPromise, getFileSizePromise, ]); if (!localURIResult || !fileSize) { return { mime: null, mediaType: null }; } const { localURI, path } = localURIResult; const readFileStep = await readFileHeader(localURI, fileSize); steps.push(readFileStep); const { mime, mediaType: baseMediaType } = readFileStep; if (!optionalFields.mediaType || !mime || !baseMediaType) { return { mime, mediaType: null }; } const { steps: getMediaTypeSteps, result: mediaType } = await getMediaTypeInfo(path, mime, baseMediaType); steps.push(...getMediaTypeSteps); return { mime, mediaType }; })(); const [localURIResult, orientation, fileSize, types] = await Promise.all([ getLocalURIPromise, getOrientationPromise, getFileSizePromise, getTypesPromise, ]); if (!localURIResult) { return { steps, result: { success: false, reason: 'no_file_path' } }; } const uri = localURIResult.localURI; if (!fileSize) { return { steps, result: { success: false, reason: 'file_stat_failed', uri }, }; } let finalURI = uri; // prefer asset URI, with one exception: // if the target URI is a file in our app local cache dir, we shouldn't // replace it because it was already preprocessed by either our media // processing logic or cropped by expo-image-picker const isFileInCacheDir = uri.includes(temporaryDirectoryPath) || uri.includes(ExpoFileSystem.cacheDirectory); if (assetURI && assetURI !== uri && !isFileInCacheDir) { finalURI = assetURI; console.log( 'fetchAssetInfo returned localURI ' + `${assetURI} when we already had ${uri}`, ); } return { steps, result: { success: true, uri: finalURI, orientation, fileSize, mime: types.mime, mediaType: types.mediaType, }, }; } async function fetchAssetInfo(mediaNativeID: string): Promise<{ steps: $ReadOnlyArray, result: { localURI: ?string, orientation: ?number }, }> { let localURI, orientation, success = false, exceptionMessage; const start = Date.now(); try { const assetInfo = await MediaLibrary.getAssetInfoAsync(mediaNativeID); success = true; localURI = assetInfo.localUri; if (Platform.OS === 'ios') { orientation = assetInfo.orientation; } else { orientation = assetInfo.exif && assetInfo.exif.Orientation; } } catch (e) { exceptionMessage = getMessageForException(e); } return { steps: [ { step: 'asset_info_fetch', success, exceptionMessage, time: Date.now() - start, localURI, orientation, }, ], result: { localURI, orientation, }, }; } async function fetchFileSize(uri: string): Promise<{ steps: $ReadOnlyArray, result: ?number, }> { let fileSize, success = false, exceptionMessage; const statStart = Date.now(); try { const result = await filesystem.stat(uri); success = true; fileSize = result.size; } catch (e) { exceptionMessage = getMessageForException(e); } return { steps: [ { step: 'stat_file', success, exceptionMessage, time: Date.now() - statStart, uri, fileSize, }, ], result: fileSize, }; } async function readFileHeader( localURI: string, fileSize: number, ): Promise { const fetchBytes = Math.min(fileSize, bytesNeededForFileTypeCheck); const start = Date.now(); let fileData, success = false, exceptionMessage; try { fileData = await filesystem.read(localURI, fetchBytes, 0, 'base64'); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } let mime, mediaType; if (fileData) { const utf8 = base64.decode(fileData); const intArray = stringToIntArray(utf8); ({ mime, mediaType } = fileInfoFromData(intArray)); } return { step: 'read_file_header', success, exceptionMessage, time: Date.now() - start, uri: localURI, mime, mediaType, }; } async function getMediaTypeInfo( path: string, mime: string, baseMediaType: MediaType, ): Promise<{ steps: $ReadOnlyArray, result: ?MediaType, }> { if (!mediaConfig[mime] || mediaConfig[mime].mediaType !== 'photo_or_video') { return { steps: [], result: baseMediaType }; } let hasMultipleFrames, success = false, exceptionMessage; const start = Date.now(); try { hasMultipleFrames = await ffmpeg.hasMultipleFrames(path); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } const steps = [ { step: 'frame_count', success, exceptionMessage, time: Date.now() - start, path, mime, hasMultipleFrames, }, ]; const result = hasMultipleFrames ? 'video' : 'photo'; return { steps, result }; } async function disposeTempFile( path: string, ): Promise { let success = false, exceptionMessage; const start = Date.now(); try { await filesystem.unlink(path); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'dispose_temporary_file', success, exceptionMessage, time: Date.now() - start, path, }; } async function mkdir(path: string): Promise { let success = false, exceptionMessage; const start = Date.now(); try { await filesystem.mkdir(path); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'make_directory', success, exceptionMessage, time: Date.now() - start, path, }; } async function androidScanFile( path: string, ): Promise { invariant(Platform.OS === 'android', 'androidScanFile only works on Android'); let success = false, exceptionMessage; const start = Date.now(); try { await filesystem.scanFile(path); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'android_scan_file', success, exceptionMessage, time: Date.now() - start, path, }; } async function fetchFileHash( path: string, ): Promise { let hash, exceptionMessage; const start = Date.now(); try { hash = await filesystem.hash(path, 'md5'); } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'fetch_file_hash', success: !!hash, exceptionMessage, time: Date.now() - start, path, hash, }; } async function copyFile( source: string, destination: string, ): Promise { let success = false, exceptionMessage; const start = Date.now(); try { await filesystem.copyFile(source, destination); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'copy_file', success, exceptionMessage, time: Date.now() - start, source, destination, }; } const temporaryDirectoryPath: string = Platform.select({ ios: filesystem.TemporaryDirectoryPath, default: `${filesystem.TemporaryDirectoryPath}/`, }); export { fetchAssetInfo, fetchFileInfo, temporaryDirectoryPath, disposeTempFile, mkdir, androidScanFile, fetchFileHash, copyFile, }; diff --git a/native/media/image-utils.js b/native/media/image-utils.js index c1d834d15..e8e01d82c 100644 --- a/native/media/image-utils.js +++ b/native/media/image-utils.js @@ -1,118 +1,118 @@ // @flow import * as ImageManipulator from 'expo-image-manipulator'; import { getImageProcessingPlan } from 'lib/media/image-utils.js'; import type { Dimensions, MediaMissionStep, MediaMissionFailure, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { generateThumbhashStep } from './media-utils.js'; type ProcessImageInfo = { uri: string, dimensions: Dimensions, mime: string, fileSize: number, orientation: ?number, }; type ProcessImageResponse = { success: true, uri: string, mime: string, dimensions: Dimensions, thumbHash: ?string, }; async function processImage(input: ProcessImageInfo): Promise<{ steps: $ReadOnlyArray, result: MediaMissionFailure | ProcessImageResponse, }> { - const steps = []; + const steps: Array = []; let { uri, dimensions, mime } = input; const { fileSize, orientation } = input; const plan = getImageProcessingPlan({ inputMIME: mime, inputDimensions: dimensions, inputFileSize: fileSize, inputOrientation: orientation, }); if (plan.action === 'none') { const thumbhashStep = await generateThumbhashStep(uri); steps.push(thumbhashStep); const { thumbHash } = thumbhashStep; return { steps, result: { success: true, uri, dimensions, mime, thumbHash }, }; } const { targetMIME, compressionRatio, fitInside } = plan; const transforms = []; if (fitInside) { const fitInsideRatio = fitInside.width / fitInside.height; if (dimensions.width / dimensions.height > fitInsideRatio) { transforms.push({ resize: { width: fitInside.width } }); } else { transforms.push({ resize: { height: fitInside.height } }); } } const format = targetMIME === 'image/png' ? ImageManipulator.SaveFormat.PNG : ImageManipulator.SaveFormat.JPEG; const saveConfig = { format, compress: compressionRatio }; let success = false, exceptionMessage; const start = Date.now(); try { const result = await ImageManipulator.manipulateAsync( uri, transforms, saveConfig, ); success = true; uri = result.uri; mime = targetMIME; dimensions = { width: result.width, height: result.height }; } catch (e) { exceptionMessage = getMessageForException(e); } steps.push({ step: 'photo_manipulation', manipulation: { transforms, saveConfig }, success, exceptionMessage, time: Date.now() - start, newMIME: success ? mime : null, newDimensions: success ? dimensions : null, newURI: success ? uri : null, }); if (!success) { return { steps, result: { success: false, reason: 'photo_manipulation_failed', size: fileSize, }, }; } const thumbhashStep = await generateThumbhashStep(uri); steps.push(thumbhashStep); const { thumbHash } = thumbhashStep; return { steps, result: { success: true, uri, dimensions, mime, thumbHash }, }; } export { processImage }; diff --git a/native/media/media-utils.js b/native/media/media-utils.js index 621ff9d14..22156bd33 100644 --- a/native/media/media-utils.js +++ b/native/media/media-utils.js @@ -1,293 +1,293 @@ // @flow import invariant from 'invariant'; import { Image } from 'react-native'; import { pathFromURI, sanitizeFilename } from 'lib/media/file-utils.js'; import type { Dimensions, MediaMissionStep, MediaMissionFailure, NativeMediaSelection, GenerateThumbhashMediaMissionStep, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { fetchFileInfo } from './file-utils.js'; import { processImage } from './image-utils.js'; import { saveMedia } from './save-media.js'; import { processVideo } from './video-utils.js'; import { generateThumbHash } from '../utils/thumbhash-module.js'; type MediaProcessConfig = { +hasWiFi: boolean, // Blocks return until we can confirm result has the correct MIME +finalFileHeaderCheck?: boolean, +onTranscodingProgress?: (percent: number) => void, }; type SharedMediaResult = { +success: true, +uploadURI: string, +shouldDisposePath: ?string, +filename: string, +mime: string, +dimensions: Dimensions, +thumbHash: ?string, }; export type MediaResult = | { +mediaType: 'photo', ...SharedMediaResult } | { +mediaType: 'video', ...SharedMediaResult, +uploadThumbnailURI: string, +loop: boolean, } | { +mediaType: 'encrypted_photo', ...SharedMediaResult, +blobHash: string, +encryptionKey: string, } | { +mediaType: 'encrypted_video', ...SharedMediaResult, +blobHash: string, +encryptionKey: string, +thumbnailBlobHash: string, +thumbnailEncryptionKey: string, +uploadThumbnailURI: string, +loop: boolean, }; function processMedia( selection: NativeMediaSelection, config: MediaProcessConfig, ): { resultPromise: Promise, reportPromise: Promise<$ReadOnlyArray>, } { let resolveResult; const sendResult = result => { if (resolveResult) { resolveResult(result); } }; const reportPromise = innerProcessMedia(selection, config, sendResult); const resultPromise = new Promise(resolve => { resolveResult = resolve; }); return { reportPromise, resultPromise }; } async function innerProcessMedia( selection: NativeMediaSelection, config: MediaProcessConfig, sendResult: (result: MediaMissionFailure | MediaResult) => void, ): Promise<$ReadOnlyArray> { let initialURI = null, uploadURI = null, uploadThumbnailURI = null, dimensions = selection.dimensions, mediaType = null, mime = null, loop = false, resultReturned = false, thumbHash = null; const returnResult = (failure?: MediaMissionFailure) => { invariant( !resultReturned, 'returnResult called twice in innerProcessMedia', ); resultReturned = true; if (failure) { sendResult(failure); return; } invariant( uploadURI && mime && mediaType, 'missing required fields in returnResult', ); const shouldDisposePath = initialURI !== uploadURI ? pathFromURI(uploadURI) : null; const filename = sanitizeFilename(selection.filename, mime); if (mediaType === 'video') { invariant(uploadThumbnailURI, 'video should have uploadThumbnailURI'); sendResult({ success: true, uploadURI, uploadThumbnailURI, shouldDisposePath, filename, mime, mediaType, dimensions, loop, thumbHash, }); } else { sendResult({ success: true, uploadURI, shouldDisposePath, filename, mime, mediaType, dimensions, thumbHash, }); } }; - const steps = [], + const steps: Array = [], completeBeforeFinish = []; const finish = async (failure?: MediaMissionFailure) => { if (!resultReturned) { returnResult(failure); } await Promise.all(completeBeforeFinish); return steps; }; if (selection.captureTime && selection.retries === 0) { const { uri } = selection; invariant( pathFromURI(uri), `captured URI ${uri} should use file:// scheme`, ); completeBeforeFinish.push( (async () => { const { reportPromise } = saveMedia(uri); const saveMediaSteps = await reportPromise; steps.push(...saveMediaSteps); })(), ); } const possiblyPhoto = selection.step.startsWith('photo_'); const mediaNativeID = selection.mediaNativeID ? selection.mediaNativeID : null; const { steps: fileInfoSteps, result: fileInfoResult } = await fetchFileInfo( selection.uri, { mediaNativeID }, { orientation: possiblyPhoto, mime: true, mediaType: true, }, ); steps.push(...fileInfoSteps); if (!fileInfoResult.success) { return await finish(fileInfoResult); } const { orientation, fileSize } = fileInfoResult; ({ uri: initialURI, mime, mediaType } = fileInfoResult); if (!mime || !mediaType) { return await finish({ success: false, reason: 'media_type_fetch_failed', detectedMIME: mime, }); } if (mediaType === 'video') { const { steps: videoSteps, result: videoResult } = await processVideo( { uri: initialURI, mime, filename: selection.filename, fileSize, dimensions, hasWiFi: config.hasWiFi, }, { onTranscodingProgress: config.onTranscodingProgress, }, ); steps.push(...videoSteps); if (!videoResult.success) { return await finish(videoResult); } ({ uri: uploadURI, thumbnailURI: uploadThumbnailURI, mime, dimensions, loop, thumbHash, } = videoResult); } else if (mediaType === 'photo') { const { steps: imageSteps, result: imageResult } = await processImage({ uri: initialURI, dimensions, mime, fileSize, orientation, }); steps.push(...imageSteps); if (!imageResult.success) { return await finish(imageResult); } ({ uri: uploadURI, mime, dimensions, thumbHash } = imageResult); } else { invariant(false, `unknown mediaType ${mediaType}`); } if (uploadURI === initialURI) { return await finish(); } if (!config.finalFileHeaderCheck) { returnResult(); } const { steps: finalFileInfoSteps, result: finalFileInfoResult } = await fetchFileInfo(uploadURI, undefined, { mime: true }); steps.push(...finalFileInfoSteps); if (!finalFileInfoResult.success) { return await finish(finalFileInfoResult); } if (finalFileInfoResult.mime && finalFileInfoResult.mime !== mime) { return await finish({ success: false, reason: 'mime_type_mismatch', reportedMediaType: mediaType, reportedMIME: mime, detectedMIME: finalFileInfoResult.mime, }); } return await finish(); } function getDimensions(uri: string): Promise { return new Promise((resolve, reject) => { Image.getSize( uri, (width: number, height: number) => resolve({ height, width }), reject, ); }); } async function generateThumbhashStep( uri: string, ): Promise { let thumbHash, exceptionMessage; try { thumbHash = await generateThumbHash(uri); } catch (err) { exceptionMessage = getMessageForException(err); } return { step: 'generate_thumbhash', success: !!thumbHash && !exceptionMessage, exceptionMessage, thumbHash, }; } export { processMedia, getDimensions, generateThumbhashStep }; diff --git a/native/media/save-media.js b/native/media/save-media.js index 1f97c05c9..455174e94 100644 --- a/native/media/save-media.js +++ b/native/media/save-media.js @@ -1,467 +1,469 @@ // @flow import * as MediaLibrary from 'expo-media-library'; import invariant from 'invariant'; import * as React from 'react'; import { Platform, PermissionsAndroid } from 'react-native'; import filesystem from 'react-native-fs'; import { useDispatch } from 'react-redux'; import { queueReportsActionType } from 'lib/actions/report-actions.js'; import { readableFilename, pathFromURI } from 'lib/media/file-utils.js'; import { isLocalUploadID } from 'lib/media/media-utils.js'; import type { MediaMissionStep, MediaMissionResult, MediaMissionFailure, } from 'lib/types/media-types.js'; import { reportTypes, type ClientMediaMissionReportCreationRequest, } from 'lib/types/report-types.js'; import { getConfig } from 'lib/utils/config.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { promiseAll } from 'lib/utils/promises.js'; import { generateReportID, useIsReportEnabled, } from 'lib/utils/report-utils.js'; import { fetchBlob } from './blob-utils.js'; import { fetchAssetInfo, fetchFileInfo, disposeTempFile, mkdir, androidScanFile, fetchFileHash, copyFile, temporaryDirectoryPath, } from './file-utils.js'; import { getMediaLibraryIdentifier } from './identifier-utils.js'; import { displayActionResultModal } from '../navigation/action-result-modal.js'; import { requestAndroidPermission } from '../utils/android-permissions.js'; export type IntentionalSaveMedia = ( uri: string, ids: { uploadID: string, messageServerID: ?string, messageLocalID: ?string, }, ) => Promise; function useIntentionalSaveMedia(): IntentionalSaveMedia { const dispatch = useDispatch(); const mediaReportsEnabled = useIsReportEnabled('mediaReports'); return React.useCallback( async ( uri: string, ids: { uploadID: string, messageServerID: ?string, messageLocalID: ?string, }, ) => { const start = Date.now(); - const steps = [{ step: 'save_media', uri, time: start }]; + const steps: Array = [ + { step: 'save_media', uri, time: start }, + ]; const { resultPromise, reportPromise } = saveMedia(uri, 'request'); const result = await resultPromise; const userTime = Date.now() - start; let message; if (result.success) { message = 'saved!'; } else if (result.reason === 'save_unsupported') { const os = Platform.select({ ios: 'iOS', android: 'Android', default: Platform.OS, }); message = `saving media is unsupported on ${os}`; } else if (result.reason === 'missing_permission') { message = 'don’t have permission :('; } else if ( result.reason === 'resolve_failed' || result.reason === 'data_uri_failed' ) { message = 'failed to resolve :('; } else if (result.reason === 'fetch_failed') { message = 'failed to download :('; } else { message = 'failed to save :('; } displayActionResultModal(message); if (!mediaReportsEnabled) { return; } const reportSteps = await reportPromise; steps.push(...reportSteps); const totalTime = Date.now() - start; const mediaMission = { steps, result, userTime, totalTime }; const { uploadID, messageServerID, messageLocalID } = ids; const uploadIDIsLocal = isLocalUploadID(uploadID); const report: ClientMediaMissionReportCreationRequest = { type: reportTypes.MEDIA_MISSION, time: Date.now(), platformDetails: getConfig().platformDetails, mediaMission, uploadServerID: uploadIDIsLocal ? undefined : uploadID, uploadLocalID: uploadIDIsLocal ? uploadID : undefined, messageServerID, messageLocalID, id: generateReportID(), }; dispatch({ type: queueReportsActionType, payload: { reports: [report] }, }); }, [dispatch, mediaReportsEnabled], ); } type Permissions = 'check' | 'request'; function saveMedia( uri: string, permissions?: Permissions = 'check', ): { resultPromise: Promise, reportPromise: Promise<$ReadOnlyArray>, } { let resolveResult; const sendResult = result => { if (resolveResult) { resolveResult(result); } }; const reportPromise = innerSaveMedia(uri, permissions, sendResult); const resultPromise = new Promise(resolve => { resolveResult = resolve; }); return { reportPromise, resultPromise }; } async function innerSaveMedia( uri: string, permissions: Permissions, sendResult: (result: MediaMissionResult) => void, ): Promise<$ReadOnlyArray> { if (Platform.OS === 'android') { return await saveMediaAndroid(uri, permissions, sendResult); } else if (Platform.OS === 'ios') { return await saveMediaIOS(uri, sendResult); } else { sendResult({ success: false, reason: 'save_unsupported' }); return []; } } const androidSavePermission = PermissionsAndroid.PERMISSIONS.WRITE_EXTERNAL_STORAGE; // On Android, we save the media to our own Comm folder in the // Pictures directory, and then trigger the media scanner to pick it up async function saveMediaAndroid( inputURI: string, permissions: Permissions, sendResult: (result: MediaMissionResult) => void, ): Promise<$ReadOnlyArray> { - const steps = []; + const steps: Array = []; let hasPermission = false, permissionCheckExceptionMessage; const permissionCheckStart = Date.now(); try { hasPermission = await requestAndroidPermission( androidSavePermission, 'throw', ); } catch (e) { permissionCheckExceptionMessage = getMessageForException(e); } steps.push({ step: 'permissions_check', success: hasPermission, exceptionMessage: permissionCheckExceptionMessage, time: Date.now() - permissionCheckStart, platform: Platform.OS, permissions: [androidSavePermission], }); if (!hasPermission) { sendResult({ success: false, reason: 'missing_permission' }); return steps; } const promises = []; let success = true; const saveFolder = `${filesystem.PicturesDirectoryPath}/Comm/`; promises.push( (async () => { const makeDirectoryStep = await mkdir(saveFolder); if (!makeDirectoryStep.success) { success = false; sendResult({ success, reason: 'make_directory_failed' }); } steps.push(makeDirectoryStep); })(), ); let uri = inputURI; let tempFile, mime; if (uri.startsWith('http')) { promises.push( (async () => { const { result: tempSaveResult, steps: tempSaveSteps } = await saveRemoteMediaToDisk(uri, temporaryDirectoryPath); steps.push(...tempSaveSteps); if (!tempSaveResult.success) { success = false; sendResult(tempSaveResult); } else { tempFile = tempSaveResult.path; uri = `file://${tempFile}`; mime = tempSaveResult.mime; } })(), ); } await Promise.all(promises); if (!success) { return steps; } const { result: copyResult, steps: copySteps } = await copyToSortedDirectory( uri, saveFolder, mime, ); steps.push(...copySteps); if (!copyResult.success) { sendResult(copyResult); return steps; } sendResult({ success: true }); const postResultPromises = []; postResultPromises.push( (async () => { const scanFileStep = await androidScanFile(copyResult.path); steps.push(scanFileStep); })(), ); if (tempFile) { postResultPromises.push( (async (file: string) => { const disposeStep = await disposeTempFile(file); steps.push(disposeStep); })(tempFile), ); } await Promise.all(postResultPromises); return steps; } // On iOS, we save the media to the camera roll async function saveMediaIOS( inputURI: string, sendResult: (result: MediaMissionResult) => void, ): Promise<$ReadOnlyArray> { - const steps = []; + const steps: Array = []; let uri = inputURI; let tempFile; if (uri.startsWith('http')) { const { result: tempSaveResult, steps: tempSaveSteps } = await saveRemoteMediaToDisk(uri, temporaryDirectoryPath); steps.push(...tempSaveSteps); if (!tempSaveResult.success) { sendResult(tempSaveResult); return steps; } tempFile = tempSaveResult.path; uri = `file://${tempFile}`; } else if (!uri.startsWith('file://')) { const mediaNativeID = getMediaLibraryIdentifier(uri); if (mediaNativeID) { const { result: fetchAssetInfoResult, steps: fetchAssetInfoSteps } = await fetchAssetInfo(mediaNativeID); steps.push(...fetchAssetInfoSteps); const { localURI } = fetchAssetInfoResult; if (localURI) { uri = localURI; } } } if (!uri.startsWith('file://')) { sendResult({ success: false, reason: 'resolve_failed', uri }); return steps; } let success = false, exceptionMessage; const start = Date.now(); try { await MediaLibrary.saveToLibraryAsync(uri); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } steps.push({ step: 'ios_save_to_library', success, exceptionMessage, time: Date.now() - start, uri, }); if (success) { sendResult({ success: true }); } else { sendResult({ success: false, reason: 'save_to_library_failed', uri }); } if (tempFile) { const disposeStep = await disposeTempFile(tempFile); steps.push(disposeStep); } return steps; } type IntermediateSaveResult = { result: { success: true, path: string, mime: string } | MediaMissionFailure, steps: $ReadOnlyArray, }; async function saveRemoteMediaToDisk( inputURI: string, directory: string, // should end with a / ): Promise { - const steps = []; + const steps: Array = []; const { result: fetchBlobResult, steps: fetchBlobSteps } = await fetchBlob( inputURI, ); steps.push(...fetchBlobSteps); if (!fetchBlobResult.success) { return { result: fetchBlobResult, steps }; } const { mime, base64 } = fetchBlobResult; const tempName = readableFilename('', mime); if (!tempName) { return { result: { success: false, reason: 'mime_check_failed', mime }, steps, }; } const tempPath = `${directory}tempsave.${tempName}`; const start = Date.now(); let success = false, exceptionMessage; try { await filesystem.writeFile(tempPath, base64, 'base64'); success = true; } catch (e) { exceptionMessage = getMessageForException(e); } steps.push({ step: 'write_file', success, exceptionMessage, time: Date.now() - start, path: tempPath, length: base64.length, }); if (!success) { return { result: { success: false, reason: 'write_file_failed' }, steps }; } return { result: { success: true, path: tempPath, mime }, steps }; } async function copyToSortedDirectory( localURI: string, directory: string, // should end with a / inputMIME: ?string, ): Promise { - const steps = []; + const steps: Array = []; const path = pathFromURI(localURI); if (!path) { return { result: { success: false, reason: 'resolve_failed', uri: localURI }, steps, }; } let mime = inputMIME; const promises = {}; promises.hashStep = fetchFileHash(path); if (!mime) { promises.fileInfoResult = fetchFileInfo(localURI, undefined, { mime: true, }); } const { hashStep, fileInfoResult } = await promiseAll(promises); steps.push(hashStep); if (!hashStep.success) { return { result: { success: false, reason: 'fetch_file_hash_failed' }, steps, }; } const { hash } = hashStep; invariant(hash, 'hash should be truthy if hashStep.success is truthy'); if (fileInfoResult) { steps.push(...fileInfoResult.steps); if (fileInfoResult.result.success && fileInfoResult.result.mime) { ({ mime } = fileInfoResult.result); } } if (!mime) { return { result: { success: false, reason: 'mime_check_failed', mime }, steps, }; } const name = readableFilename(hash, mime); if (!name) { return { result: { success: false, reason: 'mime_check_failed', mime }, steps, }; } const newPath = `${directory}${name}`; const copyStep = await copyFile(path, newPath); steps.push(copyStep); if (!copyStep.success) { return { result: { success: false, reason: 'copy_file_failed' }, steps, }; } return { result: { success: true, path: newPath, mime }, steps, }; } export { useIntentionalSaveMedia, saveMedia }; diff --git a/native/media/video-utils.js b/native/media/video-utils.js index 249bb1b45..88e7c54cb 100644 --- a/native/media/video-utils.js +++ b/native/media/video-utils.js @@ -1,294 +1,294 @@ // @flow import invariant from 'invariant'; import { Platform } from 'react-native'; import filesystem from 'react-native-fs'; import { mediaConfig, pathFromURI } from 'lib/media/file-utils.js'; import { getVideoProcessingPlan } from 'lib/media/video-utils.js'; import type { ProcessPlan } from 'lib/media/video-utils.js'; import type { MediaMissionStep, MediaMissionFailure, VideoProbeMediaMissionStep, TranscodeVideoMediaMissionStep, VideoGenerateThumbnailMediaMissionStep, Dimensions, } from 'lib/types/media-types.js'; import { getMessageForException } from 'lib/utils/errors.js'; import { ffmpeg } from './ffmpeg.js'; import { temporaryDirectoryPath } from './file-utils.js'; import { generateThumbhashStep } from './media-utils.js'; // These are some numbers I sorta kinda made up // We should try to calculate them on a per-device basis const uploadSpeeds = Object.freeze({ wifi: 4096, // in KiB/s cellular: 512, // in KiB/s }); const clientTranscodeSpeed = 1.15; // in seconds of video transcoded per second type ProcessVideoInfo = { +uri: string, +mime: string, +filename: ?string, +fileSize: number, +dimensions: Dimensions, +hasWiFi: boolean, }; type VideoProcessConfig = { +onTranscodingProgress?: (percent: number) => void, }; type ProcessVideoResponse = { +success: true, +uri: string, +thumbnailURI: string, +mime: string, +dimensions: Dimensions, +loop: boolean, +thumbHash: ?string, }; async function processVideo( input: ProcessVideoInfo, config: VideoProcessConfig, ): Promise<{ steps: $ReadOnlyArray, result: MediaMissionFailure | ProcessVideoResponse, }> { - const steps = []; + const steps: Array = []; const path = pathFromURI(input.uri); invariant(path, `could not extract path from ${input.uri}`); const initialCheckStep = await checkVideoInfo(path); steps.push(initialCheckStep); if (!initialCheckStep.success || !initialCheckStep.duration) { return { steps, result: { success: false, reason: 'video_probe_failed' } }; } const { validFormat, duration } = initialCheckStep; const plan = getVideoProcessingPlan({ inputPath: path, inputHasCorrectContainerAndCodec: validFormat, inputFileSize: input.fileSize, inputFilename: input.filename, inputMimeType: input.mime, inputDuration: duration, inputDimensions: input.dimensions, outputDirectory: temporaryDirectoryPath, // We want ffmpeg to use hardware-accelerated encoders. On iOS we can do // this using VideoToolbox, but ffmpeg on Android is still missing // MediaCodec encoding support: https://trac.ffmpeg.org/ticket/6407 outputCodec: Platform.select({ ios: 'h264_videotoolbox', //android: 'h264_mediacodec', default: 'h264', }), clientConnectionInfo: { hasWiFi: input.hasWiFi, speed: input.hasWiFi ? uploadSpeeds.wifi : uploadSpeeds.cellular, }, clientTranscodeSpeed, }); if (plan.action === 'reject') { return { steps, result: plan.failure }; } if (plan.action === 'none') { const thumbnailStep = await generateThumbnail(path, plan.thumbnailPath); steps.push(thumbnailStep); if (!thumbnailStep.success) { unlink(plan.thumbnailPath); return { steps, result: { success: false, reason: 'video_generate_thumbnail_failed' }, }; } const thumbnailURI = `file://${plan.thumbnailPath}`; const thumbhashStep = await generateThumbhashStep(thumbnailURI); steps.push(thumbhashStep); const { thumbHash } = thumbhashStep; return { steps, result: { success: true, uri: input.uri, thumbnailURI, thumbHash, mime: 'video/mp4', dimensions: input.dimensions, loop: false, }, }; } const [thumbnailStep, transcodeStep] = await Promise.all([ generateThumbnail(path, plan.thumbnailPath), transcodeVideo(plan, duration, config.onTranscodingProgress), ]); steps.push(thumbnailStep, transcodeStep); if (!thumbnailStep.success) { unlink(plan.outputPath); unlink(plan.thumbnailPath); return { steps, result: { success: false, reason: 'video_generate_thumbnail_failed', }, }; } if (!transcodeStep.success) { unlink(plan.outputPath); unlink(plan.thumbnailPath); return { steps, result: { success: false, reason: 'video_transcode_failed', }, }; } const transcodeProbeStep = await checkVideoInfo(plan.outputPath); steps.push(transcodeProbeStep); if (!transcodeProbeStep.validFormat) { unlink(plan.outputPath); unlink(plan.thumbnailPath); return { steps, result: { success: false, reason: 'video_transcode_failed' }, }; } const dimensions = transcodeProbeStep.dimensions ? transcodeProbeStep.dimensions : input.dimensions; const loop = !!( mediaConfig[input.mime] && mediaConfig[input.mime].videoConfig && mediaConfig[input.mime].videoConfig.loop ); const thumbnailURI = `file://${plan.thumbnailPath}`; const thumbhashStep = await generateThumbhashStep(thumbnailURI); steps.push(thumbhashStep); const { thumbHash } = thumbhashStep; return { steps, result: { success: true, uri: `file://${plan.outputPath}`, thumbnailURI, thumbHash, mime: 'video/mp4', dimensions, loop, }, }; } async function generateThumbnail( path: string, thumbnailPath: string, ): Promise { const thumbnailStart = Date.now(); const thumbnailReturnCode = await ffmpeg.generateThumbnail( path, thumbnailPath, ); const thumbnailGenerationSuccessful = thumbnailReturnCode === 0; return { step: 'video_generate_thumbnail', success: thumbnailGenerationSuccessful, time: Date.now() - thumbnailStart, returnCode: thumbnailReturnCode, thumbnailURI: thumbnailPath, }; } async function transcodeVideo( plan: ProcessPlan, duration: number, onProgressCallback?: number => void, ): Promise { const transcodeStart = Date.now(); let returnCode, newPath, stats, success = false, exceptionMessage; try { const { rc, lastStats } = await ffmpeg.transcodeVideo( plan.ffmpegCommand, duration, onProgressCallback, ); success = rc === 0; if (success) { returnCode = rc; newPath = plan.outputPath; stats = lastStats; } } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'video_ffmpeg_transcode', success, exceptionMessage, time: Date.now() - transcodeStart, returnCode, newPath, stats, }; } async function checkVideoInfo( path: string, ): Promise { let codec, format, dimensions, duration, success = false, validFormat = false, exceptionMessage; const start = Date.now(); try { ({ codec, format, dimensions, duration } = await ffmpeg.getVideoInfo(path)); success = true; validFormat = codec === 'h264' && format.includes('mp4'); } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'video_probe', success, exceptionMessage, time: Date.now() - start, path, validFormat, duration, codec, format, dimensions, }; } async function unlink(path: string) { try { await filesystem.unlink(path); } catch {} } function formatDuration(seconds: number): string { const mm = Math.floor(seconds / 60); const ss = (seconds % 60).toFixed(0).padStart(2, '0'); return `${mm}:${ss}`; } export { processVideo, formatDuration };