diff --git a/lib/types/media-types.js b/lib/types/media-types.js index 867ee7533..3d870c358 100644 --- a/lib/types/media-types.js +++ b/lib/types/media-types.js @@ -1,511 +1,513 @@ // @flow import type { Shape } from './core'; import { type Platform } from './device-types'; export type Dimensions = $ReadOnly<{| +height: number, +width: number, |}>; export type MediaType = 'photo' | 'video'; export type Image = {| +id: string, +uri: string, +type: 'photo', +dimensions: Dimensions, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, |}; export type Video = {| +id: string, +uri: string, +type: 'video', +dimensions: Dimensions, +loop?: boolean, + +thumbnailID: string, + +thumbnailURI: string, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, |}; export type Media = Image | Video; export type Corners = Shape<{| +topLeft: boolean, +topRight: boolean, +bottomLeft: boolean, +bottomRight: boolean, |}>; export type MediaInfo = | {| ...Image, +corners: Corners, +index: number, |} | {| ...Video, +corners: Corners, +index: number, |}; export type UploadMultimediaResult = {| +id: string, +uri: string, +dimensions: Dimensions, +mediaType: MediaType, +loop: boolean, |}; export type UpdateMultimediaMessageMediaPayload = {| +messageID: string, +currentMediaID: string, +mediaUpdate: Shape, |}; export type UploadDeletionRequest = {| +id: string, |}; export type FFmpegStatistics = {| // seconds of video being processed per second +speed: number, // total milliseconds of video processed so far +time: number, // total result file size in bytes so far +size: number, +videoQuality: number, +videoFrameNumber: number, +videoFps: number, +bitrate: number, |}; export type TranscodeVideoMediaMissionStep = {| +step: 'video_ffmpeg_transcode', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +returnCode: ?number, +newPath: ?string, +stats: ?FFmpegStatistics, |}; export type VideoGenerateThumbnailMediaMissionStep = {| +step: 'video_generate_thumbnail', +success: boolean, +time: number, // ms +returnCode: number, +thumbnailURI: string, |}; export type VideoProbeMediaMissionStep = {| +step: 'video_probe', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +validFormat: boolean, +duration: ?number, // seconds +codec: ?string, +format: ?$ReadOnlyArray, +dimensions: ?Dimensions, |}; export type ReadFileHeaderMediaMissionStep = {| +step: 'read_file_header', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +mime: ?string, +mediaType: ?MediaType, |}; export type DetermineFileTypeMediaMissionStep = {| +step: 'determine_file_type', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputFilename: string, +outputMIME: ?string, +outputMediaType: ?MediaType, +outputFilename: ?string, |}; export type FrameCountMediaMissionStep = {| +step: 'frame_count', +success: boolean, +exceptionMessage: ?string, +time: number, +path: string, +mime: string, +hasMultipleFrames: ?boolean, |}; export type DisposeTemporaryFileMediaMissionStep = {| +step: 'dispose_temporary_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type MakeDirectoryMediaMissionStep = {| +step: 'make_directory', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type AndroidScanFileMediaMissionStep = {| +step: 'android_scan_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type FetchFileHashMediaMissionStep = {| +step: 'fetch_file_hash', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +hash: ?string, |}; export type CopyFileMediaMissionStep = {| +step: 'copy_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +source: string, +destination: string, |}; export type GetOrientationMediaMissionStep = {| +step: 'exif_fetch', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +orientation: ?number, |}; export type MediaLibrarySelection = | {| +step: 'photo_library', +dimensions: Dimensions, +filename: string, +uri: string, +mediaNativeID: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |} | {| +step: 'video_library', +dimensions: Dimensions, +filename: string, +uri: string, +mediaNativeID: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, +duration: number, // seconds |}; export type PhotoCapture = {| +step: 'photo_capture', +time: number, // ms +dimensions: Dimensions, +filename: string, +uri: string, +captureTime: number, // ms timestamp +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |}; export type PhotoPaste = {| +step: 'photo_paste', +dimensions: Dimensions, +filename: string, +uri: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |}; export type NativeMediaSelection = | MediaLibrarySelection | PhotoCapture | PhotoPaste; export type MediaMissionStep = | NativeMediaSelection | {| +step: 'web_selection', +filename: string, +size: number, // in bytes +mime: string, +selectTime: number, // ms timestamp |} | {| +step: 'asset_info_fetch', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +localURI: ?string, +orientation: ?number, |} | {| +step: 'stat_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +fileSize: ?number, |} | ReadFileHeaderMediaMissionStep | DetermineFileTypeMediaMissionStep | FrameCountMediaMissionStep | {| +step: 'photo_manipulation', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +manipulation: Object, +newMIME: ?string, +newDimensions: ?Dimensions, +newURI: ?string, |} | VideoProbeMediaMissionStep | TranscodeVideoMediaMissionStep | VideoGenerateThumbnailMediaMissionStep | DisposeTemporaryFileMediaMissionStep | {| +step: 'save_media', +uri: string, +time: number, // ms timestamp |} | {| +step: 'permissions_check', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +platform: Platform, +permissions: $ReadOnlyArray, |} | MakeDirectoryMediaMissionStep | AndroidScanFileMediaMissionStep | {| +step: 'ios_save_to_library', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, |} | {| +step: 'fetch_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputURI: string, +uri: string, +size: ?number, +mime: ?string, |} | {| +step: 'data_uri_from_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +first255Chars: ?string, |} | {| +step: 'array_buffer_from_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms |} | {| +step: 'mime_check', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +mime: ?string, |} | {| +step: 'write_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +length: number, |} | FetchFileHashMediaMissionStep | CopyFileMediaMissionStep | GetOrientationMediaMissionStep | {| +step: 'preload_image', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +dimensions: ?Dimensions, |} | {| +step: 'reorient_image', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: ?string, |} | {| +step: 'upload', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputFilename: string, +outputMediaType: ?MediaType, +outputURI: ?string, +outputDimensions: ?Dimensions, +outputLoop: ?boolean, +hasWiFi?: boolean, |} | {| +step: 'wait_for_capture_uri_unload', +success: boolean, +time: number, // ms +uri: string, |}; export type MediaMissionFailure = | {| +success: false, +reason: 'no_file_path', |} | {| +success: false, +reason: 'file_stat_failed', +uri: string, |} | {| +success: false, +reason: 'photo_manipulation_failed', +size: number, // in bytes |} | {| +success: false, +reason: 'media_type_fetch_failed', +detectedMIME: ?string, |} | {| +success: false, +reason: 'mime_type_mismatch', +reportedMediaType: MediaType, +reportedMIME: string, +detectedMIME: string, |} | {| +success: false, +reason: 'http_upload_failed', +exceptionMessage: ?string, |} | {| +success: false, +reason: 'video_too_long', +duration: number, // in seconds |} | {| +success: false, +reason: 'video_probe_failed', |} | {| +success: false, +reason: 'video_transcode_failed', |} | {| +success: false, +reason: 'video_generate_thumbnail_failed', |} | {| +success: false, +reason: 'processing_exception', +time: number, // ms +exceptionMessage: ?string, |} | {| +success: false, +reason: 'save_unsupported', |} | {| +success: false, +reason: 'missing_permission', |} | {| +success: false, +reason: 'make_directory_failed', |} | {| +success: false, +reason: 'resolve_failed', +uri: string, |} | {| +success: false, +reason: 'save_to_library_failed', +uri: string, |} | {| +success: false, +reason: 'fetch_failed', |} | {| +success: false, +reason: 'data_uri_failed', |} | {| +success: false, +reason: 'array_buffer_failed', |} | {| +success: false, +reason: 'mime_check_failed', +mime: ?string, |} | {| +success: false, +reason: 'write_file_failed', |} | {| +success: false, +reason: 'fetch_file_hash_failed', |} | {| +success: false, +reason: 'copy_file_failed', |} | {| +success: false, +reason: 'exif_fetch_failed', |} | {| +success: false, +reason: 'reorient_image_failed', |} | {| +success: false, +reason: 'web_sibling_validation_failed', |}; export type MediaMissionResult = MediaMissionFailure | {| +success: true |}; export type MediaMission = {| +steps: $ReadOnlyArray, +result: MediaMissionResult, +userTime: number, +totalTime: number, |}; diff --git a/native/input/input-state-container.react.js b/native/input/input-state-container.react.js index c96d946a0..06ab032d1 100644 --- a/native/input/input-state-container.react.js +++ b/native/input/input-state-container.react.js @@ -1,1156 +1,1221 @@ // @flow import invariant from 'invariant'; import * as React from 'react'; import { Platform } from 'react-native'; import * as Upload from 'react-native-background-upload'; import { useDispatch } from 'react-redux'; import { createSelector } from 'reselect'; import { createLocalMessageActionType, sendMultimediaMessageActionTypes, sendMultimediaMessage, sendTextMessageActionTypes, sendTextMessage, } from 'lib/actions/message-actions'; import { queueReportsActionType } from 'lib/actions/report-actions'; import { uploadMultimedia, updateMultimediaMessageMediaActionType, type MultimediaUploadCallbacks, type MultimediaUploadExtras, } from 'lib/actions/upload-actions'; import { pathFromURI } from 'lib/media/file-utils'; import { isLocalUploadID, getNextLocalUploadID } from 'lib/media/media-utils'; import { videoDurationLimit } from 'lib/media/video-utils'; import { createLoadingStatusSelector, combineLoadingStatuses, } from 'lib/selectors/loading-selectors'; import { createMediaMessageInfo } from 'lib/shared/message-utils'; import { isStaff } from 'lib/shared/user-utils'; import type { UploadMultimediaResult, Media, NativeMediaSelection, MediaMissionResult, MediaMission, } from 'lib/types/media-types'; import { messageTypes, type RawMessageInfo, type RawMultimediaMessageInfo, type SendMessageResult, type SendMessagePayload, } from 'lib/types/message-types'; import type { RawImagesMessageInfo } from 'lib/types/messages/images'; import type { RawMediaMessageInfo } from 'lib/types/messages/media'; import type { RawTextMessageInfo } from 'lib/types/messages/text'; import type { Dispatch } from 'lib/types/redux-types'; import { type MediaMissionReportCreationRequest, reportTypes, } from 'lib/types/report-types'; import { type DispatchActionPromise, useServerCall, useDispatchActionPromise, } from 'lib/utils/action-utils'; import { getConfig } from 'lib/utils/config'; import { getMessageForException, cloneError } from 'lib/utils/errors'; import type { FetchJSONOptions, FetchJSONServerResponse, } from 'lib/utils/fetch-json'; import { disposeTempFile } from '../media/file-utils'; import { processMedia } from '../media/media-utils'; import { displayActionResultModal } from '../navigation/action-result-modal'; import { useSelector } from '../redux/redux-utils'; import { InputStateContext, type PendingMultimediaUploads, type MultimediaProcessingStep, } from './input-state'; type MediaIDs = | {| +type: 'photo', +localMediaID: string |} | {| +type: 'video', +localMediaID: string, +localThumbnailID: string |}; type UploadFileInput = {| +selection: NativeMediaSelection, +ids: MediaIDs, |}; type CompletedUploads = { +[localMessageID: string]: ?Set }; type BaseProps = {| +children: React.Node, |}; type Props = {| ...BaseProps, +viewerID: ?string, +nextLocalID: number, +messageStoreMessages: { [id: string]: RawMessageInfo }, +ongoingMessageCreation: boolean, +hasWiFi: boolean, +dispatch: Dispatch, +dispatchActionPromise: DispatchActionPromise, +uploadMultimedia: ( multimedia: Object, extras: MultimediaUploadExtras, callbacks: MultimediaUploadCallbacks, ) => Promise, +sendMultimediaMessage: ( threadID: string, localID: string, mediaIDs: $ReadOnlyArray, ) => Promise, +sendTextMessage: ( threadID: string, localID: string, text: string, ) => Promise, |}; type State = {| +pendingUploads: PendingMultimediaUploads, |}; class InputStateContainer extends React.PureComponent { state: State = { pendingUploads: {}, }; sendCallbacks: Array<() => void> = []; activeURIs = new Map(); replyCallbacks: Array<(message: string) => void> = []; static getCompletedUploads(props: Props, state: State): CompletedUploads { const completedUploads = {}; for (const localMessageID in state.pendingUploads) { const messagePendingUploads = state.pendingUploads[localMessageID]; const rawMessageInfo = props.messageStoreMessages[localMessageID]; if (!rawMessageInfo) { continue; } invariant( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA, `rawMessageInfo ${localMessageID} should be multimedia`, ); let allUploadsComplete = true; const completedUploadIDs = new Set(Object.keys(messagePendingUploads)); for (const singleMedia of rawMessageInfo.media) { if (isLocalUploadID(singleMedia.id)) { allUploadsComplete = false; completedUploadIDs.delete(singleMedia.id); } } if (allUploadsComplete) { completedUploads[localMessageID] = null; } else if (completedUploadIDs.size > 0) { completedUploads[localMessageID] = completedUploadIDs; } } return completedUploads; } componentDidUpdate(prevProps: Props, prevState: State) { if (this.props.viewerID !== prevProps.viewerID) { this.setState({ pendingUploads: {} }); return; } const currentlyComplete = InputStateContainer.getCompletedUploads( this.props, this.state, ); const previouslyComplete = InputStateContainer.getCompletedUploads( prevProps, prevState, ); const newPendingUploads = {}; let pendingUploadsChanged = false; const readyMessageIDs = []; for (const localMessageID in this.state.pendingUploads) { const messagePendingUploads = this.state.pendingUploads[localMessageID]; const prevRawMessageInfo = prevProps.messageStoreMessages[localMessageID]; const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; const completedUploadIDs = currentlyComplete[localMessageID]; const previouslyCompletedUploadIDs = previouslyComplete[localMessageID]; if (!rawMessageInfo && prevRawMessageInfo) { pendingUploadsChanged = true; continue; } else if (completedUploadIDs === null) { // All of this message's uploads have been completed newPendingUploads[localMessageID] = {}; if (previouslyCompletedUploadIDs !== null) { readyMessageIDs.push(localMessageID); pendingUploadsChanged = true; } continue; } else if (!completedUploadIDs) { // Nothing has been completed newPendingUploads[localMessageID] = messagePendingUploads; continue; } const newUploads = {}; let uploadsChanged = false; for (const localUploadID in messagePendingUploads) { if (!completedUploadIDs.has(localUploadID)) { newUploads[localUploadID] = messagePendingUploads[localUploadID]; } else if ( !previouslyCompletedUploadIDs || !previouslyCompletedUploadIDs.has(localUploadID) ) { uploadsChanged = true; } } if (uploadsChanged) { pendingUploadsChanged = true; newPendingUploads[localMessageID] = newUploads; } else { newPendingUploads[localMessageID] = messagePendingUploads; } } if (pendingUploadsChanged) { this.setState({ pendingUploads: newPendingUploads }); } for (const localMessageID of readyMessageIDs) { const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; if (!rawMessageInfo) { continue; } invariant( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA, `rawMessageInfo ${localMessageID} should be multimedia`, ); this.dispatchMultimediaMessageAction(rawMessageInfo); } } dispatchMultimediaMessageAction(messageInfo: RawMultimediaMessageInfo) { this.props.dispatchActionPromise( sendMultimediaMessageActionTypes, this.sendMultimediaMessageAction(messageInfo), undefined, messageInfo, ); } async sendMultimediaMessageAction( messageInfo: RawMultimediaMessageInfo, ): Promise { const { localID, threadID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const mediaIDs = []; for (const { id } of messageInfo.media) { mediaIDs.push(id); } try { const result = await this.props.sendMultimediaMessage( threadID, localID, mediaIDs, ); return { localID, serverID: result.id, threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = localID; copy.threadID = threadID; throw copy; } } inputStateSelector = createSelector( (state: State) => state.pendingUploads, (pendingUploads: PendingMultimediaUploads) => ({ pendingUploads, sendTextMessage: this.sendTextMessage, sendMultimediaMessage: this.sendMultimediaMessage, addReply: this.addReply, addReplyListener: this.addReplyListener, removeReplyListener: this.removeReplyListener, messageHasUploadFailure: this.messageHasUploadFailure, retryMessage: this.retryMessage, registerSendCallback: this.registerSendCallback, unregisterSendCallback: this.unregisterSendCallback, uploadInProgress: this.uploadInProgress, reportURIDisplayed: this.reportURIDisplayed, }), ); uploadInProgress = () => { if (this.props.ongoingMessageCreation) { return true; } for (const localMessageID in this.state.pendingUploads) { const messagePendingUploads = this.state.pendingUploads[localMessageID]; for (const localUploadID in messagePendingUploads) { const { failed } = messagePendingUploads[localUploadID]; if (!failed) { return true; } } } return false; }; sendTextMessage = (messageInfo: RawTextMessageInfo) => { this.sendCallbacks.forEach((callback) => callback()); this.props.dispatchActionPromise( sendTextMessageActionTypes, this.sendTextMessageAction(messageInfo), undefined, messageInfo, ); }; async sendTextMessageAction( messageInfo: RawTextMessageInfo, ): Promise { try { const { localID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const result = await this.props.sendTextMessage( messageInfo.threadID, localID, messageInfo.text, ); return { localID, serverID: result.id, threadID: messageInfo.threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = messageInfo.localID; copy.threadID = messageInfo.threadID; throw copy; } } sendMultimediaMessage = async ( threadID: string, selections: $ReadOnlyArray, ) => { this.sendCallbacks.forEach((callback) => callback()); const localMessageID = `local${this.props.nextLocalID}`; const uploadFileInputs = [], media = []; for (const selection of selections) { const localMediaID = getNextLocalUploadID(); let ids; if (selection.step === 'photo_library') { media.push({ id: localMediaID, uri: selection.uri, type: 'photo', dimensions: selection.dimensions, localMediaSelection: selection, }); ids = { type: 'photo', localMediaID }; } else if (selection.step === 'photo_capture') { media.push({ id: localMediaID, uri: selection.uri, type: 'photo', dimensions: selection.dimensions, localMediaSelection: selection, }); ids = { type: 'photo', localMediaID }; } else if (selection.step === 'photo_paste') { media.push({ id: localMediaID, uri: selection.uri, type: 'photo', dimensions: selection.dimensions, localMediaSelection: selection, }); ids = { type: 'photo', localMediaID }; } const localThumbnailID = getNextLocalUploadID(); if (selection.step === 'video_library') { media.push({ id: localMediaID, uri: selection.uri, type: 'video', dimensions: selection.dimensions, localMediaSelection: selection, loop: false, + thumbnailID: localThumbnailID, + thumbnailURI: selection.uri, }); ids = { type: 'video', localMediaID, localThumbnailID }; } invariant(ids, `unexpected MediaSelection ${selection.step}`); uploadFileInputs.push({ selection, ids }); } const pendingUploads = {}; for (const uploadFileInput of uploadFileInputs) { const { localMediaID } = uploadFileInput.ids; pendingUploads[localMediaID] = { failed: null, progressPercent: 0, + processingStep: null, }; if (uploadFileInput.ids.type === 'video') { const { localThumbnailID } = uploadFileInput.ids; pendingUploads[localThumbnailID] = { failed: null, progressPercent: 0, + processingStep: null, }; } } this.setState( (prevState) => { return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: pendingUploads, }, }; }, () => { const creatorID = this.props.viewerID; invariant(creatorID, 'need viewer ID in order to send a message'); const messageInfo = createMediaMessageInfo({ localID: localMessageID, threadID, creatorID, media, }); this.props.dispatch({ type: createLocalMessageActionType, payload: messageInfo, }); }, ); await this.uploadFiles(localMessageID, uploadFileInputs); }; async uploadFiles( localMessageID: string, uploadFileInputs: $ReadOnlyArray, ) { const results = await Promise.all( uploadFileInputs.map((uploadFileInput) => this.uploadFile(localMessageID, uploadFileInput), ), ); const errors = [...new Set(results.filter(Boolean))]; if (errors.length > 0) { displayActionResultModal(errors.join(', ') + ' :('); } } async uploadFile( localMessageID: string, uploadFileInput: UploadFileInput, ): Promise { const { ids, selection } = uploadFileInput; const localID = ids.localMediaID; const start = selection.sendTime; const steps = [selection]; let serverID; let userTime; let errorMessage; let reportPromise; const finish = async (result: MediaMissionResult) => { if (reportPromise) { const finalSteps = await reportPromise; steps.push(...finalSteps); } const totalTime = Date.now() - start; userTime = userTime ? userTime : totalTime; this.queueMediaMissionReport( { localID, localMessageID, serverID }, { steps, result, totalTime, userTime }, ); return errorMessage; }; const fail = (message: string) => { errorMessage = message; this.handleUploadFailure(localMessageID, localID, message); userTime = Date.now() - start; }; let processedMedia; const processingStart = Date.now(); try { const processMediaReturn = processMedia( selection, this.mediaProcessConfig(localMessageID, localID), ); reportPromise = processMediaReturn.reportPromise; const processResult = await processMediaReturn.resultPromise; if (!processResult.success) { const message = processResult.reason === 'video_too_long' ? `can't do vids longer than ${videoDurationLimit}min` : 'processing failed'; fail(message); return await finish(processResult); } processedMedia = processResult; } catch (e) { fail('processing failed'); return await finish({ success: false, reason: 'processing_exception', time: Date.now() - processingStart, exceptionMessage: getMessageForException(e), }); } const { uploadURI, shouldDisposePath, filename, mime } = processedMedia; const { hasWiFi } = this.props; const uploadStart = Date.now(); let uploadExceptionMessage, uploadResult, mediaMissionResult; try { uploadResult = await this.props.uploadMultimedia( { uri: uploadURI, name: filename, type: mime }, { ...processedMedia.dimensions, loop: processedMedia.loop }, { onProgress: (percent: number) => this.setProgress(localMessageID, localID, 'uploading', percent), uploadBlob: this.uploadBlob, }, ); mediaMissionResult = { success: true }; } catch (e) { uploadExceptionMessage = getMessageForException(e); fail('upload failed'); mediaMissionResult = { success: false, reason: 'http_upload_failed', exceptionMessage: uploadExceptionMessage, }; } if (uploadResult) { const { id, mediaType, uri, dimensions, loop } = uploadResult; serverID = id; // When we dispatch this action, it updates Redux and triggers the // componentDidUpdate in this class. componentDidUpdate will handle // calling dispatchMultimediaMessageAction once all the uploads are // complete, and does not wait until this function concludes. this.props.dispatch({ type: updateMultimediaMessageMediaActionType, payload: { messageID: localMessageID, currentMediaID: localID, mediaUpdate: { id, type: mediaType, uri, dimensions, localMediaSelection: undefined, loop, }, }, }); userTime = Date.now() - start; } const processSteps = await reportPromise; reportPromise = null; steps.push(...processSteps); steps.push({ step: 'upload', success: !!uploadResult, exceptionMessage: uploadExceptionMessage, time: Date.now() - uploadStart, inputFilename: filename, outputMediaType: uploadResult && uploadResult.mediaType, outputURI: uploadResult && uploadResult.uri, outputDimensions: uploadResult && uploadResult.dimensions, outputLoop: uploadResult && uploadResult.loop, hasWiFi, }); const promises = []; if (shouldDisposePath) { // If processMedia needed to do any transcoding before upload, we dispose // of the resultant temporary file here. Since the transcoded temporary // file is only used for upload, we can dispose of it after processMedia // (reportPromise) and the upload are complete promises.push( (async () => { const disposeStep = await disposeTempFile(shouldDisposePath); steps.push(disposeStep); })(), ); } if (selection.captureTime || selection.step === 'photo_paste') { // If we are uploading a newly captured photo, we dispose of the original // file here. Note that we try to save photo captures to the camera roll // if we have permission. Even if we fail, this temporary file isn't // visible to the user, so there's no point in keeping it around. Since // the initial URI is used in rendering paths, we have to wait for it to // be replaced with the remote URI before we can dispose. Check out the // Multimedia component to see how the URIs get switched out. const captureURI = selection.uri; promises.push( (async () => { const { steps: clearSteps, result: capturePath, } = await this.waitForCaptureURIUnload(captureURI); steps.push(...clearSteps); if (!capturePath) { return; } const disposeStep = await disposeTempFile(capturePath); steps.push(disposeStep); })(), ); } await Promise.all(promises); return await finish(mediaMissionResult); } mediaProcessConfig(localMessageID: string, localID: string) { const { hasWiFi, viewerID } = this.props; const onTranscodingProgress = (percent: number) => { this.setProgress(localMessageID, localID, 'transcoding', percent); }; if (__DEV__ || (viewerID && isStaff(viewerID))) { return { hasWiFi, finalFileHeaderCheck: true, onTranscodingProgress, }; } return { hasWiFi, onTranscodingProgress }; } setProgress( localMessageID: string, localUploadID: string, processingStep: MultimediaProcessingStep, progressPercent: number, ) { this.setState((prevState) => { const pendingUploads = prevState.pendingUploads[localMessageID]; if (!pendingUploads) { return {}; } const pendingUpload = pendingUploads[localUploadID]; if (!pendingUpload) { return {}; } const newOutOfHundred = Math.floor(progressPercent * 100); const oldOutOfHundred = Math.floor(pendingUpload.progressPercent * 100); if (newOutOfHundred === oldOutOfHundred) { return {}; } const newPendingUploads = { ...pendingUploads, [localUploadID]: { ...pendingUpload, progressPercent, processingStep, }, }; return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: newPendingUploads, }, }; }); } uploadBlob = async ( url: string, cookie: ?string, sessionID: ?string, input: { [key: string]: mixed }, options?: ?FetchJSONOptions, ): Promise => { invariant( cookie && input.multimedia && Array.isArray(input.multimedia) && input.multimedia.length === 1 && input.multimedia[0] && typeof input.multimedia[0] === 'object', 'InputStateContainer.uploadBlob sent incorrect input', ); const { uri, name, type } = input.multimedia[0]; invariant( typeof uri === 'string' && typeof name === 'string' && typeof type === 'string', 'InputStateContainer.uploadBlob sent incorrect input', ); const parameters = {}; parameters.cookie = cookie; parameters.filename = name; for (const key in input) { if ( key === 'multimedia' || key === 'cookie' || key === 'sessionID' || key === 'filename' ) { continue; } const value = input[key]; invariant( typeof value === 'string', 'blobUpload calls can only handle string values for non-multimedia keys', ); parameters[key] = value; } let path = uri; if (Platform.OS === 'android') { const resolvedPath = pathFromURI(uri); if (resolvedPath) { path = resolvedPath; } } const uploadID = await Upload.startUpload({ url, path, type: 'multipart', headers: { Accept: 'application/json', }, field: 'multimedia', parameters, }); if (options && options.abortHandler) { options.abortHandler(() => { Upload.cancelUpload(uploadID); }); } return await new Promise((resolve, reject) => { Upload.addListener('error', uploadID, (data) => { reject(data.error); }); Upload.addListener('cancelled', uploadID, () => { reject(new Error('request aborted')); }); Upload.addListener('completed', uploadID, (data) => { try { resolve(JSON.parse(data.responseBody)); } catch (e) { reject(e); } }); if (options && options.onProgress) { const { onProgress } = options; Upload.addListener('progress', uploadID, (data) => onProgress(data.progress / 100), ); } }); }; handleUploadFailure( localMessageID: string, localUploadID: string, message: string, ) { this.setState((prevState) => { const uploads = prevState.pendingUploads[localMessageID]; const upload = uploads[localUploadID]; if (!upload) { // The upload has been completed before it failed return {}; } return { pendingUploads: { ...prevState.pendingUploads, [localMessageID]: { ...uploads, [localUploadID]: { ...upload, failed: message, progressPercent: 0, }, }, }, }; }); } queueMediaMissionReport( ids: {| localID: string, localMessageID: string, serverID: ?string |}, mediaMission: MediaMission, ) { const report: MediaMissionReportCreationRequest = { type: reportTypes.MEDIA_MISSION, time: Date.now(), platformDetails: getConfig().platformDetails, mediaMission, uploadServerID: ids.serverID, uploadLocalID: ids.localID, messageLocalID: ids.localMessageID, }; this.props.dispatch({ type: queueReportsActionType, payload: { reports: [report], }, }); } messageHasUploadFailure = (localMessageID: string) => { const pendingUploads = this.state.pendingUploads[localMessageID]; if (!pendingUploads) { return false; } for (const localUploadID in pendingUploads) { const { failed } = pendingUploads[localUploadID]; if (failed) { return true; } } return false; }; addReply = (message: string) => { this.replyCallbacks.forEach((addReplyCallback) => addReplyCallback(message), ); }; addReplyListener = (callbackReply: (message: string) => void) => { this.replyCallbacks.push(callbackReply); }; removeReplyListener = (callbackReply: (message: string) => void) => { this.replyCallbacks = this.replyCallbacks.filter( (candidate) => candidate !== callbackReply, ); }; retryTextMessage = (rawMessageInfo: RawTextMessageInfo) => { this.sendTextMessage({ ...rawMessageInfo, time: Date.now(), }); }; retryMultimediaMessage = async ( rawMessageInfo: RawMultimediaMessageInfo, localMessageID: string, ) => { let pendingUploads = this.state.pendingUploads[localMessageID]; if (!pendingUploads) { pendingUploads = {}; } const now = Date.now(); const updateMedia = (media: $ReadOnlyArray): T[] => media.map((singleMedia) => { - const oldID = singleMedia.id; - if (!isLocalUploadID(oldID)) { - // already uploaded - return singleMedia; + let updatedMedia = singleMedia; + + const oldMediaID = updatedMedia.id; + if ( + // not complete + isLocalUploadID(oldMediaID) && + // not still ongoing + (!pendingUploads[oldMediaID] || pendingUploads[oldMediaID].failed) + ) { + // If we have an incomplete upload that isn't in pendingUploads, that + // indicates the app has restarted. We'll reassign a new localID to + // avoid collisions. Note that this isn't necessary for the message ID + // since the localID reducer prevents collisions there + const mediaID = pendingUploads[oldMediaID] + ? oldMediaID + : getNextLocalUploadID(); + if (updatedMedia.type === 'photo') { + updatedMedia = { + type: 'photo', + ...updatedMedia, + id: mediaID, + }; + } else { + updatedMedia = { + type: 'video', + ...updatedMedia, + id: mediaID, + }; + } } - if (pendingUploads[oldID] && !pendingUploads[oldID].failed) { - // still being uploaded - return singleMedia; + + if (updatedMedia.type === 'video') { + const oldThumbnailID = updatedMedia.thumbnailID; + invariant(oldThumbnailID, 'oldThumbnailID not null or undefined'); + if ( + // not complete + isLocalUploadID(oldThumbnailID) && + // not still ongoing + (!pendingUploads[oldThumbnailID] || + pendingUploads[oldThumbnailID].failed) + ) { + const thumbnailID = pendingUploads[oldThumbnailID] + ? oldThumbnailID + : getNextLocalUploadID(); + updatedMedia = { + ...updatedMedia, + thumbnailID, + }; + } } - // If we have an incomplete upload that isn't in pendingUploads, that - // indicates the app has restarted. We'll reassign a new localID to - // avoid collisions. Note that this isn't necessary for the message ID - // since the localID reducer prevents collisions there - const id = pendingUploads[oldID] ? oldID : getNextLocalUploadID(); + if (updatedMedia === singleMedia) { + return singleMedia; + } - const oldSelection = singleMedia.localMediaSelection; + const oldSelection = updatedMedia.localMediaSelection; invariant( oldSelection, 'localMediaSelection should be set on locally created Media', ); const retries = oldSelection.retries ? oldSelection.retries + 1 : 1; // We switch for Flow let selection; if (oldSelection.step === 'photo_capture') { selection = { ...oldSelection, sendTime: now, retries }; } else if (oldSelection.step === 'photo_library') { selection = { ...oldSelection, sendTime: now, retries }; } else if (oldSelection.step === 'photo_paste') { selection = { ...oldSelection, sendTime: now, retries }; } else { selection = { ...oldSelection, sendTime: now, retries }; } - if (singleMedia.type === 'photo') { + if (updatedMedia.type === 'photo') { return { type: 'photo', - ...singleMedia, - id, + ...updatedMedia, localMediaSelection: selection, }; } else { return { type: 'video', - ...singleMedia, - id, + ...updatedMedia, localMediaSelection: selection, }; } }); let newRawMessageInfo; // This conditional is for Flow if (rawMessageInfo.type === messageTypes.MULTIMEDIA) { newRawMessageInfo = ({ ...rawMessageInfo, time: now, media: updateMedia(rawMessageInfo.media), }: RawMediaMessageInfo); } else if (rawMessageInfo.type === messageTypes.IMAGES) { newRawMessageInfo = ({ ...rawMessageInfo, time: now, media: updateMedia(rawMessageInfo.media), }: RawImagesMessageInfo); } else { invariant(false, `rawMessageInfo ${localMessageID} should be multimedia`); } const incompleteMedia: Media[] = []; for (const singleMedia of newRawMessageInfo.media) { if (isLocalUploadID(singleMedia.id)) { incompleteMedia.push(singleMedia); } } if (incompleteMedia.length === 0) { this.dispatchMultimediaMessageAction(newRawMessageInfo); this.setState((prevState) => ({ pendingUploads: { ...prevState.pendingUploads, [localMessageID]: {}, }, })); return; } const retryMedia = incompleteMedia.filter( ({ id }) => !pendingUploads[id] || pendingUploads[id].failed, ); if (retryMedia.length === 0) { // All media are already in the process of being uploaded return; } // We're not actually starting the send here, // we just use this action to update the message in Redux this.props.dispatch({ type: sendMultimediaMessageActionTypes.started, payload: newRawMessageInfo, }); // We clear out the failed status on individual media here, // which makes the UI show pending status instead of error messages - for (const { id } of retryMedia) { - pendingUploads[id] = { + for (const singleMedia of retryMedia) { + pendingUploads[singleMedia.id] = { failed: null, progressPercent: 0, processingStep: null, }; + if (singleMedia.type === 'video') { + const { thumbnailID } = singleMedia; + invariant(thumbnailID, 'thumbnailID not null or undefined'); + pendingUploads[thumbnailID] = { + failed: null, + progressPercent: 0, + processingStep: null, + }; + } } this.setState((prevState) => ({ pendingUploads: { ...prevState.pendingUploads, [localMessageID]: pendingUploads, }, })); const uploadFileInputs = retryMedia.map((singleMedia) => { - const { id, localMediaSelection } = singleMedia; invariant( - localMediaSelection && localMediaSelection.step !== 'video_library', + singleMedia.localMediaSelection, 'localMediaSelection should be set on locally created Media', ); + + let ids; + if (singleMedia.type === 'photo') { + ids = { type: 'photo', localMediaID: singleMedia.id }; + } else { + invariant( + singleMedia.thumbnailID, + 'singleMedia.thumbnailID should be set for videos', + ); + ids = { + type: 'video', + localMediaID: singleMedia.id, + localThumbnailID: singleMedia.thumbnailID, + }; + } + return { - selection: localMediaSelection, - ids: { type: 'photo', localMediaID: id }, + selection: singleMedia.localMediaSelection, + ids, }; }); await this.uploadFiles(localMessageID, uploadFileInputs); }; retryMessage = async (localMessageID: string) => { const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; invariant(rawMessageInfo, `rawMessageInfo ${localMessageID} should exist`); if (rawMessageInfo.type === messageTypes.TEXT) { this.retryTextMessage(rawMessageInfo); } else if ( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA ) { await this.retryMultimediaMessage(rawMessageInfo, localMessageID); } }; registerSendCallback = (callback: () => void) => { this.sendCallbacks.push(callback); }; unregisterSendCallback = (callback: () => void) => { this.sendCallbacks = this.sendCallbacks.filter( (candidate) => candidate !== callback, ); }; reportURIDisplayed = (uri: string, loaded: boolean) => { const prevActiveURI = this.activeURIs.get(uri); const curCount = prevActiveURI && prevActiveURI.count; const prevCount = curCount ? curCount : 0; const count = loaded ? prevCount + 1 : prevCount - 1; const prevOnClear = prevActiveURI && prevActiveURI.onClear; const onClear = prevOnClear ? prevOnClear : []; const activeURI = { count, onClear }; if (count) { this.activeURIs.set(uri, activeURI); return; } this.activeURIs.delete(uri); for (const callback of onClear) { callback(); } }; waitForCaptureURIUnload(uri: string) { const start = Date.now(); const path = pathFromURI(uri); if (!path) { return Promise.resolve({ result: null, steps: [ { step: 'wait_for_capture_uri_unload', success: false, time: Date.now() - start, uri, }, ], }); } const getResult = () => ({ result: path, steps: [ { step: 'wait_for_capture_uri_unload', success: true, time: Date.now() - start, uri, }, ], }); const activeURI = this.activeURIs.get(uri); if (!activeURI) { return Promise.resolve(getResult()); } return new Promise((resolve) => { const finish = () => resolve(getResult()); const newActiveURI = { ...activeURI, onClear: [...activeURI.onClear, finish], }; this.activeURIs.set(uri, newActiveURI); }); } render() { const inputState = this.inputStateSelector(this.state); return ( {this.props.children} ); } } const mediaCreationLoadingStatusSelector = createLoadingStatusSelector( sendMultimediaMessageActionTypes, ); const textCreationLoadingStatusSelector = createLoadingStatusSelector( sendTextMessageActionTypes, ); export default React.memo(function ConnectedInputStateContainer( props: BaseProps, ) { const viewerID = useSelector( (state) => state.currentUserInfo && state.currentUserInfo.id, ); const nextLocalID = useSelector((state) => state.nextLocalID); const messageStoreMessages = useSelector( (state) => state.messageStore.messages, ); const ongoingMessageCreation = useSelector( (state) => combineLoadingStatuses( mediaCreationLoadingStatusSelector(state), textCreationLoadingStatusSelector(state), ) === 'loading', ); const hasWiFi = useSelector((state) => state.connectivity.hasWiFi); const callUploadMultimedia = useServerCall(uploadMultimedia); const callSendMultimediaMessage = useServerCall(sendMultimediaMessage); const callSendTextMessage = useServerCall(sendTextMessage); const dispatchActionPromise = useDispatchActionPromise(); const dispatch = useDispatch(); return ( ); }); diff --git a/server/src/fetchers/upload-fetchers.js b/server/src/fetchers/upload-fetchers.js index 2a54563f2..8a67db834 100644 --- a/server/src/fetchers/upload-fetchers.js +++ b/server/src/fetchers/upload-fetchers.js @@ -1,119 +1,121 @@ // @flow import type { Media } from 'lib/types/media-types'; import { ServerError } from 'lib/utils/errors'; import urlFacts from '../../facts/url'; import { dbQuery, SQL } from '../database/database'; import type { Viewer } from '../session/viewer'; const { baseDomain, basePath } = urlFacts; type UploadInfo = {| content: Buffer, mime: string, |}; async function fetchUpload( viewer: Viewer, id: string, secret: string, ): Promise { const query = SQL` SELECT content, mime FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { content, mime } = row; return { content, mime }; } async function fetchUploadChunk( id: string, secret: string, pos: number, len: number, ): Promise { // We use pos + 1 because SQL is 1-indexed whereas js is 0-indexed const query = SQL` SELECT SUBSTRING(content, ${pos + 1}, ${len}) AS content, mime FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { content, mime } = row; return { content, mime, }; } // Returns total size in bytes. async function getUploadSize(id: string, secret: string): Promise { const query = SQL` SELECT LENGTH(content) AS length FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { length } = row; return length; } function getUploadURL(id: string, secret: string) { return `${baseDomain}${basePath}upload/${id}/${secret}`; } function mediaFromRow(row: Object): Media { const { uploadType: type, uploadSecret: secret } = row; const { width, height, loop } = row.uploadExtra; const id = row.uploadID.toString(); const dimensions = { width, height }; const uri = getUploadURL(id, secret); if (type === 'photo') { return { id, type: 'photo', uri, dimensions }; } else if (loop) { + // $FlowFixMe add thumbnailID, thumbnailURI once they're in DB return { id, type: 'video', uri, dimensions, loop }; } else { + // $FlowFixMe add thumbnailID, thumbnailURI once they're in DB return { id, type: 'video', uri, dimensions }; } } async function fetchMedia( viewer: Viewer, mediaIDs: $ReadOnlyArray, ): Promise<$ReadOnlyArray> { const query = SQL` SELECT id AS uploadID, secret AS uploadSecret, type AS uploadType, extra AS uploadExtra FROM uploads WHERE id IN (${mediaIDs}) AND uploader = ${viewer.id} AND container IS NULL `; const [result] = await dbQuery(query); return result.map(mediaFromRow); } export { fetchUpload, fetchUploadChunk, getUploadSize, getUploadURL, mediaFromRow, fetchMedia, }; diff --git a/web/input/input-state-container.react.js b/web/input/input-state-container.react.js index 0dd31d800..523a3abb3 100644 --- a/web/input/input-state-container.react.js +++ b/web/input/input-state-container.react.js @@ -1,1061 +1,1055 @@ // @flow import { detect as detectBrowser } from 'detect-browser'; import invariant from 'invariant'; import _groupBy from 'lodash/fp/groupBy'; import _keyBy from 'lodash/fp/keyBy'; import _omit from 'lodash/fp/omit'; import _partition from 'lodash/fp/partition'; import _sortBy from 'lodash/fp/sortBy'; import _memoize from 'lodash/memoize'; import * as React from 'react'; import { useDispatch } from 'react-redux'; import { createSelector } from 'reselect'; import { createLocalMessageActionType, sendMultimediaMessageActionTypes, sendMultimediaMessage, sendTextMessageActionTypes, sendTextMessage, } from 'lib/actions/message-actions'; import { queueReportsActionType } from 'lib/actions/report-actions'; import { uploadMultimedia, updateMultimediaMessageMediaActionType, deleteUpload, type MultimediaUploadCallbacks, type MultimediaUploadExtras, } from 'lib/actions/upload-actions'; import { createMediaMessageInfo } from 'lib/shared/message-utils'; import type { UploadMultimediaResult, MediaMissionStep, MediaMissionFailure, MediaMissionResult, MediaMission, } from 'lib/types/media-types'; import { messageTypes, type RawMessageInfo, type RawMultimediaMessageInfo, type SendMessageResult, type SendMessagePayload, } from 'lib/types/message-types'; import type { RawImagesMessageInfo } from 'lib/types/messages/images'; import type { RawMediaMessageInfo } from 'lib/types/messages/media'; import type { RawTextMessageInfo } from 'lib/types/messages/text'; import type { Dispatch } from 'lib/types/redux-types'; import { reportTypes } from 'lib/types/report-types'; import { type DispatchActionPromise, useServerCall, useDispatchActionPromise, } from 'lib/utils/action-utils'; import { getConfig } from 'lib/utils/config'; import { getMessageForException, cloneError } from 'lib/utils/errors'; import { validateFile, preloadImage } from '../media/media-utils'; import InvalidUploadModal from '../modals/chat/invalid-upload.react'; import { useSelector } from '../redux/redux-utils'; import { type PendingMultimediaUpload, InputStateContext } from './input-state'; let nextLocalUploadID = 0; type BaseProps = {| +children: React.Node, +setModal: (modal: ?React.Node) => void, |}; type Props = {| ...BaseProps, +activeChatThreadID: ?string, +viewerID: ?string, +messageStoreMessages: { [id: string]: RawMessageInfo }, +exifRotate: boolean, +dispatch: Dispatch, +dispatchActionPromise: DispatchActionPromise, +uploadMultimedia: ( multimedia: Object, extras: MultimediaUploadExtras, callbacks: MultimediaUploadCallbacks, ) => Promise, +deleteUpload: (id: string) => Promise, +sendMultimediaMessage: ( threadID: string, localID: string, mediaIDs: $ReadOnlyArray, ) => Promise, +sendTextMessage: ( threadID: string, localID: string, text: string, ) => Promise, |}; type State = {| +pendingUploads: { [threadID: string]: { [localUploadID: string]: PendingMultimediaUpload }, }, +drafts: { [threadID: string]: string }, |}; class InputStateContainer extends React.PureComponent { state: State = { pendingUploads: {}, drafts: {}, }; replyCallbacks: Array<(message: string) => void> = []; static completedMessageIDs(state: State) { const completed = new Map(); for (const threadID in state.pendingUploads) { const pendingUploads = state.pendingUploads[threadID]; for (const localUploadID in pendingUploads) { const upload = pendingUploads[localUploadID]; const { messageID, serverID, failed } = upload; if (!messageID || !messageID.startsWith('local')) { continue; } if (!serverID || failed) { completed.set(messageID, false); continue; } if (completed.get(messageID) === undefined) { completed.set(messageID, true); } } } const messageIDs = new Set(); for (const [messageID, isCompleted] of completed) { if (isCompleted) { messageIDs.add(messageID); } } return messageIDs; } componentDidUpdate(prevProps: Props, prevState: State) { if (this.props.viewerID !== prevProps.viewerID) { this.setState({ pendingUploads: {} }); return; } const previouslyAssignedMessageIDs = new Set(); for (const threadID in prevState.pendingUploads) { const pendingUploads = prevState.pendingUploads[threadID]; for (const localUploadID in pendingUploads) { const { messageID } = pendingUploads[localUploadID]; if (messageID) { previouslyAssignedMessageIDs.add(messageID); } } } const newlyAssignedUploads = new Map(); for (const threadID in this.state.pendingUploads) { const pendingUploads = this.state.pendingUploads[threadID]; for (const localUploadID in pendingUploads) { const upload = pendingUploads[localUploadID]; const { messageID } = upload; if ( !messageID || !messageID.startsWith('local') || previouslyAssignedMessageIDs.has(messageID) ) { continue; } let assignedUploads = newlyAssignedUploads.get(messageID); if (!assignedUploads) { assignedUploads = { threadID, uploads: [] }; newlyAssignedUploads.set(messageID, assignedUploads); } assignedUploads.uploads.push(upload); } } const newMessageInfos = new Map(); for (const [messageID, assignedUploads] of newlyAssignedUploads) { const { uploads, threadID } = assignedUploads; const creatorID = this.props.viewerID; invariant(creatorID, 'need viewer ID in order to send a message'); const media = uploads.map( - ({ localID, serverID, uri, mediaType, dimensions, loop }) => { + ({ localID, serverID, uri, mediaType, dimensions }) => { // We can get into this state where dimensions are null if the user is // uploading a file type that the browser can't render. In that case // we fake the dimensions here while we wait for the server to tell us // the true dimensions. We actually don't use the dimensions on the // web side currently, but if we ever change that (for instance if we // want to render a properly sized loading overlay like we do on // native), 0,0 is probably a good default. const shimmedDimensions = dimensions ? dimensions : { height: 0, width: 0 }; - if (mediaType === 'photo') { - return { - id: serverID ? serverID : localID, - uri, - type: 'photo', - dimensions: shimmedDimensions, - }; - } else { - return { - id: serverID ? serverID : localID, - uri, - type: 'video', - dimensions: shimmedDimensions, - loop, - }; - } + invariant( + mediaType === 'photo', + "web InputStateContainer can't handle video", + ); + return { + id: serverID ? serverID : localID, + uri, + type: 'photo', + dimensions: shimmedDimensions, + }; }, ); const messageInfo = createMediaMessageInfo({ localID: messageID, threadID, creatorID, media, }); newMessageInfos.set(messageID, messageInfo); } const currentlyCompleted = InputStateContainer.completedMessageIDs( this.state, ); const previouslyCompleted = InputStateContainer.completedMessageIDs( prevState, ); for (const messageID of currentlyCompleted) { if (previouslyCompleted.has(messageID)) { continue; } let rawMessageInfo = newMessageInfos.get(messageID); if (rawMessageInfo) { newMessageInfos.delete(messageID); } else { rawMessageInfo = this.getRawMultimediaMessageInfo(messageID); } this.sendMultimediaMessage(rawMessageInfo); } for (const [, messageInfo] of newMessageInfos) { this.props.dispatch({ type: createLocalMessageActionType, payload: messageInfo, }); } } getRawMultimediaMessageInfo( localMessageID: string, ): RawMultimediaMessageInfo { const rawMessageInfo = this.props.messageStoreMessages[localMessageID]; invariant(rawMessageInfo, `rawMessageInfo ${localMessageID} should exist`); invariant( rawMessageInfo.type === messageTypes.IMAGES || rawMessageInfo.type === messageTypes.MULTIMEDIA, `rawMessageInfo ${localMessageID} should be multimedia`, ); return rawMessageInfo; } sendMultimediaMessage(messageInfo: RawMultimediaMessageInfo) { this.props.dispatchActionPromise( sendMultimediaMessageActionTypes, this.sendMultimediaMessageAction(messageInfo), undefined, messageInfo, ); } async sendMultimediaMessageAction( messageInfo: RawMultimediaMessageInfo, ): Promise { const { localID, threadID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const mediaIDs = []; for (const { id } of messageInfo.media) { mediaIDs.push(id); } try { const result = await this.props.sendMultimediaMessage( threadID, localID, mediaIDs, ); this.setState((prevState) => { const prevUploads = prevState.pendingUploads[threadID]; const newUploads = {}; for (const localUploadID in prevUploads) { const upload = prevUploads[localUploadID]; if (upload.messageID !== localID) { newUploads[localUploadID] = upload; } else if (!upload.uriIsReal) { newUploads[localUploadID] = { ...upload, messageID: result.id, }; } } return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newUploads, }, }; }); return { localID, serverID: result.id, threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = localID; copy.threadID = threadID; throw copy; } } inputStateSelector = _memoize((threadID: string) => createSelector( (state: State) => state.pendingUploads[threadID], (state: State) => state.drafts[threadID], ( pendingUploads: ?{ [localUploadID: string]: PendingMultimediaUpload }, draft: ?string, ) => { let threadPendingUploads = []; const assignedUploads = {}; if (pendingUploads) { const [uploadsWithMessageIDs, uploadsWithoutMessageIDs] = _partition( 'messageID', )(pendingUploads); threadPendingUploads = _sortBy('localID')(uploadsWithoutMessageIDs); const threadAssignedUploads = _groupBy('messageID')( uploadsWithMessageIDs, ); for (const messageID in threadAssignedUploads) { // lodash libdefs don't return $ReadOnlyArray assignedUploads[messageID] = [...threadAssignedUploads[messageID]]; } } return { pendingUploads: threadPendingUploads, assignedUploads, draft: draft ? draft : '', appendFiles: (files: $ReadOnlyArray) => this.appendFiles(threadID, files), cancelPendingUpload: (localUploadID: string) => this.cancelPendingUpload(threadID, localUploadID), sendTextMessage: (messageInfo: RawTextMessageInfo) => this.sendTextMessage(messageInfo), createMultimediaMessage: (localID: number) => this.createMultimediaMessage(threadID, localID), setDraft: (newDraft: string) => this.setDraft(threadID, newDraft), messageHasUploadFailure: (localMessageID: string) => this.messageHasUploadFailure(assignedUploads[localMessageID]), retryMultimediaMessage: (localMessageID: string) => this.retryMultimediaMessage( threadID, localMessageID, assignedUploads[localMessageID], ), addReply: (message: string) => this.addReply(message), addReplyListener: this.addReplyListener, removeReplyListener: this.removeReplyListener, }; }, ), ); async appendFiles( threadID: string, files: $ReadOnlyArray, ): Promise { const selectionTime = Date.now(); const { setModal } = this.props; const appendResults = await Promise.all( files.map((file) => this.appendFile(file, selectionTime)), ); if (appendResults.some(({ result }) => !result.success)) { setModal(); const time = Date.now() - selectionTime; const reports = []; for (const appendResult of appendResults) { const { steps } = appendResult; let { result } = appendResult; let uploadLocalID; if (result.success) { uploadLocalID = result.pendingUpload.localID; result = { success: false, reason: 'web_sibling_validation_failed' }; } const mediaMission = { steps, result, userTime: time, totalTime: time }; reports.push({ mediaMission, uploadLocalID }); } this.queueMediaMissionReports(reports); return false; } const newUploads = appendResults.map(({ result }) => { invariant(result.success, 'any failed validation should be caught above'); return result.pendingUpload; }); const newUploadsObject = _keyBy('localID')(newUploads); this.setState( (prevState) => { const prevUploads = prevState.pendingUploads[threadID]; const mergedUploads = prevUploads ? { ...prevUploads, ...newUploadsObject } : newUploadsObject; return { pendingUploads: { ...prevState.pendingUploads, [threadID]: mergedUploads, }, }; }, () => this.uploadFiles(threadID, newUploads), ); return true; } async appendFile( file: File, selectTime: number, ): Promise<{ steps: $ReadOnlyArray, result: | MediaMissionFailure | {| success: true, pendingUpload: PendingMultimediaUpload |}, }> { const steps = [ { step: 'web_selection', filename: file.name, size: file.size, mime: file.type, selectTime, }, ]; let response; const validationStart = Date.now(); try { response = await validateFile(file, this.props.exifRotate); } catch (e) { return { steps, result: { success: false, reason: 'processing_exception', time: Date.now() - validationStart, exceptionMessage: getMessageForException(e), }, }; } const { steps: validationSteps, result } = response; steps.push(...validationSteps); if (!result.success) { return { steps, result }; } const { uri, file: fixedFile, mediaType, dimensions } = result; return { steps, result: { success: true, pendingUpload: { localID: `localUpload${nextLocalUploadID++}`, serverID: null, messageID: null, failed: null, file: fixedFile, mediaType, dimensions, uri, loop: false, uriIsReal: false, progressPercent: 0, abort: null, steps, selectTime, }, }, }; } uploadFiles( threadID: string, uploads: $ReadOnlyArray, ) { return Promise.all( uploads.map((upload) => this.uploadFile(threadID, upload)), ); } async uploadFile(threadID: string, upload: PendingMultimediaUpload) { const { selectTime, localID } = upload; const steps = [...upload.steps]; let userTime; const sendReport = (missionResult: MediaMissionResult) => { const latestUpload = this.state.pendingUploads[threadID][localID]; invariant( latestUpload, `pendingUpload ${localID} for ${threadID} missing in sendReport`, ); const { serverID, messageID } = latestUpload; const totalTime = Date.now() - selectTime; userTime = userTime ? userTime : totalTime; const mission = { steps, result: missionResult, totalTime, userTime }; this.queueMediaMissionReports([ { mediaMission: mission, uploadLocalID: localID, uploadServerID: serverID, messageLocalID: messageID, }, ]); }; let uploadResult, uploadExceptionMessage; const uploadStart = Date.now(); try { uploadResult = await this.props.uploadMultimedia( upload.file, { ...upload.dimensions, loop: false }, { onProgress: (percent: number) => this.setProgress(threadID, localID, percent), abortHandler: (abort: () => void) => this.handleAbortCallback(threadID, localID, abort), }, ); } catch (e) { uploadExceptionMessage = getMessageForException(e); this.handleUploadFailure(threadID, localID, e); } userTime = Date.now() - selectTime; steps.push({ step: 'upload', success: !!uploadResult, exceptionMessage: uploadExceptionMessage, time: Date.now() - uploadStart, inputFilename: upload.file.name, outputMediaType: uploadResult && uploadResult.mediaType, outputURI: uploadResult && uploadResult.uri, outputDimensions: uploadResult && uploadResult.dimensions, outputLoop: uploadResult && uploadResult.loop, }); if (!uploadResult) { sendReport({ success: false, reason: 'http_upload_failed', exceptionMessage: uploadExceptionMessage, }); return; } const result = uploadResult; const uploadAfterSuccess = this.state.pendingUploads[threadID][localID]; invariant( uploadAfterSuccess, `pendingUpload ${localID}/${result.id} for ${threadID} missing ` + `after upload`, ); if (uploadAfterSuccess.messageID) { this.props.dispatch({ type: updateMultimediaMessageMediaActionType, payload: { messageID: uploadAfterSuccess.messageID, currentMediaID: localID, mediaUpdate: { id: result.id, }, }, }); } this.setState((prevState) => { const uploads = prevState.pendingUploads[threadID]; const currentUpload = uploads[localID]; invariant( currentUpload, `pendingUpload ${localID}/${result.id} for ${threadID} ` + `missing while assigning serverID`, ); return { pendingUploads: { ...prevState.pendingUploads, [threadID]: { ...uploads, [localID]: { ...currentUpload, serverID: result.id, abort: null, }, }, }, }; }); const { steps: preloadSteps } = await preloadImage(result.uri); steps.push(...preloadSteps); sendReport({ success: true }); const uploadAfterPreload = this.state.pendingUploads[threadID][localID]; invariant( uploadAfterPreload, `pendingUpload ${localID}/${result.id} for ${threadID} missing ` + `after preload`, ); if (uploadAfterPreload.messageID) { const { mediaType, uri, dimensions, loop } = result; this.props.dispatch({ type: updateMultimediaMessageMediaActionType, payload: { messageID: uploadAfterPreload.messageID, currentMediaID: uploadAfterPreload.serverID ? uploadAfterPreload.serverID : uploadAfterPreload.localID, mediaUpdate: { type: mediaType, uri, dimensions, loop }, }, }); } this.setState((prevState) => { const uploads = prevState.pendingUploads[threadID]; const currentUpload = uploads[localID]; invariant( currentUpload, `pendingUpload ${localID}/${result.id} for ${threadID} ` + `missing while assigning URI`, ); const { messageID } = currentUpload; if (messageID && !messageID.startsWith('local')) { const newPendingUploads = _omit([localID])(uploads); return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newPendingUploads, }, }; } return { pendingUploads: { ...prevState.pendingUploads, [threadID]: { ...uploads, [localID]: { ...currentUpload, uri: result.uri, mediaType: result.mediaType, dimensions: result.dimensions, uriIsReal: true, loop: result.loop, }, }, }, }; }); } handleAbortCallback( threadID: string, localUploadID: string, abort: () => void, ) { this.setState((prevState) => { const uploads = prevState.pendingUploads[threadID]; const upload = uploads[localUploadID]; if (!upload) { // The upload has been cancelled before we were even handed the // abort function. We should immediately abort. abort(); } return { pendingUploads: { ...prevState.pendingUploads, [threadID]: { ...uploads, [localUploadID]: { ...upload, abort, }, }, }, }; }); } handleUploadFailure(threadID: string, localUploadID: string, e: any) { this.setState((prevState) => { const uploads = prevState.pendingUploads[threadID]; const upload = uploads[localUploadID]; if (!upload || !upload.abort || upload.serverID) { // The upload has been cancelled or completed before it failed return {}; } const failed = e instanceof Error && e.message ? e.message : 'failed'; return { pendingUploads: { ...prevState.pendingUploads, [threadID]: { ...uploads, [localUploadID]: { ...upload, failed, progressPercent: 0, abort: null, }, }, }, }; }); } queueMediaMissionReports( partials: $ReadOnlyArray<{| mediaMission: MediaMission, uploadLocalID?: ?string, uploadServerID?: ?string, messageLocalID?: ?string, |}>, ) { const reports = partials.map( ({ mediaMission, uploadLocalID, uploadServerID, messageLocalID }) => ({ type: reportTypes.MEDIA_MISSION, time: Date.now(), platformDetails: getConfig().platformDetails, mediaMission, uploadServerID, uploadLocalID, messageLocalID, }), ); this.props.dispatch({ type: queueReportsActionType, payload: { reports } }); } cancelPendingUpload(threadID: string, localUploadID: string) { let revokeURL, abortRequest; this.setState( (prevState) => { const currentPendingUploads = prevState.pendingUploads[threadID]; if (!currentPendingUploads) { return {}; } const pendingUpload = currentPendingUploads[localUploadID]; if (!pendingUpload) { return {}; } if (!pendingUpload.uriIsReal) { revokeURL = pendingUpload.uri; } if (pendingUpload.abort) { abortRequest = pendingUpload.abort; } if (pendingUpload.serverID) { this.props.deleteUpload(pendingUpload.serverID); } const newPendingUploads = _omit([localUploadID])(currentPendingUploads); return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newPendingUploads, }, }; }, () => { if (revokeURL) { URL.revokeObjectURL(revokeURL); } if (abortRequest) { abortRequest(); } }, ); } sendTextMessage(messageInfo: RawTextMessageInfo) { this.props.dispatchActionPromise( sendTextMessageActionTypes, this.sendTextMessageAction(messageInfo), undefined, messageInfo, ); } async sendTextMessageAction( messageInfo: RawTextMessageInfo, ): Promise { try { const { localID } = messageInfo; invariant( localID !== null && localID !== undefined, 'localID should be set', ); const result = await this.props.sendTextMessage( messageInfo.threadID, localID, messageInfo.text, ); return { localID, serverID: result.id, threadID: messageInfo.threadID, time: result.time, interface: result.interface, }; } catch (e) { const copy = cloneError(e); copy.localID = messageInfo.localID; copy.threadID = messageInfo.threadID; throw copy; } } // Creates a MultimediaMessage from the unassigned pending uploads, // if there are any createMultimediaMessage(threadID: string, localID: number) { const localMessageID = `local${localID}`; this.setState((prevState) => { const currentPendingUploads = prevState.pendingUploads[threadID]; if (!currentPendingUploads) { return {}; } const newPendingUploads = {}; let uploadAssigned = false; for (const localUploadID in currentPendingUploads) { const upload = currentPendingUploads[localUploadID]; if (upload.messageID) { newPendingUploads[localUploadID] = upload; } else { const newUpload = { ...upload, messageID: localMessageID, }; uploadAssigned = true; newPendingUploads[localUploadID] = newUpload; } } if (!uploadAssigned) { return {}; } return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newPendingUploads, }, }; }); } setDraft(threadID: string, draft: string) { this.setState((prevState) => ({ drafts: { ...prevState.drafts, [threadID]: draft, }, })); } setProgress( threadID: string, localUploadID: string, progressPercent: number, ) { this.setState((prevState) => { const pendingUploads = prevState.pendingUploads[threadID]; if (!pendingUploads) { return {}; } const pendingUpload = pendingUploads[localUploadID]; if (!pendingUpload) { return {}; } const newPendingUploads = { ...pendingUploads, [localUploadID]: { ...pendingUpload, progressPercent, }, }; return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newPendingUploads, }, }; }); } messageHasUploadFailure( pendingUploads: ?$ReadOnlyArray, ) { if (!pendingUploads) { return false; } return pendingUploads.some((upload) => upload.failed); } retryMultimediaMessage( threadID: string, localMessageID: string, pendingUploads: ?$ReadOnlyArray, ) { const rawMessageInfo = this.getRawMultimediaMessageInfo(localMessageID); let newRawMessageInfo; // This conditional is for Flow if (rawMessageInfo.type === messageTypes.MULTIMEDIA) { newRawMessageInfo = ({ ...rawMessageInfo, time: Date.now(), }: RawMediaMessageInfo); } else { newRawMessageInfo = ({ ...rawMessageInfo, time: Date.now(), }: RawImagesMessageInfo); } const completed = InputStateContainer.completedMessageIDs(this.state); if (completed.has(localMessageID)) { this.sendMultimediaMessage(newRawMessageInfo); return; } if (!pendingUploads) { return; } // We're not actually starting the send here, // we just use this action to update the message's timestamp in Redux this.props.dispatch({ type: sendMultimediaMessageActionTypes.started, payload: newRawMessageInfo, }); const uploadIDsToRetry = new Set(); const uploadsToRetry = []; for (const pendingUpload of pendingUploads) { const { serverID, messageID, localID, abort } = pendingUpload; if (serverID || messageID !== localMessageID) { continue; } if (abort) { abort(); } uploadIDsToRetry.add(localID); uploadsToRetry.push(pendingUpload); } this.setState((prevState) => { const prevPendingUploads = prevState.pendingUploads[threadID]; if (!prevPendingUploads) { return {}; } const newPendingUploads = {}; let pendingUploadChanged = false; for (const localID in prevPendingUploads) { const pendingUpload = prevPendingUploads[localID]; if (uploadIDsToRetry.has(localID) && !pendingUpload.serverID) { newPendingUploads[localID] = { ...pendingUpload, failed: null, progressPercent: 0, abort: null, }; pendingUploadChanged = true; } else { newPendingUploads[localID] = pendingUpload; } } if (!pendingUploadChanged) { return {}; } return { pendingUploads: { ...prevState.pendingUploads, [threadID]: newPendingUploads, }, }; }); this.uploadFiles(threadID, uploadsToRetry); } addReply = (message: string) => { this.replyCallbacks.forEach((addReplyCallback) => addReplyCallback(message), ); }; addReplyListener = (callbackReply: (message: string) => void) => { this.replyCallbacks.push(callbackReply); }; removeReplyListener = (callbackReply: (message: string) => void) => { this.replyCallbacks = this.replyCallbacks.filter( (candidate) => candidate !== callbackReply, ); }; render() { const { activeChatThreadID } = this.props; const inputState = activeChatThreadID ? this.inputStateSelector(activeChatThreadID)(this.state) : null; return ( {this.props.children} ); } } export default React.memo(function ConnectedInputStateContainer( props: BaseProps, ) { const exifRotate = useSelector((state) => { const browser = detectBrowser(state.userAgent); return !browser || (browser.name !== 'safari' && browser.name !== 'chrome'); }); const activeChatThreadID = useSelector( (state) => state.navInfo.activeChatThreadID, ); const viewerID = useSelector( (state) => state.currentUserInfo && state.currentUserInfo.id, ); const messageStoreMessages = useSelector( (state) => state.messageStore.messages, ); const callUploadMultimedia = useServerCall(uploadMultimedia); const callDeleteUpload = useServerCall(deleteUpload); const callSendMultimediaMessage = useServerCall(sendMultimediaMessage); const callSendTextMessage = useServerCall(sendTextMessage); const dispatch = useDispatch(); const dispatchActionPromise = useDispatchActionPromise(); return ( ); });