diff --git a/native/media/media-utils.js b/native/media/media-utils.js index 8351a4ae5..e637f9e48 100644 --- a/native/media/media-utils.js +++ b/native/media/media-utils.js @@ -1,228 +1,228 @@ // @flow import invariant from 'invariant'; import { Image } from 'react-native'; import { pathFromURI, readableFilename } from 'lib/media/file-utils'; import type { Dimensions, MediaType, MediaMissionStep, MediaMissionFailure, NativeMediaSelection, } from 'lib/types/media-types'; import { fetchFileInfo } from './file-utils'; import { processImage } from './image-utils'; import { saveMedia } from './save-media'; import { processVideo } from './video-utils'; type MediaProcessConfig = {| - hasWiFi: boolean, + +hasWiFi: boolean, // Blocks return until we can confirm result has the correct MIME - finalFileHeaderCheck?: boolean, - onTranscodingProgress: (percent: number) => void, + +finalFileHeaderCheck?: boolean, + +onTranscodingProgress: (percent: number) => void, |}; type MediaResult = {| - success: true, - uploadURI: string, - shouldDisposePath: ?string, - filename: string, - mime: string, - mediaType: MediaType, - dimensions: Dimensions, - loop: boolean, + +success: true, + +uploadURI: string, + +shouldDisposePath: ?string, + +filename: string, + +mime: string, + +mediaType: MediaType, + +dimensions: Dimensions, + +loop: boolean, |}; function processMedia( selection: NativeMediaSelection, config: MediaProcessConfig, ): {| resultPromise: Promise, reportPromise: Promise<$ReadOnlyArray>, |} { let resolveResult; const sendResult = (result) => { if (resolveResult) { resolveResult(result); } }; const reportPromise = innerProcessMedia(selection, config, sendResult); const resultPromise = new Promise((resolve) => { resolveResult = resolve; }); return { reportPromise, resultPromise }; } async function innerProcessMedia( selection: NativeMediaSelection, config: MediaProcessConfig, sendResult: (result: MediaMissionFailure | MediaResult) => void, ): Promise<$ReadOnlyArray> { let initialURI = null, uploadURI = null, dimensions = selection.dimensions, mediaType = null, mime = null, loop = false, resultReturned = false; const returnResult = (failure?: MediaMissionFailure) => { invariant( !resultReturned, 'returnResult called twice in innerProcessMedia', ); resultReturned = true; if (failure) { sendResult(failure); return; } invariant( uploadURI && mime && mediaType, 'missing required fields in returnResult', ); const shouldDisposePath = initialURI !== uploadURI ? pathFromURI(uploadURI) : null; const filename = readableFilename(selection.filename, mime); invariant(filename, `could not construct filename for ${mime}`); sendResult({ success: true, uploadURI, shouldDisposePath, filename, mime, mediaType, dimensions, loop, }); }; const steps = [], completeBeforeFinish = []; const finish = async (failure?: MediaMissionFailure) => { if (!resultReturned) { returnResult(failure); } await Promise.all(completeBeforeFinish); return steps; }; if (selection.captureTime && selection.retries === 0) { const { uri } = selection; invariant( pathFromURI(uri), `captured URI ${uri} should use file:// scheme`, ); completeBeforeFinish.push( (async () => { const { reportPromise } = saveMedia(uri); const saveMediaSteps = await reportPromise; steps.push(...saveMediaSteps); })(), ); } const possiblyPhoto = selection.step.startsWith('photo_'); const mediaNativeID = selection.mediaNativeID ? selection.mediaNativeID : null; const { steps: fileInfoSteps, result: fileInfoResult } = await fetchFileInfo( selection.uri, { mediaNativeID }, { orientation: possiblyPhoto, mime: true, mediaType: true, }, ); steps.push(...fileInfoSteps); if (!fileInfoResult.success) { return await finish(fileInfoResult); } const { orientation, fileSize } = fileInfoResult; ({ uri: initialURI, mime, mediaType } = fileInfoResult); if (!mime || !mediaType) { return await finish({ success: false, reason: 'media_type_fetch_failed', detectedMIME: mime, }); } if (mediaType === 'video') { const { steps: videoSteps, result: videoResult } = await processVideo( { uri: initialURI, mime, filename: selection.filename, fileSize, dimensions, hasWiFi: config.hasWiFi, }, { onTranscodingProgress: config.onTranscodingProgress, }, ); steps.push(...videoSteps); if (!videoResult.success) { return await finish(videoResult); } ({ uri: uploadURI, mime, dimensions, loop } = videoResult); } else if (mediaType === 'photo') { const { steps: imageSteps, result: imageResult } = await processImage({ uri: initialURI, dimensions, mime, fileSize, orientation, }); steps.push(...imageSteps); if (!imageResult.success) { return await finish(imageResult); } ({ uri: uploadURI, mime, dimensions } = imageResult); } else { invariant(false, `unknown mediaType ${mediaType}`); } if (uploadURI === initialURI) { return await finish(); } if (!config.finalFileHeaderCheck) { returnResult(); } const { steps: finalFileInfoSteps, result: finalFileInfoResult, } = await fetchFileInfo(uploadURI, undefined, { mime: true }); steps.push(...finalFileInfoSteps); if (!finalFileInfoResult.success) { return await finish(finalFileInfoResult); } if (finalFileInfoResult.mime && finalFileInfoResult.mime !== mime) { return await finish({ success: false, reason: 'mime_type_mismatch', reportedMediaType: mediaType, reportedMIME: mime, detectedMIME: finalFileInfoResult.mime, }); } return await finish(); } function getDimensions(uri: string): Promise { return new Promise((resolve, reject) => { Image.getSize( uri, (width: number, height: number) => resolve({ height, width }), reject, ); }); } export { processMedia, getDimensions }; diff --git a/native/media/video-utils.js b/native/media/video-utils.js index 8d76100f3..65ebe6445 100644 --- a/native/media/video-utils.js +++ b/native/media/video-utils.js @@ -1,224 +1,224 @@ // @flow import invariant from 'invariant'; import { Platform } from 'react-native'; import filesystem from 'react-native-fs'; import { mediaConfig, pathFromURI } from 'lib/media/file-utils'; import { getVideoProcessingPlan } from 'lib/media/video-utils'; import type { MediaMissionStep, MediaMissionFailure, VideoProbeMediaMissionStep, Dimensions, } from 'lib/types/media-types'; import { getMessageForException } from 'lib/utils/errors'; import { ffmpeg } from './ffmpeg'; // These are some numbers I sorta kinda made up // We should try to calculate them on a per-device basis const uploadSpeeds = Object.freeze({ wifi: 4096, // in KiB/s cellular: 512, // in KiB/s }); const clientTranscodeSpeed = 1.15; // in seconds of video transcoded per second type ProcessVideoInfo = {| - uri: string, - mime: string, - filename: string, - fileSize: number, - dimensions: Dimensions, - hasWiFi: boolean, + +uri: string, + +mime: string, + +filename: string, + +fileSize: number, + +dimensions: Dimensions, + +hasWiFi: boolean, |}; type VideoProcessConfig = {| +onTranscodingProgress: (percent: number) => void, |}; type ProcessVideoResponse = {| - success: true, - uri: string, - mime: string, - dimensions: Dimensions, - loop: boolean, + +success: true, + +uri: string, + +mime: string, + +dimensions: Dimensions, + +loop: boolean, |}; async function processVideo( input: ProcessVideoInfo, config: VideoProcessConfig, ): Promise<{| steps: $ReadOnlyArray, result: MediaMissionFailure | ProcessVideoResponse, |}> { const steps = []; const path = pathFromURI(input.uri); invariant(path, `could not extract path from ${input.uri}`); const initialCheckStep = await checkVideoInfo(path); steps.push(initialCheckStep); if (!initialCheckStep.success || !initialCheckStep.duration) { return { steps, result: { success: false, reason: 'video_probe_failed' } }; } const { validFormat, duration } = initialCheckStep; const plan = getVideoProcessingPlan({ inputPath: path, inputHasCorrectContainerAndCodec: validFormat, inputFileSize: input.fileSize, inputFilename: input.filename, inputDuration: duration, inputDimensions: input.dimensions, outputDirectory: Platform.select({ ios: filesystem.TemporaryDirectoryPath, default: `${filesystem.TemporaryDirectoryPath}/`, }), // We want ffmpeg to use hardware-accelerated encoders. On iOS we can do // this using VideoToolbox, but ffmpeg on Android is still missing // MediaCodec encoding support: https://trac.ffmpeg.org/ticket/6407 outputCodec: Platform.select({ ios: 'h264_videotoolbox', //android: 'h264_mediacodec', default: 'h264', }), clientConnectionInfo: { hasWiFi: input.hasWiFi, speed: input.hasWiFi ? uploadSpeeds.wifi : uploadSpeeds.cellular, }, clientTranscodeSpeed, }); if (plan.action === 'reject') { return { steps, result: plan.failure }; } if (plan.action === 'none') { return { steps, result: { success: true, uri: input.uri, mime: 'video/mp4', dimensions: input.dimensions, loop: false, }, }; } const { outputPath, ffmpegCommand } = plan; let returnCode, newPath, stats, success = false, exceptionMessage; const start = Date.now(); try { const { rc, lastStats } = await ffmpeg.transcodeVideo( ffmpegCommand, duration, config.onTranscodingProgress, ); success = rc === 0; if (success) { returnCode = rc; newPath = outputPath; stats = lastStats; } } catch (e) { exceptionMessage = getMessageForException(e); } if (!success) { unlink(outputPath); } steps.push({ step: 'video_ffmpeg_transcode', success, exceptionMessage, time: Date.now() - start, returnCode, newPath, stats, }); if (!success) { return { steps, result: { success: false, reason: 'video_transcode_failed' }, }; } const transcodeProbeStep = await checkVideoInfo(outputPath); steps.push(transcodeProbeStep); if (!transcodeProbeStep.validFormat) { unlink(outputPath); return { steps, result: { success: false, reason: 'video_transcode_failed' }, }; } const dimensions = transcodeProbeStep.dimensions ? transcodeProbeStep.dimensions : input.dimensions; const loop = !!( mediaConfig[input.mime] && mediaConfig[input.mime].videoConfig && mediaConfig[input.mime].videoConfig.loop ); return { steps, result: { success: true, uri: `file://${outputPath}`, mime: 'video/mp4', dimensions, loop, }, }; } async function checkVideoInfo( path: string, ): Promise { let codec, format, dimensions, duration, success = false, validFormat = false, exceptionMessage; const start = Date.now(); try { ({ codec, format, dimensions, duration } = await ffmpeg.getVideoInfo(path)); success = true; validFormat = codec === 'h264' && format.includes('mp4'); } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'video_probe', success, exceptionMessage, time: Date.now() - start, path, validFormat, duration, codec, format, dimensions, }; } async function unlink(path: string) { try { await filesystem.unlink(path); } catch {} } function formatDuration(seconds: number) { const mm = Math.floor(seconds / 60); const ss = (seconds % 60).toFixed(0).padStart(2, '0'); return `${mm}:${ss}`; } export { processVideo, formatDuration };