diff --git a/lib/media/video-utils.js b/lib/media/video-utils.js index 0d8260537..7d32d8527 100644 --- a/lib/media/video-utils.js +++ b/lib/media/video-utils.js @@ -1,145 +1,145 @@ // @flow import invariant from 'invariant'; import type { Dimensions, MediaMissionFailure } from '../types/media-types'; import { getUUID } from '../utils/uuid'; import { replaceExtension } from './file-utils'; import { maxDimensions } from './media-utils'; const { height: maxHeight, width: maxWidth } = maxDimensions; const estimatedResultBitrate = 0.35; // in MiB/s type Input = {| inputPath: string, inputHasCorrectContainerAndCodec: boolean, inputFileSize: number, // in bytes inputFilename: string, inputDuration: number, inputDimensions: Dimensions, outputDirectory: string, outputCodec: string, clientConnectionInfo?: {| hasWiFi: boolean, speed: number, // in kilobytes per second |}, clientTranscodeSpeed?: number, // in input video seconds per second |}; -type ProcessPlan = {| +export type ProcessPlan = {| action: 'process', outputPath: string, ffmpegCommand: string, |}; type Plan = | {| action: 'none' |} | {| action: 'reject', failure: MediaMissionFailure |} | ProcessPlan; function getVideoProcessingPlan(input: Input): Plan { const { inputPath, inputHasCorrectContainerAndCodec, inputFileSize, inputFilename, inputDuration, inputDimensions, outputDirectory, outputCodec, clientConnectionInfo, clientTranscodeSpeed, } = input; if (inputDuration > videoDurationLimit * 60) { return { action: 'reject', failure: { success: false, reason: 'video_too_long', duration: inputDuration, }, }; } if (inputHasCorrectContainerAndCodec) { if (inputFileSize < 1e7) { return { action: 'none' }; } if (clientConnectionInfo && clientTranscodeSpeed) { const rawUploadTime = inputFileSize / 1024 / clientConnectionInfo.speed; // in seconds const transcodeTime = inputDuration / clientTranscodeSpeed; // in seconds const estimatedResultFileSize = inputDuration * estimatedResultBitrate * 1024; // in KiB const transcodedUploadTime = estimatedResultFileSize / clientConnectionInfo.speed; // in seconds const fullProcessTime = transcodeTime + transcodedUploadTime; if ( (clientConnectionInfo.hasWiFi && rawUploadTime < fullProcessTime) || (inputFileSize < 1e8 && rawUploadTime * 2 < fullProcessTime) ) { return { action: 'none' }; } } } const outputFilename = replaceExtension( `transcode.${getUUID()}.${inputFilename}`, 'mp4', ); const outputPath = `${outputDirectory}${outputFilename}`; let quality, speed, scale; if (outputCodec === 'h264') { const { floor, min, max, log2 } = Math; const crf = floor(min(5, max(0, log2(inputDuration / 5)))) + 23; quality = `-crf ${crf}`; speed = '-preset ultrafast'; scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`; } else if (outputCodec === 'h264_videotoolbox') { quality = '-profile:v baseline'; speed = '-realtime 1'; const { width, height } = inputDimensions; scale = ''; const exceedsDimensions = width > maxWidth || height > maxHeight; if (exceedsDimensions && width / height > maxWidth / maxHeight) { scale = `-vf scale=${maxWidth}:-1`; } else if (exceedsDimensions) { scale = `-vf scale=-1:${maxHeight}`; } } else { invariant(false, `unrecognized outputCodec ${outputCodec}`); } const ffmpegCommand = `-i ${inputPath} ` + `-c:a copy -c:v ${outputCodec} ` + `${quality} ` + '-vsync 2 -r 30 ' + `${scale} ` + `${speed} ` + '-movflags +faststart ' + '-pix_fmt yuv420p ' + '-v quiet ' + outputPath; return { action: 'process', outputPath, ffmpegCommand }; } function getHasMultipleFramesProbeCommand(path: string) { const ffprobeCommand = '-v error ' + '-count_frames ' + '-select_streams v:0 ' + '-show_entries stream=nb_read_frames ' + '-of default=nokey=1:noprint_wrappers=1 ' + '-read_intervals "%+#2" ' + path; return ffprobeCommand; } const videoDurationLimit = 3; // in minutes export { getVideoProcessingPlan, getHasMultipleFramesProbeCommand, videoDurationLimit, }; diff --git a/lib/types/media-types.js b/lib/types/media-types.js index cf10bf800..11a347614 100644 --- a/lib/types/media-types.js +++ b/lib/types/media-types.js @@ -1,496 +1,498 @@ // @flow import type { Shape } from './core'; import { type Platform } from './device-types'; export type Dimensions = $ReadOnly<{| +height: number, +width: number, |}>; export type MediaType = 'photo' | 'video'; export type Image = {| +id: string, +uri: string, +type: 'photo', +dimensions: Dimensions, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, |}; export type Video = {| +id: string, +uri: string, +type: 'video', +dimensions: Dimensions, +loop?: boolean, // stored on native only during creation in case retry needed after state lost +localMediaSelection?: NativeMediaSelection, |}; export type Media = Image | Video; export type Corners = Shape<{| +topLeft: boolean, +topRight: boolean, +bottomLeft: boolean, +bottomRight: boolean, |}>; export type MediaInfo = | {| ...Image, +corners: Corners, +index: number, |} | {| ...Video, +corners: Corners, +index: number, |}; export type UploadMultimediaResult = {| +id: string, +uri: string, +dimensions: Dimensions, +mediaType: MediaType, +loop: boolean, |}; export type UpdateMultimediaMessageMediaPayload = {| +messageID: string, +currentMediaID: string, +mediaUpdate: Shape, |}; export type UploadDeletionRequest = {| +id: string, |}; export type FFmpegStatistics = {| // seconds of video being processed per second +speed: number, // total milliseconds of video processed so far +time: number, // total result file size in bytes so far +size: number, +videoQuality: number, +videoFrameNumber: number, +videoFps: number, +bitrate: number, |}; +export type TranscodeVideoMediaMissionStep = {| + +step: 'video_ffmpeg_transcode', + +success: boolean, + +exceptionMessage: ?string, + +time: number, // ms + +returnCode: ?number, + +newPath: ?string, + +stats: ?FFmpegStatistics, +|}; + export type VideoProbeMediaMissionStep = {| +step: 'video_probe', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +validFormat: boolean, +duration: ?number, // seconds +codec: ?string, +format: ?$ReadOnlyArray, +dimensions: ?Dimensions, |}; export type ReadFileHeaderMediaMissionStep = {| +step: 'read_file_header', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +mime: ?string, +mediaType: ?MediaType, |}; export type DetermineFileTypeMediaMissionStep = {| +step: 'determine_file_type', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputFilename: string, +outputMIME: ?string, +outputMediaType: ?MediaType, +outputFilename: ?string, |}; export type FrameCountMediaMissionStep = {| +step: 'frame_count', +success: boolean, +exceptionMessage: ?string, +time: number, +path: string, +mime: string, +hasMultipleFrames: ?boolean, |}; export type DisposeTemporaryFileMediaMissionStep = {| +step: 'dispose_temporary_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type MakeDirectoryMediaMissionStep = {| +step: 'make_directory', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type AndroidScanFileMediaMissionStep = {| +step: 'android_scan_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, |}; export type FetchFileHashMediaMissionStep = {| +step: 'fetch_file_hash', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +hash: ?string, |}; export type CopyFileMediaMissionStep = {| +step: 'copy_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +source: string, +destination: string, |}; export type GetOrientationMediaMissionStep = {| +step: 'exif_fetch', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +orientation: ?number, |}; export type MediaLibrarySelection = | {| +step: 'photo_library', +dimensions: Dimensions, +filename: string, +uri: string, +mediaNativeID: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |} | {| +step: 'video_library', +dimensions: Dimensions, +filename: string, +uri: string, +mediaNativeID: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, +duration: number, // seconds |}; export type PhotoCapture = {| +step: 'photo_capture', +time: number, // ms +dimensions: Dimensions, +filename: string, +uri: string, +captureTime: number, // ms timestamp +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |}; export type PhotoPaste = {| +step: 'photo_paste', +dimensions: Dimensions, +filename: string, +uri: string, +selectTime: number, // ms timestamp +sendTime: number, // ms timestamp +retries: number, |}; export type NativeMediaSelection = | MediaLibrarySelection | PhotoCapture | PhotoPaste; export type MediaMissionStep = | NativeMediaSelection | {| +step: 'web_selection', +filename: string, +size: number, // in bytes +mime: string, +selectTime: number, // ms timestamp |} | {| +step: 'asset_info_fetch', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +localURI: ?string, +orientation: ?number, |} | {| +step: 'stat_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +fileSize: ?number, |} | ReadFileHeaderMediaMissionStep | DetermineFileTypeMediaMissionStep | FrameCountMediaMissionStep | {| +step: 'photo_manipulation', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +manipulation: Object, +newMIME: ?string, +newDimensions: ?Dimensions, +newURI: ?string, |} | VideoProbeMediaMissionStep - | {| - +step: 'video_ffmpeg_transcode', - +success: boolean, - +exceptionMessage: ?string, - +time: number, // ms - +returnCode: ?number, - +newPath: ?string, - +stats: ?FFmpegStatistics, - |} + | TranscodeVideoMediaMissionStep | DisposeTemporaryFileMediaMissionStep | {| +step: 'save_media', +uri: string, +time: number, // ms timestamp |} | {| +step: 'permissions_check', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +platform: Platform, +permissions: $ReadOnlyArray, |} | MakeDirectoryMediaMissionStep | AndroidScanFileMediaMissionStep | {| +step: 'ios_save_to_library', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, |} | {| +step: 'fetch_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputURI: string, +uri: string, +size: ?number, +mime: ?string, |} | {| +step: 'data_uri_from_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +first255Chars: ?string, |} | {| +step: 'array_buffer_from_blob', +success: boolean, +exceptionMessage: ?string, +time: number, // ms |} | {| +step: 'mime_check', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +mime: ?string, |} | {| +step: 'write_file', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +path: string, +length: number, |} | FetchFileHashMediaMissionStep | CopyFileMediaMissionStep | GetOrientationMediaMissionStep | {| +step: 'preload_image', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: string, +dimensions: ?Dimensions, |} | {| +step: 'reorient_image', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +uri: ?string, |} | {| +step: 'upload', +success: boolean, +exceptionMessage: ?string, +time: number, // ms +inputFilename: string, +outputMediaType: ?MediaType, +outputURI: ?string, +outputDimensions: ?Dimensions, +outputLoop: ?boolean, +hasWiFi?: boolean, |} | {| +step: 'wait_for_capture_uri_unload', +success: boolean, +time: number, // ms +uri: string, |}; export type MediaMissionFailure = | {| +success: false, +reason: 'no_file_path', |} | {| +success: false, +reason: 'file_stat_failed', +uri: string, |} | {| +success: false, +reason: 'photo_manipulation_failed', +size: number, // in bytes |} | {| +success: false, +reason: 'media_type_fetch_failed', +detectedMIME: ?string, |} | {| +success: false, +reason: 'mime_type_mismatch', +reportedMediaType: MediaType, +reportedMIME: string, +detectedMIME: string, |} | {| +success: false, +reason: 'http_upload_failed', +exceptionMessage: ?string, |} | {| +success: false, +reason: 'video_too_long', +duration: number, // in seconds |} | {| +success: false, +reason: 'video_probe_failed', |} | {| +success: false, +reason: 'video_transcode_failed', |} | {| +success: false, +reason: 'processing_exception', +time: number, // ms +exceptionMessage: ?string, |} | {| +success: false, +reason: 'save_unsupported', |} | {| +success: false, +reason: 'missing_permission', |} | {| +success: false, +reason: 'make_directory_failed', |} | {| +success: false, +reason: 'resolve_failed', +uri: string, |} | {| +success: false, +reason: 'save_to_library_failed', +uri: string, |} | {| +success: false, +reason: 'fetch_failed', |} | {| +success: false, +reason: 'data_uri_failed', |} | {| +success: false, +reason: 'array_buffer_failed', |} | {| +success: false, +reason: 'mime_check_failed', +mime: ?string, |} | {| +success: false, +reason: 'write_file_failed', |} | {| +success: false, +reason: 'fetch_file_hash_failed', |} | {| +success: false, +reason: 'copy_file_failed', |} | {| +success: false, +reason: 'exif_fetch_failed', |} | {| +success: false, +reason: 'reorient_image_failed', |} | {| +success: false, +reason: 'web_sibling_validation_failed', |}; export type MediaMissionResult = MediaMissionFailure | {| +success: true |}; export type MediaMission = {| +steps: $ReadOnlyArray, +result: MediaMissionResult, +userTime: number, +totalTime: number, |}; diff --git a/native/media/video-utils.js b/native/media/video-utils.js index d9f545c64..fe6f37ea1 100644 --- a/native/media/video-utils.js +++ b/native/media/video-utils.js @@ -1,222 +1,236 @@ // @flow import invariant from 'invariant'; import { Platform } from 'react-native'; import filesystem from 'react-native-fs'; import { mediaConfig, pathFromURI } from 'lib/media/file-utils'; import { getVideoProcessingPlan } from 'lib/media/video-utils'; +import type { ProcessPlan } from 'lib/media/video-utils'; import type { MediaMissionStep, MediaMissionFailure, VideoProbeMediaMissionStep, + TranscodeVideoMediaMissionStep, Dimensions, } from 'lib/types/media-types'; import { getMessageForException } from 'lib/utils/errors'; import { ffmpeg } from './ffmpeg'; import { temporaryDirectoryPath } from './file-utils'; // These are some numbers I sorta kinda made up // We should try to calculate them on a per-device basis const uploadSpeeds = Object.freeze({ wifi: 4096, // in KiB/s cellular: 512, // in KiB/s }); const clientTranscodeSpeed = 1.15; // in seconds of video transcoded per second type ProcessVideoInfo = {| +uri: string, +mime: string, +filename: string, +fileSize: number, +dimensions: Dimensions, +hasWiFi: boolean, |}; type VideoProcessConfig = {| +onTranscodingProgress: (percent: number) => void, |}; type ProcessVideoResponse = {| +success: true, +uri: string, +mime: string, +dimensions: Dimensions, +loop: boolean, |}; async function processVideo( input: ProcessVideoInfo, config: VideoProcessConfig, ): Promise<{| steps: $ReadOnlyArray, result: MediaMissionFailure | ProcessVideoResponse, |}> { const steps = []; const path = pathFromURI(input.uri); invariant(path, `could not extract path from ${input.uri}`); const initialCheckStep = await checkVideoInfo(path); steps.push(initialCheckStep); if (!initialCheckStep.success || !initialCheckStep.duration) { return { steps, result: { success: false, reason: 'video_probe_failed' } }; } const { validFormat, duration } = initialCheckStep; const plan = getVideoProcessingPlan({ inputPath: path, inputHasCorrectContainerAndCodec: validFormat, inputFileSize: input.fileSize, inputFilename: input.filename, inputDuration: duration, inputDimensions: input.dimensions, outputDirectory: temporaryDirectoryPath, // We want ffmpeg to use hardware-accelerated encoders. On iOS we can do // this using VideoToolbox, but ffmpeg on Android is still missing // MediaCodec encoding support: https://trac.ffmpeg.org/ticket/6407 outputCodec: Platform.select({ ios: 'h264_videotoolbox', //android: 'h264_mediacodec', default: 'h264', }), clientConnectionInfo: { hasWiFi: input.hasWiFi, speed: input.hasWiFi ? uploadSpeeds.wifi : uploadSpeeds.cellular, }, clientTranscodeSpeed, }); if (plan.action === 'reject') { return { steps, result: plan.failure }; } if (plan.action === 'none') { return { steps, result: { success: true, uri: input.uri, mime: 'video/mp4', dimensions: input.dimensions, loop: false, }, }; } - const { outputPath, ffmpegCommand } = plan; + const { outputPath } = plan; - let returnCode, - newPath, - stats, - success = false, - exceptionMessage; - const start = Date.now(); - try { - const { rc, lastStats } = await ffmpeg.transcodeVideo( - ffmpegCommand, - duration, - config.onTranscodingProgress, - ); - success = rc === 0; - if (success) { - returnCode = rc; - newPath = outputPath; - stats = lastStats; - } - } catch (e) { - exceptionMessage = getMessageForException(e); - } - if (!success) { - unlink(outputPath); - } - - steps.push({ - step: 'video_ffmpeg_transcode', - success, - exceptionMessage, - time: Date.now() - start, - returnCode, - newPath, - stats, - }); - - if (!success) { + const transcodeStep = await transcodeVideo( + plan, + duration, + config.onTranscodingProgress, + ); + steps.push(transcodeStep); + if (!transcodeStep.success) { return { steps, result: { success: false, reason: 'video_transcode_failed' }, }; } const transcodeProbeStep = await checkVideoInfo(outputPath); steps.push(transcodeProbeStep); if (!transcodeProbeStep.validFormat) { unlink(outputPath); return { steps, result: { success: false, reason: 'video_transcode_failed' }, }; } const dimensions = transcodeProbeStep.dimensions ? transcodeProbeStep.dimensions : input.dimensions; const loop = !!( mediaConfig[input.mime] && mediaConfig[input.mime].videoConfig && mediaConfig[input.mime].videoConfig.loop ); return { steps, result: { success: true, uri: `file://${outputPath}`, mime: 'video/mp4', dimensions, loop, }, }; } +async function transcodeVideo( + plan: ProcessPlan, + duration: number, + onProgressCallback: (number) => void, +): Promise { + const transcodeStart = Date.now(); + let returnCode, + newPath, + stats, + success = false, + exceptionMessage; + try { + const { rc, lastStats } = await ffmpeg.transcodeVideo( + plan.ffmpegCommand, + duration, + onProgressCallback, + ); + success = rc === 0; + if (success) { + returnCode = rc; + newPath = plan.outputPath; + stats = lastStats; + } + } catch (e) { + exceptionMessage = getMessageForException(e); + } + if (!success) { + unlink(plan.outputPath); + } + + return { + step: 'video_ffmpeg_transcode', + success, + exceptionMessage, + time: Date.now() - transcodeStart, + returnCode, + newPath, + stats, + }; +} + async function checkVideoInfo( path: string, ): Promise { let codec, format, dimensions, duration, success = false, validFormat = false, exceptionMessage; const start = Date.now(); try { ({ codec, format, dimensions, duration } = await ffmpeg.getVideoInfo(path)); success = true; validFormat = codec === 'h264' && format.includes('mp4'); } catch (e) { exceptionMessage = getMessageForException(e); } return { step: 'video_probe', success, exceptionMessage, time: Date.now() - start, path, validFormat, duration, codec, format, dimensions, }; } async function unlink(path: string) { try { await filesystem.unlink(path); } catch {} } function formatDuration(seconds: number) { const mm = Math.floor(seconds / 60); const ss = (seconds % 60).toFixed(0).padStart(2, '0'); return `${mm}:${ss}`; } export { processVideo, formatDuration };