Page MenuHomePhabricator
Differential D12971

[backup] Add endpoint to delete user data
ClosedPublic
Actions

Authored by bartek on Aug 5 2024, 4:49 AM.
Tags
None
Referenced Files
Unknown Object (File)
Tue, Mar 25, 3:52 AM
Unknown Object (File)
Sat, Mar 15, 3:18 AM
Unknown Object (File)
Wed, Mar 12, 6:02 AM
Unknown Object (File)
Feb 18 2025, 5:48 AM
Unknown Object (File)
Feb 17 2025, 10:35 AM
Unknown Object (File)
Feb 17 2025, 10:35 AM
Unknown Object (File)
Feb 17 2025, 10:34 AM
Unknown Object (File)
Feb 17 2025, 10:34 AM
View All 55 Files
Subscribers
ashoat
tomek

Details

Reviewers
kamil
marcin
varun
Commits
rCOMM9fc405d0fda4: [backup] Add endpoint to delete user data
Summary

Added a DELETE /user_data/{user_id} Backup service endpoint that takes care of deleting all service user data.

Depends on D12970

Test Plan

Created two backups for user, and some logs (used Commtest to generate data). Called the endpoint in Postman

  • Verified that DDB data was deleted
  • Verified that blobs were removed from Blob service
  • Verified that only service-to-service token is accepted. HTTP 403 otherwise

Diff Detail

Repository
rCOMM Comm
Lint
Lint Not Applicable
Unit
Tests Not Applicable

Event Timeline

bartek created this revision.Aug 5 2024, 4:49 AM
bartek held this revision as a draft.
Herald added subscribers: tomek, ashoat. · View Herald TranscriptAug 5 2024, 4:49 AM
bartek added a child revision: D12972: [identity] Add config for Backup service URL.Aug 5 2024, 4:51 AM
bartek published this revision for review.Aug 5 2024, 5:08 AM
kamil accepted this revision.Aug 5 2024, 5:53 AM
kamil added inline comments.
services/backup/src/http/handlers/user_data.rs
30 ↗(On Diff #43088)

I would log userID here

This revision is now accepted and ready to land.Aug 5 2024, 5:53 AM
bartek added inline comments.Aug 5 2024, 5:56 AM
services/backup/src/http/handlers/user_data.rs
30 ↗(On Diff #43088)

We consider userID as sensitive data in logs

This revision was landed with ongoing or failed builds.Aug 5 2024, 6:27 AM
Closed by commit rCOMM9fc405d0fda4: [backup] Add endpoint to delete user data (authored by bartek). · Explain Why
This revision was automatically updated to reflect the committed changes.
bartek added a commit: rCOMM9fc405d0fda4: [backup] Add endpoint to delete user data.
Harbormaster completed remote builds in B30908: Diff 43088.Aug 5 2024, 6:48 AM

Revision Contents
Changeset List

PathSize
services/
backup/
src/
database/
18 lines
http/
handlers/
38 lines
9 lines

Diff 43124

View Options

services/backup/src/database/mod.rs

Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Linesfunction getVideoProcessingPlan(input: Input): Plan {
} }
const outputFilename = replaceExtension( const outputFilename = replaceExtension(
`transcode.${uuid}.${sanitizedFilename}`, `transcode.${uuid}.${sanitizedFilename}`,
'mp4', 'mp4',
); );
const outputPath = `${outputDirectory}${outputFilename}`; const outputPath = `${outputDirectory}${outputFilename}`;
let quality, speed, scale; let hardwareAcceleration, quality, speed, scale, pixelFormat;
if (outputCodec === 'h264') { if (outputCodec === 'h264_mediacodec') {
const { floor, min, max, log2 } = Math; hardwareAcceleration = 'mediacodec';
const crf = floor(min(5, max(0, log2(inputDuration / 5)))) + 23; quality = '';
quality = `-crf ${crf}`; speed = '';
speed = '-preset ultrafast';
scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`; scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`;
pixelFormat = '';
} else if (outputCodec === 'h264_videotoolbox') { } else if (outputCodec === 'h264_videotoolbox') {
hardwareAcceleration = 'videotoolbox';
quality = '-profile:v baseline'; quality = '-profile:v baseline';
speed = '-realtime 1'; speed = '-realtime 1';
const { width, height } = inputDimensions; const { width, height } = inputDimensions;
scale = ''; scale = '';
const exceedsDimensions = width > maxWidth || height > maxHeight; const exceedsDimensions = width > maxWidth || height > maxHeight;
if (exceedsDimensions && width / height > maxWidth / maxHeight) { if (exceedsDimensions && width / height > maxWidth / maxHeight) {
scale = `-vf scale=${maxWidth}:-1`; scale = `-vf scale=${maxWidth}:-1`;
} else if (exceedsDimensions) { } else if (exceedsDimensions) {
scale = `-vf scale=-1:${maxHeight}`; scale = `-vf scale=-1:${maxHeight}`;
} }
pixelFormat = '-pix_fmt yuv420p';
} else { } else {
invariant(false, `unrecognized outputCodec ${outputCodec}`); invariant(false, `unrecognized outputCodec ${outputCodec}`);
} }
const ffmpegCommand = const ffmpegCommand =
`-hwaccel ${hardwareAcceleration} ` +
`-i ${inputPath} ` + `-i ${inputPath} ` +
`-c:a copy -c:v ${outputCodec} ` + `-c:a copy -c:v ${outputCodec} ` +
`${quality} ` + `${quality} ` +
'-vsync 2 -r 30 ' + '-fps_mode cfr -r 30 ' +
`${scale} ` + `${scale} ` +
`${speed} ` + `${speed} ` +
'-movflags +faststart ' + '-movflags +faststart ' +
'-pix_fmt yuv420p ' + `${pixelFormat} ` +
'-v quiet ' + '-v quiet ' +
outputPath; outputPath;
return { action: 'process', thumbnailPath, outputPath, ffmpegCommand }; return { action: 'process', thumbnailPath, outputPath, ffmpegCommand };
} }
function getHasMultipleFramesProbeCommand(path: string): string { function getHasMultipleFramesProbeCommand(path: string): string {
const ffprobeCommand = const ffprobeCommand =
Show All 17 Lines
View Options

services/backup/src/http/handlers/user_data.rs

Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Linesfunction getVideoProcessingPlan(input: Input): Plan {
} }
const outputFilename = replaceExtension( const outputFilename = replaceExtension(
`transcode.${uuid}.${sanitizedFilename}`, `transcode.${uuid}.${sanitizedFilename}`,
'mp4', 'mp4',
); );
const outputPath = `${outputDirectory}${outputFilename}`; const outputPath = `${outputDirectory}${outputFilename}`;
let quality, speed, scale; let hardwareAcceleration, quality, speed, scale, pixelFormat;
if (outputCodec === 'h264') { if (outputCodec === 'h264_mediacodec') {
const { floor, min, max, log2 } = Math; hardwareAcceleration = 'mediacodec';
const crf = floor(min(5, max(0, log2(inputDuration / 5)))) + 23; quality = '';
quality = `-crf ${crf}`; speed = '';
speed = '-preset ultrafast';
scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`; scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`;
pixelFormat = '';
} else if (outputCodec === 'h264_videotoolbox') { } else if (outputCodec === 'h264_videotoolbox') {
hardwareAcceleration = 'videotoolbox';
quality = '-profile:v baseline'; quality = '-profile:v baseline';
speed = '-realtime 1'; speed = '-realtime 1';
const { width, height } = inputDimensions; const { width, height } = inputDimensions;
scale = ''; scale = '';
const exceedsDimensions = width > maxWidth || height > maxHeight; const exceedsDimensions = width > maxWidth || height > maxHeight;
if (exceedsDimensions && width / height > maxWidth / maxHeight) { if (exceedsDimensions && width / height > maxWidth / maxHeight) {
scale = `-vf scale=${maxWidth}:-1`; scale = `-vf scale=${maxWidth}:-1`;
} else if (exceedsDimensions) { } else if (exceedsDimensions) {
scale = `-vf scale=-1:${maxHeight}`; scale = `-vf scale=-1:${maxHeight}`;
} }
pixelFormat = '-pix_fmt yuv420p';
} else { } else {
invariant(false, `unrecognized outputCodec ${outputCodec}`); invariant(false, `unrecognized outputCodec ${outputCodec}`);
} }
const ffmpegCommand = const ffmpegCommand =
`-hwaccel ${hardwareAcceleration} ` +
`-i ${inputPath} ` + `-i ${inputPath} ` +
`-c:a copy -c:v ${outputCodec} ` + `-c:a copy -c:v ${outputCodec} ` +
`${quality} ` + `${quality} ` +
'-vsync 2 -r 30 ' + '-fps_mode cfr -r 30 ' +
`${scale} ` + `${scale} ` +
`${speed} ` + `${speed} ` +
'-movflags +faststart ' + '-movflags +faststart ' +
'-pix_fmt yuv420p ' + `${pixelFormat} ` +
'-v quiet ' + '-v quiet ' +
outputPath; outputPath;
return { action: 'process', thumbnailPath, outputPath, ffmpegCommand }; return { action: 'process', thumbnailPath, outputPath, ffmpegCommand };
} }
function getHasMultipleFramesProbeCommand(path: string): string { function getHasMultipleFramesProbeCommand(path: string): string {
const ffprobeCommand = const ffprobeCommand =
Show All 17 Lines
View Options

services/backup/src/http/mod.rs

Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Linesfunction getVideoProcessingPlan(input: Input): Plan {
} }
const outputFilename = replaceExtension( const outputFilename = replaceExtension(
`transcode.${uuid}.${sanitizedFilename}`, `transcode.${uuid}.${sanitizedFilename}`,
'mp4', 'mp4',
); );
const outputPath = `${outputDirectory}${outputFilename}`; const outputPath = `${outputDirectory}${outputFilename}`;
let quality, speed, scale; let hardwareAcceleration, quality, speed, scale, pixelFormat;
if (outputCodec === 'h264') { if (outputCodec === 'h264_mediacodec') {
const { floor, min, max, log2 } = Math; hardwareAcceleration = 'mediacodec';
const crf = floor(min(5, max(0, log2(inputDuration / 5)))) + 23; quality = '';
quality = `-crf ${crf}`; speed = '';
speed = '-preset ultrafast';
scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`; scale = `-vf scale=${maxWidth}:${maxHeight}:force_original_aspect_ratio=decrease`;
pixelFormat = '';
} else if (outputCodec === 'h264_videotoolbox') { } else if (outputCodec === 'h264_videotoolbox') {
hardwareAcceleration = 'videotoolbox';
quality = '-profile:v baseline'; quality = '-profile:v baseline';
speed = '-realtime 1'; speed = '-realtime 1';
const { width, height } = inputDimensions; const { width, height } = inputDimensions;
scale = ''; scale = '';
const exceedsDimensions = width > maxWidth || height > maxHeight; const exceedsDimensions = width > maxWidth || height > maxHeight;
if (exceedsDimensions && width / height > maxWidth / maxHeight) { if (exceedsDimensions && width / height > maxWidth / maxHeight) {
scale = `-vf scale=${maxWidth}:-1`; scale = `-vf scale=${maxWidth}:-1`;
} else if (exceedsDimensions) { } else if (exceedsDimensions) {
scale = `-vf scale=-1:${maxHeight}`; scale = `-vf scale=-1:${maxHeight}`;
} }
pixelFormat = '-pix_fmt yuv420p';
} else { } else {
invariant(false, `unrecognized outputCodec ${outputCodec}`); invariant(false, `unrecognized outputCodec ${outputCodec}`);
} }
const ffmpegCommand = const ffmpegCommand =
`-hwaccel ${hardwareAcceleration} ` +
`-i ${inputPath} ` + `-i ${inputPath} ` +
`-c:a copy -c:v ${outputCodec} ` + `-c:a copy -c:v ${outputCodec} ` +
`${quality} ` + `${quality} ` +
'-vsync 2 -r 30 ' + '-fps_mode cfr -r 30 ' +
`${scale} ` + `${scale} ` +
`${speed} ` + `${speed} ` +
'-movflags +faststart ' + '-movflags +faststart ' +
'-pix_fmt yuv420p ' + `${pixelFormat} ` +
'-v quiet ' + '-v quiet ' +
outputPath; outputPath;
return { action: 'process', thumbnailPath, outputPath, ffmpegCommand }; return { action: 'process', thumbnailPath, outputPath, ffmpegCommand };
} }
function getHasMultipleFramesProbeCommand(path: string): string { function getHasMultipleFramesProbeCommand(path: string): string {
const ffprobeCommand = const ffprobeCommand =
Show All 17 Lines