Paths

Table of Contentst

-
keyserver/src/responders/
-
src/
-
responders/
-
redux-state-responders.js
-
web/redux/
-
redux/
-
initial-state-gate.js

[web] start using threads from database
ClosedPublic
Actions

Authored by kamil on Nov 23 2023, 6:06 AM.

Details

Reviewers

tomek
michal

Commits

rCOMM65c3cc5b137b: [web] start using threads from database

Summary

Migrate threads to DB when needed
Use threads from DB and avoid fetching from keyserver when data is present.

Depends on D9956

Test Plan

Test plan was done twice, in Chrome and Safari:

Check if migrating works
Check if threads are not fetched from keyserver when there are in DB
Modify thread and reload app
Modify thread on different account
Close app, modify thread on different account and open app (test if checkpointing works)

I tested this on the account with ~4k threads, I didn't notice any perf downgrade, moreover, I think initial app load was faster when reading threads from DB.

Diff Detail

Repository

rCOMM Comm

Lint

Lint Not Applicable

Unit

Tests Not Applicable

Event Timeline

kamil created this revision.Nov 23 2023, 6:06 AM

kamil held this revision as a draft.

Herald added a subscriber: ashoat. · View Herald TranscriptNov 23 2023, 6:06 AM

Harbormaster completed remote builds in B24376: Diff 33569.Nov 23 2023, 6:26 AM

kamil published this revision for review.Nov 23 2023, 6:36 AM

ashoat added inline comments.Nov 23 2023, 7:28 AM

web/redux/initial-state-gate.js
33 ↗	(On Diff #33569)	I think your IDE added this

tomek accepted this revision.Nov 24 2023, 3:55 AM

tomek added inline comments.

keyserver/src/responders/redux-state-responders.js
95 ↗	(On Diff #33569)	Can we use `canUseDatabaseOnWeb` from https://phab.comm.dev/D9948?
161 ↗	(On Diff #33569)	I'm wondering if it would be beneficial to differentiate between empty thread infos and excluding them. So in this case, instead of returning an empty object, we would return e.g. `null`. Would that be useful?
web/redux/initial-state-gate.js
65 ↗	(On Diff #33569)	Should we check the length just like in https://phab.comm.dev/D9954?
72 ↗	(On Diff #33569)	Can we use `canUseDatabaseOnWeb`?

This revision is now accepted and ready to land.Nov 24 2023, 3:55 AM

kamil mentioned this in D9951: [web/lib] move reading and populating data to state gate.Nov 27 2023, 3:32 AM

kamil mentioned this in D9954: [web] processing ThreadStore ops on worker.Nov 27 2023, 5:31 AM

address review

keyserver/src/responders/redux-state-responders.js
161 ↗	(On Diff #33569)	I don't think that improves or changes anything, there is no advance in the current state.
web/redux/initial-state-gate.js
33 ↗	(On Diff #33569)	ahhh... IDE did it again 😠
65 ↗	(On Diff #33569)	This is the result of the function from D9954. So the check if inside `getClientStore`
72 ↗	(On Diff #33569)	sure, that was my idea but I messed something while creating diff

fix condition, add check if user is logged in

This revision was landed with ongoing or failed builds.Nov 27 2023, 6:47 AM

Closed by commit rCOMM65c3cc5b137b: [web] start using threads from database (authored by kamil). · Explain Why

This revision was automatically updated to reflect the committed changes.

kamil added a commit: rCOMM65c3cc5b137b: [web] start using threads from database.

Harbormaster failed remote builds in B24510: Diff 33716!Nov 27 2023, 7:48 AM

Harbormaster completed remote builds in B24515: Diff 33721.Nov 27 2023, 8:41 AM

Revision Contents
Changeset List

Path

Size

keyserver/

src/

responders/

redux-state-responders.js

8 lines

web/

redux/

initial-state-gate.js

48 lines

Diff 33738

View Options

keyserver/src/responders/redux-state-responders.js

// @flow		// @flow

import invariant from 'invariant';		import invariant from 'invariant';
import _groupBy from 'lodash/fp/groupBy.js';		import _groupBy from 'lodash/fp/groupBy.js';
import _keyBy from 'lodash/fp/keyBy.js';		import _keyBy from 'lodash/fp/keyBy.js';
import _omit from 'lodash/fp/omit.js';		import _omit from 'lodash/fp/omit.js';
import _partition from 'lodash/fp/partition.js';		import _partition from 'lodash/fp/partition.js';
import _sortBy from 'lodash/fp/sortBy.js';		import _sortBy from 'lodash/fp/sortBy.js';
import _memoize from 'lodash/memoize.js';		import _memoize from 'lodash/memoize.js';
import _throttle from 'lodash/throttle.js';
import * as React from 'react';		import * as React from 'react';
import { useDispatch } from 'react-redux';		import { useDispatch } from 'react-redux';
import { createSelector } from 'reselect';		import { createSelector } from 'reselect';
import uuid from 'uuid';

import {		import {
createLocalMessageActionType,		createLocalMessageActionType,
sendMultimediaMessageActionTypes,		sendMultimediaMessageActionTypes,
legacySendMultimediaMessage,		legacySendMultimediaMessage,
sendTextMessageActionTypes,		sendTextMessageActionTypes,
sendTextMessage,		sendTextMessage,
} from 'lib/actions/message-actions.js';		} from 'lib/actions/message-actions.js';
import { queueReportsActionType } from 'lib/actions/report-actions.js';		import { queueReportsActionType } from 'lib/actions/report-actions.js';
import { newThread } from 'lib/actions/thread-actions.js';		import { newThread } from 'lib/actions/thread-actions.js';
import {		import {
uploadMultimedia,		uploadMultimedia,
uploadMediaMetadata,		uploadMediaMetadata,
updateMultimediaMessageMediaActionType,		updateMultimediaMessageMediaActionType,
deleteUpload,		deleteUpload,
		blobServiceUpload,
type MultimediaUploadCallbacks,		type MultimediaUploadCallbacks,
type MultimediaUploadExtras,		type MultimediaUploadExtras,
		type BlobServiceUploadAction,
} from 'lib/actions/upload-actions.js';		} from 'lib/actions/upload-actions.js';
import {		import {
useModalContext,		useModalContext,
type PushModal,		type PushModal,
} from 'lib/components/modal-provider.react.js';		} from 'lib/components/modal-provider.react.js';
import blobService from 'lib/facts/blob-service.js';		import blobService from 'lib/facts/blob-service.js';
import commStaffCommunity from 'lib/facts/comm-staff-community.js';		import commStaffCommunity from 'lib/facts/comm-staff-community.js';
import { getNextLocalUploadID } from 'lib/media/media-utils.js';		import { getNextLocalUploadID } from 'lib/media/media-utils.js';
Show All 15 Lines
import type { CalendarQuery } from 'lib/types/entry-types.js';		import type { CalendarQuery } from 'lib/types/entry-types.js';
import type {		import type {
UploadMultimediaResult,		UploadMultimediaResult,
UploadMediaMetadataRequest,		UploadMediaMetadataRequest,
MediaMissionStep,		MediaMissionStep,
MediaMissionFailure,		MediaMissionFailure,
MediaMissionResult,		MediaMissionResult,
MediaMission,		MediaMission,
Dimensions,
} from 'lib/types/media-types.js';		} from 'lib/types/media-types.js';
import { messageTypes } from 'lib/types/message-types-enum.js';		import { messageTypes } from 'lib/types/message-types-enum.js';
import {		import {
type RawMessageInfo,		type RawMessageInfo,
type RawMultimediaMessageInfo,		type RawMultimediaMessageInfo,
type SendMessageResult,		type SendMessageResult,
type SendMessagePayload,		type SendMessagePayload,
} from 'lib/types/message-types.js';		} from 'lib/types/message-types.js';
Show All 9 Lines	import {
type ThreadInfo,		type ThreadInfo,
type RawThreadInfo,		type RawThreadInfo,
} from 'lib/types/thread-types.js';		} from 'lib/types/thread-types.js';
import {		import {
type DispatchActionPromise,		type DispatchActionPromise,
useServerCall,		useServerCall,
useDispatchActionPromise,		useDispatchActionPromise,
} from 'lib/utils/action-utils.js';		} from 'lib/utils/action-utils.js';
import { toBase64URL } from 'lib/utils/base64.js';
import {		import {
makeBlobServiceEndpointURL,		makeBlobServiceEndpointURL,
isBlobServiceURI,		isBlobServiceURI,
blobHashFromBlobServiceURI,		blobHashFromBlobServiceURI,
} from 'lib/utils/blob-service.js';		} from 'lib/utils/blob-service.js';
import type { CallServerEndpointOptions } from 'lib/utils/call-server-endpoint.js';
import { getConfig } from 'lib/utils/config.js';		import { getConfig } from 'lib/utils/config.js';
import { getMessageForException, cloneError } from 'lib/utils/errors.js';		import { getMessageForException, cloneError } from 'lib/utils/errors.js';
import { generateReportID } from 'lib/utils/report-utils.js';		import { generateReportID } from 'lib/utils/report-utils.js';
import { handleHTTPResponseError } from 'lib/utils/services-utils.js';

import {		import {
type PendingMultimediaUpload,		type PendingMultimediaUpload,
type TypeaheadState,		type TypeaheadState,
InputStateContext,		InputStateContext,
} from './input-state.js';		} from './input-state.js';
import { encryptFile } from '../media/encryption-utils.js';		import { encryptFile } from '../media/encryption-utils.js';
import { generateThumbHash } from '../media/image-utils.js';		import { generateThumbHash } from '../media/image-utils.js';
Show All 21 Lines	type Props = {
+dispatch: Dispatch,		+dispatch: Dispatch,
+dispatchActionPromise: DispatchActionPromise,		+dispatchActionPromise: DispatchActionPromise,
+calendarQuery: () => CalendarQuery,		+calendarQuery: () => CalendarQuery,
+uploadMultimedia: (		+uploadMultimedia: (
multimedia: Object,		multimedia: Object,
extras: MultimediaUploadExtras,		extras: MultimediaUploadExtras,
callbacks: MultimediaUploadCallbacks,		callbacks: MultimediaUploadCallbacks,
) => Promise<UploadMultimediaResult>,		) => Promise<UploadMultimediaResult>,
		+blobServiceUpload: BlobServiceUploadAction,
+uploadMediaMetadata: (		+uploadMediaMetadata: (
input: UploadMediaMetadataRequest,		input: UploadMediaMetadataRequest,
) => Promise<UploadMultimediaResult>,		) => Promise<UploadMultimediaResult>,
+deleteUpload: (id: string) => Promise<void>,		+deleteUpload: (id: string) => Promise<void>,
+sendMultimediaMessage: (		+sendMultimediaMessage: (
threadID: string,		threadID: string,
localID: string,		localID: string,
mediaIDs: $ReadOnlyArray<string>,		mediaIDs: $ReadOnlyArray<string>,
▲ Show 20 Lines • Show All 729 Lines • ▼ Show 20 Lines	try {
(upload.mediaType === 'encrypted_photo' \|\|		(upload.mediaType === 'encrypted_photo' \|\|
upload.mediaType === 'encrypted_video')		upload.mediaType === 'encrypted_video')
) {		) {
const { blobHash, dimensions, thumbHash } = upload;		const { blobHash, dimensions, thumbHash } = upload;
invariant(		invariant(
encryptionKey && blobHash && dimensions,		encryptionKey && blobHash && dimensions,
'incomplete encrypted upload',		'incomplete encrypted upload',
);		);
uploadResult = await this.blobServiceUpload(		uploadResult = await this.props.blobServiceUpload({
{		input: {
		blobInput: {
		type: 'file',
file: upload.file,		file: upload.file,
		},
blobHash,		blobHash,
encryptionKey,		encryptionKey,
dimensions,		dimensions,
loop: false,		loop: false,
...(thumbHash ? { thumbHash } : undefined),		thumbHash,
},		},
{ ...callbacks },		callbacks,
);		});
} else {		} else {
let uploadExtras = {		let uploadExtras = {
...upload.dimensions,		...upload.dimensions,
loop: false,		loop: false,
thumbHash: upload.thumbHash,		thumbHash: upload.thumbHash,
};		};
if (encryptionKey) {		if (encryptionKey) {
uploadExtras = { ...uploadExtras, encryptionKey };		uploadExtras = { ...uploadExtras, encryptionKey };
▲ Show 20 Lines • Show All 159 Lines • ▼ Show 20 Lines	this.setState(prevState => {
loop: result.loop,		loop: result.loop,
},		},
},		},
},		},
};		};
});		});
}		}

async blobServiceUpload(
input: {
+file: File,
+blobHash: string,
+encryptionKey: string,
+dimensions: Dimensions,
+loop?: boolean,
+thumbHash?: string,
},
options?: ?CallServerEndpointOptions,
): Promise<{
...UploadMultimediaResult,
+blobHolder: string,
}> {
const blobHolder = uuid.v4();
const blobHash = toBase64URL(input.blobHash);

// 1. Assign new holder for blob with given blobHash
let blobAlreadyExists: boolean;
try {
const assignHolderEndpoint = blobService.httpEndpoints.ASSIGN_HOLDER;
const assignHolderResponse = await fetch(
makeBlobServiceEndpointURL(assignHolderEndpoint),
{
method: assignHolderEndpoint.method,
body: JSON.stringify({
holder: blobHolder,
blob_hash: blobHash,
}),
headers: {
'content-type': 'application/json',
},
},
);

handleHTTPResponseError(assignHolderResponse);

const { data_exists: dataExistsResponse } =
await assignHolderResponse.json();
blobAlreadyExists = dataExistsResponse;
} catch (e) {
throw new Error(
`Failed to assign holder: ${
getMessageForException(e) ?? 'unknown error'
}`,
);
}

// 2. Upload blob contents if blob doesn't exist
if (!blobAlreadyExists) {
const formData = new FormData();
formData.append('blob_hash', blobHash);
formData.append('blob_data', input.file);

const xhr = new XMLHttpRequest();
const uploadEndpoint = blobService.httpEndpoints.UPLOAD_BLOB;
xhr.open(
uploadEndpoint.method,
makeBlobServiceEndpointURL(uploadEndpoint),
);
if (options?.timeout) {
xhr.timeout = options.timeout;
}
if (options && options.onProgress) {
const { onProgress } = options;
xhr.upload.onprogress = _throttle(
({ loaded, total }) => onProgress(loaded / total),
50,
);
}

let failed = false;
const responsePromise = new Promise((resolve, reject) => {
xhr.onload = () => {
if (failed) {
return;
}
resolve();
};
xhr.onabort = () => {
failed = true;
reject(new Error('request aborted'));
};
xhr.onerror = event => {
failed = true;
reject(event);
};
if (options && options.timeout) {
xhr.ontimeout = event => {
failed = true;
reject(event);
};
}
if (options && options.abortHandler) {
options.abortHandler(() => {
failed = true;
reject(new Error('request aborted'));
xhr.abort();
});
}
});

if (!failed) {
xhr.send(formData);
}
await responsePromise;
}

// 3. Send upload metadata to the keyserver, return response
const result = await this.props.uploadMediaMetadata({
...input.dimensions,
loop: input.loop ?? false,
blobHolder,
blobHash,
encryptionKey: input.encryptionKey,
mimeType: input.file.type,
filename: input.file.name,
thumbHash: input.thumbHash,
});
return { ...result, blobHolder };
}

handleAbortCallback(		handleAbortCallback(
threadID: string,		threadID: string,
localUploadID: string,		localUploadID: string,
abort: () => void,		abort: () => void,
) {		) {
this.setState(prevState => {		this.setState(prevState => {
const newThreadID = this.getRealizedOrPendingThreadID(threadID);		const newThreadID = this.getRealizedOrPendingThreadID(threadID);
const uploads = prevState.pendingUploads[newThreadID];		const uploads = prevState.pendingUploads[newThreadID];
▲ Show 20 Lines • Show All 533 Lines • ▼ Show 20 Lines	React.memo<BaseProps>(function ConnectedInputStateContainer(props) {
const threadStoreThreadInfos = useSelector(		const threadStoreThreadInfos = useSelector(
state => state.threadStore.threadInfos,		state => state.threadStore.threadInfos,
);		);
const pendingToRealizedThreadIDs = useSelector(state =>		const pendingToRealizedThreadIDs = useSelector(state =>
pendingToRealizedThreadIDsSelector(state.threadStore.threadInfos),		pendingToRealizedThreadIDsSelector(state.threadStore.threadInfos),
);		);
const calendarQuery = useSelector(nonThreadCalendarQuery);		const calendarQuery = useSelector(nonThreadCalendarQuery);
const callUploadMultimedia = useServerCall(uploadMultimedia);		const callUploadMultimedia = useServerCall(uploadMultimedia);
		const callBlobServiceUpload = useServerCall(blobServiceUpload);
const callUploadMediaMetadata = useServerCall(uploadMediaMetadata);		const callUploadMediaMetadata = useServerCall(uploadMediaMetadata);
const callDeleteUpload = useServerCall(deleteUpload);		const callDeleteUpload = useServerCall(deleteUpload);
const callSendMultimediaMessage = useServerCall(		const callSendMultimediaMessage = useServerCall(
legacySendMultimediaMessage,		legacySendMultimediaMessage,
);		);
const callSendTextMessage = useServerCall(sendTextMessage);		const callSendTextMessage = useServerCall(sendTextMessage);
const callNewThread = useServerCall(newThread);		const callNewThread = useServerCall(newThread);
const dispatch = useDispatch();		const dispatch = useDispatch();
Show All 24 Lines	return (
drafts={drafts}		drafts={drafts}
viewerID={viewerID}		viewerID={viewerID}
messageStoreMessages={messageStoreMessages}		messageStoreMessages={messageStoreMessages}
threadStoreThreadInfos={threadStoreThreadInfos}		threadStoreThreadInfos={threadStoreThreadInfos}
pendingRealizedThreadIDs={pendingToRealizedThreadIDs}		pendingRealizedThreadIDs={pendingToRealizedThreadIDs}
calendarQuery={calendarQuery}		calendarQuery={calendarQuery}
uploadMultimedia={callUploadMultimedia}		uploadMultimedia={callUploadMultimedia}
uploadMediaMetadata={callUploadMediaMetadata}		uploadMediaMetadata={callUploadMediaMetadata}
		blobServiceUpload={callBlobServiceUpload}
deleteUpload={callDeleteUpload}		deleteUpload={callDeleteUpload}
sendMultimediaMessage={callSendMultimediaMessage}		sendMultimediaMessage={callSendMultimediaMessage}
sendTextMessage={callSendTextMessage}		sendTextMessage={callSendTextMessage}
newThread={callNewThread}		newThread={callNewThread}
dispatch={dispatch}		dispatch={dispatch}
dispatchActionPromise={dispatchActionPromise}		dispatchActionPromise={dispatchActionPromise}
pushModal={modalContext.pushModal}		pushModal={modalContext.pushModal}
sendCallbacks={sendCallbacks}		sendCallbacks={sendCallbacks}
registerSendCallback={registerSendCallback}		registerSendCallback={registerSendCallback}
unregisterSendCallback={unregisterSendCallback}		unregisterSendCallback={unregisterSendCallback}
textMessageCreationSideEffectsFunc={textMessageCreationSideEffectsFunc}		textMessageCreationSideEffectsFunc={textMessageCreationSideEffectsFunc}
/>		/>
);		);
});		});

export default ConnectedInputStateContainer;		export default ConnectedInputStateContainer;