Differential D7117 Diff 24463 keyserver/src/database/search-utils.js

Changeset View

Standalone View

keyserver/src/database/search-utils.js

Show All 29 Lines	function segmentAndStem(message: string): string {
}		}

return stemmed_segments		return stemmed_segments
.filter(segment => !segment.match(whiteSpacesRegex))		.filter(segment => !segment.match(whiteSpacesRegex))
.join(' ');		.join(' ');
}		}

async function processMessagesForSearch(		async function processMessagesForSearch(
messages: $ReadOnlyArray<RawMessageInfo>,		messages: $ReadOnlyArray<RawMessageInfo \| ProcessedForSearchRow>,
): Promise<void> {		): Promise<void> {
const processedMessages = [];		const processedMessages = [];

for (const msg of messages) {		for (const msg of messages) {
if (		if (
msg.type !== messageTypes.TEXT &&		msg.type !== messageTypes.TEXT &&
msg.type !== messageTypes.EDIT_MESSAGE		msg.type !== messageTypes.EDIT_MESSAGE
) {		) {
Show All 17 Lines	await dbQuery(SQL`
INSERT INTO message_search (original_message_id, message_id, processed_content)		INSERT INTO message_search (original_message_id, message_id, processed_content)
VALUES ${processedMessages}		VALUES ${processedMessages}
ON DUPLICATE KEY UPDATE		ON DUPLICATE KEY UPDATE
message_id = VALUE(message_id),		message_id = VALUE(message_id),
processed_content = VALUE(processed_content);		processed_content = VALUE(processed_content);
`);		`);
}		}

export { processMessagesForSearch, segmentAndStem, stopwords };		type ProcessedForSearchRowText = {
		+type: 0,
		+id: string,
		+text: string,
		};
		type ProcessedForSearchRowEdit = {
		+type: 20,
		+id: string,
		+targetMessageID: string,
		+text: string,
		};
		type ProcessedForSearchRow =
		\| ProcessedForSearchRowText
		\| ProcessedForSearchRowEdit;

		function processRowsForSearch(
		rows: $ReadOnlyArray<any>,
		): $ReadOnlyArray<ProcessedForSearchRow> {
		const results = [];
		for (const row of rows) {
		if (row.type === messageTypes.TEXT) {
		results.push({ type: row.type, id: row.id, text: row.content });
		} else if (row.type === messageTypes.EDIT_MESSAGE) {
		results.push({
		type: row.type,
		id: row.id,
		targetMessageID: row.target_message,
		text: row.content,
		});
		}
		}
		return results;
		}

		const pageSize = 1000;

		async function processMessagesInDBForSearch(): Promise<void> {
		let lastID = 0;

		while (true) {
		const [messages] = await dbQuery(SQL`
		SELECT id, type, content, target_message
		FROM messages
		WHERE (type = ${messageTypes.TEXT} OR type = ${messageTypes.EDIT_MESSAGE})
		AND id > ${lastID}
		ORDER BY id
		LIMIT ${pageSize}
		`);

		if (messages.length === 0) {
		break;
		}

		const processedRows = processRowsForSearch(messages);

		await processMessagesForSearch(processedRows);

		if (messages.length < pageSize) {
		break;
		}
		lastID = messages[messages.length - 1].id;
		}
		}

		export {
		processMessagesForSearch,
		processMessagesInDBForSearch,
		segmentAndStem,
		stopwords,
		};