Differential D7117 Diff 24075 keyserver/src/database/search-utils.js

Changeset View

Standalone View

keyserver/src/database/search-utils.js

// @flow		// @flow

import natural from 'natural';		import natural from 'natural';

import type { RawMessageInfo } from 'lib/types/message-types';		import type { RawMessageInfo } from 'lib/types/message-types';
import { messageTypes } from 'lib/types/message-types.js';		import { messageTypes } from 'lib/types/message-types.js';

import { dbQuery, SQL } from '../database/database.js';		import { dbQuery, SQL } from '../database/database.js';

async function processMessagesForSearch(		async function processMessagesForSearch(
messages: $ReadOnlyArray<RawMessageInfo>,		messages: $ReadOnlyArray<RawMessageInfo \| ProcessedForSearchRow>,
): Promise<void> {		): Promise<void> {
const processedMessages = [];		const processedMessages = [];

for (const msg of messages) {		for (const msg of messages) {
if (		if (
msg.type !== messageTypes.TEXT &&		msg.type !== messageTypes.TEXT &&
msg.type !== messageTypes.EDIT_MESSAGE		msg.type !== messageTypes.EDIT_MESSAGE
) {		) {
Show All 20 Lines	await dbQuery(SQL`
INSERT INTO search (original_message_id, message_id, processed_content)		INSERT INTO search (original_message_id, message_id, processed_content)
VALUES ${processedMessages}		VALUES ${processedMessages}
ON DUPLICATE KEY UPDATE		ON DUPLICATE KEY UPDATE
message_id = VALUE(message_id),		message_id = VALUE(message_id),
processed_content = VALUE(processed_content);		processed_content = VALUE(processed_content);
`);		`);
}		}

export { processMessagesForSearch };		type ProcessedForSearchRowText = {
		+type: 0,
		+id: string,
		+text: string,
		};
		type ProcessedForSearchRowEdit = {
		+type: 20,
		+id: string,
		+targetMessageID: string,
		+text: string,
		};
		type ProcessedForSearchRow =
		\| ProcessedForSearchRowText
		\| ProcessedForSearchRowEdit;

		function processRowsForSearch(
		rows: $ReadOnlyArray<any>,
		): $ReadOnlyArray<ProcessedForSearchRow> {
		const results = [];
		for (const row of rows) {
		if (row.type === messageTypes.TEXT) {
		results.push({ type: row.type, id: row.id, text: row.content });
		} else if (row.type === messageTypes.EDIT_MESSAGE) {
		results.push({
		type: row.type,
		id: row.id,
		targetMessageID: row.target_message,
		text: row.content,
		});
		}
		}
		return results;
		}

		const pageSize = 1001;

		async function processMessagesInDBForSearch(): Promise<void> {
		let lastID = 0;
		let messages = [];

		do {
		[messages] = await dbQuery(SQL`
		SELECT id, type, content, target_message
		FROM messages
		WHERE (type = ${messageTypes.TEXT} OR type = ${messageTypes.EDIT_MESSAGE})
		AND id > ${lastID}
		ORDER BY id
		LIMIT ${pageSize}
		`);

		const truncatedMessages =
		messages.length < pageSize ? messages : messages.slice(0, -1);

		const processedRows = processRowsForSearch(truncatedMessages);

		await processMessagesForSearch(processedRows);

		lastID = truncatedMessages[truncatedMessages.length - 1].id;
		} while (messages.length === pageSize);
		}

		export { processMessagesForSearch, processMessagesInDBForSearch };