diff --git a/keyserver/src/database/migration-config.js b/keyserver/src/database/migration-config.js --- a/keyserver/src/database/migration-config.js +++ b/keyserver/src/database/migration-config.js @@ -5,6 +5,7 @@ import { policyTypes } from 'lib/facts/policies.js'; import { dbQuery, SQL } from '../database/database.js'; +import { processMessagesInDBForSearch } from '../database/search-utils.js'; import { updateRolesAndPermissionsForAllThreads } from '../updaters/thread-permission-updaters.js'; const migrations: $ReadOnlyMap Promise> = new Map([ @@ -294,6 +295,7 @@ ); }, ], + [24, processMessagesInDBForSearch], ]); const newDatabaseVersion: number = Math.max(...migrations.keys()); diff --git a/keyserver/src/database/search-utils.js b/keyserver/src/database/search-utils.js --- a/keyserver/src/database/search-utils.js +++ b/keyserver/src/database/search-utils.js @@ -35,7 +35,7 @@ } async function processMessagesForSearch( - messages: $ReadOnlyArray, + messages: $ReadOnlyArray, ): Promise { const processedMessages = []; @@ -69,4 +69,73 @@ `); } -export { processMessagesForSearch, segmentAndStem, stopwords }; +type ProcessedForSearchRowText = { + +type: 0, + +id: string, + +text: string, +}; +type ProcessedForSearchRowEdit = { + +type: 20, + +id: string, + +targetMessageID: string, + +text: string, +}; +type ProcessedForSearchRow = + | ProcessedForSearchRowText + | ProcessedForSearchRowEdit; + +function processRowsForSearch( + rows: $ReadOnlyArray, +): $ReadOnlyArray { + const results = []; + for (const row of rows) { + if (row.type === messageTypes.TEXT) { + results.push({ type: row.type, id: row.id, text: row.content }); + } else if (row.type === messageTypes.EDIT_MESSAGE) { + results.push({ + type: row.type, + id: row.id, + targetMessageID: row.target_message, + text: row.content, + }); + } + } + return results; +} + +const pageSize = 1000; + +async function processMessagesInDBForSearch(): Promise { + let lastID = 0; + + while (true) { + const [messages] = await dbQuery(SQL` + SELECT id, type, content, target_message + FROM messages + WHERE (type = ${messageTypes.TEXT} OR type = ${messageTypes.EDIT_MESSAGE}) + AND id > ${lastID} + ORDER BY id + LIMIT ${pageSize} + `); + + if (messages.length === 0) { + break; + } + + const processedRows = processRowsForSearch(messages); + + await processMessagesForSearch(processedRows); + + if (messages.length < pageSize) { + break; + } + lastID = messages[messages.length - 1].id; + } +} + +export { + processMessagesForSearch, + processMessagesInDBForSearch, + segmentAndStem, + stopwords, +};