Page MenuHomePhorge
Files F33241401

D7117.1768570263.diff
No OneTemporary
Actions

Size
2 KB
Referenced Files
None
Subscribers
None

D7117.1768570263.diff
View Options

diff --git a/keyserver/src/database/migration-config.js b/keyserver/src/database/migration-config.js
--- a/keyserver/src/database/migration-config.js
+++ b/keyserver/src/database/migration-config.js
@@ -5,6 +5,7 @@
import { policyTypes } from 'lib/facts/policies.js';
import { dbQuery, SQL } from '../database/database.js';
+import { processMessagesInDBForSearch } from '../database/search-utils.js';
import { updateRolesAndPermissionsForAllThreads } from '../updaters/thread-permission-updaters.js';
const migrations: $ReadOnlyMap<number, () => Promise<void>> = new Map([
@@ -296,6 +297,7 @@
);
},
],
+ [26, processMessagesInDBForSearch],
]);
const newDatabaseVersion: number = Math.max(...migrations.keys());
diff --git a/keyserver/src/database/search-utils.js b/keyserver/src/database/search-utils.js
--- a/keyserver/src/database/search-utils.js
+++ b/keyserver/src/database/search-utils.js
@@ -36,7 +36,7 @@
}
async function processMessagesForSearch(
- messages: $ReadOnlyArray<RawMessageInfo>,
+ messages: $ReadOnlyArray<RawMessageInfo | ProcessedForSearchRow>,
): Promise<void> {
const processedMessages = [];
@@ -70,4 +70,73 @@
`);
}
-export { processMessagesForSearch, segmentAndStem, stopwords };
+type ProcessedForSearchRowText = {
+ +type: 0,
+ +id: string,
+ +text: string,
+};
+type ProcessedForSearchRowEdit = {
+ +type: 20,
+ +id: string,
+ +targetMessageID: string,
+ +text: string,
+};
+type ProcessedForSearchRow =
+ | ProcessedForSearchRowText
+ | ProcessedForSearchRowEdit;
+
+function processRowsForSearch(
+ rows: $ReadOnlyArray<any>,
+): $ReadOnlyArray<ProcessedForSearchRow> {
+ const results = [];
+ for (const row of rows) {
+ if (row.type === messageTypes.TEXT) {
+ results.push({ type: row.type, id: row.id, text: row.content });
+ } else if (row.type === messageTypes.EDIT_MESSAGE) {
+ results.push({
+ type: row.type,
+ id: row.id,
+ targetMessageID: row.target_message,
+ text: row.content,
+ });
+ }
+ }
+ return results;
+}
+
+const pageSize = 1000;
+
+async function processMessagesInDBForSearch(): Promise<void> {
+ let lastID = 0;
+
+ while (true) {
+ const [messages] = await dbQuery(SQL`
+ SELECT id, type, content, target_message
+ FROM messages
+ WHERE (type = ${messageTypes.TEXT} OR type = ${messageTypes.EDIT_MESSAGE})
+ AND id > ${lastID}
+ ORDER BY id
+ LIMIT ${pageSize}
+ `);
+
+ if (messages.length === 0) {
+ break;
+ }
+
+ const processedRows = processRowsForSearch(messages);
+
+ await processMessagesForSearch(processedRows);
+
+ if (messages.length < pageSize) {
+ break;
+ }
+ lastID = messages[messages.length - 1].id;
+ }
+}
+
+export {
+ processMessagesForSearch,
+ processMessagesInDBForSearch,
+ segmentAndStem,
+ stopwords,
+};

File Metadata

Mime Type
text/plain
Expires
Fri, Jan 16, 1:31 PM (8 h, 33 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
5944511
Default Alt Text
D7117.1768570263.diff (2 KB)

Event Timeline