Changeset View
Changeset View
Standalone View
Standalone View
keyserver/src/database/search-utils.js
Show All 30 Lines | function segmentAndStem(message: string): string { | ||||
} | } | ||||
return stemmedSegments | return stemmedSegments | ||||
.filter(segment => !segment.match(whiteSpacesRegex)) | .filter(segment => !segment.match(whiteSpacesRegex)) | ||||
.join(' '); | .join(' '); | ||||
} | } | ||||
async function processMessagesForSearch( | async function processMessagesForSearch( | ||||
messages: $ReadOnlyArray<RawMessageInfo>, | messages: $ReadOnlyArray<RawMessageInfo | ProcessedForSearchRow>, | ||||
): Promise<void> { | ): Promise<void> { | ||||
const processedMessages = []; | const processedMessages = []; | ||||
for (const msg of messages) { | for (const msg of messages) { | ||||
if ( | if ( | ||||
msg.type !== messageTypes.TEXT && | msg.type !== messageTypes.TEXT && | ||||
msg.type !== messageTypes.EDIT_MESSAGE | msg.type !== messageTypes.EDIT_MESSAGE | ||||
) { | ) { | ||||
Show All 17 Lines | await dbQuery(SQL` | ||||
INSERT INTO message_search (original_message_id, message_id, processed_content) | INSERT INTO message_search (original_message_id, message_id, processed_content) | ||||
VALUES ${processedMessages} | VALUES ${processedMessages} | ||||
ON DUPLICATE KEY UPDATE | ON DUPLICATE KEY UPDATE | ||||
message_id = VALUE(message_id), | message_id = VALUE(message_id), | ||||
processed_content = VALUE(processed_content); | processed_content = VALUE(processed_content); | ||||
`); | `); | ||||
} | } | ||||
export { processMessagesForSearch, segmentAndStem, stopwords }; | type ProcessedForSearchRowText = { | ||||
+type: 0, | |||||
+id: string, | |||||
+text: string, | |||||
}; | |||||
type ProcessedForSearchRowEdit = { | |||||
+type: 20, | |||||
+id: string, | |||||
+targetMessageID: string, | |||||
+text: string, | |||||
}; | |||||
type ProcessedForSearchRow = | |||||
| ProcessedForSearchRowText | |||||
| ProcessedForSearchRowEdit; | |||||
function processRowsForSearch( | |||||
rows: $ReadOnlyArray<any>, | |||||
): $ReadOnlyArray<ProcessedForSearchRow> { | |||||
const results = []; | |||||
for (const row of rows) { | |||||
if (row.type === messageTypes.TEXT) { | |||||
results.push({ type: row.type, id: row.id, text: row.content }); | |||||
} else if (row.type === messageTypes.EDIT_MESSAGE) { | |||||
results.push({ | |||||
type: row.type, | |||||
id: row.id, | |||||
targetMessageID: row.target_message, | |||||
text: row.content, | |||||
}); | |||||
} | |||||
} | |||||
return results; | |||||
} | |||||
const pageSize = 1000; | |||||
async function processMessagesInDBForSearch(): Promise<void> { | |||||
let lastID = 0; | |||||
while (true) { | |||||
const [messages] = await dbQuery(SQL` | |||||
SELECT id, type, content, target_message | |||||
FROM messages | |||||
WHERE (type = ${messageTypes.TEXT} OR type = ${messageTypes.EDIT_MESSAGE}) | |||||
AND id > ${lastID} | |||||
ORDER BY id | |||||
LIMIT ${pageSize} | |||||
`); | |||||
if (messages.length === 0) { | |||||
break; | |||||
} | |||||
const processedRows = processRowsForSearch(messages); | |||||
await processMessagesForSearch(processedRows); | |||||
if (messages.length < pageSize) { | |||||
break; | |||||
} | |||||
lastID = messages[messages.length - 1].id; | |||||
} | |||||
} | |||||
export { | |||||
processMessagesForSearch, | |||||
processMessagesInDBForSearch, | |||||
segmentAndStem, | |||||
stopwords, | |||||
}; |