diff --git a/keyserver/src/cron/cron.js b/keyserver/src/cron/cron.js --- a/keyserver/src/cron/cron.js +++ b/keyserver/src/cron/cron.js @@ -5,7 +5,10 @@ import cluster from 'cluster'; import schedule from 'node-schedule'; -import { verifyMemoryUsage } from 'lib/utils/olm-memory-utils.js'; +import { + getOlmMemory, + compareAndLogOlmMemory, +} from 'lib/utils/olm-memory-utils.js'; import { backupDB } from './backups.js'; import { createDailyUpdatesThread } from './daily-updates.js'; @@ -86,6 +89,7 @@ schedule.scheduleJob( '0 5 * * *', // every day at 5:00 AM in the keyserver's timezone async () => { + const memBefore = getOlmMemory(); try { await updateIdentityReservedUsernames(); } catch (e) { @@ -94,12 +98,15 @@ 'identity service', e, ); + } finally { + compareAndLogOlmMemory(memBefore, 'reserved usernames cronjob'); } }, ); schedule.scheduleJob( '0 0 * * *', // every day at midnight in the keyserver's timezone async () => { + const memBefore = getOlmMemory(); try { await fetchCallUpdateOlmAccount( 'content', @@ -112,12 +119,15 @@ ); } catch (e) { console.warn('encountered error while trying to validate prekeys', e); + } finally { + compareAndLogOlmMemory(memBefore, 'prekey upload cronjob'); } }, ); schedule.scheduleJob( '0 2 * * *', // every day at 2:00 AM in the keyserver's timezone async () => { + const memBefore = getOlmMemory(); try { await synchronizeInviteLinksWithBlobs(); } catch (e) { @@ -125,14 +135,25 @@ 'encountered an error while trying to synchronize invite links with blobs', e, ); + } finally { + compareAndLogOlmMemory(memBefore, 'invite links cronjob'); } }, ); schedule.scheduleJob( '0,15,30,45 * * * *', // every 15 minutes async () => { - await olm.init(); - verifyMemoryUsage('cronjob'); + const memBefore = getOlmMemory(); + try { + await olm.init(); + } catch (e) { + console.warn( + 'encountered an error while executing olm init cron job', + e, + ); + } finally { + compareAndLogOlmMemory(memBefore, 'olm init cronjob'); + } }, ); } diff --git a/keyserver/src/socket/tunnelbroker.js b/keyserver/src/socket/tunnelbroker.js --- a/keyserver/src/socket/tunnelbroker.js +++ b/keyserver/src/socket/tunnelbroker.js @@ -44,6 +44,10 @@ convertObjToBytes, } from 'lib/utils/conversion-utils.js'; import { getMessageForException } from 'lib/utils/errors.js'; +import { + compareAndLogOlmMemory, + getOlmMemory, +} from 'lib/utils/olm-memory-utils.js'; import sleep from 'lib/utils/sleep.js'; import { @@ -387,8 +391,15 @@ refreshOneTimeKeys: (numberOfKeys: number) => void = numberOfKeys => { const oldOneTimeKeysPromise = this.oneTimeKeysPromise; this.oneTimeKeysPromise = (async () => { - await oldOneTimeKeysPromise; - await uploadNewOneTimeKeys(numberOfKeys); + const memBefore = getOlmMemory(); + try { + await oldOneTimeKeysPromise; + await uploadNewOneTimeKeys(numberOfKeys); + } catch (e) { + console.error('Encountered error when trying to upload new OTKs:', e); + } finally { + compareAndLogOlmMemory(memBefore, 'otk refresh'); + } })(); }; diff --git a/lib/utils/olm-memory-utils.js b/lib/utils/olm-memory-utils.js --- a/lib/utils/olm-memory-utils.js +++ b/lib/utils/olm-memory-utils.js @@ -10,13 +10,18 @@ if (olmTotalMemory === null && olmUsedMemory === null) { olmTotalMemory = olm.get_total_memory(); olmUsedMemory = olm.get_used_memory(); + console.error( + `Olm first time memory check - Total: ${olmTotalMemory ?? -1}, Used: ${ + olmUsedMemory ?? -1 + }`, + ); return; } const currentTotalMemory = olm.get_total_memory(); if (currentTotalMemory !== olmTotalMemory) { console.error( - `Olm's total memory changed from ${olmTotalMemory ?? 0} ` + + `Olm's total memory changed from ${olmTotalMemory ?? -1} ` + `to ${currentTotalMemory} after executing ${method} method`, ); olmTotalMemory = currentTotalMemory; @@ -25,7 +30,7 @@ const currentUsedMemory = olm.get_used_memory(); if (currentUsedMemory !== olmUsedMemory) { console.error( - `Olm's used memory changed from ${olmUsedMemory ?? 0} ` + + `Olm's used memory changed from ${olmUsedMemory ?? -1} ` + `to ${currentUsedMemory} after executing ${method} method`, ); olmUsedMemory = currentUsedMemory; @@ -35,4 +40,36 @@ } } -export { verifyMemoryUsage }; +type OlmMemory = { + +total: ?number, + +used: ?number, +}; + +function getOlmMemory(): OlmMemory { + try { + const total = olm.get_total_memory(); + const used = olm.get_used_memory(); + return { total, used }; + } catch (e) { + console.error('Encountered error in getOlmMemory:', e); + return { total: null, used: null }; + } +} + +function compareAndLogOlmMemory(previous: OlmMemory, method: string) { + const current = getOlmMemory(); + if (current.total !== previous.total) { + console.error( + `Olm's total memory changed from ${previous.total ?? -1} ` + + `to ${current.total ?? -1} during execution of ${method} method`, + ); + } + if (current.used !== previous.used) { + console.error( + `Olm's used memory changed from ${previous.used ?? -1} ` + + `to ${current.used ?? -1} during execution of ${method} method`, + ); + } +} + +export { verifyMemoryUsage, getOlmMemory, compareAndLogOlmMemory };