diff --git a/keyserver/src/cron/cron.js b/keyserver/src/cron/cron.js index 1ad4bb55d..d8b7420f7 100644 --- a/keyserver/src/cron/cron.js +++ b/keyserver/src/cron/cron.js @@ -1,168 +1,182 @@ // @flow import type { Account as OlmAccount } from '@commapp/olm'; import cluster from 'cluster'; import schedule from 'node-schedule'; import { backupDB } from './backups.js'; import { createDailyUpdatesThread } from './daily-updates.js'; +import { postMetrics } from './metrics.js'; import { postLeaderboard } from './phab-leaderboard.js'; import { updateAndReloadGeoipDB } from './update-geoip-db.js'; import { updateIdentityReservedUsernames } from './update-identity-reserved-usernames.js'; import { deleteOrphanedActivity } from '../deleters/activity-deleters.js'; import { deleteExpiredCookies } from '../deleters/cookie-deleters.js'; import { deleteOrphanedDays } from '../deleters/day-deleters.js'; import { deleteOrphanedEntries } from '../deleters/entry-deleters.js'; import { deleteOrphanedInviteLinks } from '../deleters/link-deleters.js'; import { deleteOrphanedMemberships } from '../deleters/membership-deleters.js'; import { deleteOrphanedMessages } from '../deleters/message-deleters.js'; import { deleteOrphanedNotifs } from '../deleters/notif-deleters.js'; import { deleteOrphanedRevisions } from '../deleters/revision-deleters.js'; import { deleteOrphanedRoles } from '../deleters/role-deleters.js'; import { deleteOrphanedSessions, deleteOldWebSessions, } from '../deleters/session-deleters.js'; import { deleteStaleSIWENonceEntries } from '../deleters/siwe-nonce-deleters.js'; import { deleteInaccessibleThreads } from '../deleters/thread-deleters.js'; import { deleteExpiredUpdates } from '../deleters/update-deleters.js'; import { deleteUnassignedUploads } from '../deleters/upload-deleters.js'; import { fetchCallUpdateOlmAccount } from '../updaters/olm-account-updater.js'; import { validateAndUploadAccountPrekeys } from '../utils/olm-utils.js'; import { isPrimaryNode, isAuxiliaryNode, } from '../utils/primary-secondary-utils.js'; import { synchronizeInviteLinksWithBlobs } from '../utils/synchronize-invite-links-with-blobs.js'; const { RUN_COMM_TEAM_DEV_SCRIPTS } = process.env; if (cluster.isMaster) { schedule.scheduleJob( '0 3 ? * 0', // every Sunday at 3:00 AM in the keyserver's timezone async () => { try { await updateAndReloadGeoipDB(); } catch (e) { console.warn( 'encountered error while trying to update GeoIP database', e, ); } }, ); if (isPrimaryNode) { schedule.scheduleJob( '30 3 * * *', // every day at 3:30 AM in the keyserver's timezone async () => { try { // Do everything one at a time to reduce load since we're in no hurry, // and since some queries depend on previous ones. await deleteExpiredCookies(); await deleteInaccessibleThreads(); await deleteOrphanedMemberships(); await deleteOrphanedDays(); await deleteOrphanedEntries(); await deleteOrphanedRevisions(); await deleteOrphanedRoles(); await deleteOrphanedMessages(); await deleteOrphanedActivity(); await deleteOrphanedNotifs(); await deleteOrphanedSessions(); await deleteOldWebSessions(); await deleteExpiredUpdates(); await deleteUnassignedUploads(); await deleteStaleSIWENonceEntries(); await deleteOrphanedInviteLinks(); } catch (e) { console.warn('encountered error while trying to clean database', e); } }, ); schedule.scheduleJob( '0 5 * * *', // every day at 5:00 AM in the keyserver's timezone async () => { try { await updateIdentityReservedUsernames(); } catch (e) { console.warn( 'encountered error while trying to update reserved usernames on ' + 'identity service', e, ); } }, ); schedule.scheduleJob( '0 0 * * *', // every day at midnight in the keyserver's timezone async () => { try { await fetchCallUpdateOlmAccount( 'content', (contentAccount: OlmAccount) => fetchCallUpdateOlmAccount( 'notifications', (notifAccount: OlmAccount) => validateAndUploadAccountPrekeys(contentAccount, notifAccount), ), ); } catch (e) { console.warn('encountered error while trying to validate prekeys', e); } }, ); schedule.scheduleJob( '0 2 * * *', // every day at 2:00 AM in the keyserver's timezone async () => { try { await synchronizeInviteLinksWithBlobs(); } catch (e) { console.warn( 'encountered an error while trying to synchronize invite links with blobs', e, ); } }, ); } if (isPrimaryNode || isAuxiliaryNode) { schedule.scheduleJob( '0 */4 * * *', // every four hours async () => { try { await backupDB(); } catch (e) { console.warn('encountered error while trying to backup database', e); } }, ); } if (RUN_COMM_TEAM_DEV_SCRIPTS && (isPrimaryNode || isAuxiliaryNode)) { schedule.scheduleJob( '0 0 * * *', // every day at midnight in the keyserver's timezone async () => { try { await createDailyUpdatesThread(); } catch (e) { console.warn( 'encountered error while trying to create daily updates thread', e, ); } }, ); schedule.scheduleJob( '0 0 8 * *', // 8th of every month at midnight in the keyserver's timezone async () => { try { await postLeaderboard(); } catch (e) { console.warn( 'encountered error while trying to post Phabricator leaderboard', e, ); } }, ); + schedule.scheduleJob( + '0 6 * * *', // every day at 6:00 AM in the keyserver's timezone + async () => { + try { + await postMetrics(); + } catch (e) { + console.warn( + 'encountered error while trying to post product metrics', + e, + ); + } + }, + ); } } diff --git a/keyserver/src/cron/metrics.js b/keyserver/src/cron/metrics.js new file mode 100644 index 000000000..953774b13 --- /dev/null +++ b/keyserver/src/cron/metrics.js @@ -0,0 +1,127 @@ +// @flow + +import bots from 'lib/facts/bots.js'; +import { messageTypes } from 'lib/types/message-types-enum.js'; + +import createMessages from '../creators/message-creator.js'; +import { dbQuery, SQL } from '../database/database.js'; +import { createScriptViewer } from '../session/scripts.js'; + +const metricsChannel = '80820870'; +const millisecondsPerDay = 24 * 60 * 60 * 1000; + +async function postMetrics() { + if (!process.env.RUN_COMM_TEAM_DEV_SCRIPTS) { + // This is a job that the Comm internal team uses + return; + } + + const oneDayAgo = Date.now() - millisecondsPerDay; + const thirtyDaysAgo = Date.now() - millisecondsPerDay * 30; + const [ + dailyActives, + monthlyActives, + oneWeekRetention, + twoWeekRetention, + retentionSinceLaunch, + ] = await Promise.all([ + getActiveCountSince(oneDayAgo), + getActiveCountSince(thirtyDaysAgo), + getRetention(7), + getRetention(14), + getRetentionSinceLaunch(), + ]); + + const metrics = { + 'DAUs': dailyActives, + 'MAUs': monthlyActives, + 'D7': oneWeekRetention, + 'D14': twoWeekRetention, + 'retention since launch': retentionSinceLaunch, + }; + const today = new Date().toLocaleString('default', { + day: 'numeric', + month: 'long', + year: 'numeric', + }); + const metricText = + `### Metrics for ${today}\n` + + '```\n' + + `${JSON.stringify(metrics, undefined, 2)}\n` + + '```'; + + const viewer = createScriptViewer(bots.commbot.userID); + const messageDatas = [ + { + type: messageTypes.TEXT, + threadID: metricsChannel, + creatorID: bots.commbot.userID, + time: Date.now(), + text: metricText, + }, + ]; + await createMessages(viewer, messageDatas); +} + +async function getActiveCountSince(time: number): Promise { + const [result] = await dbQuery(SQL` + SELECT COUNT(DISTINCT u.id) AS count + FROM users u + LEFT JOIN cookies c ON c.user = u.id + WHERE last_used IS NOT NULL AND last_used > ${time} + `); + const [row] = result; + return row.count; +} + +// Of the users that created their account N days ago, +// how many were active in the last day? +type RetentionResult = { +retainedCount: number, +totalCount: number }; +async function getRetention(daysAgo: number): Promise { + const startOfNDaysAgo = Date.now() - millisecondsPerDay * daysAgo; + const endOfNDaysAgo = Date.now() - millisecondsPerDay * (daysAgo - 1); + const [result] = await dbQuery(SQL` + SELECT u.id, MAX(c.last_used) AS lastUsed + FROM users u + LEFT JOIN cookies c ON c.user = u.id + WHERE u.creation_time >= ${startOfNDaysAgo} + AND u.creation_time < ${endOfNDaysAgo} + GROUP BY u.id + `); + + const totalCount = result.length; + + const oneDayAgo = Date.now() - millisecondsPerDay; + const retainedCount = result.filter( + ({ lastUsed }) => lastUsed > oneDayAgo, + ).length; + + return { retainedCount, totalCount }; +} + +// We're measuring users that signed up in the 7 days following launch. +// They count as retained if they've used Comm in the last day. +async function getRetentionSinceLaunch(): Promise { + const launchDate = new Date('2024-10-03'); + const launchDaysAgo = Math.ceil( + (Date.now() - launchDate.getTime()) / millisecondsPerDay, + ); + + const retentionPromises: Array> = []; + for (let i = 0; i < 7; i++) { + retentionPromises.push(getRetention(launchDaysAgo - i)); + } + + const totalRetentionResults = { + retainedCount: 0, + totalCount: 0, + }; + const retentionResults = await Promise.all(retentionPromises); + for (const retentionResult of retentionResults) { + totalRetentionResults.retainedCount += retentionResult.retainedCount; + totalRetentionResults.totalCount += retentionResult.totalCount; + } + return totalRetentionResults; +} + +export { postMetrics };