diff --git a/keyserver/src/cron/compare-users.js b/keyserver/src/cron/compare-users.js new file mode 100644 index 000000000..cca750238 --- /dev/null +++ b/keyserver/src/cron/compare-users.js @@ -0,0 +1,30 @@ +// @flow + +import { deleteCookies } from '../deleters/cookie-deleters.js'; +import { fetchNativeCookieIDsForUserIDs } from '../fetchers/cookie-fetchers.js'; +import { fetchAllUserIDs } from '../fetchers/user-fetchers.js'; + +async function compareMySQLUsersToIdentityService(): Promise { + // eslint-disable-next-line no-unused-vars + const allUserIDs = await fetchAllUserIDs(); + // next we need to query identity service for two things: + // 1. users in identity that aren't here + // 2. users here that aren't in identity + const userMissingFromKeyserver = []; + const userMissingFromIdentity = []; + if (userMissingFromKeyserver.length > 0) { + console.warn( + "found users in identity service that aren't in MySQL! " + + JSON.stringify(userMissingFromKeyserver), + ); + } + if (userMissingFromIdentity.length === 0) { + return; + } + const cookieIDs = await fetchNativeCookieIDsForUserIDs( + userMissingFromIdentity, + ); + await deleteCookies(cookieIDs); +} + +export { compareMySQLUsersToIdentityService }; diff --git a/keyserver/src/cron/cron.js b/keyserver/src/cron/cron.js index 01f2379d2..ff88244f2 100644 --- a/keyserver/src/cron/cron.js +++ b/keyserver/src/cron/cron.js @@ -1,93 +1,108 @@ // @flow import cluster from 'cluster'; import schedule from 'node-schedule'; import { backupDB } from './backups.js'; +import { compareMySQLUsersToIdentityService } from './compare-users.js'; import { createDailyUpdatesThread } from './daily-updates.js'; import { updateAndReloadGeoipDB } from './update-geoip-db.js'; import { deleteOrphanedActivity } from '../deleters/activity-deleters.js'; import { deleteExpiredCookies } from '../deleters/cookie-deleters.js'; import { deleteOrphanedDays } from '../deleters/day-deleters.js'; import { deleteOrphanedEntries } from '../deleters/entry-deleters.js'; import { deleteOrphanedMemberships } from '../deleters/membership-deleters.js'; import { deleteOrphanedMessages } from '../deleters/message-deleters.js'; import { deleteOrphanedNotifs } from '../deleters/notif-deleters.js'; import { deleteOrphanedRevisions } from '../deleters/revision-deleters.js'; import { deleteOrphanedRoles } from '../deleters/role-deleters.js'; import { deleteOrphanedSessions, deleteOldWebSessions, } from '../deleters/session-deleters.js'; import { deleteStaleSIWENonceEntries } from '../deleters/siwe-nonce-deleters.js'; import { deleteInaccessibleThreads } from '../deleters/thread-deleters.js'; import { deleteExpiredUpdates } from '../deleters/update-deleters.js'; import { deleteUnassignedUploads } from '../deleters/upload-deleters.js'; if (cluster.isMaster) { schedule.scheduleJob( '30 3 * * *', // every day at 3:30 AM Pacific Time async () => { try { // Do everything one at a time to reduce load since we're in no hurry, // and since some queries depend on previous ones. await deleteExpiredCookies(); await deleteInaccessibleThreads(); await deleteOrphanedMemberships(); await deleteOrphanedDays(); await deleteOrphanedEntries(); await deleteOrphanedRevisions(); await deleteOrphanedRoles(); await deleteOrphanedMessages(); await deleteOrphanedActivity(); await deleteOrphanedNotifs(); await deleteOrphanedSessions(); await deleteOldWebSessions(); await deleteExpiredUpdates(); await deleteUnassignedUploads(); await deleteStaleSIWENonceEntries(); } catch (e) { console.warn('encountered error while trying to clean database', e); } }, ); schedule.scheduleJob( '0 */4 * * *', // every four hours async () => { try { await backupDB(); } catch (e) { console.warn('encountered error while trying to backup database', e); } }, ); schedule.scheduleJob( '0 3 ? * 0', // every Sunday at 3:00 AM GMT async () => { try { await updateAndReloadGeoipDB(); } catch (e) { console.warn( 'encountered error while trying to update GeoIP database', e, ); } }, ); schedule.scheduleJob( '0 0 * * *', // every day at midnight GMT async () => { try { if (process.env.RUN_COMM_TEAM_DEV_SCRIPTS) { // This is a job that the Comm internal team uses await createDailyUpdatesThread(); } } catch (e) { console.warn( 'encountered error while trying to create daily updates thread', e, ); } }, ); + schedule.scheduleJob( + '0 5 * * *', // every day at 5:00 AM GMT + async () => { + try { + await compareMySQLUsersToIdentityService(); + } catch (e) { + console.warn( + 'encountered error while trying to compare users table with ' + + 'identity service', + e, + ); + } + }, + ); } diff --git a/keyserver/src/deleters/cookie-deleters.js b/keyserver/src/deleters/cookie-deleters.js index 02b180d27..e9ae4c4b4 100644 --- a/keyserver/src/deleters/cookie-deleters.js +++ b/keyserver/src/deleters/cookie-deleters.js @@ -1,41 +1,46 @@ // @flow import invariant from 'invariant'; import { cookieLifetime } from 'lib/types/session-types.js'; import { dbQuery, SQL, mergeOrConditions } from '../database/database.js'; import type { SQLStatementType } from '../database/types.js'; async function deleteCookiesByConditions( conditions: $ReadOnlyArray, ) { invariant(conditions.length > 0, 'no conditions specified'); const conditionClause = mergeOrConditions(conditions); const query = SQL` DELETE c, i, s, si, u, iu, fo FROM cookies c LEFT JOIN ids i ON i.id = c.id LEFT JOIN sessions s ON s.cookie = c.id LEFT JOIN ids si ON si.id = s.id LEFT JOIN updates u ON u.target = c.id OR u.target = s.id LEFT JOIN ids iu ON iu.id = u.id LEFT JOIN focused fo ON fo.session = c.id OR fo.session = s.id WHERE `; query.append(conditionClause); await dbQuery(query); } async function deleteCookie(cookieID: string): Promise { const condition = SQL`c.id = ${cookieID}`; await deleteCookiesByConditions([condition]); } +async function deleteCookies(cookieIDs: $ReadOnlyArray): Promise { + const condition = SQL`c.id IN (${cookieIDs})`; + await deleteCookiesByConditions([condition]); +} + async function deleteExpiredCookies(): Promise { const earliestInvalidLastUpdate = Date.now() - cookieLifetime; const condition = SQL`c.last_used <= ${earliestInvalidLastUpdate}`; await deleteCookiesByConditions([condition]); } -export { deleteCookie, deleteExpiredCookies }; +export { deleteCookie, deleteCookies, deleteExpiredCookies };