diff --git a/keyserver/src/cron/backups.js b/keyserver/src/cron/backups.js index 6dff25a3a..1fd0c4d22 100644 --- a/keyserver/src/cron/backups.js +++ b/keyserver/src/cron/backups.js @@ -1,216 +1,216 @@ // @flow import childProcess from 'child_process'; import dateFormat from 'dateformat'; import fs from 'fs'; import invariant from 'invariant'; import { ReReadable } from 'rereadable-stream'; import { PassThrough } from 'stream'; import { promisify } from 'util'; import zlib from 'zlib'; import { getDBConfig, type DBConfig } from '../database/db-config'; import { importJSON } from '../utils/import-json'; const readdir = promisify(fs.readdir); const lstat = promisify(fs.lstat); const unlink = promisify(fs.unlink); async function backupDB() { const [backupConfig, dbConfig] = await Promise.all([ - importJSON('facts/backups'), + importJSON({ folder: 'facts', name: 'backups' }), getDBConfig(), ]); if (!backupConfig || !backupConfig.enabled) { return; } const dateString = dateFormat('yyyy-mm-dd-HH:MM'); const filename = `comm.${dateString}.sql.gz`; const filePath = `${backupConfig.directory}/${filename}`; const rawStream = new PassThrough(); (async () => { try { await mysqldump(dbConfig, filename, rawStream, ['--no-data'], { end: false, }); } catch {} try { const ignoreReports = `--ignore-table=${dbConfig.database}.reports`; await mysqldump(dbConfig, filename, rawStream, [ '--no-create-info', ignoreReports, ]); } catch { rawStream.end(); } })(); const gzippedBuffer = new ReReadable(); rawStream .on('error', (e: Error) => { console.warn(`mysqldump stdout stream emitted error for ${filename}`, e); }) .pipe(zlib.createGzip()) .on('error', (e: Error) => { console.warn(`gzip transform stream emitted error for ${filename}`, e); }) .pipe(gzippedBuffer); try { await saveBackup(filename, filePath, gzippedBuffer); } catch (e) { console.warn(`saveBackup threw for ${filename}`, e); await unlink(filePath); } } function mysqldump( dbConfig: DBConfig, filename: string, rawStream: PassThrough, extraParams: $ReadOnlyArray, pipeParams?: { end?: boolean, ... }, ): Promise { const mysqlDump = childProcess.spawn( 'mysqldump', [ '-h', dbConfig.host, '-u', dbConfig.user, `-p${dbConfig.password}`, '--single-transaction', '--no-tablespaces', ...extraParams, dbConfig.database, ], { stdio: ['ignore', 'pipe', 'ignore'], }, ); const extraParamsString = extraParams.join(' '); return new Promise((resolve, reject) => { mysqlDump.on('error', (e: Error) => { console.warn( `error trying to spawn mysqldump ${extraParamsString} for ${filename}`, e, ); reject(e); }); mysqlDump.on('exit', (code: number | null, signal: string | null) => { if (signal !== null && signal !== undefined) { console.warn( `mysqldump ${extraParamsString} received signal ${signal} for ` + filename, ); reject(new Error(`mysqldump ${JSON.stringify({ code, signal })}`)); } else if (code !== null && code !== 0) { console.warn( `mysqldump ${extraParamsString} exited with code ${code} for ` + filename, ); reject(new Error(`mysqldump ${JSON.stringify({ code, signal })}`)); } resolve(); }); mysqlDump.stdout.pipe(rawStream, pipeParams); }); } async function saveBackup( filename: string, filePath: string, gzippedBuffer: ReReadable, retries: number = 2, ): Promise { try { await trySaveBackup(filename, filePath, gzippedBuffer); } catch (saveError) { if (saveError.code !== 'ENOSPC') { throw saveError; } if (!retries) { throw saveError; } try { await deleteOldestBackup(); } catch (deleteError) { if (deleteError.message === 'no_backups_left') { throw saveError; } else { throw deleteError; } } await saveBackup(filename, filePath, gzippedBuffer, retries - 1); } } const backupWatchFrequency = 60 * 1000; function trySaveBackup( filename: string, filePath: string, gzippedBuffer: ReReadable, ): Promise { const timeoutObject: { timeout: ?TimeoutID } = { timeout: null }; const setBackupTimeout = (alreadyWaited: number) => { timeoutObject.timeout = setTimeout(() => { const nowWaited = alreadyWaited + backupWatchFrequency; console.log( `writing backup for ${filename} has taken ${nowWaited}ms so far`, ); setBackupTimeout(nowWaited); }, backupWatchFrequency); }; setBackupTimeout(0); const writeStream = fs.createWriteStream(filePath); return new Promise((resolve, reject) => { gzippedBuffer .rewind() .pipe(writeStream) .on('finish', () => { clearTimeout(timeoutObject.timeout); resolve(); }) .on('error', (e: Error) => { clearTimeout(timeoutObject.timeout); console.warn(`write stream emitted error for ${filename}`, e); reject(e); }); }); } async function deleteOldestBackup() { - const backupConfig = await importJSON('facts/backups'); + const backupConfig = await importJSON({ folder: 'facts', name: 'backups' }); invariant(backupConfig, 'backupConfig should be non-null'); const files = await readdir(backupConfig.directory); let oldestFile; for (const file of files) { if (!file.endsWith('.sql.gz') || !file.startsWith('comm.')) { continue; } const stat = await lstat(`${backupConfig.directory}/${file}`); if (stat.isDirectory()) { continue; } if (!oldestFile || stat.mtime < oldestFile.mtime) { oldestFile = { file, mtime: stat.mtime }; } } if (!oldestFile) { throw new Error('no_backups_left'); } try { await unlink(`${backupConfig.directory}/${oldestFile.file}`); } catch (e) { // Check if it's already been deleted if (e.code !== 'ENOENT') { throw e; } } } export { backupDB }; diff --git a/keyserver/src/cron/update-geoip-db.js b/keyserver/src/cron/update-geoip-db.js index ce84032d7..7e2281bb3 100644 --- a/keyserver/src/cron/update-geoip-db.js +++ b/keyserver/src/cron/update-geoip-db.js @@ -1,59 +1,62 @@ // @flow import childProcess from 'child_process'; import cluster from 'cluster'; import geoip from 'geoip-lite'; import { handleAsyncPromise } from '../responders/handlers'; import { importJSON } from '../utils/import-json'; async function updateGeoipDB(): Promise { - const geoipLicense = await importJSON('secrets/geoip_license'); + const geoipLicense = await importJSON({ + folder: 'secrets', + name: 'geoip_license', + }); if (!geoipLicense) { console.log('no keyserver/secrets/geoip_license.json so skipping update'); return; } await spawnUpdater(geoipLicense); } function spawnUpdater(geoipLicense: { key: string }): Promise { const spawned = childProcess.spawn(process.execPath, [ '../node_modules/geoip-lite/scripts/updatedb.js', `license_key=${geoipLicense.key}`, ]); return new Promise((resolve, reject) => { spawned.on('error', reject); spawned.on('exit', () => resolve()); }); } function reloadGeoipDB(): Promise { return new Promise(resolve => geoip.reloadData(resolve)); } type IPCMessage = { type: 'geoip_reload', }; const reloadMessage: IPCMessage = { type: 'geoip_reload' }; async function updateAndReloadGeoipDB(): Promise { await updateGeoipDB(); await reloadGeoipDB(); if (!cluster.isMaster) { return; } for (const id in cluster.workers) { cluster.workers[Number(id)].send(reloadMessage); } } if (!cluster.isMaster) { process.on('message', (ipcMessage: IPCMessage) => { if (ipcMessage.type === 'geoip_reload') { handleAsyncPromise(reloadGeoipDB()); } }); } export { updateGeoipDB, updateAndReloadGeoipDB }; diff --git a/keyserver/src/database/db-config.js b/keyserver/src/database/db-config.js index 45292d271..36bd45a81 100644 --- a/keyserver/src/database/db-config.js +++ b/keyserver/src/database/db-config.js @@ -1,38 +1,41 @@ // @flow import invariant from 'invariant'; import { importJSON } from '../utils/import-json'; export type DBConfig = { +host: string, +user: string, +password: string, +database: string, }; let dbConfig; async function getDBConfig(): Promise { if (dbConfig !== undefined) { return dbConfig; } if ( process.env.COMM_MYSQL_DATABASE && process.env.COMM_MYSQL_USER && process.env.COMM_MYSQL_PASSWORD ) { dbConfig = { host: process.env.COMM_MYSQL_HOST || 'localhost', user: process.env.COMM_MYSQL_USER, password: process.env.COMM_MYSQL_PASSWORD, database: process.env.COMM_MYSQL_DATABASE, }; } else { - const importedDBConfig = await importJSON('secrets/db_config'); + const importedDBConfig = await importJSON({ + folder: 'secrets', + name: 'db_config', + }); invariant(importedDBConfig, 'DB config missing'); dbConfig = importedDBConfig; } return dbConfig; } export { getDBConfig }; diff --git a/keyserver/src/push/providers.js b/keyserver/src/push/providers.js index fd33d1bf9..ff5191326 100644 --- a/keyserver/src/push/providers.js +++ b/keyserver/src/push/providers.js @@ -1,91 +1,91 @@ // @flow import apn from '@parse/node-apn'; import type { Provider as APNProvider } from '@parse/node-apn'; import fcmAdmin from 'firebase-admin'; import type { FirebaseApp } from 'firebase-admin'; import invariant from 'invariant'; import { importJSON } from '../utils/import-json'; type APNPushProfile = 'apn_config' | 'comm_apn_config'; function getAPNPushProfileForCodeVersion(codeVersion: ?number): APNPushProfile { return codeVersion && codeVersion >= 87 ? 'comm_apn_config' : 'apn_config'; } type FCMPushProfile = 'fcm_config' | 'comm_fcm_config'; function getFCMPushProfileForCodeVersion(codeVersion: ?number): FCMPushProfile { return codeVersion && codeVersion >= 87 ? 'comm_fcm_config' : 'fcm_config'; } const cachedAPNProviders = new Map(); async function getAPNProvider(profile: APNPushProfile): Promise { const provider = cachedAPNProviders.get(profile); if (provider !== undefined) { return provider; } try { - const apnConfig = await importJSON(`secrets/${profile}`); + const apnConfig = await importJSON({ folder: 'secrets', name: profile }); invariant(apnConfig, `APN config missing for ${profile}`); if (!cachedAPNProviders.has(profile)) { cachedAPNProviders.set(profile, new apn.Provider(apnConfig)); } } catch { if (!cachedAPNProviders.has(profile)) { cachedAPNProviders.set(profile, null); } } return cachedAPNProviders.get(profile); } const cachedFCMProviders = new Map(); async function getFCMProvider(profile: FCMPushProfile): Promise { const provider = cachedFCMProviders.get(profile); if (provider !== undefined) { return provider; } try { - const fcmConfig = await importJSON(`secrets/${profile}`); + const fcmConfig = await importJSON({ folder: 'secrets', name: profile }); invariant(fcmConfig, `FCM config missed for ${profile}`); if (!cachedFCMProviders.has(profile)) { cachedFCMProviders.set( profile, fcmAdmin.initializeApp( { credential: fcmAdmin.credential.cert(fcmConfig), }, profile, ), ); } } catch { if (!cachedFCMProviders.has(profile)) { cachedFCMProviders.set(profile, null); } } return cachedFCMProviders.get(profile); } function endFirebase() { fcmAdmin.apps?.forEach(app => app?.delete()); } function endAPNs() { for (const provider of cachedAPNProviders.values()) { provider?.shutdown(); } } function getAPNsNotificationTopic(codeVersion: ?number): string { return codeVersion && codeVersion >= 87 ? 'app.comm' : 'org.squadcal.app'; } export { getAPNPushProfileForCodeVersion, getFCMPushProfileForCodeVersion, getAPNProvider, getFCMProvider, endFirebase, endAPNs, getAPNsNotificationTopic, }; diff --git a/keyserver/src/utils/import-json.js b/keyserver/src/utils/import-json.js index 781f9a694..b890c731a 100644 --- a/keyserver/src/utils/import-json.js +++ b/keyserver/src/utils/import-json.js @@ -1,34 +1,53 @@ // @flow +type ConfigName = { + +folder: 'secrets' | 'facts', + +name: string, +}; + +function getKeyForConfigName(configName: ConfigName): string { + return `${configName.folder}_${configName.name}`; +} + +function getPathForConfigName(configName: ConfigName): string { + return `${configName.folder}/${configName.name}.json`; +} + const cachedJSON = new Map(); -async function importJSON(path: string): Promise { - const cached = cachedJSON.get(path); +async function importJSON(configName: ConfigName): Promise { + const key = getKeyForConfigName(configName); + const cached = cachedJSON.get(key); if (cached !== undefined) { return cached; } - const json = await getJSON(path); - if (!cachedJSON.has(path)) { - cachedJSON.set(path, json); + const json = await getJSON(configName); + if (!cachedJSON.has(key)) { + cachedJSON.set(key, json); } - return cachedJSON.get(path); + return cachedJSON.get(key); } -async function getJSON(path: string): Promise { - const fromEnv = process.env[`COMM_JSONCONFIG_${path}`]; +async function getJSON(configName: ConfigName): Promise { + const key = getKeyForConfigName(configName); + const fromEnv = process.env[`COMM_JSONCONFIG_${key}`]; if (fromEnv) { try { return JSON.parse(fromEnv); } catch (e) { - console.log(`failed to parse JSON from env for ${path}`, e); + console.log( + `failed to parse JSON from env for ${JSON.stringify(configName)}`, + e, + ); } } + const path = getPathForConfigName(configName); try { // $FlowFixMe const importedJSON = await import(`../../${path}`); return importedJSON.default; } catch { return null; } } export { importJSON }; diff --git a/keyserver/src/utils/olm-utils.js b/keyserver/src/utils/olm-utils.js index 67eaf6026..e6bc9ec0b 100644 --- a/keyserver/src/utils/olm-utils.js +++ b/keyserver/src/utils/olm-utils.js @@ -1,18 +1,18 @@ // @flow import invariant from 'invariant'; import { importJSON } from './import-json'; type OlmConfig = { +picklingKey: string, +pickledAccount: string, }; async function getOlmConfig(): Promise { - const olmConfig = await importJSON('secrets/olm_config'); + const olmConfig = await importJSON({ folder: 'secrets', name: 'olm_config' }); invariant(olmConfig, 'OLM config missing'); return olmConfig; } export { getOlmConfig }; diff --git a/keyserver/src/utils/urls.js b/keyserver/src/utils/urls.js index f2fa36f02..f7dc5f94e 100644 --- a/keyserver/src/utils/urls.js +++ b/keyserver/src/utils/urls.js @@ -1,91 +1,94 @@ // @flow import invariant from 'invariant'; import { values } from 'lib/utils/objects'; import { importJSON } from './import-json'; export type AppURLFacts = { +baseDomain: string, +basePath: string, +https: boolean, +baseRoutePath: string, }; const sitesObj = Object.freeze({ a: 'landing', b: 'commapp', c: 'squadcal', }); export type Site = $Values; const sites: $ReadOnlyArray = values(sitesObj); const cachedURLFacts = new Map(); async function fetchURLFacts(site: Site): Promise { const existing = cachedURLFacts.get(site); if (existing !== undefined) { return existing; } - const urlFacts: ?AppURLFacts = await importJSON(`facts/${site}_url`); + const urlFacts: ?AppURLFacts = await importJSON({ + folder: 'facts', + name: `${site}_url`, + }); cachedURLFacts.set(site, urlFacts); return urlFacts; } async function prefetchAllURLFacts() { await Promise.all(sites.map(fetchURLFacts)); } function getSquadCalURLFacts(): ?AppURLFacts { return cachedURLFacts.get('squadcal'); } function getCommAppURLFacts(): ?AppURLFacts { return cachedURLFacts.get('commapp'); } function getAndAssertCommAppURLFacts(): AppURLFacts { const urlFacts = getCommAppURLFacts(); invariant(urlFacts, 'keyserver/facts/commapp_url.json missing'); return urlFacts; } function getAppURLFactsFromRequestURL(url: string): AppURLFacts { const commURLFacts = getCommAppURLFacts(); if (commURLFacts && url.startsWith(commURLFacts.baseRoutePath)) { return commURLFacts; } const squadCalURLFacts = getSquadCalURLFacts(); if (squadCalURLFacts) { return squadCalURLFacts; } invariant(false, 'request received but no URL facts are present'); } function getLandingURLFacts(): ?AppURLFacts { return cachedURLFacts.get('landing'); } function getAndAssertLandingURLFacts(): AppURLFacts { const urlFacts = getLandingURLFacts(); invariant(urlFacts, 'keyserver/facts/landing_url.json missing'); return urlFacts; } function clientPathFromRouterPath( routerPath: string, urlFacts: AppURLFacts, ): string { const { basePath } = urlFacts; return basePath + routerPath; } export { prefetchAllURLFacts, getSquadCalURLFacts, getCommAppURLFacts, getAndAssertCommAppURLFacts, getLandingURLFacts, getAndAssertLandingURLFacts, getAppURLFactsFromRequestURL, clientPathFromRouterPath, };