diff --git a/keyserver/src/creators/report-creator.js b/keyserver/src/creators/report-creator.js index a1af1c020..86c4bac3f 100644 --- a/keyserver/src/creators/report-creator.js +++ b/keyserver/src/creators/report-creator.js @@ -1,238 +1,238 @@ // @flow import _isEqual from 'lodash/fp/isEqual'; import bots from 'lib/facts/bots'; import { filterRawEntryInfosByCalendarQuery, serverEntryInfosObject, } from 'lib/shared/entry-utils'; import { messageTypes } from 'lib/types/message-types'; import { type ReportCreationRequest, type ReportCreationResponse, type ThreadInconsistencyReportCreationRequest, type EntryInconsistencyReportCreationRequest, type UserInconsistencyReportCreationRequest, reportTypes, } from 'lib/types/report-types'; import { values } from 'lib/utils/objects'; import { sanitizeReduxReport, type ReduxCrashReport, } from 'lib/utils/sanitization'; import { dbQuery, SQL } from '../database/database'; import { fetchUsername } from '../fetchers/user-fetchers'; import { handleAsyncPromise } from '../responders/handlers'; import { createBotViewer } from '../session/bots'; import type { Viewer } from '../session/viewer'; -import { getCommAppURLFacts } from '../utils/urls'; +import { getAndAssertCommAppURLFacts } from '../utils/urls'; import createIDs from './id-creator'; import createMessages from './message-creator'; const { commbot } = bots; async function createReport( viewer: Viewer, request: ReportCreationRequest, ): Promise { const shouldIgnore = await ignoreReport(viewer, request); if (shouldIgnore) { return null; } const [id] = await createIDs('reports', 1); let type, report, time; if (request.type === reportTypes.THREAD_INCONSISTENCY) { ({ type, time, ...report } = request); time = time ? time : Date.now(); } else if (request.type === reportTypes.ENTRY_INCONSISTENCY) { ({ type, time, ...report } = request); } else if (request.type === reportTypes.MEDIA_MISSION) { ({ type, time, ...report } = request); } else if (request.type === reportTypes.USER_INCONSISTENCY) { ({ type, time, ...report } = request); } else { ({ type, ...report } = request); time = Date.now(); const redactedReduxReport: ReduxCrashReport = sanitizeReduxReport({ preloadedState: report.preloadedState, currentState: report.currentState, actions: report.actions, }); report = { ...report, ...redactedReduxReport, }; } const row = [ id, viewer.id, type, request.platformDetails.platform, JSON.stringify(report), time, ]; const query = SQL` INSERT INTO reports (id, user, type, platform, report, creation_time) VALUES ${[row]} `; await dbQuery(query); handleAsyncPromise(sendSquadbotMessage(viewer, request, id)); return { id }; } async function sendSquadbotMessage( viewer: Viewer, request: ReportCreationRequest, reportID: string, ): Promise { const canGenerateMessage = getSquadbotMessage(request, reportID, null); if (!canGenerateMessage) { return; } const username = await fetchUsername(viewer.id); const message = getSquadbotMessage(request, reportID, username); if (!message) { return; } const time = Date.now(); await createMessages(createBotViewer(commbot.userID), [ { type: messageTypes.TEXT, threadID: commbot.staffThreadID, creatorID: commbot.userID, time, text: message, }, ]); } async function ignoreReport( viewer: Viewer, request: ReportCreationRequest, ): Promise { // The below logic is to avoid duplicate inconsistency reports if ( request.type !== reportTypes.THREAD_INCONSISTENCY && request.type !== reportTypes.ENTRY_INCONSISTENCY ) { return false; } const { type, platformDetails, time } = request; if (!time) { return false; } const { platform } = platformDetails; const query = SQL` SELECT id FROM reports WHERE user = ${viewer.id} AND type = ${type} AND platform = ${platform} AND creation_time = ${time} `; const [result] = await dbQuery(query); return result.length !== 0; } function getSquadbotMessage( request: ReportCreationRequest, reportID: string, username: ?string, ): ?string { const name = username ? username : '[null]'; const { platformDetails } = request; const { platform, codeVersion } = platformDetails; const platformString = codeVersion ? `${platform} v${codeVersion}` : platform; if (request.type === reportTypes.ERROR) { - const { baseDomain, basePath } = getCommAppURLFacts(); + const { baseDomain, basePath } = getAndAssertCommAppURLFacts(); return ( `${name} got an error :(\n` + `using ${platformString}\n` + `${baseDomain}${basePath}download_error_report/${reportID}` ); } else if (request.type === reportTypes.THREAD_INCONSISTENCY) { const nonMatchingThreadIDs = getInconsistentThreadIDsFromReport(request); const nonMatchingString = [...nonMatchingThreadIDs].join(', '); return ( `system detected inconsistency for ${name}!\n` + `using ${platformString}\n` + `occurred during ${request.action.type}\n` + `thread IDs that are inconsistent: ${nonMatchingString}` ); } else if (request.type === reportTypes.ENTRY_INCONSISTENCY) { const nonMatchingEntryIDs = getInconsistentEntryIDsFromReport(request); const nonMatchingString = [...nonMatchingEntryIDs].join(', '); return ( `system detected inconsistency for ${name}!\n` + `using ${platformString}\n` + `occurred during ${request.action.type}\n` + `entry IDs that are inconsistent: ${nonMatchingString}` ); } else if (request.type === reportTypes.USER_INCONSISTENCY) { const nonMatchingUserIDs = getInconsistentUserIDsFromReport(request); const nonMatchingString = [...nonMatchingUserIDs].join(', '); return ( `system detected inconsistency for ${name}!\n` + `using ${platformString}\n` + `occurred during ${request.action.type}\n` + `user IDs that are inconsistent: ${nonMatchingString}` ); } else if (request.type === reportTypes.MEDIA_MISSION) { const mediaMissionJSON = JSON.stringify(request.mediaMission); const success = request.mediaMission.result.success ? 'media mission success!' : 'media mission failed :('; return `${name} ${success}\n` + mediaMissionJSON; } else { return null; } } function findInconsistentObjectKeys( first: { +[id: string]: O }, second: { +[id: string]: O }, ): Set { const nonMatchingIDs = new Set(); for (const id in first) { if (!_isEqual(first[id])(second[id])) { nonMatchingIDs.add(id); } } for (const id in second) { if (!first[id]) { nonMatchingIDs.add(id); } } return nonMatchingIDs; } function getInconsistentThreadIDsFromReport( request: ThreadInconsistencyReportCreationRequest, ): Set { const { pushResult, beforeAction } = request; return findInconsistentObjectKeys(beforeAction, pushResult); } function getInconsistentEntryIDsFromReport( request: EntryInconsistencyReportCreationRequest, ): Set { const { pushResult, beforeAction, calendarQuery } = request; const filteredBeforeAction = filterRawEntryInfosByCalendarQuery( serverEntryInfosObject(values(beforeAction)), calendarQuery, ); const filteredAfterAction = filterRawEntryInfosByCalendarQuery( serverEntryInfosObject(values(pushResult)), calendarQuery, ); return findInconsistentObjectKeys(filteredBeforeAction, filteredAfterAction); } function getInconsistentUserIDsFromReport( request: UserInconsistencyReportCreationRequest, ): Set { const { beforeStateCheck, afterStateCheck } = request; return findInconsistentObjectKeys(beforeStateCheck, afterStateCheck); } export default createReport; diff --git a/keyserver/src/fetchers/upload-fetchers.js b/keyserver/src/fetchers/upload-fetchers.js index 0139f2dec..05027cf69 100644 --- a/keyserver/src/fetchers/upload-fetchers.js +++ b/keyserver/src/fetchers/upload-fetchers.js @@ -1,120 +1,120 @@ // @flow import type { Media } from 'lib/types/media-types'; import { ServerError } from 'lib/utils/errors'; import { dbQuery, SQL } from '../database/database'; import type { Viewer } from '../session/viewer'; -import { getCommAppURLFacts } from '../utils/urls'; +import { getAndAssertCommAppURLFacts } from '../utils/urls'; type UploadInfo = { content: Buffer, mime: string, }; async function fetchUpload( viewer: Viewer, id: string, secret: string, ): Promise { const query = SQL` SELECT content, mime FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { content, mime } = row; return { content, mime }; } async function fetchUploadChunk( id: string, secret: string, pos: number, len: number, ): Promise { // We use pos + 1 because SQL is 1-indexed whereas js is 0-indexed const query = SQL` SELECT SUBSTRING(content, ${pos + 1}, ${len}) AS content, mime FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { content, mime } = row; return { content, mime, }; } // Returns total size in bytes. async function getUploadSize(id: string, secret: string): Promise { const query = SQL` SELECT LENGTH(content) AS length FROM uploads WHERE id = ${id} AND secret = ${secret} `; const [result] = await dbQuery(query); if (result.length === 0) { throw new ServerError('invalid_parameters'); } const [row] = result; const { length } = row; return length; } function getUploadURL(id: string, secret: string): string { - const { baseDomain, basePath } = getCommAppURLFacts(); + const { baseDomain, basePath } = getAndAssertCommAppURLFacts(); return `${baseDomain}${basePath}upload/${id}/${secret}`; } function mediaFromRow(row: Object): Media { const { uploadType: type, uploadSecret: secret } = row; const { width, height, loop } = row.uploadExtra; const id = row.uploadID.toString(); const dimensions = { width, height }; const uri = getUploadURL(id, secret); if (type === 'photo') { return { id, type: 'photo', uri, dimensions }; } else if (loop) { // $FlowFixMe add thumbnailID, thumbnailURI once they're in DB return { id, type: 'video', uri, dimensions, loop }; } else { // $FlowFixMe add thumbnailID, thumbnailURI once they're in DB return { id, type: 'video', uri, dimensions }; } } async function fetchMedia( viewer: Viewer, mediaIDs: $ReadOnlyArray, ): Promise<$ReadOnlyArray> { const query = SQL` SELECT id AS uploadID, secret AS uploadSecret, type AS uploadType, extra AS uploadExtra FROM uploads WHERE id IN (${mediaIDs}) AND uploader = ${viewer.id} AND container IS NULL `; const [result] = await dbQuery(query); return result.map(mediaFromRow); } export { fetchUpload, fetchUploadChunk, getUploadSize, getUploadURL, mediaFromRow, fetchMedia, }; diff --git a/keyserver/src/keyserver.js b/keyserver/src/keyserver.js index 9b2c7cb9d..41e3afe6e 100644 --- a/keyserver/src/keyserver.js +++ b/keyserver/src/keyserver.js @@ -1,161 +1,165 @@ // @flow import cluster from 'cluster'; import cookieParser from 'cookie-parser'; import express from 'express'; import expressWs from 'express-ws'; import os from 'os'; import './cron/cron'; import { migrate } from './database/migrations'; import { jsonEndpoints } from './endpoints'; import { emailSubscriptionResponder } from './responders/comm-landing-responders'; import { jsonHandler, httpGetHandler, downloadHandler, htmlHandler, uploadHandler, } from './responders/handlers'; import landingHandler from './responders/landing-handler'; import { errorReportDownloadResponder } from './responders/report-responders'; import { createNewVersionResponder, markVersionDeployedResponder, } from './responders/version-responders'; import { websiteResponder } from './responders/website-responders'; import { onConnection } from './socket/socket'; import { multerProcessor, multimediaUploadResponder, uploadDownloadResponder, } from './uploads/uploads'; import { prefetchAllURLFacts, getSquadCalURLFacts, getLandingURLFacts, getCommAppURLFacts, } from './utils/urls'; (async () => { await prefetchAllURLFacts(); const squadCalBaseRoutePath = getSquadCalURLFacts()?.baseRoutePath; - const landingBaseRoutePath = getLandingURLFacts().baseRoutePath; - const commAppBaseRoutePath = getCommAppURLFacts().baseRoutePath; + const landingBaseRoutePath = getLandingURLFacts()?.baseRoutePath; + const commAppBaseRoutePath = getCommAppURLFacts()?.baseRoutePath; const compiledFolderOptions = process.env.NODE_ENV === 'development' ? undefined : { maxAge: '1y', immutable: true }; if (cluster.isMaster) { const didMigrationsSucceed: boolean = await migrate(); if (!didMigrationsSucceed) { // The following line uses exit code 2 to ensure nodemon exits // in a dev environment, instead of restarting. Context provided // in https://github.com/remy/nodemon/issues/751 process.exit(2); } const cpuCount = os.cpus().length; for (let i = 0; i < cpuCount; i++) { cluster.fork(); } cluster.on('exit', () => cluster.fork()); } else { const server = express(); expressWs(server); server.use(express.json({ limit: '50mb' })); server.use(cookieParser()); const setupAppRouter = router => { router.use('/images', express.static('images')); router.use('/fonts', express.static('fonts')); router.use('/misc', express.static('misc')); router.use( '/.well-known', express.static( '.well-known', // Necessary for apple-app-site-association file { setHeaders: res => res.setHeader('Content-Type', 'application/json'), }, ), ); router.use( '/compiled', express.static('app_compiled', compiledFolderOptions), ); router.use('/', express.static('icons')); for (const endpoint in jsonEndpoints) { // $FlowFixMe Flow thinks endpoint is string const responder = jsonEndpoints[endpoint]; const expectCookieInvalidation = endpoint === 'log_out'; router.post( `/${endpoint}`, jsonHandler(responder, expectCookieInvalidation), ); } router.get( '/create_version/:deviceType/:codeVersion', httpGetHandler(createNewVersionResponder), ); router.get( '/mark_version_deployed/:deviceType/:codeVersion', httpGetHandler(markVersionDeployedResponder), ); router.get( '/download_error_report/:reportID', downloadHandler(errorReportDownloadResponder), ); router.get( '/upload/:uploadID/:secret', downloadHandler(uploadDownloadResponder), ); // $FlowFixMe express-ws has side effects that can't be typed router.ws('/ws', onConnection); router.get('*', htmlHandler(websiteResponder)); router.post( '/upload_multimedia', multerProcessor, uploadHandler(multimediaUploadResponder), ); }; // Note - the order of router declarations matters. On prod we have // squadCalBaseRoutePath configured to '/', which means it's a catch-all. If // we call server.use on squadCalRouter first, it will catch all requests // and prevent commAppRouter and landingRouter from working correctly. So we // make sure that squadCalRouter goes last - const landingRouter = express.Router(); - landingRouter.use('/images', express.static('images')); - landingRouter.use('/fonts', express.static('fonts')); - landingRouter.use( - '/compiled', - express.static('landing_compiled', compiledFolderOptions), - ); - landingRouter.use('/', express.static('landing_icons')); - landingRouter.post('/subscribe_email', emailSubscriptionResponder); - landingRouter.get('*', landingHandler); - server.use(landingBaseRoutePath, landingRouter); - - const commAppRouter = express.Router(); - setupAppRouter(commAppRouter); - server.use(commAppBaseRoutePath, commAppRouter); + if (landingBaseRoutePath) { + const landingRouter = express.Router(); + landingRouter.use('/images', express.static('images')); + landingRouter.use('/fonts', express.static('fonts')); + landingRouter.use( + '/compiled', + express.static('landing_compiled', compiledFolderOptions), + ); + landingRouter.use('/', express.static('landing_icons')); + landingRouter.post('/subscribe_email', emailSubscriptionResponder); + landingRouter.get('*', landingHandler); + server.use(landingBaseRoutePath, landingRouter); + } + + if (commAppBaseRoutePath) { + const commAppRouter = express.Router(); + setupAppRouter(commAppRouter); + server.use(commAppBaseRoutePath, commAppRouter); + } if (squadCalBaseRoutePath) { const squadCalRouter = express.Router(); setupAppRouter(squadCalRouter); server.use(squadCalBaseRoutePath, squadCalRouter); } server.listen(parseInt(process.env.PORT, 10) || 3000, 'localhost'); } })(); diff --git a/keyserver/src/responders/landing-handler.js b/keyserver/src/responders/landing-handler.js index 87cd91c27..1d4c5b506 100644 --- a/keyserver/src/responders/landing-handler.js +++ b/keyserver/src/responders/landing-handler.js @@ -1,166 +1,170 @@ // @flow import html from 'common-tags/lib/html'; import type { $Response, $Request } from 'express'; import fs from 'fs'; import * as React from 'react'; import ReactDOMServer from 'react-dom/server'; import { promisify } from 'util'; import { type LandingSSRProps } from '../landing/landing-ssr.react'; import { waitForStream } from '../utils/json-stream'; -import { getLandingURLFacts, clientPathFromRouterPath } from '../utils/urls'; +import { + getAndAssertLandingURLFacts, + clientPathFromRouterPath, +} from '../utils/urls'; import { getMessageForException } from './utils'; async function landingHandler(req: $Request, res: $Response) { try { await landingResponder(req, res); } catch (e) { console.warn(e); if (!res.headersSent) { res.status(500).send(getMessageForException(e)); } } } const access = promisify(fs.access); const googleFontsURL = 'https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@500&family=IBM+Plex+Sans:wght@400;500&display=swap'; const iaDuoFontsURL = 'fonts/duo.css'; const localFontsURL = 'fonts/local-fonts.css'; async function getDevFontURLs(): Promise<$ReadOnlyArray> { try { await access(localFontsURL); return [localFontsURL, iaDuoFontsURL]; } catch { return [googleFontsURL, iaDuoFontsURL]; } } type AssetInfo = { +jsURL: string, +fontURLs: $ReadOnlyArray, +cssInclude: string, }; let assetInfo: ?AssetInfo = null; async function getAssetInfo() { if (assetInfo) { return assetInfo; } if (process.env.NODE_ENV === 'development') { const fontURLs = await getDevFontURLs(); assetInfo = { jsURL: 'http://localhost:8082/dev.build.js', fontURLs, cssInclude: '', }; return assetInfo; } // $FlowFixMe landing/dist doesn't always exist const { default: assets } = await import('landing/dist/assets'); assetInfo = { jsURL: `compiled/${assets.browser.js}`, fontURLs: [googleFontsURL, iaDuoFontsURL], cssInclude: html` `, }; return assetInfo; } type LandingApp = React.ComponentType; let webpackCompiledRootComponent: ?LandingApp = null; async function getWebpackCompiledRootComponentForSSR() { if (webpackCompiledRootComponent) { return webpackCompiledRootComponent; } try { // $FlowFixMe landing/dist doesn't always exist const webpackBuild = await import('landing/dist/landing.build.cjs'); webpackCompiledRootComponent = webpackBuild.default.default; return webpackCompiledRootComponent; } catch { throw new Error( 'Could not load landing.build.cjs. ' + 'Did you forget to run `yarn dev` in the landing folder?', ); } } -const urlFacts = getLandingURLFacts(); -const { basePath } = urlFacts; const { renderToNodeStream } = ReactDOMServer; async function landingResponder(req: $Request, res: $Response) { const [{ jsURL, fontURLs, cssInclude }, LandingSSR] = await Promise.all([ getAssetInfo(), getWebpackCompiledRootComponentForSSR(), ]); const fontsInclude = fontURLs .map(url => ``) .join(''); + const urlFacts = getAndAssertLandingURLFacts(); + const { basePath } = urlFacts; + // prettier-ignore res.write(html` Comm ${fontsInclude} ${cssInclude}
`); // We remove trailing slash for `react-router` const routerBasename = basePath.replace(/\/$/, ''); const clientPath = clientPathFromRouterPath(req.url, urlFacts); const reactStream = renderToNodeStream( , ); reactStream.pipe(res, { end: false }); await waitForStream(reactStream); // prettier-ignore res.end(html`
`); } export default landingHandler; diff --git a/keyserver/src/utils/urls.js b/keyserver/src/utils/urls.js index f67463872..b652b6937 100644 --- a/keyserver/src/utils/urls.js +++ b/keyserver/src/utils/urls.js @@ -1,88 +1,98 @@ // @flow import invariant from 'invariant'; import { values } from 'lib/utils/objects'; export type AppURLFacts = { +baseDomain: string, +basePath: string, +https: boolean, +baseRoutePath: string, }; const sitesObj = Object.freeze({ a: 'landing', b: 'commapp', c: 'squadcal', }); export type Site = $Values; const sites: $ReadOnlyArray = values(sitesObj); const cachedURLFacts = new Map(); async function fetchURLFacts(site: Site): Promise { const cached = cachedURLFacts.get(site); if (cached !== undefined) { return cached; } try { // $FlowFixMe const urlFacts = await import(`../../facts/${site}_url`); if (!cachedURLFacts.has(site)) { cachedURLFacts.set(site, urlFacts.default); } } catch { if (!cachedURLFacts.has(site)) { cachedURLFacts.set(site, null); } } return cachedURLFacts.get(site); } async function prefetchAllURLFacts() { await Promise.all(sites.map(fetchURLFacts)); } function getSquadCalURLFacts(): ?AppURLFacts { return cachedURLFacts.get('squadcal'); } -function getCommAppURLFacts(): AppURLFacts { - const urlFacts = cachedURLFacts.get('commapp'); +function getCommAppURLFacts(): ?AppURLFacts { + return cachedURLFacts.get('commapp'); +} + +function getAndAssertCommAppURLFacts(): AppURLFacts { + const urlFacts = getCommAppURLFacts(); invariant(urlFacts, 'keyserver/facts/commapp_url.json missing'); return urlFacts; } function getAppURLFactsFromRequestURL(url: string): AppURLFacts { const commURLFacts = getCommAppURLFacts(); if (commURLFacts && url.startsWith(commURLFacts.baseRoutePath)) { return commURLFacts; } const squadCalURLFacts = getSquadCalURLFacts(); if (squadCalURLFacts) { return squadCalURLFacts; } invariant(false, 'request received but no URL facts are present'); } -function getLandingURLFacts(): AppURLFacts { - const urlFacts = cachedURLFacts.get('landing'); +function getLandingURLFacts(): ?AppURLFacts { + return cachedURLFacts.get('landing'); +} + +function getAndAssertLandingURLFacts(): AppURLFacts { + const urlFacts = getLandingURLFacts(); invariant(urlFacts, 'keyserver/facts/landing_url.json missing'); return urlFacts; } function clientPathFromRouterPath( routerPath: string, urlFacts: AppURLFacts, ): string { const { basePath } = urlFacts; return basePath + routerPath; } export { prefetchAllURLFacts, getSquadCalURLFacts, getCommAppURLFacts, + getAndAssertCommAppURLFacts, getLandingURLFacts, + getAndAssertLandingURLFacts, getAppURLFactsFromRequestURL, clientPathFromRouterPath, };