diff --git a/.eslintignore b/.eslintignore --- a/.eslintignore +++ b/.eslintignore @@ -6,17 +6,17 @@ web/dist web/flow-typed web/node_modules -server/app_compiled -server/landing_compiled -server/dist -server/secrets -server/facts -server/fonts -server/flow-typed -server/node_modules -server/src/landing -server/src/lib -server/src/web +keyserver/app_compiled +keyserver/landing_compiled +keyserver/dist +keyserver/secrets +keyserver/facts +keyserver/fonts +keyserver/flow-typed +keyserver/node_modules +keyserver/src/landing +keyserver/src/lib +keyserver/src/web native/flow-typed native/node_modules native/codegen/dist diff --git a/.eslintrc.json b/.eslintrc.json --- a/.eslintrc.json +++ b/.eslintrc.json @@ -51,6 +51,6 @@ "version": "detect" }, "import/ignore": ["react-native"], - "import/internal-regex": "^(lib|native|server|web)/" + "import/internal-regex": "^(lib|native|keyserver|web)/" } } diff --git a/.github/workflows/android_ci.yml b/.github/workflows/android_ci.yml --- a/.github/workflows/android_ci.yml +++ b/.github/workflows/android_ci.yml @@ -7,7 +7,7 @@ - 'landing/**' - 'web/**' - 'docs/**' - - 'server/**' + - 'keyserver/**' jobs: build: diff --git a/.github/workflows/eslint_flow_jest.yml b/.github/workflows/eslint_flow_jest.yml --- a/.github/workflows/eslint_flow_jest.yml +++ b/.github/workflows/eslint_flow_jest.yml @@ -23,8 +23,8 @@ working-directory: ./lib run: ./node_modules/.bin/flow - - name: '[server] flow' - working-directory: ./server + - name: '[keyserver] flow' + working-directory: ./keyserver run: | mkdir secrets touch secrets/db_config.json diff --git a/.github/workflows/ios_ci.yml b/.github/workflows/ios_ci.yml --- a/.github/workflows/ios_ci.yml +++ b/.github/workflows/ios_ci.yml @@ -7,7 +7,7 @@ - 'landing/**' - 'web/**' - 'docs/**' - - 'server/**' + - 'keyserver/**' jobs: build: diff --git a/.github/workflows/services_ci.yml b/.github/workflows/services_ci.yml --- a/.github/workflows/services_ci.yml +++ b/.github/workflows/services_ci.yml @@ -7,7 +7,7 @@ - 'landing/**' - 'web/**' - 'docs/**' - - 'server/**' + - 'keyserver/**' jobs: build: diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -6,9 +6,9 @@ native/cpp/CommonCpp/CryptoTools/opaque-ke-cxx/target web/node_modules web/dist -server/dist -server/node_modules -server/secrets -server/facts +keyserver/dist +keyserver/node_modules +keyserver/secrets +keyserver/facts .eslintcache .vscode diff --git a/.lintstagedrc.js b/.lintstagedrc.js --- a/.lintstagedrc.js +++ b/.lintstagedrc.js @@ -22,8 +22,8 @@ '{native,lib}/**/*.js': function nativeFlow(files) { return 'yarn workspace native flow --quiet'; }, - '{server,web,lib}/**/*.js': function serverFlow(files) { - return 'yarn workspace server flow --quiet'; + '{keyserver,web,lib}/**/*.js': function keyServerFlow(files) { + return 'yarn workspace keyserver flow --quiet'; }, '{landing,lib}/**/*.js': function landingFlow(files) { return 'yarn workspace landing flow --quiet'; diff --git a/.prettierignore b/.prettierignore --- a/.prettierignore +++ b/.prettierignore @@ -3,17 +3,17 @@ lib/flow-typed web/dist web/flow-typed -server/app_compiled -server/landing_compiled -server/dist -server/secrets -server/facts -server/images -server/fonts -server/flow-typed -server/src/landing -server/src/lib -server/src/web +keyserver/app_compiled +keyserver/landing_compiled +keyserver/dist +keyserver/secrets +keyserver/facts +keyserver/images +keyserver/fonts +keyserver/flow-typed +keyserver/src/landing +keyserver/src/lib +keyserver/src/web native/android native/flow-typed native/ios diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The whole project is written in Flow-typed Javascript. The code is organized in a monorepo structure using Yarn Workspaces. - `native` contains the code for the React Native app, which supports both iOS and Android. -- `server` contains the code for the Node/Express server. +- `keyserver` contains the code for the Node/Express server. - `web` contains the code for the React desktop website. - `landing` contains the code for the [Comm landing page](https://comm.app). - `lib` contains code that is shared across multiple other workspaces, including most of the Redux stack that is shared across native/web. diff --git a/keyserver/loader.mjs b/keyserver/loader.mjs new file mode 100644 --- /dev/null +++ b/keyserver/loader.mjs @@ -0,0 +1,19 @@ +// @flow +const localPackages = ['landing', 'lib', 'web']; + +async function resolve(specifier, context, defaultResolve) { + const defaultResult = defaultResolve(specifier, context, defaultResolve); + + // Special hack to use Babel-transpiled lib and web + if (localPackages.some(pkg => specifier.startsWith(`${pkg}/`))) { + const url = defaultResult.url.replace( + specifier, + `keyserver/dist/${specifier}`, + ); + return { url }; + } + + return defaultResult; +} + +export { resolve }; diff --git a/keyserver/package.json b/keyserver/package.json new file mode 100644 --- /dev/null +++ b/keyserver/package.json @@ -0,0 +1,98 @@ +{ + "name": "keyserver", + "version": "0.0.1", + "type": "module", + "private": true, + "license": "BSD-3-Clause", + "main": "dist/keyserver", + "scripts": { + "clean": "rm -rf dist/ && rm -rf node_modules/ && mkdir dist", + "babel-build": "yarn --silent babel src/ --out-dir dist/ --config-file ./babel.config.cjs --verbose --ignore 'src/landing/flow-typed','src/landing/node_modules','src/landing/package.json','src/lib/flow-typed','src/lib/node_modules','src/lib/package.json','src/web/flow-typed','src/web/node_modules','src/web/package.json','src/web/dist','src/web/webpack.config.js','src/web/account-bar.react.js','src/web/app.react.js','src/web/calendar','src/web/chat','src/web/flow','src/web/loading-indicator.react.js','src/web/modals','src/web/root.js','src/web/router-history.js','src/web/script.js','src/web/selectors/chat-selectors.js','src/web/selectors/entry-selectors.js','src/web/splash','src/web/vector-utils.js','src/web/vectors.react.js'", + "rsync": "rsync -rLpmuv --exclude '*/package.json' --exclude '*/node_modules/*' --include '*.json' --include '*.cjs' --exclude '*.*' src/ dist/", + "prod-build": "yarn babel-build && yarn rsync && yarn update-geoip", + "update-geoip": "yarn script dist/scripts/update-geoip.js", + "prod": "node --trace-warnings --experimental-json-modules --loader=./loader.mjs --experimental-specifier-resolution=node dist/keyserver", + "dev-rsync": "yarn --silent chokidar --initial --silent -s 'src/**/*.json' 'src/**/*.cjs' -c 'yarn rsync > /dev/null 2>&1'", + "dev": "yarn concurrently --names=\"BABEL,RSYNC,NODEM\" -c \"bgBlue.bold,bgMagenta.bold,bgGreen.bold\" \"yarn babel-build --watch\" \"yarn dev-rsync\" \". bash/source-nvm.sh && NODE_ENV=development nodemon -e js,json,cjs --watch dist --experimental-json-modules --loader=./loader.mjs --experimental-specifier-resolution=node dist/keyserver\"", + "script": ". bash/source-nvm.sh && NODE_ENV=development node --experimental-json-modules --loader=./loader.mjs --experimental-specifier-resolution=node", + "test": "jest" + }, + "devDependencies": { + "@babel/cli": "^7.13.14", + "@babel/core": "^7.13.14", + "@babel/node": "^7.13.13", + "@babel/plugin-proposal-class-properties": "^7.13.0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", + "@babel/plugin-proposal-object-rest-spread": "^7.13.8", + "@babel/plugin-proposal-optional-chaining": "^7.13.12", + "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-transform-runtime": "^7.13.10", + "@babel/preset-env": "^7.13.12", + "@babel/preset-flow": "^7.13.13", + "@babel/preset-react": "^7.13.13", + "babel-jest": "^26.6.3", + "chokidar-cli": "^2.1.0", + "concurrently": "^5.3.0", + "flow-bin": "^0.158.0", + "flow-typed": "^3.2.1", + "jest": "^26.6.3", + "nodemon": "^2.0.4" + }, + "dependencies": { + "@babel/runtime": "^7.13.10", + "@grpc/grpc-js": "^1.4.6", + "@matrix-org/olm": "3.2.4", + "@parse/node-apn": "^3.2.0", + "@vingle/bmp-js": "^0.2.5", + "JSONStream": "^1.3.5", + "common-tags": "^1.7.2", + "cookie-parser": "^1.4.3", + "dateformat": "^3.0.3", + "express": "^4.17.1", + "express-ws": "^4.0.0", + "firebase-admin": "^10.0.1", + "geoip-lite": "^1.4.0", + "invariant": "^2.2.4", + "landing": "0.0.1", + "lib": "0.0.1", + "lodash": "^4.17.21", + "multer": "^1.4.1", + "mysql2": "^1.5.1", + "node-schedule": "^1.3.0", + "nodemailer": "^6.6.1", + "react": "17.0.2", + "react-dom": "17.0.2", + "react-html-email": "^3.0.0", + "react-redux": "^7.1.1", + "react-router": "^5.2.0", + "redis": "^3.1.1", + "redux": "^4.0.4", + "replacestream": "^4.0.3", + "rereadable-stream": "^1.4.5", + "sharp": "^0.29.3", + "sql-template-strings": "^2.2.2", + "stream-combiner": "^0.2.2", + "tcomb": "^3.2.29", + "twin-bcrypt": "^2.1.1", + "uuid": "^3.3.3", + "web": "0.0.1" + }, + "optionalDependencies": { + "bufferutil": "^4.0.5", + "utf-8-validate": "^5.0.7" + }, + "nodemonConfig": { + "delay": "200" + }, + "jest": { + "roots": [ + "/src" + ], + "transform": { + "\\.js$": "babel-jest" + }, + "transformIgnorePatterns": [ + "/node_modules/(?!@babel/runtime)" + ] + } +} diff --git a/keyserver/src/cron/update-geoip-db.js b/keyserver/src/cron/update-geoip-db.js new file mode 100644 --- /dev/null +++ b/keyserver/src/cron/update-geoip-db.js @@ -0,0 +1,77 @@ +// @flow + +import childProcess from 'child_process'; +import cluster from 'cluster'; +import geoip from 'geoip-lite'; + +import { handleAsyncPromise } from '../responders/handlers'; + +let cachedGeoipLicense = undefined; +async function getGeoipLicense() { + if (cachedGeoipLicense !== undefined) { + return cachedGeoipLicense; + } + try { + // $FlowFixMe + const geoipLicenseImport = await import('../../secrets/geoip_license'); + if (cachedGeoipLicense === undefined) { + cachedGeoipLicense = geoipLicenseImport.default; + } + } catch { + if (cachedGeoipLicense === undefined) { + cachedGeoipLicense = null; + } + } + return cachedGeoipLicense; +} + +async function updateGeoipDB(): Promise { + const geoipLicense = await getGeoipLicense(); + if (!geoipLicense) { + console.log('no keyserver/secrets/geoip_license.json so skipping update'); + return; + } + await spawnUpdater(geoipLicense); +} + +function spawnUpdater(geoipLicense: { key: string }): Promise { + const spawned = childProcess.spawn(process.execPath, [ + '../node_modules/geoip-lite/scripts/updatedb.js', + `license_key=${geoipLicense.key}`, + ]); + return new Promise((resolve, reject) => { + spawned.on('error', reject); + spawned.on('exit', () => resolve()); + }); +} + +function reloadGeoipDB(): Promise { + return new Promise(resolve => geoip.reloadData(resolve)); +} + +type IPCMessage = { + type: 'geoip_reload', +}; +const reloadMessage: IPCMessage = { type: 'geoip_reload' }; + +async function updateAndReloadGeoipDB(): Promise { + await updateGeoipDB(); + await reloadGeoipDB(); + + if (!cluster.isMaster) { + return; + } + for (const id in cluster.workers) { + cluster.workers[Number(id)].send(reloadMessage); + } +} + +if (!cluster.isMaster) { + process.on('message', (ipcMessage: IPCMessage) => { + if (ipcMessage.type === 'geoip_reload') { + handleAsyncPromise(reloadGeoipDB()); + } + }); +} + +export { updateGeoipDB, updateAndReloadGeoipDB }; diff --git a/keyserver/src/push/utils.js b/keyserver/src/push/utils.js new file mode 100644 --- /dev/null +++ b/keyserver/src/push/utils.js @@ -0,0 +1,199 @@ +// @flow + +import apn from '@parse/node-apn'; +import type { ResponseFailure } from '@parse/node-apn'; +import type { FirebaseApp, FirebaseError } from 'firebase-admin'; +import invariant from 'invariant'; + +import { threadSubscriptions } from 'lib/types/subscription-types'; +import { threadPermissions } from 'lib/types/thread-types'; + +import { dbQuery, SQL } from '../database/database'; +import { + getAPNPushProfileForCodeVersion, + getFCMPushProfileForCodeVersion, + getAPNProvider, + getFCMProvider, +} from './providers'; + +const fcmTokenInvalidationErrors = new Set([ + 'messaging/registration-token-not-registered', + 'messaging/invalid-registration-token', +]); +const apnTokenInvalidationErrorCode = 410; +const apnBadRequestErrorCode = 400; +const apnBadTokenErrorString = 'BadDeviceToken'; + +type APNPushResult = + | { +success: true } + | { + +errors: $ReadOnlyArray, + +invalidTokens?: $ReadOnlyArray, + }; +async function apnPush({ + notification, + deviceTokens, + codeVersion, +}: { + +notification: apn.Notification, + +deviceTokens: $ReadOnlyArray, + +codeVersion: ?number, +}): Promise { + const pushProfile = getAPNPushProfileForCodeVersion(codeVersion); + const apnProvider = await getAPNProvider(pushProfile); + if (!apnProvider && process.env.NODE_ENV === 'development') { + console.log(`no keyserver/secrets/${pushProfile}.json so ignoring notifs`); + return { success: true }; + } + invariant(apnProvider, `keyserver/secrets/${pushProfile}.json should exist`); + const result = await apnProvider.send(notification, deviceTokens); + const errors = []; + const invalidTokens = []; + for (const error of result.failed) { + errors.push(error); + /* eslint-disable eqeqeq */ + if ( + error.status == apnTokenInvalidationErrorCode || + (error.status == apnBadRequestErrorCode && + error.response.reason === apnBadTokenErrorString) + ) { + invalidTokens.push(error.device); + } + /* eslint-enable eqeqeq */ + } + if (invalidTokens.length > 0) { + return { errors, invalidTokens }; + } else if (errors.length > 0) { + return { errors }; + } else { + return { success: true }; + } +} + +type FCMPushResult = { + +success?: true, + +fcmIDs?: $ReadOnlyArray, + +errors?: $ReadOnlyArray, + +invalidTokens?: $ReadOnlyArray, +}; +async function fcmPush({ + notification, + deviceTokens, + collapseKey, + codeVersion, +}: { + +notification: Object, + +deviceTokens: $ReadOnlyArray, + +codeVersion: ?number, + +collapseKey?: ?string, +}): Promise { + const pushProfile = getFCMPushProfileForCodeVersion(codeVersion); + const fcmProvider = await getFCMProvider(pushProfile); + if (!fcmProvider && process.env.NODE_ENV === 'development') { + console.log(`no keyserver/secrets/${pushProfile}.json so ignoring notifs`); + return { success: true }; + } + invariant(fcmProvider, `keyserver/secrets/${pushProfile}.json should exist`); + const options: Object = { + priority: 'high', + }; + if (collapseKey) { + options.collapseKey = collapseKey; + } + // firebase-admin is extremely barebones and has a lot of missing or poorly + // thought-out functionality. One of the issues is that if you send a + // multicast messages and one of the device tokens is invalid, the resultant + // won't explain which of the device tokens is invalid. So we're forced to + // avoid the multicast functionality and call it once per deviceToken. + const promises = []; + for (const deviceToken of deviceTokens) { + promises.push( + fcmSinglePush(fcmProvider, notification, deviceToken, options), + ); + } + const pushResults = await Promise.all(promises); + + const errors = []; + const ids = []; + const invalidTokens = []; + for (let i = 0; i < pushResults.length; i++) { + const pushResult = pushResults[i]; + for (const error of pushResult.errors) { + errors.push(error); + if (fcmTokenInvalidationErrors.has(error.errorInfo.code)) { + invalidTokens.push(deviceTokens[i]); + } + } + for (const id of pushResult.fcmIDs) { + ids.push(id); + } + } + + const result = {}; + if (ids.length > 0) { + result.fcmIDs = ids; + } + if (errors.length > 0) { + result.errors = errors; + } else { + result.success = true; + } + if (invalidTokens.length > 0) { + result.invalidTokens = invalidTokens; + } + return { ...result }; +} + +async function fcmSinglePush( + provider: FirebaseApp, + notification: Object, + deviceToken: string, + options: Object, +) { + try { + const deliveryResult = await provider + .messaging() + .sendToDevice(deviceToken, notification, options); + const errors = []; + const ids = []; + for (const fcmResult of deliveryResult.results) { + if (fcmResult.error) { + errors.push(fcmResult.error); + } else if (fcmResult.messageId) { + ids.push(fcmResult.messageId); + } + } + return { fcmIDs: ids, errors }; + } catch (e) { + return { fcmIDs: [], errors: [e] }; + } +} + +async function getUnreadCounts( + userIDs: string[], +): Promise<{ [userID: string]: number }> { + const visPermissionExtractString = `$.${threadPermissions.VISIBLE}.value`; + const notificationExtractString = `$.${threadSubscriptions.home}`; + const query = SQL` + SELECT user, COUNT(thread) AS unread_count + FROM memberships + WHERE user IN (${userIDs}) AND last_message > last_read_message + AND role > 0 + AND JSON_EXTRACT(permissions, ${visPermissionExtractString}) + AND JSON_EXTRACT(subscription, ${notificationExtractString}) + GROUP BY user + `; + const [result] = await dbQuery(query); + const usersToUnreadCounts = {}; + for (const row of result) { + usersToUnreadCounts[row.user.toString()] = row.unread_count; + } + for (const userID of userIDs) { + if (usersToUnreadCounts[userID] === undefined) { + usersToUnreadCounts[userID] = 0; + } + } + return usersToUnreadCounts; +} + +export { apnPush, fcmPush, getUnreadCounts }; diff --git a/keyserver/src/scripts/generate-olm-config.js b/keyserver/src/scripts/generate-olm-config.js new file mode 100644 --- /dev/null +++ b/keyserver/src/scripts/generate-olm-config.js @@ -0,0 +1,39 @@ +// @flow + +import olm from '@matrix-org/olm'; +import fs from 'fs'; +import path from 'path'; +import uuid from 'uuid'; + +import { main } from './utils'; + +const olmConfigRelativePath = './secrets/olm_config.json'; + +async function generateOlmConfig() { + await olm.init(); + const account = new olm.Account(); + account.create(); + const picklingKey = uuid.v4(); + const pickledAccount = account.pickle(picklingKey); + + const olmConfig = { + picklingKey: picklingKey, + pickledAccount: pickledAccount, + }; + const scriptWorkingDirectory = path.resolve(); + + if (!scriptWorkingDirectory.endsWith('comm/keyserver')) { + throw new Error( + 'Script must be run in keyserver directory in comm project.', + ); + } + + const olmConfigFilePath = path.join( + scriptWorkingDirectory, + olmConfigRelativePath, + ); + + fs.writeFileSync(olmConfigFilePath, JSON.stringify(olmConfig)); +} + +main([generateOlmConfig]); diff --git a/landing/webpack.config.cjs b/landing/webpack.config.cjs --- a/landing/webpack.config.cjs +++ b/landing/webpack.config.cjs @@ -18,7 +18,7 @@ }, resolve: { alias: { - '../images': path.resolve('../server/images'), + '../images': path.resolve('../keyserver/images'), }, }, }; diff --git a/native/.flowconfig b/native/.flowconfig --- a/native/.flowconfig +++ b/native/.flowconfig @@ -11,7 +11,7 @@ .*/Libraries/Utilities/LoadingView.js .*/comm/web/.* -.*/comm/server/.* +.*/comm/keyserver/.* .*/android/app/build/.* diff --git a/package.json b/package.json --- a/package.json +++ b/package.json @@ -5,11 +5,11 @@ "lib", "web", "native", - "server", + "keyserver", "landing" ], "scripts": { - "clean": "yarn workspace lib clean && yarn workspace web clean && yarn workspace native clean && yarn workspace server clean && yarn workspace landing clean && rm -rf node_modules/", + "clean": "yarn workspace lib clean && yarn workspace web clean && yarn workspace native clean && yarn workspace keyserver clean && yarn workspace landing clean && rm -rf node_modules/", "cleaninstall": "yarn clean && yarn", "eslint": "eslint .", "eslint:fix": "eslint --fix .", diff --git a/web/webpack.config.cjs b/web/webpack.config.cjs --- a/web/webpack.config.cjs +++ b/web/webpack.config.cjs @@ -18,7 +18,7 @@ }, resolve: { alias: { - '../images': path.resolve('../server/images'), + '../images': path.resolve('../keyserver/images'), }, }, }; @@ -52,7 +52,7 @@ const baseNodeServerRenderingConfig = { externals: ['react', 'react-dom', 'react-redux'], entry: { - server: ['./app.react.js'], + keyserver: ['./app.react.js'], }, output: { filename: 'app.build.cjs',