diff --git a/keyserver/Dockerfile b/keyserver/Dockerfile index 4b43a1746..a8354a19c 100644 --- a/keyserver/Dockerfile +++ b/keyserver/Dockerfile @@ -1,198 +1,206 @@ FROM node:16.18.0-bullseye #------------------------------------------------------------------------------- # STEP 0: SET UP USER # Set up Linux user and group for the container #------------------------------------------------------------------------------- # We use bind mounts for our backups folder, which means Docker on Linux will # blindly match the UID/GID for the backups folder on the container with the # host. In order to make sure the container is able to create backups with the # right UID/GID, we need to do two things: # 1. Make sure that the user that runs the Docker container on the host has # permissions to write to the backups folder on the host. We rely on the host # to configure this properly # 2. Make sure we're running this container with the same UID/GID that the host # is using, so the UID/GID show up correctly on both sides of the bind mount # To handle 2 correctly, we have the host pass the UID/GID with which they're # running the container. Our approach is based on this one: # https://github.com/mhart/alpine-node/issues/48#issuecomment-430902787 ARG HOST_UID ARG HOST_GID ARG COMM_JSONCONFIG_secrets_alchemy ARG COMM_JSONCONFIG_secrets_walletconnect ARG COMM_JSONCONFIG_secrets_geoip_license USER root RUN \ if [ -z "`getent group $HOST_GID`" ]; then \ addgroup --system --gid $HOST_GID comm; \ else \ groupmod --new-name comm `getent group $HOST_GID | cut -d: -f1`; \ fi && \ if [ -z "`getent passwd $HOST_UID`" ]; then \ adduser --system --uid $HOST_UID --ingroup comm --shell /bin/bash comm; \ else \ usermod --login comm --gid $HOST_GID --home /home/comm --move-home \ `getent passwd $HOST_UID | cut -d: -f1`; \ fi #------------------------------------------------------------------------------- # STEP 1: INSTALL PREREQS # Install prereqs first so we don't have to reinstall them if anything changes #------------------------------------------------------------------------------- # We need to add the MariaDB repo to apt in order to install mariadb-client RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup \ && chmod +x mariadb_repo_setup \ && ./mariadb_repo_setup \ && rm mariadb_repo_setup # We need rsync in the prod-build yarn script # We need mariadb-client so we can use mysqldump for backups # We need cmake to install protobuf (prereq for rust-node-addon) RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rsync \ mariadb-client \ cmake \ && rm -rf /var/lib/apt/lists/* # Install protobuf manually to ensure that we have the correct version COPY scripts/install_protobuf.sh scripts/ RUN cd scripts && ./install_protobuf.sh #------------------------------------------------------------------------------- # STEP 2: DEVOLVE PRIVILEGES # Create another user to run the rest of the commands #------------------------------------------------------------------------------- USER comm WORKDIR /home/comm/app #------------------------------------------------------------------------------- # STEP 3: SET UP MYSQL BACKUPS # Prepare the system to properly handle mysqldump backups #------------------------------------------------------------------------------- # Prepare the directory that will hold the backups RUN mkdir /home/comm/backups #------------------------------------------------------------------------------- # STEP 4: SET UP CARGO (RUST PACKAGE MANAGER) # We use Cargo to build pre-compiled Node.js addons in Rust #------------------------------------------------------------------------------- # Install Rust and add Cargo's bin directory to the $PATH environment variable RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH /home/comm/.cargo/bin:$PATH #------------------------------------------------------------------------------- # STEP 5: SET UP NVM # We use nvm to make sure we're running the right Node version #------------------------------------------------------------------------------- # First we install nvm ENV NVM_DIR /home/comm/.nvm RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh \ | bash # Then we use nvm to install the right version of Node. We call this early so # Docker build caching saves us from re-downloading Node when any file changes COPY --chown=comm keyserver/.nvmrc keyserver/ COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/ RUN cd keyserver && . bash/source-nvm.sh #------------------------------------------------------------------------------- # STEP 6: YARN CLEANINSTALL # We run yarn cleaninstall before copying most of the files in for build caching #------------------------------------------------------------------------------- # Copy in package.json files, yarn.lock files, and relevant installation scripts COPY --chown=comm package.json yarn.lock postinstall.sh ./ COPY --chown=comm keyserver/package.json keyserver/.flowconfig keyserver/ COPY --chown=comm lib/package.json lib/.flowconfig lib/ COPY --chown=comm web/package.json web/.flowconfig web/ COPY --chown=comm native/package.json native/.flowconfig native/ COPY --chown=comm landing/package.json landing/.flowconfig landing/ COPY --chown=comm desktop/package.json desktop/ COPY --chown=comm desktop/addons/windows-pushnotifications/package.json \ desktop/addons/windows-pushnotifications/ COPY --chown=comm keyserver/addons/rust-node-addon/package.json \ keyserver/addons/rust-node-addon/install_ci_deps.sh \ keyserver/addons/rust-node-addon/postinstall.sh \ keyserver/addons/rust-node-addon/ COPY --chown=comm native/expo-modules/android-lifecycle/package.json \ native/expo-modules/android-lifecycle/ COPY --chown=comm native/expo-modules/aes-crypto/package.json \ native/expo-modules/aes-crypto/ COPY --chown=comm services/electron-update-server/package.json \ services/electron-update-server/ # Create empty Rust library and copy in Cargo.toml file RUN cargo init keyserver/addons/rust-node-addon --lib COPY --chown=comm keyserver/addons/rust-node-addon/Cargo.toml \ keyserver/addons/rust-node-addon/ # Copy in local dependencies of rust-node-addon COPY --chown=comm shared/comm-opaque shared/comm-opaque/ COPY --chown=comm shared/comm-opaque2 shared/comm-opaque2/ COPY --chown=comm shared/tunnelbroker-client shared/tunnelbroker-client/ # Copy protobuf files as a dependency for the shared client libraries COPY --chown=comm shared/protos shared/protos/ # Copy in files needed for patch-package COPY --chown=comm patches patches/ # Actually run yarn RUN yarn cleaninstall #------------------------------------------------------------------------------- -# STEP 7: WEBPACK BUILD +# STEP 7: GEOIP UPDATE +# We update the GeoIP database for mapping from IP address to timezone +#------------------------------------------------------------------------------- + +COPY --chown=comm keyserver/bash/docker-update-geoip.sh keyserver/bash/ +RUN cd keyserver && bash/docker-update-geoip.sh + +#------------------------------------------------------------------------------- +# STEP 8: WEBPACK BUILD # We do this first so Docker doesn't rebuild when only keyserver files change #------------------------------------------------------------------------------- # These are needed for babel-build-comm-config COPY --chown=comm keyserver/src keyserver/src COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/source-nvm.sh COPY --chown=comm keyserver/babel.config.cjs keyserver/babel.config.cjs COPY --chown=comm lib lib/ COPY --chown=comm landing landing/ RUN yarn workspace landing prod COPY --chown=comm web web/ RUN yarn workspace web prod #------------------------------------------------------------------------------- -# STEP 8: COPY IN SOURCE FILES +# STEP 9: COPY IN SOURCE FILES # We run this later so the above layers are cached if only source files change #------------------------------------------------------------------------------- COPY --chown=comm . . #------------------------------------------------------------------------------- -# STEP 9: BUILD NODE ADDON +# STEP 10: BUILD NODE ADDON # Now that source files have been copied in, build rust-node-addon #------------------------------------------------------------------------------- RUN yarn workspace rust-node-addon build #------------------------------------------------------------------------------- -# STEP 10: RUN BUILD SCRIPTS +# STEP 11: RUN BUILD SCRIPTS # We need to populate keyserver/dist, among other things #------------------------------------------------------------------------------- # Babel transpilation of keyserver src RUN yarn workspace keyserver prod-build #------------------------------------------------------------------------------- -# STEP 11: RUN THE SERVER +# STEP 12: RUN THE SERVER # Actually run the Node.js keyserver using nvm #------------------------------------------------------------------------------- EXPOSE 3000 WORKDIR /home/comm/app/keyserver CMD bash/run-prod.sh diff --git a/keyserver/bash/docker-update-geoip.sh b/keyserver/bash/docker-update-geoip.sh new file mode 100755 index 000000000..789a7a728 --- /dev/null +++ b/keyserver/bash/docker-update-geoip.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# run as: Docker container user +# run from: anywhere + +set -eo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd -P) + +if [[ -n "${COMM_JSONCONFIG_secrets_geoip_license-}" ]]; then + # shellcheck source=/dev/null + . "${SCRIPT_DIR}/source-nvm.sh" + node "${SCRIPT_DIR}/../node_modules/geoip-lite/scripts/updatedb.js" \ + license_key="$( + # shellcheck disable=SC2001 + echo "${COMM_JSONCONFIG_secrets_geoip_license}" | + sed 's/{\"key\":\"\([a-zA-Z0-9]*\)\"}/\1/' + )" +fi diff --git a/keyserver/package.json b/keyserver/package.json index d5998400b..9e5b414ef 100644 --- a/keyserver/package.json +++ b/keyserver/package.json @@ -1,109 +1,108 @@ { "name": "keyserver", "version": "0.0.1", "type": "module", "private": true, "license": "BSD-3-Clause", "main": "dist/keyserver", "scripts": { "clean": "rm -rf dist/ && rm -rf node_modules/ && mkdir dist", "babel-build-comm-config": ". bash/source-nvm.sh && yarn --silent babel src/lib/utils/comm-config.js --out-dir dist/lib/utils/ --config-file ./babel.config.cjs", "babel-build": ". bash/source-nvm.sh && yarn --silent babel src/ --out-dir dist/ --config-file ./babel.config.cjs --verbose --ignore 'src/landing/flow-typed','src/landing/node_modules','src/landing/package.json','src/lib/flow-typed','src/lib/node_modules','src/lib/package.json','src/web/flow-typed','src/web/node_modules','src/web/package.json','src/web/dist','src/web/webpack.config.js','src/web/account-bar.react.js','src/web/app.react.js','src/web/calendar','src/web/chat','src/web/flow','src/web/loading-indicator.react.js','src/web/modals','src/web/root.js','src/web/router-history.js','src/web/script.js','src/web/selectors/chat-selectors.js','src/web/selectors/entry-selectors.js','src/web/splash','src/web/vector-utils.js','src/web/vectors.react.js'", "rsync": "rsync -rLpmuv --exclude '*/package.json' --exclude '*/node_modules/*' --include '*.json' --include '*.cjs' --include '*.node' --exclude '*.*' src/ dist/", - "prod-build": "yarn babel-build && yarn rsync && yarn update-geoip", - "update-geoip": "yarn script dist/scripts/update-geoip.js", + "prod-build": "yarn babel-build && yarn rsync", "prod": "node --trace-warnings --loader=./loader.mjs dist/keyserver", "dev-rsync": "yarn --silent chokidar --initial --silent -s 'src/**/*.json' 'src/**/*.cjs' -c 'yarn rsync > /dev/null 2>&1'", "dev": ". bash/source-nvm.sh && yarn concurrently --names=\"BABEL,RSYNC,NODEM\" -c \"bgBlue.bold,bgMagenta.bold,bgGreen.bold\" \"yarn babel-build --source-maps --watch\" \"yarn dev-rsync\" \". bash/source-nvm.sh && NODE_ENV=development nodemon -e js,json,cjs --watch dist --no-warnings=ExperimentalWarning --loader=./loader.mjs dist/keyserver\"", "script": ". bash/source-nvm.sh && NODE_ENV=development node --loader=./loader.mjs", "test": "jest" }, "devDependencies": { "@babel/cli": "^7.13.14", "@babel/core": "^7.13.14", "@babel/node": "^7.13.13", "@babel/plugin-proposal-class-properties": "^7.13.0", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", "@babel/plugin-proposal-object-rest-spread": "^7.13.8", "@babel/plugin-proposal-optional-chaining": "^7.13.12", "@babel/plugin-syntax-dynamic-import": "^7.8.3", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", "@babel/preset-flow": "^7.13.13", "@babel/preset-react": "^7.13.13", "babel-jest": "^26.6.3", "chokidar-cli": "^2.1.0", "concurrently": "^5.3.0", "flow-bin": "^0.182.0", "flow-typed": "^3.2.1", "internal-ip": "4.3.0", "jest": "^26.6.3", "nodemon": "^2.0.4" }, "dependencies": { "@babel/runtime": "^7.13.10", "@commapp/olm": "0.0.4", "@parse/node-apn": "^3.2.0", "@vingle/bmp-js": "^0.2.5", "JSONStream": "^1.3.5", "common-tags": "^1.7.2", "cookie-parser": "^1.4.3", "dateformat": "^3.0.3", "detect-browser": "^4.0.4", "ethers": "^5.7.2", "express": "^4.17.3", "express-ws": "^4.0.0", "firebase-admin": "^10.1.0", "geoip-lite": "^1.4.5", "invariant": "^2.2.4", "landing": "0.0.1", "lib": "0.0.1", "lodash": "^4.17.21", "multer": "^1.4.1", "mysql2": "^2.3.3", "natural": "^6.2.0", "node-fetch": "^2.6.9", "node-schedule": "^2.1.0", "nodemailer": "^6.6.1", "react": "18.1.0", "react-dom": "18.1.0", "react-html-email": "^3.0.0", "react-redux": "^7.1.1", "react-router": "^5.2.0", "redis": "^3.1.1", "redux": "^4.0.4", "replacestream": "^4.0.3", "rereadable-stream": "^1.4.5", "rust-node-addon": "0.0.1", "sharp": "^0.30.5", "siwe": "^1.1.6", "sql-template-strings": "^2.2.2", "stream-combiner": "^0.2.2", "tcomb": "^3.2.29", "twin-bcrypt": "^2.1.1", "uuid": "^3.3.3", "web": "0.0.1", "web-push": "^3.5.0" }, "optionalDependencies": { "bufferutil": "^4.0.5", "utf-8-validate": "^5.0.7" }, "nodemonConfig": { "delay": "200" }, "jest": { "roots": [ "/src" ], "transform": { "\\.js$": "babel-jest" }, "transformIgnorePatterns": [ "/node_modules/(?!@babel/runtime)" ], "setupFiles": [ "/jest-setup.js" ] } } diff --git a/keyserver/src/cron/update-geoip-db.js b/keyserver/src/cron/update-geoip-db.js index b40e70b09..3c7a56eba 100644 --- a/keyserver/src/cron/update-geoip-db.js +++ b/keyserver/src/cron/update-geoip-db.js @@ -1,63 +1,63 @@ // @flow import childProcess from 'child_process'; import cluster from 'cluster'; import geoip from 'geoip-lite'; import { getCommConfig } from 'lib/utils/comm-config.js'; import { handleAsyncPromise } from '../responders/handlers.js'; async function updateGeoipDB(): Promise { const geoipLicense = await getCommConfig({ folder: 'secrets', name: 'geoip_license', }); if (!geoipLicense) { console.log('no keyserver/secrets/geoip_license.json so skipping update'); return; } await spawnUpdater(geoipLicense); } function spawnUpdater(geoipLicense: { key: string }): Promise { const spawned = childProcess.spawn(process.execPath, [ '../node_modules/geoip-lite/scripts/updatedb.js', `license_key=${geoipLicense.key}`, ]); return new Promise((resolve, reject) => { spawned.on('error', reject); spawned.on('exit', () => resolve()); }); } function reloadGeoipDB(): Promise { return new Promise(resolve => geoip.reloadData(resolve)); } type IPCMessage = { type: 'geoip_reload', }; const reloadMessage: IPCMessage = { type: 'geoip_reload' }; async function updateAndReloadGeoipDB(): Promise { await updateGeoipDB(); await reloadGeoipDB(); if (!cluster.isMaster) { return; } for (const id in cluster.workers) { cluster.workers[Number(id)].send(reloadMessage); } } if (!cluster.isMaster) { process.on('message', (ipcMessage: IPCMessage) => { if (ipcMessage.type === 'geoip_reload') { handleAsyncPromise(reloadGeoipDB()); } }); } -export { updateGeoipDB, updateAndReloadGeoipDB }; +export { updateAndReloadGeoipDB }; diff --git a/keyserver/src/scripts/update-geoip.js b/keyserver/src/scripts/update-geoip.js deleted file mode 100644 index 655175c95..000000000 --- a/keyserver/src/scripts/update-geoip.js +++ /dev/null @@ -1,16 +0,0 @@ -// @flow - -import { endScript } from './utils.js'; -import { updateGeoipDB } from '../cron/update-geoip-db.js'; - -async function main() { - try { - await updateGeoipDB(); - endScript(); - } catch (e) { - endScript(); - console.warn(e); - } -} - -main();