diff --git a/keyserver/Dockerfile b/keyserver/Dockerfile index aa5bf690d..fdf4f9365 100644 --- a/keyserver/Dockerfile +++ b/keyserver/Dockerfile @@ -1,189 +1,195 @@ FROM node:16.18.0-bullseye #------------------------------------------------------------------------------- # STEP 0: SET UP USER # Set up Linux user and group for the container #------------------------------------------------------------------------------- # We use bind mounts for our backups folder, which means Docker on Linux will # blindly match the UID/GID for the backups folder on the container with the # host. In order to make sure the container is able to create backups with the # right UID/GID, we need to do two things: # 1. Make sure that the user that runs the Docker container on the host has # permissions to write to the backups folder on the host. We rely on the host # to configure this properly # 2. Make sure we're running this container with the same UID/GID that the host # is using, so the UID/GID show up correctly on both sides of the bind mount # To handle 2 correctly, we have the host pass the UID/GID with which they're # running the container. Our approach is based on this one: # https://github.com/mhart/alpine-node/issues/48#issuecomment-430902787 ARG HOST_UID ARG HOST_GID ARG COMM_ALCHEMY_KEY ARG COMM_WALLETCONNECT_KEY ARG COMM_JSONCONFIG_secrets_geoip_license USER root RUN \ if [ -z "`getent group $HOST_GID`" ]; then \ addgroup --system --gid $HOST_GID comm; \ else \ groupmod --new-name comm `getent group $HOST_GID | cut -d: -f1`; \ fi && \ if [ -z "`getent passwd $HOST_UID`" ]; then \ adduser --system --uid $HOST_UID --ingroup comm --shell /bin/bash comm; \ else \ usermod --login comm --gid $HOST_GID --home /home/comm --move-home \ `getent passwd $HOST_UID | cut -d: -f1`; \ fi #------------------------------------------------------------------------------- # STEP 1: INSTALL PREREQS # Install prereqs first so we don't have to reinstall them if anything changes #------------------------------------------------------------------------------- # We need to add the MariaDB repo to apt in order to install mariadb-client RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup \ && chmod +x mariadb_repo_setup \ && ./mariadb_repo_setup \ && rm mariadb_repo_setup # We need rsync in the prod-build yarn script # We need mariadb-client so we can use mysqldump for backups # We need cmake to install protobuf (prereq for rust-node-addon) RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rsync \ mariadb-client \ cmake \ && rm -rf /var/lib/apt/lists/* # Install protobuf manually to ensure that we have the correct version COPY scripts/install_protobuf.sh scripts/ RUN cd scripts && ./install_protobuf.sh #------------------------------------------------------------------------------- # STEP 2: DEVOLVE PRIVILEGES # Create another user to run the rest of the commands #------------------------------------------------------------------------------- USER comm WORKDIR /home/comm/app #------------------------------------------------------------------------------- # STEP 3: SET UP MYSQL BACKUPS # Prepare the system to properly handle mysqldump backups #------------------------------------------------------------------------------- # Prepare the directory that will hold the backups RUN mkdir /home/comm/backups #------------------------------------------------------------------------------- # STEP 4: SET UP CARGO (RUST PACKAGE MANAGER) # We use Cargo to build pre-compiled Node.js addons in Rust #------------------------------------------------------------------------------- # Install Rust and add Cargo's bin directory to the $PATH environment variable RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH /home/comm/.cargo/bin:$PATH #------------------------------------------------------------------------------- # STEP 5: SET UP NVM # We use nvm to make sure we're running the right Node version #------------------------------------------------------------------------------- # First we install nvm ENV NVM_DIR /home/comm/.nvm RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh \ | bash # Then we use nvm to install the right version of Node. We call this early so # Docker build caching saves us from re-downloading Node when any file changes COPY --chown=comm keyserver/.nvmrc keyserver/ COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/ RUN cd keyserver && . bash/source-nvm.sh #------------------------------------------------------------------------------- # STEP 6: YARN CLEANINSTALL # We run yarn cleaninstall before copying most of the files in for build caching #------------------------------------------------------------------------------- # Copy in package.json files, yarn.lock files, and relevant installation scripts COPY --chown=comm package.json yarn.lock postinstall.sh ./ COPY --chown=comm keyserver/package.json keyserver/.flowconfig keyserver/ COPY --chown=comm lib/package.json lib/.flowconfig lib/ COPY --chown=comm web/package.json web/.flowconfig web/ COPY --chown=comm native/package.json native/.flowconfig native/ COPY --chown=comm landing/package.json landing/.flowconfig landing/ COPY --chown=comm desktop/package.json desktop/ COPY --chown=comm keyserver/addons/rust-node-addon/package.json \ keyserver/addons/rust-node-addon/install_ci_deps.sh \ keyserver/addons/rust-node-addon/postinstall.sh \ keyserver/addons/rust-node-addon/ COPY --chown=comm native/expo-modules/android-lifecycle/package.json \ native/expo-modules/android-lifecycle/ COPY --chown=comm native/expo-modules/aes-crypto/package.json \ native/expo-modules/aes-crypto/ COPY --chown=comm services/electron-update-server/package.json \ services/electron-update-server/ # Create empty Rust library and copy in Cargo.toml file RUN cargo init keyserver/addons/rust-node-addon --lib COPY --chown=comm keyserver/addons/rust-node-addon/Cargo.toml \ keyserver/addons/rust-node-addon/ # Copy in local dependencies of rust-node-addon COPY --chown=comm shared/comm-opaque shared/comm-opaque/ COPY --chown=comm shared/tunnelbroker-client shared/tunnelbroker-client/ # Copy protobuf files as a dependency for the shared client libraries COPY --chown=comm shared/protos shared/protos/ # Copy in files needed for patch-package COPY --chown=comm patches patches/ # Actually run yarn RUN yarn cleaninstall #------------------------------------------------------------------------------- # STEP 7: WEBPACK BUILD # We do this first so Docker doesn't rebuild when only keyserver files change #------------------------------------------------------------------------------- +# These are needed for babel-build-comm-config +COPY --chown=comm keyserver/src keyserver/src +COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/source-nvm.sh +COPY --chown=comm keyserver/babel.config.cjs keyserver/babel.config.cjs + COPY --chown=comm lib lib/ + COPY --chown=comm landing landing/ RUN yarn workspace landing prod COPY --chown=comm web web/ RUN yarn workspace web prod #------------------------------------------------------------------------------- # STEP 8: COPY IN SOURCE FILES # We run this later so the above layers are cached if only source files change #------------------------------------------------------------------------------- COPY --chown=comm . . #------------------------------------------------------------------------------- # STEP 9: BUILD NODE ADDON # Now that source files have been copied in, build rust-node-addon #------------------------------------------------------------------------------- RUN yarn workspace rust-node-addon build #------------------------------------------------------------------------------- # STEP 10: RUN BUILD SCRIPTS # We need to populate keyserver/dist, among other things #------------------------------------------------------------------------------- # Babel transpilation of keyserver src RUN yarn workspace keyserver prod-build #------------------------------------------------------------------------------- # STEP 11: RUN THE SERVER # Actually run the Node.js keyserver using nvm #------------------------------------------------------------------------------- EXPOSE 3000 WORKDIR /home/comm/app/keyserver CMD bash/run-prod.sh diff --git a/keyserver/package.json b/keyserver/package.json index 7b27638c8..7bbd92acd 100644 --- a/keyserver/package.json +++ b/keyserver/package.json @@ -1,104 +1,105 @@ { "name": "keyserver", "version": "0.0.1", "type": "module", "private": true, "license": "BSD-3-Clause", "main": "dist/keyserver", "scripts": { "clean": "rm -rf dist/ && rm -rf node_modules/ && mkdir dist", + "babel-build-comm-config": ". bash/source-nvm.sh && yarn --silent babel src/lib/utils/comm-config.js --out-dir dist/lib/utils/ --config-file ./babel.config.cjs", "babel-build": ". bash/source-nvm.sh && yarn --silent babel src/ --out-dir dist/ --config-file ./babel.config.cjs --verbose --ignore 'src/landing/flow-typed','src/landing/node_modules','src/landing/package.json','src/lib/flow-typed','src/lib/node_modules','src/lib/package.json','src/web/flow-typed','src/web/node_modules','src/web/package.json','src/web/dist','src/web/webpack.config.js','src/web/account-bar.react.js','src/web/app.react.js','src/web/calendar','src/web/chat','src/web/flow','src/web/loading-indicator.react.js','src/web/modals','src/web/root.js','src/web/router-history.js','src/web/script.js','src/web/selectors/chat-selectors.js','src/web/selectors/entry-selectors.js','src/web/splash','src/web/vector-utils.js','src/web/vectors.react.js'", "rsync": "rsync -rLpmuv --exclude '*/package.json' --exclude '*/node_modules/*' --include '*.json' --include '*.cjs' --include '*.node' --exclude '*.*' src/ dist/", "prod-build": "yarn babel-build && yarn rsync && yarn update-geoip", "update-geoip": "yarn script dist/scripts/update-geoip.js", "prod": "node --trace-warnings --loader=./loader.mjs dist/keyserver", "dev-rsync": "yarn --silent chokidar --initial --silent -s 'src/**/*.json' 'src/**/*.cjs' -c 'yarn rsync > /dev/null 2>&1'", "dev": ". bash/source-nvm.sh && yarn concurrently --names=\"BABEL,RSYNC,NODEM\" -c \"bgBlue.bold,bgMagenta.bold,bgGreen.bold\" \"yarn babel-build --source-maps --watch\" \"yarn dev-rsync\" \". bash/source-nvm.sh && NODE_ENV=development nodemon -e js,json,cjs --watch dist --no-warnings=ExperimentalWarning --loader=./loader.mjs dist/keyserver\"", "script": ". bash/source-nvm.sh && NODE_ENV=development node --loader=./loader.mjs", "test": "jest" }, "devDependencies": { "@babel/cli": "^7.13.14", "@babel/core": "^7.13.14", "@babel/node": "^7.13.13", "@babel/plugin-proposal-class-properties": "^7.13.0", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", "@babel/plugin-proposal-object-rest-spread": "^7.13.8", "@babel/plugin-proposal-optional-chaining": "^7.13.12", "@babel/plugin-syntax-dynamic-import": "^7.8.3", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", "@babel/preset-flow": "^7.13.13", "@babel/preset-react": "^7.13.13", "babel-jest": "^26.6.3", "chokidar-cli": "^2.1.0", "concurrently": "^5.3.0", "flow-bin": "^0.182.0", "flow-typed": "^3.2.1", "internal-ip": "4.3.0", "jest": "^26.6.3", "nodemon": "^2.0.4" }, "dependencies": { "@babel/runtime": "^7.13.10", "@matrix-org/olm": "3.2.14", "@parse/node-apn": "^3.2.0", "@vingle/bmp-js": "^0.2.5", "JSONStream": "^1.3.5", "common-tags": "^1.7.2", "cookie-parser": "^1.4.3", "dateformat": "^3.0.3", "detect-browser": "^4.0.4", "ethers": "^5.7.2", "express": "^4.17.3", "express-ws": "^4.0.0", "firebase-admin": "^10.1.0", "geoip-lite": "^1.4.5", "invariant": "^2.2.4", "landing": "0.0.1", "lib": "0.0.1", "lodash": "^4.17.21", "multer": "^1.4.1", "mysql2": "^2.3.3", "natural": "^6.2.0", "node-schedule": "^2.1.0", "nodemailer": "^6.6.1", "react": "18.1.0", "react-dom": "18.1.0", "react-html-email": "^3.0.0", "react-redux": "^7.1.1", "react-router": "^5.2.0", "redis": "^3.1.1", "redux": "^4.0.4", "replacestream": "^4.0.3", "rereadable-stream": "^1.4.5", "rust-node-addon": "0.0.1", "sharp": "^0.30.5", "siwe": "^1.1.6", "sql-template-strings": "^2.2.2", "stream-combiner": "^0.2.2", "tcomb": "^3.2.29", "twin-bcrypt": "^2.1.1", "uuid": "^3.3.3", "web": "0.0.1", "web-push": "^3.5.0" }, "optionalDependencies": { "bufferutil": "^4.0.5", "utf-8-validate": "^5.0.7" }, "nodemonConfig": { "delay": "200" }, "jest": { "roots": [ "/src" ], "transform": { "\\.js$": "babel-jest" }, "transformIgnorePatterns": [ "/node_modules/(?!@babel/runtime)" ] } } diff --git a/landing/package.json b/landing/package.json index f7afe4362..3a1d27723 100644 --- a/landing/package.json +++ b/landing/package.json @@ -1,55 +1,55 @@ { "name": "landing", "version": "0.0.1", "type": "module", "private": true, "license": "BSD-3-Clause", "scripts": { "clean": "rm -rf dist/ && rm -rf node_modules/", - "dev": "yarn concurrently --names=\"NODESSR,BROWSER\" -c \"bgBlue.bold,bgMagenta.bold\" \"yarn webpack --config webpack.config.cjs --config-name=server --watch\" \"yarn webpack-dev-server --config webpack.config.cjs --config-name=browser\"", - "prod": "yarn webpack --config webpack.config.cjs --env prod --progress" + "dev": "yarn workspace keyserver babel-build-comm-config && yarn concurrently --names=\"NODESSR,BROWSER\" -c \"bgBlue.bold,bgMagenta.bold\" \"yarn webpack --config webpack.config.cjs --config-name=server --watch\" \"yarn webpack-dev-server --config webpack.config.cjs --config-name=browser\"", + "prod": "yarn workspace keyserver babel-build-comm-config && yarn webpack --config webpack.config.cjs --env prod --progress" }, "devDependencies": { "@babel/core": "^7.13.14", "@babel/plugin-proposal-class-properties": "^7.13.0", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", "@babel/plugin-proposal-object-rest-spread": "^7.13.8", "@babel/plugin-proposal-optional-chaining": "^7.13.12", "@babel/plugin-transform-react-constant-elements": "^7.13.13", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", "@babel/preset-flow": "^7.13.13", "@babel/preset-react": "^7.13.13", "babel-plugin-transform-remove-console": "^6.9.4", "concurrently": "^5.3.0", "flow-bin": "^0.182.0", "flow-typed": "^3.2.1", "webpack-cli": "^5.0.1", "webpack-dev-server": "^4.11.1", "webpack-manifest-plugin": "^5.0.0" }, "dependencies": { "@babel/runtime": "^7.13.10", "@fortawesome/fontawesome-svg-core": "1.2.25", "@fortawesome/free-brands-svg-icons": "5.15.4", "@fortawesome/free-regular-svg-icons": "5.11.2", "@fortawesome/free-solid-svg-icons": "5.11.2", "@fortawesome/react-fontawesome": "0.1.5", "@lottiefiles/lottie-interactivity": "^0.1.4", "@lottiefiles/lottie-player": "^1.0.3", "@rainbow-me/rainbowkit": "^0.8.1", "classnames": "^2.2.5", "core-js": "^3.6.5", "ethers": "^5.7.2", "invariant": "^2.2.4", "isomorphic-fetch": "^3.0.0", "lib": "0.0.1", "lodash": "^4.17.21", "react": "18.1.0", "react-dom": "18.1.0", "react-router": "^5.2.0", "react-router-dom": "^5.2.0", "react-text-loop": "^2.3.0", "wagmi": "^0.8.10" } } diff --git a/lib/utils/comm-config.js b/lib/utils/comm-config.js index 0ec739e70..c6d54385a 100644 --- a/lib/utils/comm-config.js +++ b/lib/utils/comm-config.js @@ -1,64 +1,71 @@ // @flow import fs from 'fs'; +import path from 'path'; import { promisify } from 'util'; const readFile = promisify(fs.readFile); type ConfigName = { +folder: 'secrets' | 'facts', +name: string, }; function getKeyForConfigName(configName: ConfigName): string { return `${configName.folder}_${configName.name}`; } function getPathForConfigName(configName: ConfigName): string { return `${configName.folder}/${configName.name}.json`; } const cachedJSON = new Map(); // This function checks for an env var named COMM_JSONCONFIG_{folder}_{name} // If it doesn't find one, it then looks for keyserver/{folder}/{name}.json // In both cases, it expects to find a JSON blob async function getCommConfig(configName: ConfigName): Promise { const key = getKeyForConfigName(configName); const cached = cachedJSON.get(key); if (cached !== undefined) { return cached; } const json = await getJSON(configName); if (!cachedJSON.has(key)) { cachedJSON.set(key, json); } return cachedJSON.get(key); } async function getJSON(configName: ConfigName): Promise { const key = getKeyForConfigName(configName); const fromEnv = process.env[`COMM_JSONCONFIG_${key}`]; if (fromEnv) { try { return JSON.parse(fromEnv); } catch (e) { console.log( `failed to parse JSON from env for ${JSON.stringify(configName)}`, e, ); } } - const path = getPathForConfigName(configName); + + let filePath = getPathForConfigName(configName); + const cwd = path.resolve(); + if (!cwd.endsWith('/keyserver')) { + filePath = `../keyserver/${filePath}`; + } + try { - const pathString = await readFile(path, 'utf8'); + const pathString = await readFile(filePath, 'utf8'); return JSON.parse(pathString); } catch (e) { if (e.code !== 'ENOENT') { - console.log(`Failed to read JSON from ${path}`, e); + console.log(`Failed to read JSON from ${filePath}`, e); } return null; } } export { getCommConfig }; diff --git a/web/package.json b/web/package.json index 627c83f4b..af9bad23b 100644 --- a/web/package.json +++ b/web/package.json @@ -1,108 +1,108 @@ { "name": "web", "version": "0.0.1", "type": "module", "private": true, "license": "BSD-3-Clause", "scripts": { "clean": "rm -rf dist/ && rm -rf node_modules/", - "dev": "yarn concurrently --names=\"NODESSR,BROWSER,WORKERS\" -c \"bgBlue.bold,bgMagenta.bold,bgCyan.bold\" \"yarn webpack --config webpack.config.cjs --config-name=server --watch\" \"yarn webpack-dev-server --config webpack.config.cjs --config-name=browser\" \"yarn webpack --config webpack.config.cjs --config-name=webworkers --watch\"", - "prod": "yarn webpack --config webpack.config.cjs --env prod --progress", + "dev": "yarn workspace keyserver babel-build-comm-config && yarn concurrently --names=\"NODESSR,BROWSER,WORKERS\" -c \"bgBlue.bold,bgMagenta.bold,bgCyan.bold\" \"yarn webpack --config webpack.config.cjs --config-name=server --watch\" \"yarn webpack-dev-server --config webpack.config.cjs --config-name=browser\" \"yarn webpack --config webpack.config.cjs --config-name=webworkers --watch\"", + "prod": "yarn workspace keyserver babel-build-comm-config && yarn webpack --config webpack.config.cjs --env prod --progress", "test": "jest" }, "devDependencies": { "@babel/core": "^7.13.14", "@babel/plugin-proposal-class-properties": "^7.13.0", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", "@babel/plugin-proposal-object-rest-spread": "^7.13.8", "@babel/plugin-proposal-optional-chaining": "^7.13.12", "@babel/plugin-transform-react-constant-elements": "^7.13.13", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", "@babel/preset-flow": "^7.13.13", "@babel/preset-react": "^7.13.13", "babel-jest": "^26.6.3", "babel-plugin-transform-remove-console": "^6.9.4", "concurrently": "^5.3.0", "copy-webpack-plugin": "^11.0.0", "flow-bin": "^0.182.0", "flow-typed": "^3.2.1", "identity-obj-proxy": "^3.0.0", "jest": "^26.6.3", "webpack-cli": "^5.0.1", "webpack-dev-server": "^4.11.1", "webpack-manifest-plugin": "^5.0.0" }, "dependencies": { "@babel/runtime": "^7.13.10", "@commapp/opaque-ke-wasm": "npm:@commapp/opaque-ke-wasm@^0.0.3", "@emoji-mart/data": "^1.1.2", "@emoji-mart/react": "^1.1.1", "@fortawesome/fontawesome-svg-core": "1.2.25", "@fortawesome/free-regular-svg-icons": "5.11.2", "@fortawesome/free-solid-svg-icons": "5.11.2", "@fortawesome/react-fontawesome": "0.1.5", "@matrix-org/olm": "3.2.14", "@rainbow-me/rainbowkit": "^0.8.1", "basscss": "8.0.2", "classnames": "^2.2.5", "core-js": "^3.6.5", "dateformat": "^3.0.3", "detect-browser": "^4.0.4", "emoji-mart": "^5.5.2", "ethers": "^5.7.2", "exif-js": "^2.3.0", "google-protobuf": "^3.21.2", "grpc-web": "^1.4.2", "history": "^4.6.3", "invariant": "^2.2.4", "is-svg": "^4.3.0", "isomorphic-fetch": "^3.0.0", "lib": "0.0.1", "localforage": "^1.10.0", "lodash": "^4.17.21", "react": "18.1.0", "react-circular-progressbar": "^2.0.2", "react-color": "^2.13.0", "react-dnd": "^11.1.3", "react-dnd-html5-backend": "^11.1.3", "react-dom": "18.1.0", "react-feather": "^2.0.3", "react-icomoon": "^2.5.7", "react-icons": "^4.4.0", "react-redux": "^7.1.1", "react-router": "^5.2.0", "react-router-dom": "^5.2.0", "react-switch": "^7.0.0", "react-timeago": "^5.2.0", "redux": "^4.0.4", "redux-devtools-extension": "^2.13.2", "redux-persist": "^6.0.0", "redux-thunk": "^2.2.0", "reselect": "^4.0.0", "simple-markdown": "^0.7.2", "siwe": "^1.1.6", "sql.js": "^1.8.0", "tinycolor2": "^1.4.1", "uuid": "^3.3.3", "visibilityjs": "^2.0.2", "wagmi": "^0.8.10" }, "jest": { "roots": [ "" ], "transform": { "\\.js$": "babel-jest" }, "transformIgnorePatterns": [ "/node_modules/(?!@babel/runtime)" ], "moduleNameMapper": { "\\.(css)$": "identity-obj-proxy" }, "setupFiles": [ "/jest-setup.js" ] } }