diff --git a/docs/nix_keyserver_deployment.md b/docs/nix_keyserver_deployment.md index eeca5b063..cf1c005a3 100644 --- a/docs/nix_keyserver_deployment.md +++ b/docs/nix_keyserver_deployment.md @@ -1,72 +1,75 @@ # Keyserver Deployment Deploying the keyserver requires configuring it, building its Docker image, and deploying that image with Docker Compose. ## Configuration In order for the keyserver to interface with dependencies, host the landing page, and host the Comm web application, the following must be added to `keyserver/.env`: ``` # Mandatory COMM_DATABASE_DATABASE=comm COMM_DATABASE_USER= COMM_DATABASE_PASSWORD= COMM_JSONCONFIG_secrets_user_credentials='{"username":"","password":""}' COMM_JSONCONFIG_facts_landing_url='{"baseDomain":"http://localhost","basePath":"/commlanding/","baseRoutePath":"/commlanding/","https":false}' COMM_JSONCONFIG_facts_webapp_url='{"baseDomain":"http://localhost:3000","basePath":"/webapp/","https":false,"baseRoutePath":"/webapp/","proxy":"none"}' COMM_JSONCONFIG_facts_keyserver_url='{"baseDomain":"http://localhost:3000","basePath":"/keyserver/","baseRoutePath":"/keyserver/","https":false,"proxy":"none"}' COMM_JSONCONFIG_facts_webapp_cors='{"domain": "http://localhost:3000"}' # Required to connect to production Identity service COMM_JSONCONFIG_secrets_identity_service_config="{\"identitySocketAddr\":\"https://identity.commtechnologies.org:50054\"}" # Required for ETH login COMM_JSONCONFIG_secrets_alchemy='{"key":""}' COMM_JSONCONFIG_secrets_walletconnect='{"key":""}' +# Required for Farcaster login +COMM_JSONCONFIG_secrets_neynar='{"key":""}' + # Example backup configuration that stores up to 10 GiB of backups in /home/comm/backups COMM_JSONCONFIG_facts_backups='{"enabled":true,"directory":"/home/comm/backups","maxDirSizeMiB":10240}' ``` ### MariaDB configuration - `COMM_DATABASE_DATABASE`: Specifies the name of the database the keyserver will use. - `COMM_DATABASE_USER`: The username the keyserver uses to connect to MariaDB. Replace `` with your desired username. - `COMM_DATABASE_PASSWORD`: Corresponding password for the above user. Replace `` with your desired password. ### Identity service configuration - `COMM_JSONCONFIG_secrets_user_credentials`: Credentials for authenticating against the Identity service. Replace `` and `` with any values. In the future, they will need to be actual credentials registered with the Identity service. - `COMM_JSONCONFIG_secrets_identity_service_config`: Socket address for the Identity service. If omitted, the keyserver will try to connect to a local instance of the Identity service. ### ETH login configuration - `COMM_JSONCONFIG_secrets_alchemy`: Alchemy key used for Ethereum Name Service (ENS) resolution and retrieving ETH public keys. Replace `` with your actual key. - `COMM_JSONCONFIG_secrets_walletconnect`: WalletConnect key used to enable Sign-In with Ethereum (SIWE). Replace `` with your actual key. ### URL configuration - `COMM_JSONCONFIG_facts_keyserver_url`: Your keyserver needs to know what its externally-facing URL is in order to construct links. It also needs to know if it’s being proxied to that externally-facing URL, and what the internal route path is. - `baseDomain`: Externally-facing domain. Used for constructing links. - `basePath`: Externally-facing path. Used for constructing links. - `baseRoutePath`: Internally-facing path. Same as basePath if no proxy. If there’s a proxy, this is the local path (e.g. http://localhost:3000/landing would correspond with /landing/) - `proxy`: `"none" | "apache"` Determines which request headers to use for HTTPS validation and IP address timezone detection. - `https`: If true, checks request headers to validate that HTTPS is in use. ### CORS configuration - `COMM_JSONCONFIG_facts_webapp_cors`: Your keyserver needs to be able to include CORS headers with the domain where the comm web application is hosted. - `domain`: Domain where the web application is hosted. ### Backup configuration - `COMM_JSONCONFIG_facts_backups`: Specifies whether to enable backups, where to store them, and the max size of the backups directory. ## Building & deploying Once configured, the keyserver can be built and deployed by simply running: ```bash cd keyserver ./bash/dc.sh up --build ``` diff --git a/keyserver/Dockerfile b/keyserver/Dockerfile index e183aa1b7..e020179b1 100644 --- a/keyserver/Dockerfile +++ b/keyserver/Dockerfile @@ -1,224 +1,225 @@ FROM node:20.10.0-bullseye #------------------------------------------------------------------------------- # STEP 0: SET UP USER # Set up Linux user and group for the container #------------------------------------------------------------------------------- # We use bind mounts for our backups folder, which means Docker on Linux will # blindly match the UID/GID for the backups folder on the container with the # host. In order to make sure the container is able to create backups with the # right UID/GID, we need to do two things: # 1. Make sure that the user that runs the Docker container on the host has # permissions to write to the backups folder on the host. We rely on the host # to configure this properly # 2. Make sure we're running this container with the same UID/GID that the host # is using, so the UID/GID show up correctly on both sides of the bind mount # To handle 2 correctly, we have the host pass the UID/GID with which they're # running the container. Our approach is based on this one: # https://github.com/mhart/alpine-node/issues/48#issuecomment-430902787 ARG HOST_UID ARG HOST_GID ARG COMM_JSONCONFIG_secrets_alchemy ARG COMM_JSONCONFIG_secrets_walletconnect +ARG COMM_JSONCONFIG_secrets_neynar ARG COMM_JSONCONFIG_secrets_geoip_license USER root RUN \ if [ -z "`getent group $HOST_GID`" ]; then \ addgroup --system --gid $HOST_GID comm; \ else \ groupmod --new-name comm `getent group $HOST_GID | cut -d: -f1`; \ fi && \ if [ -z "`getent passwd $HOST_UID`" ]; then \ adduser --system --uid $HOST_UID --ingroup comm --shell /bin/bash comm; \ else \ usermod --login comm --gid $HOST_GID --home /home/comm --move-home \ `getent passwd $HOST_UID | cut -d: -f1`; \ fi #------------------------------------------------------------------------------- # STEP 1: INSTALL PREREQS # Install prereqs first so we don't have to reinstall them if anything changes #------------------------------------------------------------------------------- # We need to add the MariaDB repo to apt in order to install mariadb-client RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup \ && chmod +x mariadb_repo_setup \ && ./mariadb_repo_setup \ && rm mariadb_repo_setup # We need rsync in the prod-build yarn script # We need mariadb-client so we can use mysqldump for backups # We need cmake to install protobuf (prereq for rust-node-addon) # We need binaryen for wasm-opt tool used for WASM optimization RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rsync \ mariadb-client \ cmake \ binaryen \ && rm -rf /var/lib/apt/lists/* # Install protobuf manually to ensure that we have the correct version COPY scripts/install_protobuf.sh scripts/ RUN cd scripts && ./install_protobuf.sh #------------------------------------------------------------------------------- # STEP 2: DEVOLVE PRIVILEGES # Create another user to run the rest of the commands #------------------------------------------------------------------------------- USER comm WORKDIR /home/comm/app #------------------------------------------------------------------------------- # STEP 3: SET UP MYSQL BACKUPS # Prepare the system to properly handle mysqldump backups #------------------------------------------------------------------------------- # Prepare the directory that will hold the backups RUN mkdir /home/comm/backups #------------------------------------------------------------------------------- # STEP 4: SET UP CARGO (RUST PACKAGE MANAGER) # We use Cargo to build pre-compiled Node.js addons in Rust #------------------------------------------------------------------------------- # Install Rust and add Cargo's bin directory to the $PATH environment variable RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH /home/comm/.cargo/bin:$PATH #------------------------------------------------------------------------------- # STEP 5: SET UP NVM # We use nvm to make sure we're running the right Node version #------------------------------------------------------------------------------- # First we install nvm ENV NVM_DIR /home/comm/.nvm RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh \ | bash # Then we use nvm to install the right version of Node. We call this early so # Docker build caching saves us from re-downloading Node when any file changes COPY --chown=comm keyserver/.nvmrc keyserver/ COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/ RUN cd keyserver && . bash/source-nvm.sh #------------------------------------------------------------------------------- # STEP 6: YARN CLEANINSTALL # We run yarn cleaninstall before copying most of the files in for build caching #------------------------------------------------------------------------------- # Copy in package.json files, yarn.lock files, and relevant installation scripts COPY --chown=comm package.json yarn.lock postinstall.sh ./ COPY --chown=comm keyserver/package.json keyserver/.flowconfig keyserver/ COPY --chown=comm lib/package.json lib/.flowconfig lib/ COPY --chown=comm web/package.json web/.flowconfig web/ COPY --chown=comm web/scripts/postinstall.sh web/scripts/run-wasmpack.sh \ web/scripts/ COPY --chown=comm native/package.json native/.flowconfig native/ COPY --chown=comm landing/package.json landing/.flowconfig landing/ COPY --chown=comm desktop/package.json desktop/ COPY --chown=comm desktop/addons/windows-pushnotifications/package.json \ desktop/addons/windows-pushnotifications/ COPY --chown=comm keyserver/addons/rust-node-addon/package.json \ keyserver/addons/rust-node-addon/install_ci_deps.sh \ keyserver/addons/rust-node-addon/postinstall.sh \ keyserver/addons/rust-node-addon/ COPY --chown=comm native/expo-modules/comm-expo-package/package.json \ native/expo-modules/comm-expo-package/ COPY --chown=comm services/electron-update-server/package.json \ services/electron-update-server/ # Create empty Rust library and copy in Cargo.toml file RUN cargo init keyserver/addons/rust-node-addon --lib COPY --chown=comm keyserver/addons/rust-node-addon/Cargo.toml \ keyserver/addons/rust-node-addon/ # Create empty Rust library for WASM backup client and copy in Cargo.toml file RUN cargo init web/backup-client-wasm/ --lib COPY --chown=comm web/backup-client-wasm/Cargo.toml \ web/backup-client-wasm/ # Copy in local dependencies of rust-node-addon COPY --chown=comm shared/comm-opaque2 shared/comm-opaque2/ # Copy protobuf files as a dependency for the shared client libraries COPY --chown=comm shared/protos shared/protos/ # Copy identity service gRPC client COPY --chown=comm shared/grpc_clients shared/grpc_clients/ # Copy shared backup client COPY --chown=comm shared/backup_client shared/backup_client/ # Copy shared lib COPY --chown=comm shared/comm-lib shared/comm-lib/ # Copy in files needed for patch-package COPY --chown=comm patches patches/ # Actually run yarn RUN yarn cleaninstall #------------------------------------------------------------------------------- # STEP 7: GEOIP UPDATE # We update the GeoIP database for mapping from IP address to timezone #------------------------------------------------------------------------------- COPY --chown=comm keyserver/bash/docker-update-geoip.sh keyserver/bash/ RUN cd keyserver && bash/docker-update-geoip.sh #------------------------------------------------------------------------------- # STEP 8: WEBPACK BUILD # We do this first so Docker doesn't rebuild when only keyserver files change #------------------------------------------------------------------------------- # These are needed for babel-build-comm-config COPY --chown=comm keyserver/src keyserver/src COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/source-nvm.sh COPY --chown=comm keyserver/.babelrc.cjs keyserver/.babelrc.cjs COPY --chown=comm lib lib/ COPY --chown=comm landing landing/ RUN yarn workspace landing prod COPY --chown=comm web web/ # Build WASM backup client no that source files are copied in RUN yarn workspace web build-backup-client-wasm RUN yarn workspace web prod #------------------------------------------------------------------------------- # STEP 9: COPY IN SOURCE FILES # We run this later so the above layers are cached if only source files change #------------------------------------------------------------------------------- COPY --chown=comm . . #------------------------------------------------------------------------------- # STEP 10: BUILD NODE ADDON # Now that source files have been copied in, build rust-node-addon #------------------------------------------------------------------------------- RUN yarn workspace rust-node-addon build #------------------------------------------------------------------------------- # STEP 11: RUN BUILD SCRIPTS # We need to populate keyserver/dist, among other things #------------------------------------------------------------------------------- # Babel transpilation of keyserver src RUN yarn workspace keyserver prod-build #------------------------------------------------------------------------------- # STEP 12: RUN THE SERVER # Actually run the Node.js keyserver using nvm #------------------------------------------------------------------------------- EXPOSE 3000 WORKDIR /home/comm/app/keyserver CMD bash/run-prod.sh diff --git a/keyserver/docker-compose.yml b/keyserver/docker-compose.yml index b9a0528e8..1b123f0b8 100644 --- a/keyserver/docker-compose.yml +++ b/keyserver/docker-compose.yml @@ -1,57 +1,58 @@ version: '3.9' services: node: build: dockerfile: keyserver/Dockerfile context: ../ args: - HOST_UID=${HOST_UID} - HOST_GID=${HOST_GID} - COMM_JSONCONFIG_secrets_alchemy=${COMM_JSONCONFIG_secrets_alchemy} - COMM_JSONCONFIG_secrets_walletconnect=${COMM_JSONCONFIG_secrets_walletconnect} + - COMM_JSONCONFIG_secrets_neynar=${COMM_JSONCONFIG_secrets_neynar} - COMM_JSONCONFIG_secrets_geoip_license=${COMM_JSONCONFIG_secrets_geoip_license} image: commapp/node-keyserver:1.0 restart: always ports: - '3000:3000' env_file: - .env environment: - REDIS_URL=redis://cache - COMM_LISTEN_ADDR=0.0.0.0 - COMM_DATABASE_HOST=${COMM_DATABASE_HOST:-database} - COMM_DATABASE_DATABASE - COMM_DATABASE_USER - COMM_DATABASE_PASSWORD - COMM_DATABASE_TYPE=mariadb10.8 depends_on: - cache - database database: image: mariadb:10.8.3-jammy restart: always expose: - '3306' command: > --performance-schema --max-allowed-packet=256M --local-infile=0 --sql-mode=STRICT_ALL_TABLES --innodb-buffer-pool-size=1600M --innodb-ft-min-token-size=1 --innodb-ft-enable-stopword=0 environment: - MARIADB_RANDOM_ROOT_PASSWORD=yes - MARIADB_DATABASE=$COMM_DATABASE_DATABASE - MARIADB_USER=$COMM_DATABASE_USER - MARIADB_PASSWORD=$COMM_DATABASE_PASSWORD volumes: - mysqldata:/var/lib/mysql cache: image: redis:6.2.6-bullseye restart: always expose: - '6379' command: redis-server --loglevel warning volumes: mysqldata: diff --git a/lib/webpack/shared.cjs b/lib/webpack/shared.cjs index 6ad5f8b03..01cd5cfdd 100644 --- a/lib/webpack/shared.cjs +++ b/lib/webpack/shared.cjs @@ -1,303 +1,308 @@ const ReactRefreshWebpackPlugin = require('@pmmmwh/react-refresh-webpack-plugin'); const { CleanWebpackPlugin } = require('clean-webpack-plugin'); const CssMinimizerPlugin = require('css-minimizer-webpack-plugin'); const MiniCssExtractPlugin = require('mini-css-extract-plugin'); const TerserPlugin = require('terser-webpack-plugin'); const webpack = require('webpack'); async function getConfig(configName) { const { getCommConfig } = await import( '../../keyserver/dist/lib/utils/comm-config.js' ); return await getCommConfig(configName); } const sharedPlugins = [ new webpack.optimize.LimitChunkCountPlugin({ maxChunks: 1, }), ]; const cssLoader = { loader: 'css-loader', options: { modules: { mode: 'local', localIdentName: '[path][name]__[local]--[contenthash:base64:5]', }, }, }; const cssExtractLoader = { loader: MiniCssExtractPlugin.loader, }; const styleLoader = { loader: 'style-loader', }; function getBabelRule(babelConfig) { return { test: /\.js$/, exclude: /node_modules\/(?!lib)/, loader: 'babel-loader', options: babelConfig, }; } function getBrowserBabelRule(babelConfig) { const babelRule = getBabelRule(babelConfig); return { ...babelRule, options: { ...babelRule.options, presets: [ ...babelRule.options.presets, [ '@babel/preset-env', { targets: 'defaults', useBuiltIns: 'usage', corejs: '3.6', }, ], ], }, }; } const imageRule = { test: /\.(png|svg)$/, type: 'asset/inline', }; const typographyRule = { test: /\.(woff2|woff)$/, type: 'asset/inline', }; function createBaseBrowserConfig(baseConfig) { return { ...baseConfig, name: 'browser', optimization: { minimizer: [new TerserPlugin(), new CssMinimizerPlugin()], }, plugins: [ ...(baseConfig.plugins ?? []), ...sharedPlugins, new webpack.ProvidePlugin({ Buffer: ['buffer', 'Buffer'], }), new CleanWebpackPlugin({ cleanOnceBeforeBuildPatterns: [], }), ], node: { global: true, }, }; } async function getConfigs() { - const [alchemySecret, walletConnectSecret] = await Promise.all([ + const [alchemySecret, walletConnectSecret, neynarSecret] = await Promise.all([ getConfig({ folder: 'secrets', name: 'alchemy' }), getConfig({ folder: 'secrets', name: 'walletconnect' }), + getConfig({ folder: 'secrets', name: 'neynar' }), ]); const alchemyKey = alchemySecret?.key; const walletConnectKey = walletConnectSecret?.key; - return { alchemyKey, walletConnectKey }; + const neynarKey = neynarSecret?.key; + return { alchemyKey, walletConnectKey, neynarKey }; } async function createProdBrowserConfig(baseConfig, babelConfig, envVars) { const browserConfig = createBaseBrowserConfig(baseConfig); const babelRule = getBrowserBabelRule(babelConfig); - const { alchemyKey, walletConnectKey } = await getConfigs(); + const { alchemyKey, walletConnectKey, neynarKey } = await getConfigs(); return { ...browserConfig, mode: 'production', plugins: [ ...browserConfig.plugins, new webpack.DefinePlugin({ 'process.env': { NODE_ENV: JSON.stringify('production'), BROWSER: true, COMM_ALCHEMY_KEY: JSON.stringify(alchemyKey), COMM_WALLETCONNECT_KEY: JSON.stringify(walletConnectKey), + COMM_NEYNAR_KEY: JSON.stringify(neynarKey), ...envVars, }, }), new MiniCssExtractPlugin({ filename: 'prod.[contenthash:12].build.css', }), ], module: { rules: [ { ...babelRule, options: { ...babelRule.options, plugins: [ ...babelRule.options.plugins, '@babel/plugin-transform-react-constant-elements', ['transform-remove-console', { exclude: ['error', 'warn'] }], ], }, }, { test: /\.css$/, exclude: /node_modules\/.*\.css$/, use: [ cssExtractLoader, { ...cssLoader, options: { ...cssLoader.options, url: false, }, }, ], }, { test: /node_modules\/.*\.css$/, sideEffects: true, use: [ cssExtractLoader, { ...cssLoader, options: { ...cssLoader.options, url: false, modules: false, }, }, ], }, ], }, }; } async function createDevBrowserConfig(baseConfig, babelConfig, envVars) { const browserConfig = createBaseBrowserConfig(baseConfig); const babelRule = getBrowserBabelRule(babelConfig); - const { alchemyKey, walletConnectKey } = await getConfigs(); + const { alchemyKey, walletConnectKey, neynarKey } = await getConfigs(); return { ...browserConfig, mode: 'development', plugins: [ ...browserConfig.plugins, new webpack.DefinePlugin({ 'process.env': { NODE_ENV: JSON.stringify('development'), BROWSER: true, COMM_ALCHEMY_KEY: JSON.stringify(alchemyKey), COMM_WALLETCONNECT_KEY: JSON.stringify(walletConnectKey), + COMM_NEYNAR_KEY: JSON.stringify(neynarKey), ...envVars, }, }), new ReactRefreshWebpackPlugin(), ], module: { rules: [ { ...babelRule, options: { ...babelRule.options, plugins: [ require.resolve('react-refresh/babel'), ...babelRule.options.plugins, ], }, }, imageRule, typographyRule, { test: /\.css$/, exclude: /node_modules\/.*\.css$/, use: [styleLoader, cssLoader], }, { test: /node_modules\/.*\.css$/, sideEffects: true, use: [ styleLoader, { ...cssLoader, options: { ...cssLoader.options, modules: false, }, }, ], }, ], }, devtool: 'eval-cheap-module-source-map', }; } async function createNodeServerRenderingConfig(baseConfig, babelConfig) { - const { alchemyKey, walletConnectKey } = await getConfigs(); + const { alchemyKey, walletConnectKey, neynarKey } = await getConfigs(); return { ...baseConfig, name: 'server', target: 'node', module: { rules: [ getBabelRule(babelConfig), { test: /\.css$/, use: { ...cssLoader, options: { ...cssLoader.options, modules: { ...cssLoader.options.modules, exportOnlyLocals: true, }, }, }, }, ], }, plugins: [ ...sharedPlugins, new webpack.DefinePlugin({ 'process.env': { COMM_ALCHEMY_KEY: JSON.stringify(alchemyKey), COMM_WALLETCONNECT_KEY: JSON.stringify(walletConnectKey), + COMM_NEYNAR_KEY: JSON.stringify(neynarKey), }, }), ], }; } function createWebWorkersConfig(env, baseConfig, babelConfig) { return { ...baseConfig, name: 'webworkers', target: 'webworker', mode: env.prod ? 'production' : 'development', module: { rules: [getBrowserBabelRule(babelConfig)], }, plugins: [ ...sharedPlugins, ...baseConfig.plugins, new webpack.DefinePlugin({ 'process.env': { NODE_ENV: JSON.stringify(env.prod ? 'production' : 'development'), BROWSER: true, }, }), new webpack.ProvidePlugin({ Buffer: ['buffer', 'Buffer'], }), ], }; } module.exports = { createProdBrowserConfig, createDevBrowserConfig, createNodeServerRenderingConfig, createWebWorkersConfig, };