diff --git a/keyserver/.nvmrc b/keyserver/.nvmrc index 94f5f26dc..b460d6f2d 100644 --- a/keyserver/.nvmrc +++ b/keyserver/.nvmrc @@ -1 +1 @@ -16.18.0 +18.12.1 diff --git a/keyserver/Dockerfile b/keyserver/Dockerfile index 0fdd19573..c6392a643 100644 --- a/keyserver/Dockerfile +++ b/keyserver/Dockerfile @@ -1,202 +1,202 @@ -FROM node:16.18.0-bullseye +FROM node:18.12.1-bullseye #------------------------------------------------------------------------------- # STEP 0: SET UP USER # Set up Linux user and group for the container #------------------------------------------------------------------------------- # We use bind mounts for our backups folder, which means Docker on Linux will # blindly match the UID/GID for the backups folder on the container with the # host. In order to make sure the container is able to create backups with the # right UID/GID, we need to do two things: # 1. Make sure that the user that runs the Docker container on the host has # permissions to write to the backups folder on the host. We rely on the host # to configure this properly # 2. Make sure we're running this container with the same UID/GID that the host # is using, so the UID/GID show up correctly on both sides of the bind mount # To handle 2 correctly, we have the host pass the UID/GID with which they're # running the container. Our approach is based on this one: # https://github.com/mhart/alpine-node/issues/48#issuecomment-430902787 ARG HOST_UID ARG HOST_GID ARG COMM_JSONCONFIG_secrets_alchemy ARG COMM_JSONCONFIG_secrets_walletconnect ARG COMM_JSONCONFIG_secrets_geoip_license USER root RUN \ if [ -z "`getent group $HOST_GID`" ]; then \ addgroup --system --gid $HOST_GID comm; \ else \ groupmod --new-name comm `getent group $HOST_GID | cut -d: -f1`; \ fi && \ if [ -z "`getent passwd $HOST_UID`" ]; then \ adduser --system --uid $HOST_UID --ingroup comm --shell /bin/bash comm; \ else \ usermod --login comm --gid $HOST_GID --home /home/comm --move-home \ `getent passwd $HOST_UID | cut -d: -f1`; \ fi #------------------------------------------------------------------------------- # STEP 1: INSTALL PREREQS # Install prereqs first so we don't have to reinstall them if anything changes #------------------------------------------------------------------------------- # We need to add the MariaDB repo to apt in order to install mariadb-client RUN wget https://downloads.mariadb.com/MariaDB/mariadb_repo_setup \ && chmod +x mariadb_repo_setup \ && ./mariadb_repo_setup \ && rm mariadb_repo_setup # We need rsync in the prod-build yarn script # We need mariadb-client so we can use mysqldump for backups # We need cmake to install protobuf (prereq for rust-node-addon) RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ rsync \ mariadb-client \ cmake \ && rm -rf /var/lib/apt/lists/* # Install protobuf manually to ensure that we have the correct version COPY scripts/install_protobuf.sh scripts/ RUN cd scripts && ./install_protobuf.sh #------------------------------------------------------------------------------- # STEP 2: DEVOLVE PRIVILEGES # Create another user to run the rest of the commands #------------------------------------------------------------------------------- USER comm WORKDIR /home/comm/app #------------------------------------------------------------------------------- # STEP 3: SET UP MYSQL BACKUPS # Prepare the system to properly handle mysqldump backups #------------------------------------------------------------------------------- # Prepare the directory that will hold the backups RUN mkdir /home/comm/backups #------------------------------------------------------------------------------- # STEP 4: SET UP CARGO (RUST PACKAGE MANAGER) # We use Cargo to build pre-compiled Node.js addons in Rust #------------------------------------------------------------------------------- # Install Rust and add Cargo's bin directory to the $PATH environment variable RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH /home/comm/.cargo/bin:$PATH #------------------------------------------------------------------------------- # STEP 5: SET UP NVM # We use nvm to make sure we're running the right Node version #------------------------------------------------------------------------------- # First we install nvm ENV NVM_DIR /home/comm/.nvm RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh \ | bash # Then we use nvm to install the right version of Node. We call this early so # Docker build caching saves us from re-downloading Node when any file changes COPY --chown=comm keyserver/.nvmrc keyserver/ COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/ RUN cd keyserver && . bash/source-nvm.sh #------------------------------------------------------------------------------- # STEP 6: YARN CLEANINSTALL # We run yarn cleaninstall before copying most of the files in for build caching #------------------------------------------------------------------------------- # Copy in package.json files, yarn.lock files, and relevant installation scripts COPY --chown=comm package.json yarn.lock postinstall.sh ./ COPY --chown=comm keyserver/package.json keyserver/.flowconfig keyserver/ COPY --chown=comm lib/package.json lib/.flowconfig lib/ COPY --chown=comm web/package.json web/.flowconfig web/ COPY --chown=comm native/package.json native/.flowconfig native/ COPY --chown=comm landing/package.json landing/.flowconfig landing/ COPY --chown=comm desktop/package.json desktop/ COPY --chown=comm desktop/addons/windows-pushnotifications/package.json \ desktop/addons/windows-pushnotifications/ COPY --chown=comm keyserver/addons/rust-node-addon/package.json \ keyserver/addons/rust-node-addon/install_ci_deps.sh \ keyserver/addons/rust-node-addon/postinstall.sh \ keyserver/addons/rust-node-addon/ COPY --chown=comm native/expo-modules/comm-expo-package/package.json \ native/expo-modules/comm-expo-package/ COPY --chown=comm services/electron-update-server/package.json \ services/electron-update-server/ # Create empty Rust library and copy in Cargo.toml file RUN cargo init keyserver/addons/rust-node-addon --lib COPY --chown=comm keyserver/addons/rust-node-addon/Cargo.toml \ keyserver/addons/rust-node-addon/ # Copy in local dependencies of rust-node-addon COPY --chown=comm shared/comm-opaque2 shared/comm-opaque2/ # Copy protobuf files as a dependency for the shared client libraries COPY --chown=comm shared/protos shared/protos/ # Copy in files needed for patch-package COPY --chown=comm patches patches/ # Actually run yarn RUN yarn cleaninstall #------------------------------------------------------------------------------- # STEP 7: GEOIP UPDATE # We update the GeoIP database for mapping from IP address to timezone #------------------------------------------------------------------------------- COPY --chown=comm keyserver/bash/docker-update-geoip.sh keyserver/bash/ RUN cd keyserver && bash/docker-update-geoip.sh #------------------------------------------------------------------------------- # STEP 8: WEBPACK BUILD # We do this first so Docker doesn't rebuild when only keyserver files change #------------------------------------------------------------------------------- # These are needed for babel-build-comm-config COPY --chown=comm keyserver/src keyserver/src COPY --chown=comm keyserver/bash/source-nvm.sh keyserver/bash/source-nvm.sh COPY --chown=comm keyserver/babel.config.cjs keyserver/babel.config.cjs COPY --chown=comm lib lib/ COPY --chown=comm landing landing/ RUN yarn workspace landing prod COPY --chown=comm web web/ RUN yarn workspace web prod #------------------------------------------------------------------------------- # STEP 9: COPY IN SOURCE FILES # We run this later so the above layers are cached if only source files change #------------------------------------------------------------------------------- COPY --chown=comm . . #------------------------------------------------------------------------------- # STEP 10: BUILD NODE ADDON # Now that source files have been copied in, build rust-node-addon #------------------------------------------------------------------------------- RUN yarn workspace rust-node-addon build #------------------------------------------------------------------------------- # STEP 11: RUN BUILD SCRIPTS # We need to populate keyserver/dist, among other things #------------------------------------------------------------------------------- # Babel transpilation of keyserver src RUN yarn workspace keyserver prod-build #------------------------------------------------------------------------------- # STEP 12: RUN THE SERVER # Actually run the Node.js keyserver using nvm #------------------------------------------------------------------------------- EXPOSE 3000 WORKDIR /home/comm/app/keyserver CMD bash/run-prod.sh diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix index c32a2c57c..2f799846f 100644 --- a/nix/dev-shell.nix +++ b/nix/dev-shell.nix @@ -1,161 +1,161 @@ { mkShell , stdenv , lib , amqp-cpp , awscli2 , arcanist , aws-sdk-cpp , better-prompt , boost , bundler , c-ares_cmake-config , cargo-udeps , cmake , cmake-format , cocoapods , corrosion , darwin , double-conversion , emscripten , folly , fmt , glog , grpc , gtest , libiconv , libuv , localstack , mariadb , mariadb-up -, nodejs-16_x-openssl_1_1 +, nodejs , olm , openjdk11 , openssl , pkg-config , protobuf_3_15_cmake , python3 , rabbitmq-server , redis , redis-up , rustup , shellcheck , sops , sqlite , terraform , rustfmt , wasm-pack , yarn , protoc-gen-grpc-web }: mkShell { # programs which are meant to be executed should go here nativeBuildInputs = [ # generic development or tools arcanist awscli2 shellcheck sops terraform emscripten # android openjdk11 # node development mariadb - nodejs-16_x-openssl_1_1 + nodejs yarn python3 redis wasm-pack protoc-gen-grpc-web # native dependencies # C/CXX toolchains are already brought in with mkShell # Identity Service rustfmt rustup cargo-udeps # Tunnelbroker + CMake amqp-cpp c-ares_cmake-config cmake cmake-format # linting libuv localstack pkg-config protobuf_3_15_cmake grpc rabbitmq-server # runtime service ] ++ lib.optionals stdenv.isDarwin [ cocoapods # needed for ios bundler ]; # include any libraries buildInputs buildInputs = [ # protobuf exposes both a library and a command # thus should appear in both inputs protobuf_3_15_cmake aws-sdk-cpp # tunnelbroker corrosion # tunnelbroker double-conversion # tunnelbroker glog # tunnelbroker gtest # testing services folly # cpp tools fmt # needed for folly boost # needed for folly olm # needed for CryptoTools sqlite # needed for sqlite_orm openssl # needed for grpc ] ++ lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ CoreFoundation CoreServices Security libiconv # identity service ]); JAVA_HOME = openjdk11.passthru.home; # shell commands to be ran upon entering shell shellHook = '' PRJ_ROOT=$(git rev-parse --show-toplevel) # Set development environment variable defaults source "${../scripts/source_development_defaults.sh}" # Cache development path for some use cases such as XCode "$PRJ_ROOT/scripts/save_path.sh" '' # Darwin condition can be removed once linux services are supported + lib.optionalString stdenv.isDarwin '' # Start MariaDB development services "${mariadb-up}"/bin/mariadb-up & mariadb_pid=$! "${redis-up}"/bin/redis-up & redis_pid=$! wait "$mariadb_pid" "$redis_pid" ${../scripts}/install_homebrew_macos.sh ${../scripts}/install_homebrew_deps.sh watchman '' + '' # Render default configuration for keyserver $PRJ_ROOT/scripts/create_url_facts.sh # Ensure rustup tooling is installed $PRJ_ROOT/scripts/ensure_rustup_setup.sh # Provide decent bash prompt source "${better-prompt}/bin/better-prompt" echo "Welcome to Comm dev environment! :)" ''; } diff --git a/nix/overlay.nix b/nix/overlay.nix index ac837df18..77de34548 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,90 +1,85 @@ # An overlay allows for a package set to be extended with new or modified packages # `final` refers to the package set with all overlays applied. # This allows for added or modified packages to be referenced with # all relevant changes final: # `prev` refers to the previous package set before this current overlay is applied. # This is cheaper for nix to evaluate, thus should be prefered over final when possible. prev: { # Patch aws-sdk-cpp to automatically pick up header location # specific to nixpkgs, as nixpkgs separates build-time and runtime # depencenies (a saving of 400MB in header + generated files). # In the case of c and c++, this means the header files are # located in a separate directory from the libraries. # # From a developer perspective, this avoids having to manually specify # the header location with `-DAWS_CORE_HEADER_FILE` each time # one invokes `cmake` on the command line when using # `find_package(AWSSDK COMPONENTS [comps])` # # For more information, see: # - aws-sdk-cpp issue: https://github.com/aws/aws-sdk-cpp/issues/2009 # - Nixpkgs fix: https://github.com/NixOS/nixpkgs/pull/182918 aws-sdk-cpp = (prev.aws-sdk-cpp.overrideAttrs(oldAttrs:{ postPatch = oldAttrs.postPatch + '' substituteInPlace cmake/AWSSDKConfig.cmake \ --replace 'C:/AWSSDK/''${AWSSDK_INSTALL_INCLUDEDIR}/aws/core' \ 'C:/AWSSDK/''${AWSSDK_INSTALL_INCLUDEDIR}/aws/core" "${placeholder "dev"}/include/aws/core' ''; })).override { # avoid rebuildilng all 300+ apis apis = [ "core" "s3" "dynamodb" ]; }; # add packages meant for just this repository amqp-cpp = prev.callPackage ./amqp-cpp.nix { }; arcanist = prev.callPackage ./arcanist.nix { }; better-prompt = prev.callPackage ./better-prompt.nix { }; # c-ares is used to bootstrap curl, so cmake is not available in the default # build c-ares_cmake-config = prev.c-ares.overrideAttrs(o: { nativeBuildInputs = (o.nativeBuildInputs or []) ++ [ prev.cmake ]; }); comm-blob = final.callPackage ./blob.nix { }; protobuf_3_15_cmake = prev.callPackage ./protobuf_3_15.nix { }; devShells.default = final.callPackage ./dev-shell.nix { }; devShell = final.devShells.default; localstack-down = prev.callPackage ./localstack-down.nix { }; localstack-up = prev.callPackage ./localstack-up.nix { }; # Make our version of mariadb the default everywhere mariadb = prev.mariadb_108; mariadb-up = prev.callPackage ./mariadb-up-mac.nix { }; mysql-down = prev.callPackage ./mysql-down-linux.nix { }; mysql-up = prev.callPackage ./mysql-up-linux.nix { }; redis-up = prev.callPackage ./redis-up-mac.nix { }; olm = prev.olm.overrideAttrs(oldAttrs: { # *.hh files aren't meant to be used externally # so we patch installation to add it postInstall = '' cp \ $NIX_BUILD_TOP/${oldAttrs.src.name}/include/olm/*.h* \ ''${!outputDev}/include/olm ''; }); rabbitmq-up = prev.callPackage ./rabbitmq-up-mac.nix { }; - - # Ensure that yarn is using the pinned version - yarn = prev.yarn.override (_: { - nodejs = final.nodejs-16_x-openssl_1_1; - }); }