diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix index ebf4b0d75..7da414fc6 100644 --- a/nix/dev-shell.nix +++ b/nix/dev-shell.nix @@ -1,110 +1,114 @@ { mkShell , stdenv , lib , amqp-cpp , arcanist , aws-sdk-cpp , boost , cargo , cmake , cmake-format , cocoapods , corrosion , cryptopp , darwin , double-conversion , folly , fmt , glog , grpc , libiconv , libuv +, mariadb , nodejs-16_x , olm , openjdk8 , openssl , pkg-config , protobuf_3_15_cmake , python3 , shellcheck , sqlite , watchman , rustfmt , yarn }: mkShell { # programs which are meant to be executed should go here nativeBuildInputs = [ # generic development arcanist shellcheck # node development + mariadb nodejs-16_x yarn watchman # react native python3 # native dependencies # C/CXX toolchains are already brought in with mkShell # Identity Service cargo # includes rustc rustfmt # Tunnelbroker + CMake amqp-cpp cryptopp cmake cmake-format # linting libuv pkg-config protobuf_3_15_cmake grpc ] ++ lib.optionals stdenv.isDarwin [ cocoapods # needed for ios ]; # include any libraries buildInputs buildInputs = [ # protobuf exposes both a library and a command # thus should appear in both inputs protobuf_3_15_cmake aws-sdk-cpp # tunnelbroker corrosion # tunnelbroker double-conversion # tunnelbroker glog # tunnelbroker folly # cpp tools fmt # needed for folly boost # needed for folly olm # needed for CryptoTools sqlite # needed for sqlite_orm openssl # needed for grpc ] ++ lib.optionals stdenv.isDarwin (with darwin.apple_sdk.frameworks; [ CoreFoundation CoreServices Security libiconv # identity service ]); JAVA_HOME = openjdk8.passthru.home; # shell commands to be ran upon entering shell shellHook = let - socket = "mysql-socket/mysql.sock"; + socket = "mysql.sock"; in '' if [[ "$OSTYPE" == 'linux'* ]]; then export MYSQL_UNIX_PORT=''${XDG_RUNTIME_DIR:-/run/user/$UID}/${socket} export ANDROID_SDK_ROOT=''${ANDROID_SDK_ROOT:-$HOME/Android/Sdk} fi if [[ "$OSTYPE" == 'darwin'* ]]; then # Many commands for cocoapods expect the native BSD versions of commands export PATH=/usr/bin:$PATH + MARIADB_DIR=''${XDG_DATA_HOME:-$HOME/.local/share}/MariaDB + export MYSQL_UNIX_PORT="$MARIADB_DIR"/${socket} export ANDROID_SDK_ROOT=''${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk} fi echo "Welcome to Comm dev environment! :)" ''; } diff --git a/nix/mariadb-up-mac.nix b/nix/mariadb-up-mac.nix new file mode 100644 index 000000000..9baab4619 --- /dev/null +++ b/nix/mariadb-up-mac.nix @@ -0,0 +1,130 @@ +{ lib +, gnused +, openssl +, mariadb +, writeShellApplication +, writeTextFile +}: + +let + # Use small script executed by bash to have a normal shell environment. + mariadb-entrypoint = writeShellApplication { + name = "mariadb-init"; + text = '' + MARIADB_DIR=''${XDG_DATA_HOME:-$HOME/.local/share}/MariaDB + # 'exec' allows for us to replace bash process with MariaDB + exec ${mariadb}/bin/mariadbd \ + --socket "$MARIADB_DIR"/mysql.sock \ + --datadir "$MARIADB_DIR" \ + &> "$MARIADB_DIR"/logs + ''; + }; + + mariadb-version = let + versions = lib.versions; + in "${versions.major mariadb.version}.${versions.minor mariadb.version}"; + + # Small boiler-plate text file for us to write for keyserver + db_config_template = writeTextFile { + name = "db-config"; + text = '' + { + "host": "localhost", + "user": "comm", + "password": "PASS", + "database": "comm", + "dbType": "mariadb${mariadb-version}" + } + ''; + }; + +# writeShellApplication is a "writer helper" which +# will create a shellchecked executable shell script located in $out/bin/ +# This shell script will be used to allow for impure+stateful actions +in writeShellApplication { + name = "mariadb-up"; + text = '' + # "$HOME/Library/Application Support/" is the canonical path to use + # on darwin for storing user data for installed applications. + # However, mysql and mariadb don't quote paths in the mariadbd script, + # so use XDG conventions and hope $HOME doesn't have a space. + MARIADB_DATA_HOME="''${XDG_DATA_HOME:-$HOME/.local/share}/MariaDB" + MARIADB_PIDFILE="$MARIADB_DATA_HOME"/mariadb.pid + export MYSQL_UNIX_PORT="$MARIADB_DATA_HOME"/mysql.sock + + if [[ ! -d "$MARIADB_DATA_HOME"/mysql ]]; then + # mysql directory should exist if MariaDB has been initialized + echo "Initializing MariaDB database at $MARIADB_DATA_HOME" >&2 + "${lib.getBin mariadb}/bin/mariadb-install-db" \ + --datadir="$MARIADB_DATA_HOME" \ + --auth-root-authentication-method=socket + fi + + # Check if MariaDB server was already started + set +e # allow for pgrep to not find matches + # BSD pgrep doesn't have a "count" feature, use wc then trim whitespace + mariadb_count=$(pgrep mariadbd | wc -l | xargs echo) + set -e + + if [[ "$mariadb_count" -eq "0" ]]; then + echo "Starting MariaDB server" + # No MariaDB present, start our own + # Launch in subshell so if the original terminal is closed, the process + # will be inherited instead of also being forced closed + ("${mariadb-entrypoint}/bin/mariadb-init" & + echo "$!" > "$MARIADB_PIDFILE") + + echo "Waiting for MariaDB to come up" + while [[ ! -S "$MYSQL_UNIX_PORT" ]]; do sleep 1; done + + elif [[ "$mariadb_count" -eq "1" ]]; then + + # Check if it was started by this script + running_pid="$(pgrep mariadbd)" + if [[ ! -f "$MARIADB_PIDFILE" ]] || \ + [[ "$(cat "$MARIADB_PIDFILE")" != "$running_pid" ]]; then + echo "Existing MariaDB instance found outside of nix environment" >&2 + echo "Please stop existing services and attempt 'mariadb-up' again" >&2 + exit 1 + fi + + else + echo "Many MariaDB instances found outside of nix environment" >&2 + echo "Please stop existing services and attempt 'mariadb-up' again" >&2 + exit 1 + + fi + + # Initialize comm user, database, and secrets file for MariaDB + # Connecting through socket doesn't require a password + userCount=$("${lib.getBin mariadb}/bin/mariadb" -u "$USER" \ + -Bse "SELECT COUNT(1) FROM mysql.user WHERE user = 'comm';" + ) + if [[ "$userCount" -eq 0 ]]; then + PASS=$("${lib.getBin openssl}/bin/openssl" rand -hex 6) + + echo "Creating comm user and comm database" >&2 + "${lib.getBin mariadb}/bin/mariadb" -u "$USER" \ + -Bse "CREATE DATABASE comm; + CREATE USER comm@localhost IDENTIFIED BY '$PASS'; + GRANT ALL ON "'comm.*'" TO comm@localhost;" + echo "Comm user and database has been created!" >&2 + + # Assume this was ran from git repository + PRJ_ROOT=$(git rev-parse --show-toplevel) + KEYSERVER_DB_CONFIG="$PRJ_ROOT"/keyserver/secrets/db_config.json + + echo "Writing connection information to $KEYSERVER_DB_CONFIG" >&2 + mkdir -p "$(dirname "$KEYSERVER_DB_CONFIG")" + + # It's very difficult to write json from bash, just copy a nix + # file then use sed to subsitute + cp "${db_config_template}" "$KEYSERVER_DB_CONFIG" + chmod +w "$KEYSERVER_DB_CONFIG" # Nix files are read-only + "${gnused}/bin/sed" -i -e "s|PASS|$PASS|g" "$KEYSERVER_DB_CONFIG" + fi + + echo "View MariaDB Logs: tail -f $MARIADB_DATA_HOME/logs" >&2 + echo "Kill MariaDB server: pkill mariadbd" >&2 + ''; +} diff --git a/nix/overlay.nix b/nix/overlay.nix index 80ac0e7bf..ce365c129 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,95 +1,100 @@ # An overlay allows for a package set to be extended with new or modified packages # `final` refers to the package set with all overlays applied. # This allows for added or modified packages to be referenced with # all relevant changes final: # `prev` refers to the previous package set before this current overlay is applied. # This is cheaper for nix to evaluate, thus should be prefered over final when possible. prev: { # Patch aws-sdk-cpp to automatically pick up header location # specific to nixpkgs, as nixpkgs separates build-time and runtime # depencenies (a saving of 400MB in header + generated files). # In the case of c and c++, this means the header files are # located in a separate directory from the libraries. # # From a developer perspective, this avoids having to manually specify # the header location with `-DAWS_CORE_HEADER_FILE` each time # one invokes `cmake` on the command line when using # `find_package(AWSSDK COMPONENTS [comps])` # # For more information, see: # - aws-sdk-cpp issue: https://github.com/aws/aws-sdk-cpp/issues/2009 # - Nixpkgs fix: https://github.com/NixOS/nixpkgs/pull/182918 aws-sdk-cpp = (prev.aws-sdk-cpp.overrideAttrs(oldAttrs:{ postPatch = oldAttrs.postPatch + '' substituteInPlace cmake/AWSSDKConfig.cmake \ --replace 'C:/AWSSDK/''${AWSSDK_INSTALL_INCLUDEDIR}/aws/core' \ 'C:/AWSSDK/''${AWSSDK_INSTALL_INCLUDEDIR}/aws/core" "${placeholder "dev"}/include/aws/core' ''; })).override { # avoid rebuildilng all 300+ apis apis = [ "core" "s3" "dynamodb" ]; }; # add packages meant for just this repository amqp-cpp = prev.callPackage ./amqp-cpp.nix { }; arcanist = prev.callPackage ./arcanist.nix { }; protobuf_3_15_cmake = prev.callPackage ./protobuf_3_15.nix { }; devShell = final.callPackage ./dev-shell.nix { }; + # Make our version of mariadb the default everywhere + mariadb = prev.mariadb_108; + + mariadb-up = prev.callPackage ./mariadb-up-mac.nix { }; + mysql-down = prev.callPackage ./mysql-down-linux.nix { }; mysql-up = prev.callPackage ./mysql-up-linux.nix { }; olm = prev.olm.overrideAttrs(oldAttrs: { # *.hh files aren't meant to be used externally # so we patch installation to add it postInstall = '' cp \ $NIX_BUILD_TOP/${oldAttrs.src.name}/include/olm/*.h* \ ''${!outputDev}/include/olm ''; }); # 16.14 now requires experimental import assertions syntax, pin to 16.13 # https://github.com/nodejs/node/blob/main/doc/changelogs/CHANGELOG_V16.md nodejs-16_x = prev.nodejs-16_x.overrideAttrs (oldAttrs: rec { version = "16.13.0"; name = "nodejs-${version}"; src = prev.fetchurl { url = "https://nodejs.org/dist/v${version}/node-v${version}.tar.xz"; sha256 = "sha256-MhFLPcOUXtD5X4vDO0LGjg7xjECMtWEiVyoWPZB+y8w="; }; # Nixpkgs applies two patches for 16.15. One patch is for finding headers # needed for v8 on darwin using apple_sdk 11; the other patch fixes crashes # related cache dir defaulting to using `$HOME` without asserting that # it exists. # # However, 16.13 doesn't need the second patch, as the regression which # caused it was introduced after 16.13. This ends up being a no-op. But # nix will still try to apply the patch and fail with "this patch has # already been applied". # # For more context, see (https://github.com/npm/cli/pull/5197) # # lib.head will select the first element in an array patches = [ (prev.lib.head oldAttrs.patches) ]; }); # Ensure that yarn is using the pinned version yarn = prev.yarn.override (_: { nodejs = final.nodejs-16_x; }); }