diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix --- a/nix/dev-shell.nix +++ b/nix/dev-shell.nix @@ -7,6 +7,7 @@ , boost , bundler , c-ares_cmake-config +, cargo-lambda , cargo-udeps , cmake , cmake-format @@ -74,6 +75,7 @@ # Identity Service rustfmt rustup + cargo-lambda cargo-udeps # Tunnelbroker + CMake diff --git a/services/commtest/Dockerfile b/services/commtest/Dockerfile --- a/services/commtest/Dockerfile +++ b/services/commtest/Dockerfile @@ -2,7 +2,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ build-essential cmake git libgtest-dev libssl-dev zlib1g-dev \ - gnupg software-properties-common + gnupg software-properties-common python3-pip # These steps are required to install terraform RUN wget -O- https://apt.releases.hashicorp.com/gpg | \ @@ -26,10 +26,12 @@ WORKDIR /home/root/app/commtest +# Install cargo lambda +RUN pip3 install cargo-lambda + # Install more recent version of protobuf, must be ran as root COPY scripts/install_protobuf.sh ../../scripts/install_protobuf.sh RUN ../../scripts/install_protobuf.sh - ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse # Copy actual application sources @@ -37,5 +39,7 @@ COPY services/terraform/dev ../terraform/dev COPY services/terraform/modules ../terraform/modules COPY services/commtest ./ +COPY services/search-index-lambda ../search-index-lambda + CMD ["bash", "./run-tests-ci.sh"] diff --git a/services/commtest/run-tests-ci.sh b/services/commtest/run-tests-ci.sh --- a/services/commtest/run-tests-ci.sh +++ b/services/commtest/run-tests-ci.sh @@ -8,6 +8,16 @@ aws --endpoint-url="$LOCALSTACK_ENDPOINT" "$@" } +build_lambdas() { + echo "Building lambdas..." + + pushd ../search-index-lambda >/dev/null + + cargo lambda build --arm64 --output-format zip + + popd >/dev/null +} + # Set up Localstack using Terraform reset_localstack() { echo "Resetting Localstack..." @@ -43,6 +53,9 @@ fi } +# Build lambdas for terraform +build_lambdas + # Reset localstack and then run tests reset_localstack diff --git a/services/docker-compose.yml b/services/docker-compose.yml --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -91,11 +91,12 @@ ports: - '4566:4566' environment: - - SERVICES=s3,dynamodb + - SERVICES=s3,dynamodb,lambda - LOCALSTACK_HOST=localstack:4566 - PERSISTENCE=1 volumes: - localstack:/var/lib/localstack + - "/var/run/docker.sock:/var/run/docker.sock" # RabbitMQ rabbitmq: # This version matches AWS MQ version (set in Terraform) diff --git a/services/search-index-lambda/.gitignore b/services/search-index-lambda/.gitignore new file mode 100644 diff --git a/services/terraform/dev/main.tf b/services/terraform/dev/main.tf --- a/services/terraform/dev/main.tf +++ b/services/terraform/dev/main.tf @@ -31,6 +31,8 @@ dynamic "endpoints" { for_each = local.aws_settings.override_endpoint[*] content { + lambda = endpoints.value + ec2 = endpoints.value opensearch = endpoints.value dynamodb = endpoints.value s3 = endpoints.value diff --git a/services/terraform/dev/run.sh b/services/terraform/dev/run.sh --- a/services/terraform/dev/run.sh +++ b/services/terraform/dev/run.sh @@ -2,6 +2,11 @@ set -e +cd ../../search-index-lambda +cargo lambda build --arm64 --output-format zip --release + +cd ../terraform/dev + terraform init terraform apply -auto-approve diff --git a/services/terraform/modules/shared/outputs.tf b/services/terraform/modules/shared/outputs.tf --- a/services/terraform/modules/shared/outputs.tf +++ b/services/terraform/modules/shared/outputs.tf @@ -4,6 +4,7 @@ aws_dynamodb_table.backup-service-backup, aws_dynamodb_table.reports-service-reports, aws_dynamodb_table.tunnelbroker-undelivered-messages, + aws_dynamodb_table.identity-users, ] } @@ -19,3 +20,7 @@ output "opensearch_domain_identity" { value = aws_opensearch_domain.identity-search } + +output "search_index_lambda" { + value = aws_lambda_function.search_index_lambda +} diff --git a/services/terraform/modules/shared/search_index_lambda.tf b/services/terraform/modules/shared/search_index_lambda.tf new file mode 100644 --- /dev/null +++ b/services/terraform/modules/shared/search_index_lambda.tf @@ -0,0 +1,61 @@ +variable "search_index_lambda_iam_role_arn" { + default = "arn:aws:iam::000000000000:role/lambda-role" +} + + +variable "lambda_zip_dir" { + type = string + default = "../../search-index-lambda/target/lambda/search-index-lambda" +} + +resource "aws_lambda_function" "search_index_lambda" { + function_name = "search-index-lambda-function" + filename = "${var.lambda_zip_dir}/bootstrap.zip" + source_code_hash = filebase64sha256("${var.lambda_zip_dir}/bootstrap.zip") + handler = "bootstrap" + role = var.search_index_lambda_iam_role_arn + runtime = "provided.al2" + architectures = ["arm64"] + timeout = 300 + + vpc_config { + subnet_ids = var.subnet_ids + security_group_ids = [aws_security_group.search_index_lambda.id] + } + + environment { + variables = { + RUST_BACKTRACE = "1" + OPENSEARCH_ENDPOINT = aws_opensearch_domain.identity-search.endpoint + } + } + + tracing_config { + mode = "Active" + } +} + +resource "aws_lambda_event_source_mapping" "identity_users_trigger" { + event_source_arn = aws_dynamodb_table.identity-users.stream_arn + function_name = aws_lambda_function.search_index_lambda.arn + starting_position = "LATEST" +} + +resource "aws_security_group" "search_index_lambda" { + name = "search_index_lambda_sg" + vpc_id = var.vpc_id + + egress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} + + +resource "aws_lambda_function_event_invoke_config" "search-index-lambda" { + function_name = aws_lambda_function.search_index_lambda.function_name + maximum_event_age_in_seconds = 60 + maximum_retry_attempts = 2 +} diff --git a/services/terraform/remote/aws_iam.tf b/services/terraform/remote/aws_iam.tf --- a/services/terraform/remote/aws_iam.tf +++ b/services/terraform/remote/aws_iam.tf @@ -195,13 +195,111 @@ ] } + +data "aws_iam_policy_document" "assume_identity_search_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "search_index_lambda" { + name = "search_index_lambda" + assume_role_policy = data.aws_iam_policy_document.assume_identity_search_role.json + + managed_policy_arns = [ + aws_iam_policy.manage_cloudwatch_logs.arn, + aws_iam_policy.manage_network_interface.arn, + aws_iam_policy.read_identity_users_stream.arn, + ] +} + +data "aws_iam_policy_document" "read_identity_users_stream" { + statement { + effect = "Allow" + + actions = [ + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:DescribeStream", + "dynamodb:ListStreams", + ] + resources = [ + module.shared.dynamodb_tables["identity-users"].arn, + module.shared.dynamodb_tables["identity-users"].stream_arn, + "${module.shared.dynamodb_tables["identity-users"].arn}/stream/*", + ] + } +} + +resource "aws_iam_policy" "read_identity_users_stream" { + name = "read-identity-users-stream" + path = "/" + description = "IAM policy for managing identity-users stream" + policy = data.aws_iam_policy_document.read_identity_users_stream.json +} + +data "aws_iam_policy_document" "manage_cloudwatch_logs" { + statement { + effect = "Allow" + + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + + resources = ["arn:aws:logs:*:*:*"] + } +} + +resource "aws_iam_policy" "manage_cloudwatch_logs" { + name = "manage-cloudwatch-logs" + path = "/" + description = "IAM policy for managing cloudwatch logs" + policy = data.aws_iam_policy_document.manage_cloudwatch_logs.json +} + +data "aws_iam_policy_document" "manage_network_interface" { + statement { + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface" + ] + + resources = ["*"] + } +} + +resource "aws_iam_policy" "manage_network_interface" { + name = "manage-network-interface" + path = "/" + description = "IAM policy for managing network interfaces" + policy = data.aws_iam_policy_document.manage_network_interface.json +} + + +resource "aws_iam_role_policy_attachment" "AWSLambdaVPCAccessExecutionRole" { + role = aws_iam_role.search_index_lambda.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" +} + data "aws_iam_policy_document" "opensearch_domain_access" { statement { effect = "Allow" principals { type = "*" - identifiers = [] + identifiers = ["${module.shared.search_index_lambda.arn}"] } actions = [ diff --git a/services/terraform/remote/main.tf b/services/terraform/remote/main.tf --- a/services/terraform/remote/main.tf +++ b/services/terraform/remote/main.tf @@ -52,8 +52,9 @@ source = "../modules/shared" bucket_name_suffix = local.s3_bucket_name_suffix - vpc_id = aws_vpc.default.id - cidr_block = aws_vpc.default.cidr_block + vpc_id = aws_vpc.default.id + search_index_lambda_iam_role_arn = aws_iam_role.search_index_lambda.arn + cidr_block = aws_vpc.default.cidr_block subnet_ids = [ aws_subnet.public_a.id, ] diff --git a/services/terraform/remote/run.sh b/services/terraform/remote/run.sh new file mode 100755 --- /dev/null +++ b/services/terraform/remote/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e + +cd ../../search-index-lambda +cargo lambda build --arm64 --output-format zip --release +mv ./target/lambda/search-index-lambda/bootstrap.zip . + +cd ../terraform/remote +terraform init +terraform apply