Page MenuHomePhabricator
Files F3356139

D9878.id35230.diff
No OneTemporary
Actions

D9878.id35230.diff
View Options

diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix
--- a/nix/dev-shell.nix
+++ b/nix/dev-shell.nix
@@ -9,6 +9,7 @@
, boost
, bundler
, c-ares_cmake-config
+, cargo-lambda
, cargo-udeps
, cmake
, cmake-format
@@ -77,6 +78,7 @@
# Identity Service
rustfmt
rustup
+ cargo-lambda
cargo-udeps
# Tunnelbroker + CMake
diff --git a/services/commtest/Dockerfile b/services/commtest/Dockerfile
--- a/services/commtest/Dockerfile
+++ b/services/commtest/Dockerfile
@@ -2,7 +2,7 @@
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential cmake git libgtest-dev libssl-dev zlib1g-dev \
- gnupg software-properties-common
+ gnupg software-properties-common python3-pip
# These steps are required to install terraform
RUN wget -O- https://apt.releases.hashicorp.com/gpg | \
@@ -26,10 +26,12 @@
WORKDIR /home/root/app/commtest
+# Install cargo lambda
+RUN pip3 install cargo-lambda
+
# Install more recent version of protobuf, must be ran as root
COPY scripts/install_protobuf.sh ../../scripts/install_protobuf.sh
RUN ../../scripts/install_protobuf.sh
-
ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
# Copy actual application sources
@@ -37,5 +39,7 @@
COPY services/terraform/dev ../terraform/dev
COPY services/terraform/modules ../terraform/modules
COPY services/commtest ./
+COPY services/search-index-lambda ../search-index-lambda
+
CMD ["bash", "./run-tests-ci.sh"]
diff --git a/services/commtest/run-tests-ci.sh b/services/commtest/run-tests-ci.sh
--- a/services/commtest/run-tests-ci.sh
+++ b/services/commtest/run-tests-ci.sh
@@ -8,6 +8,17 @@
aws --endpoint-url="$LOCALSTACK_ENDPOINT" "$@"
}
+build_lambdas() {
+ echo "Building lambdas..."
+
+ pushd ../search-index-lambda >/dev/null
+
+ cargo lambda build --arm64 --output-format zip
+ mv ./target/lambda/search-index-lambda/bootstrap.zip .
+
+ popd >/dev/null
+}
+
# Set up Localstack using Terraform
reset_localstack() {
echo "Resetting Localstack..."
@@ -43,6 +54,9 @@
fi
}
+# Build lambdas for terraform
+build_lambdas
+
# Reset localstack and then run tests
reset_localstack
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -91,11 +91,12 @@
ports:
- '4566:4566'
environment:
- - SERVICES=s3,dynamodb
+ - SERVICES=s3,dynamodb,lambda
- LOCALSTACK_HOST=localstack:4566
- PERSISTENCE=1
volumes:
- localstack:/var/lib/localstack
+ - "/var/run/docker.sock:/var/run/docker.sock"
# RabbitMQ
rabbitmq:
# This version matches AWS MQ version (set in Terraform)
diff --git a/services/search-index-lambda/.gitignore b/services/search-index-lambda/.gitignore
new file mode 100644
diff --git a/services/terraform/dev/main.tf b/services/terraform/dev/main.tf
--- a/services/terraform/dev/main.tf
+++ b/services/terraform/dev/main.tf
@@ -31,6 +31,8 @@
dynamic "endpoints" {
for_each = local.aws_settings.override_endpoint[*]
content {
+ lambda = endpoints.value
+ ec2 = endpoints.value
opensearch = endpoints.value
dynamodb = endpoints.value
s3 = endpoints.value
diff --git a/services/terraform/modules/shared/outputs.tf b/services/terraform/modules/shared/outputs.tf
--- a/services/terraform/modules/shared/outputs.tf
+++ b/services/terraform/modules/shared/outputs.tf
@@ -4,6 +4,7 @@
aws_dynamodb_table.backup-service-backup,
aws_dynamodb_table.reports-service-reports,
aws_dynamodb_table.tunnelbroker-undelivered-messages,
+ aws_dynamodb_table.identity-users,
]
}
@@ -19,3 +20,7 @@
output "opensearch_domain_identity" {
value = aws_opensearch_domain.identity-search
}
+
+output "search_index_lambda" {
+ value = aws_lambda_function.search_index_lambda
+}
diff --git a/services/terraform/modules/shared/search_index_lambda.tf b/services/terraform/modules/shared/search_index_lambda.tf
new file mode 100644
--- /dev/null
+++ b/services/terraform/modules/shared/search_index_lambda.tf
@@ -0,0 +1,61 @@
+variable "search_index_lambda_iam_role_arn" {
+ default = "arn:aws:iam::000000000000:role/lambda-role"
+}
+
+
+variable "lambda_zip_dir" {
+ type = string
+ default = "../../search-index-lambda"
+}
+
+resource "aws_lambda_function" "search_index_lambda" {
+ function_name = "search-index-lambda-function"
+ filename = "${var.lambda_zip_dir}/bootstrap.zip"
+ source_code_hash = filebase64sha256("${var.lambda_zip_dir}/bootstrap.zip")
+ handler = "bootstrap"
+ role = var.search_index_lambda_iam_role_arn
+ runtime = "provided.al2"
+ architectures = ["arm64"]
+ timeout = 300
+
+ vpc_config {
+ subnet_ids = var.subnet_ids
+ security_group_ids = [aws_security_group.search_index_lambda.id]
+ }
+
+ environment {
+ variables = {
+ RUST_BACKTRACE = "1"
+ OPENSEARCH_ENDPOINT = aws_opensearch_domain.identity-search.endpoint
+ }
+ }
+
+ tracing_config {
+ mode = "Active"
+ }
+}
+
+resource "aws_lambda_event_source_mapping" "identity_users_trigger" {
+ event_source_arn = aws_dynamodb_table.identity-users.stream_arn
+ function_name = aws_lambda_function.search_index_lambda.arn
+ starting_position = "LATEST"
+}
+
+resource "aws_security_group" "search_index_lambda" {
+ name = "search_index_lambda_sg"
+ vpc_id = var.vpc_id
+
+ egress {
+ from_port = 443
+ to_port = 443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+
+resource "aws_lambda_function_event_invoke_config" "search-index-lambda" {
+ function_name = aws_lambda_function.search_index_lambda.function_name
+ maximum_event_age_in_seconds = 60
+ maximum_retry_attempts = 2
+}
diff --git a/services/terraform/remote/aws_iam.tf b/services/terraform/remote/aws_iam.tf
--- a/services/terraform/remote/aws_iam.tf
+++ b/services/terraform/remote/aws_iam.tf
@@ -195,13 +195,111 @@
]
}
+
+data "aws_iam_policy_document" "assume_identity_search_role" {
+ statement {
+ effect = "Allow"
+
+ principals {
+ type = "Service"
+ identifiers = ["lambda.amazonaws.com"]
+ }
+
+ actions = ["sts:AssumeRole"]
+ }
+}
+
+resource "aws_iam_role" "search_index_lambda" {
+ name = "search_index_lambda"
+ assume_role_policy = data.aws_iam_policy_document.assume_identity_search_role.json
+
+ managed_policy_arns = [
+ aws_iam_policy.manage_cloudwatch_logs.arn,
+ aws_iam_policy.manage_network_interface.arn,
+ aws_iam_policy.read_identity_users_stream.arn,
+ ]
+}
+
+data "aws_iam_policy_document" "read_identity_users_stream" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:DescribeStream",
+ "dynamodb:ListStreams",
+ ]
+ resources = [
+ module.shared.dynamodb_tables["identity-users"].arn,
+ module.shared.dynamodb_tables["identity-users"].stream_arn,
+ "${module.shared.dynamodb_tables["identity-users"].arn}/stream/*",
+ ]
+ }
+}
+
+resource "aws_iam_policy" "read_identity_users_stream" {
+ name = "read-identity-users-stream"
+ path = "/"
+ description = "IAM policy for managing identity-users stream"
+ policy = data.aws_iam_policy_document.read_identity_users_stream.json
+}
+
+data "aws_iam_policy_document" "manage_cloudwatch_logs" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "logs:CreateLogGroup",
+ "logs:CreateLogStream",
+ "logs:PutLogEvents",
+ ]
+
+ resources = ["arn:aws:logs:*:*:*"]
+ }
+}
+
+resource "aws_iam_policy" "manage_cloudwatch_logs" {
+ name = "manage-cloudwatch-logs"
+ path = "/"
+ description = "IAM policy for managing cloudwatch logs"
+ policy = data.aws_iam_policy_document.manage_cloudwatch_logs.json
+}
+
+data "aws_iam_policy_document" "manage_network_interface" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "ec2:CreateNetworkInterface",
+ "ec2:DescribeNetworkInterfaces",
+ "ec2:DeleteNetworkInterface"
+ ]
+
+ resources = ["*"]
+ }
+}
+
+resource "aws_iam_policy" "manage_network_interface" {
+ name = "manage-network-interface"
+ path = "/"
+ description = "IAM policy for managing network interfaces"
+ policy = data.aws_iam_policy_document.manage_network_interface.json
+}
+
+
+resource "aws_iam_role_policy_attachment" "AWSLambdaVPCAccessExecutionRole" {
+ role = aws_iam_role.search_index_lambda.name
+ policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
+}
+
data "aws_iam_policy_document" "opensearch_domain_access" {
statement {
effect = "Allow"
principals {
type = "*"
- identifiers = []
+ identifiers = ["${module.shared.search_index_lambda.arn}"]
}
actions = [
diff --git a/services/terraform/remote/main.tf b/services/terraform/remote/main.tf
--- a/services/terraform/remote/main.tf
+++ b/services/terraform/remote/main.tf
@@ -52,8 +52,9 @@
source = "../modules/shared"
bucket_name_suffix = local.s3_bucket_name_suffix
- vpc_id = aws_vpc.default.id
- cidr_block = aws_vpc.default.cidr_block
+ vpc_id = aws_vpc.default.id
+ search_index_lambda_iam_role_arn = aws_iam_role.search_index_lambda.arn
+ cidr_block = aws_vpc.default.cidr_block
subnet_ids = [
aws_subnet.public_a.id,
]
diff --git a/services/terraform/remote/run.sh b/services/terraform/remote/run.sh
new file mode 100755
--- /dev/null
+++ b/services/terraform/remote/run.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd ../../search-index-lambda
+cargo lambda build --arm64 --output-format zip --release
+mv ./target/lambda/search-index-lambda/bootstrap.zip .
+
+cd ../terraform/remote
+terraform init
+terraform apply

File Metadata

Mime Type
text/plain
Expires
Sun, Nov 24, 5:05 PM (20 h, 3 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2576815
Default Alt Text
D9878.id35230.diff (9 KB)

Event Timeline