Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F3355439
D9878.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
10 KB
Referenced Files
None
Subscribers
None
D9878.diff
View Options
diff --git a/nix/dev-shell.nix b/nix/dev-shell.nix
--- a/nix/dev-shell.nix
+++ b/nix/dev-shell.nix
@@ -7,6 +7,7 @@
, boost
, bundler
, c-ares_cmake-config
+, cargo-lambda
, cargo-udeps
, cmake
, cmake-format
@@ -74,6 +75,7 @@
# Identity Service
rustfmt
rustup
+ cargo-lambda
cargo-udeps
# Tunnelbroker + CMake
diff --git a/services/commtest/Dockerfile b/services/commtest/Dockerfile
--- a/services/commtest/Dockerfile
+++ b/services/commtest/Dockerfile
@@ -2,7 +2,7 @@
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
build-essential cmake git libgtest-dev libssl-dev zlib1g-dev \
- gnupg software-properties-common
+ gnupg software-properties-common python3-pip
# These steps are required to install terraform
RUN wget -O- https://apt.releases.hashicorp.com/gpg | \
@@ -26,10 +26,12 @@
WORKDIR /home/root/app/commtest
+# Install cargo lambda
+RUN pip3 install cargo-lambda
+
# Install more recent version of protobuf, must be ran as root
COPY scripts/install_protobuf.sh ../../scripts/install_protobuf.sh
RUN ../../scripts/install_protobuf.sh
-
ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
# Copy actual application sources
@@ -37,5 +39,7 @@
COPY services/terraform/dev ../terraform/dev
COPY services/terraform/modules ../terraform/modules
COPY services/commtest ./
+COPY services/search-index-lambda ../search-index-lambda
+
CMD ["bash", "./run-tests-ci.sh"]
diff --git a/services/commtest/run-tests-ci.sh b/services/commtest/run-tests-ci.sh
--- a/services/commtest/run-tests-ci.sh
+++ b/services/commtest/run-tests-ci.sh
@@ -8,6 +8,16 @@
aws --endpoint-url="$LOCALSTACK_ENDPOINT" "$@"
}
+build_lambdas() {
+ echo "Building lambdas..."
+
+ pushd ../search-index-lambda >/dev/null
+
+ cargo lambda build --arm64 --output-format zip
+
+ popd >/dev/null
+}
+
# Set up Localstack using Terraform
reset_localstack() {
echo "Resetting Localstack..."
@@ -43,6 +53,9 @@
fi
}
+# Build lambdas for terraform
+build_lambdas
+
# Reset localstack and then run tests
reset_localstack
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -91,11 +91,12 @@
ports:
- '4566:4566'
environment:
- - SERVICES=s3,dynamodb
+ - SERVICES=s3,dynamodb,lambda
- LOCALSTACK_HOST=localstack:4566
- PERSISTENCE=1
volumes:
- localstack:/var/lib/localstack
+ - "/var/run/docker.sock:/var/run/docker.sock"
# RabbitMQ
rabbitmq:
# This version matches AWS MQ version (set in Terraform)
diff --git a/services/search-index-lambda/.gitignore b/services/search-index-lambda/.gitignore
new file mode 100644
diff --git a/services/terraform/dev/main.tf b/services/terraform/dev/main.tf
--- a/services/terraform/dev/main.tf
+++ b/services/terraform/dev/main.tf
@@ -31,6 +31,8 @@
dynamic "endpoints" {
for_each = local.aws_settings.override_endpoint[*]
content {
+ lambda = endpoints.value
+ ec2 = endpoints.value
opensearch = endpoints.value
dynamodb = endpoints.value
s3 = endpoints.value
diff --git a/services/terraform/dev/run.sh b/services/terraform/dev/run.sh
--- a/services/terraform/dev/run.sh
+++ b/services/terraform/dev/run.sh
@@ -2,6 +2,11 @@
set -e
+cd ../../search-index-lambda
+cargo lambda build --arm64 --output-format zip --release
+
+cd ../terraform/dev
+
terraform init
terraform apply -auto-approve
diff --git a/services/terraform/modules/shared/outputs.tf b/services/terraform/modules/shared/outputs.tf
--- a/services/terraform/modules/shared/outputs.tf
+++ b/services/terraform/modules/shared/outputs.tf
@@ -4,6 +4,7 @@
aws_dynamodb_table.backup-service-backup,
aws_dynamodb_table.reports-service-reports,
aws_dynamodb_table.tunnelbroker-undelivered-messages,
+ aws_dynamodb_table.identity-users,
]
}
@@ -19,3 +20,7 @@
output "opensearch_domain_identity" {
value = aws_opensearch_domain.identity-search
}
+
+output "search_index_lambda" {
+ value = aws_lambda_function.search_index_lambda
+}
diff --git a/services/terraform/modules/shared/search_index_lambda.tf b/services/terraform/modules/shared/search_index_lambda.tf
new file mode 100644
--- /dev/null
+++ b/services/terraform/modules/shared/search_index_lambda.tf
@@ -0,0 +1,61 @@
+variable "search_index_lambda_iam_role_arn" {
+ default = "arn:aws:iam::000000000000:role/lambda-role"
+}
+
+
+variable "lambda_zip_dir" {
+ type = string
+ default = "../../search-index-lambda/target/lambda/search-index-lambda"
+}
+
+resource "aws_lambda_function" "search_index_lambda" {
+ function_name = "search-index-lambda-function"
+ filename = "${var.lambda_zip_dir}/bootstrap.zip"
+ source_code_hash = filebase64sha256("${var.lambda_zip_dir}/bootstrap.zip")
+ handler = "bootstrap"
+ role = var.search_index_lambda_iam_role_arn
+ runtime = "provided.al2"
+ architectures = ["arm64"]
+ timeout = 300
+
+ vpc_config {
+ subnet_ids = var.subnet_ids
+ security_group_ids = [aws_security_group.search_index_lambda.id]
+ }
+
+ environment {
+ variables = {
+ RUST_BACKTRACE = "1"
+ OPENSEARCH_ENDPOINT = aws_opensearch_domain.identity-search.endpoint
+ }
+ }
+
+ tracing_config {
+ mode = "Active"
+ }
+}
+
+resource "aws_lambda_event_source_mapping" "identity_users_trigger" {
+ event_source_arn = aws_dynamodb_table.identity-users.stream_arn
+ function_name = aws_lambda_function.search_index_lambda.arn
+ starting_position = "LATEST"
+}
+
+resource "aws_security_group" "search_index_lambda" {
+ name = "search_index_lambda_sg"
+ vpc_id = var.vpc_id
+
+ egress {
+ from_port = 443
+ to_port = 443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+
+resource "aws_lambda_function_event_invoke_config" "search-index-lambda" {
+ function_name = aws_lambda_function.search_index_lambda.function_name
+ maximum_event_age_in_seconds = 60
+ maximum_retry_attempts = 2
+}
diff --git a/services/terraform/remote/aws_iam.tf b/services/terraform/remote/aws_iam.tf
--- a/services/terraform/remote/aws_iam.tf
+++ b/services/terraform/remote/aws_iam.tf
@@ -195,13 +195,111 @@
]
}
+
+data "aws_iam_policy_document" "assume_identity_search_role" {
+ statement {
+ effect = "Allow"
+
+ principals {
+ type = "Service"
+ identifiers = ["lambda.amazonaws.com"]
+ }
+
+ actions = ["sts:AssumeRole"]
+ }
+}
+
+resource "aws_iam_role" "search_index_lambda" {
+ name = "search_index_lambda"
+ assume_role_policy = data.aws_iam_policy_document.assume_identity_search_role.json
+
+ managed_policy_arns = [
+ aws_iam_policy.manage_cloudwatch_logs.arn,
+ aws_iam_policy.manage_network_interface.arn,
+ aws_iam_policy.read_identity_users_stream.arn,
+ ]
+}
+
+data "aws_iam_policy_document" "read_identity_users_stream" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "dynamodb:GetRecords",
+ "dynamodb:GetShardIterator",
+ "dynamodb:DescribeStream",
+ "dynamodb:ListStreams",
+ ]
+ resources = [
+ module.shared.dynamodb_tables["identity-users"].arn,
+ module.shared.dynamodb_tables["identity-users"].stream_arn,
+ "${module.shared.dynamodb_tables["identity-users"].arn}/stream/*",
+ ]
+ }
+}
+
+resource "aws_iam_policy" "read_identity_users_stream" {
+ name = "read-identity-users-stream"
+ path = "/"
+ description = "IAM policy for managing identity-users stream"
+ policy = data.aws_iam_policy_document.read_identity_users_stream.json
+}
+
+data "aws_iam_policy_document" "manage_cloudwatch_logs" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "logs:CreateLogGroup",
+ "logs:CreateLogStream",
+ "logs:PutLogEvents",
+ ]
+
+ resources = ["arn:aws:logs:*:*:*"]
+ }
+}
+
+resource "aws_iam_policy" "manage_cloudwatch_logs" {
+ name = "manage-cloudwatch-logs"
+ path = "/"
+ description = "IAM policy for managing cloudwatch logs"
+ policy = data.aws_iam_policy_document.manage_cloudwatch_logs.json
+}
+
+data "aws_iam_policy_document" "manage_network_interface" {
+ statement {
+ effect = "Allow"
+
+ actions = [
+ "ec2:CreateNetworkInterface",
+ "ec2:DescribeNetworkInterfaces",
+ "ec2:DeleteNetworkInterface"
+ ]
+
+ resources = ["*"]
+ }
+}
+
+resource "aws_iam_policy" "manage_network_interface" {
+ name = "manage-network-interface"
+ path = "/"
+ description = "IAM policy for managing network interfaces"
+ policy = data.aws_iam_policy_document.manage_network_interface.json
+}
+
+
+resource "aws_iam_role_policy_attachment" "AWSLambdaVPCAccessExecutionRole" {
+ role = aws_iam_role.search_index_lambda.name
+ policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
+}
+
data "aws_iam_policy_document" "opensearch_domain_access" {
statement {
effect = "Allow"
principals {
type = "*"
- identifiers = []
+ identifiers = ["${module.shared.search_index_lambda.arn}"]
}
actions = [
diff --git a/services/terraform/remote/main.tf b/services/terraform/remote/main.tf
--- a/services/terraform/remote/main.tf
+++ b/services/terraform/remote/main.tf
@@ -52,8 +52,9 @@
source = "../modules/shared"
bucket_name_suffix = local.s3_bucket_name_suffix
- vpc_id = aws_vpc.default.id
- cidr_block = aws_vpc.default.cidr_block
+ vpc_id = aws_vpc.default.id
+ search_index_lambda_iam_role_arn = aws_iam_role.search_index_lambda.arn
+ cidr_block = aws_vpc.default.cidr_block
subnet_ids = [
aws_subnet.public_a.id,
]
diff --git a/services/terraform/remote/run.sh b/services/terraform/remote/run.sh
new file mode 100755
--- /dev/null
+++ b/services/terraform/remote/run.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd ../../search-index-lambda
+cargo lambda build --arm64 --output-format zip --release
+
+cd ../terraform/remote
+terraform init
+terraform apply
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Nov 24, 2:31 PM (21 h, 18 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2576333
Default Alt Text
D9878.diff (10 KB)
Attached To
Mode
D9878: [terraform] [4/n] search_index_lambda terraform configuration options
Attached
Detach File
Event Timeline
Log In to Comment