diff --git a/.dockerignore b/.dockerignore index 36742b538..1707fbb4a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,42 +1,43 @@ .dockerignore .DS_Store .git .eslintcache .vscode !.vscode/extensions.json node_modules landing/node_modules landing/dist lib/node_modules native !native/package.json !native/.flowconfig !native/ios/Podfile !native/ios/pod-patch !native/cpp/CommonCpp/grpc web/node_modules web/dist keyserver/dist keyserver/node_modules keyserver/facts keyserver/secrets keyserver/*.env keyserver/*.env.* services/tunnelbroker/Dockerfile services/identity/target services/identity/Dockerfile services/backup/Dockerfile +services/blob/target services/blob/Dockerfile native/cpp/**/build services/*/build services/build services/lib/src/build shared/protos/build diff --git a/services/blob/Cargo.toml b/services/blob/Cargo.toml index d800e6912..b011a389a 100644 --- a/services/blob/Cargo.toml +++ b/services/blob/Cargo.toml @@ -1,23 +1,22 @@ [package] name = "blob" description = "Blob service" homepage = "https://comm.app" license = "BSD-3-Clause" version = "0.1.0" edition = "2021" -links = "blob" [dependencies] anyhow = "1.0" aws-config = "0.51.0" aws-sdk-dynamodb = "0.21.0" aws-sdk-s3 = "0.21.0" aws-types = "0.51.0" chrono = "0.4" prost = "0.11" tokio = { version = "1.21", features = ["rt-multi-thread"]} tokio-stream = "0.1" tonic = "0.8" [build-dependencies] tonic-build = "0.8" diff --git a/services/blob/Dockerfile b/services/blob/Dockerfile index 710476e6d..c01345784 100644 --- a/services/blob/Dockerfile +++ b/services/blob/Dockerfile @@ -1,25 +1,44 @@ -FROM commapp/services-base:1.3.2 +FROM commapp/services-base:1.3.2 as builder -RUN apt-get update && \ - apt-get install -y uuid-dev && \ - rm -rf /var/lib/apt/lists/* +ENV PATH=/root/.cargo/bin:$PATH -ARG COMM_TEST_SERVICES -ARG COMM_SERVICES_SANDBOX +# Install Curl +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y curl \ + && rm -rf /var/lib/apt/lists/* -ENV COMM_TEST_SERVICES=${COMM_TEST_SERVICES} -ENV COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} +# Install Rust +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +RUN mkdir -p /home/comm/app/blob +WORKDIR /home/comm/app/blob +RUN cargo init --bin + +# Cache build dependencies in a new layer +COPY services/blob/Cargo.toml services/blob/Cargo.lock ./ +RUN cargo build --release && rm src/*.rs -WORKDIR /transferred +# Copy actual application sources +COPY services/blob . +COPY shared/protos/blob.proto ../../shared/protos/ -COPY services/lib/docker/ scripts/ -COPY services/blob/old/ blob -COPY services/lib/src lib/src/ +# Remove the previously-built binary so that only the application itself is +# rebuilt +RUN rm ./target/release/deps/blob* +RUN cargo build --release -WORKDIR /transferred/blob +# Runner stage +FROM commapp/services-base:1.3.2 as runner -ADD shared/protos grpc +# Create a new user comm and use it to run subsequent commands +RUN useradd -m comm +USER comm -RUN ../scripts/build_service.sh +# Only copy built binary from builder stage +WORKDIR /home/comm/app/blob +COPY --from=builder /home/comm/app/blob/target/release/blob . + +ARG COMM_SERVICES_SANDBOX +ENV COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} -CMD if [ "$COMM_TEST_SERVICES" -eq 1 ]; then ../scripts/run_tests.sh; else ../scripts/run_service.sh; fi +CMD ["./blob"] diff --git a/services/blob/src/constants.rs b/services/blob/src/constants.rs index b42e40762..3a70900c0 100644 --- a/services/blob/src/constants.rs +++ b/services/blob/src/constants.rs @@ -1,46 +1,46 @@ // Assorted constants pub const GRPC_SERVER_DEFAULT_PORT: u64 = 50051; pub const AWS_REGION: &str = "us-east-2"; -pub const LOCALSTACK_URL: &str = "http://localhost:4566"; +pub const LOCALSTACK_URL: &str = "http://localstack:4566"; pub const MPSC_CHANNEL_BUFFER_CAPACITY: usize = 1; /// 4MB limit /// /// WARNING: use keeping in mind that grpc adds its own headers to messages /// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md /// so the message that actually is being sent over the network looks like this /// ``` /// [Compressed-Flag] [Message-Length] [Message] /// [Compressed-Flag] 1 byte - added by grpc /// [Message-Length] 4 bytes - added by grpc /// [Message] N bytes - actual data /// ``` /// so for every message we get 5 additional bytes of data /// as [mentioned here](https://github.com/grpc/grpc/issues/15734#issuecomment-396962671), /// gRPC stream may contain more than one message pub const GRPC_CHUNK_SIZE_LIMIT: u64 = 4 * 1024 * 1024; /// See [`GRPC_CHUNK_SIZE_LIMIT`] description for details pub const GRPC_METADATA_SIZE_PER_MESSAGE: u64 = 5; // DynamoDB constants pub const BLOB_TABLE_NAME: &str = "blob-service-blob"; pub const BLOB_TABLE_BLOB_HASH_FIELD: &str = "blobHash"; pub const BLOB_TABLE_S3_PATH_FIELD: &str = "s3Path"; pub const BLOB_TABLE_CREATED_FIELD: &str = "created"; pub const BLOB_REVERSE_INDEX_TABLE_NAME: &str = "blob-service-reverse-index"; pub const BLOB_REVERSE_INDEX_TABLE_HOLDER_FIELD: &str = "holder"; pub const BLOB_REVERSE_INDEX_TABLE_BLOB_HASH_FIELD: &str = "blobHash"; pub const BLOB_REVERSE_INDEX_TABLE_HASH_INDEX_NAME: &str = "blobHash-index"; // Environment variables pub const SANDBOX_ENV_VAR: &str = "COMM_SERVICES_SANDBOX"; // S3 constants pub const BLOB_S3_BUCKET_NAME: &str = "commapp-blob"; pub const S3_MULTIPART_UPLOAD_MINIMUM_CHUNK_SIZE: u64 = 5 * 1024 * 1024; diff --git a/services/docker-compose.yml b/services/docker-compose.yml index f37f03a1d..1c664d71a 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1,86 +1,87 @@ version: "3.9" volumes: localstack: services: # tunnelbroker tunnelbroker-server: depends_on: - localstack - rabbitmq build: dockerfile: services/tunnelbroker/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} - COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} image: commapp/tunnelbroker-server:0.2 ports: - "${COMM_SERVICES_PORT_TUNNELBROKER}:50051" volumes: - $HOME/.aws/config:/root/.aws/config:ro - $HOME/.aws/credentials:/root/.aws/credentials:ro - ./tunnelbroker/tunnelbroker.ini:/root/tunnelbroker/tunnelbroker.ini:ro - ./tunnelbroker/tunnelbroker-sandbox.ini:/root/tunnelbroker/tunnelbroker-sandbox.ini:ro # backup backup-server: depends_on: - localstack build: dockerfile: services/backup/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} - COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} image: commapp/backup-server:0.1 ports: - "${COMM_SERVICES_PORT_BACKUP}:50051" volumes: - $HOME/.aws/credentials:/root/.aws/credentials:ro # blob blob-server: depends_on: - localstack build: dockerfile: services/blob/Dockerfile context: ../ args: - COMM_TEST_SERVICES=${COMM_TEST_SERVICES} - COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} image: commapp/blob-server:0.1 ports: - "${COMM_SERVICES_PORT_BLOB}:50051" volumes: - - $HOME/.aws/credentials:/root/.aws/credentials:ro + - $HOME/.aws/config:/home/comm/.aws/config:ro + - $HOME/.aws/credentials:/home/comm/.aws/credentials:ro # identity identity-server: depends_on: - localstack build: dockerfile: services/identity/Dockerfile context: ../ image: commapp/identity-server:0.1 ports: - "${COMM_SERVICES_PORT_IDENTITY}:50051" # localstack localstack: image: localstack/localstack hostname: localstack ports: - "4566:4566" environment: - SERVICES=s3,dynamodb - DATA_DIR=/tmp/localstack - HOSTNAME_EXTERNAL=localstack volumes: - localstack:/tmp/localstack # RabbitMQ rabbitmq: image: rabbitmq:3-management hostname: rabbitmq ports: - "5672:5672" - "5671:5671" - "15672:15672" environment: - RABBITMQ_DEFAULT_USER=comm - RABBITMQ_DEFAULT_PASS=comm