diff --git a/services/aws_backup/dynamo_db_tables/backup-service-backup b/services/aws_backup/dynamo_db_tables/backup-service-backup new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/backup-service-backup @@ -0,0 +1,68 @@ +{ + "TableName": "backup-service-backup", + "KeySchema": [ + { + "AttributeName": "userID", + "KeyType": "HASH" + }, + { + "AttributeName": "backupID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "backupID", + "AttributeType": "S" + }, + { + "AttributeName": "created", + "AttributeType": "S" + }, + { + "AttributeName": "userID", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "userID-created-index-2", + "KeySchema": [ + { + "AttributeName": "userID", + "KeyType": "HASH" + }, + { + "AttributeName": "created", + "KeyType": "RANGE" + } + ], + "Projection": { + "ProjectionType": "INCLUDE", + "NonKeyAttributes": [ + "recoveryData" + ] + } + }, + { + "IndexName": "userID-created-index", + "KeySchema": [ + { + "AttributeName": "userID", + "KeyType": "HASH" + }, + { + "AttributeName": "created", + "KeyType": "RANGE" + } + ], + "Projection": { + "ProjectionType": "INCLUDE", + "NonKeyAttributes": [ + "reverseIndex" + ] + } + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/backup-service-backup-test b/services/aws_backup/dynamo_db_tables/backup-service-backup-test new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/backup-service-backup-test @@ -0,0 +1,49 @@ +{ + "TableName": "backup-service-backup-test", + "KeySchema": [ + { + "AttributeName": "userID", + "KeyType": "HASH" + }, + { + "AttributeName": "backupID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "backupID", + "AttributeType": "S" + }, + { + "AttributeName": "created", + "AttributeType": "S" + }, + { + "AttributeName": "userID", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "userID-created-index", + "KeySchema": [ + { + "AttributeName": "userID", + "KeyType": "HASH" + }, + { + "AttributeName": "created", + "KeyType": "RANGE" + } + ], + "Projection": { + "ProjectionType": "INCLUDE", + "NonKeyAttributes": [ + "recoveryData" + ] + } + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/backup-service-log b/services/aws_backup/dynamo_db_tables/backup-service-log new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/backup-service-log @@ -0,0 +1,24 @@ +{ + "TableName": "backup-service-log", + "KeySchema": [ + { + "AttributeName": "backupID", + "KeyType": "HASH" + }, + { + "AttributeName": "logID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "backupID", + "AttributeType": "S" + }, + { + "AttributeName": "logID", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/backup-service-log-test b/services/aws_backup/dynamo_db_tables/backup-service-log-test new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/backup-service-log-test @@ -0,0 +1,24 @@ +{ + "TableName": "backup-service-log-test", + "KeySchema": [ + { + "AttributeName": "backupID", + "KeyType": "HASH" + }, + { + "AttributeName": "logID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "backupID", + "AttributeType": "S" + }, + { + "AttributeName": "logID", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/blob-service-blob b/services/aws_backup/dynamo_db_tables/blob-service-blob new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/blob-service-blob @@ -0,0 +1,16 @@ +{ + "TableName": "blob-service-blob", + "KeySchema": [ + { + "AttributeName": "blobHash", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "blobHash", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/blob-service-blob-test b/services/aws_backup/dynamo_db_tables/blob-service-blob-test new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/blob-service-blob-test @@ -0,0 +1,16 @@ +{ + "TableName": "blob-service-blob-test", + "KeySchema": [ + { + "AttributeName": "blobHash", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "blobHash", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/blob-service-reverse-index b/services/aws_backup/dynamo_db_tables/blob-service-reverse-index new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/blob-service-reverse-index @@ -0,0 +1,34 @@ +{ + "TableName": "blob-service-reverse-index", + "KeySchema": [ + { + "AttributeName": "holder", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "blobHash", + "AttributeType": "S" + }, + { + "AttributeName": "holder", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "blobHash-index", + "KeySchema": [ + { + "AttributeName": "blobHash", + "KeyType": "HASH" + } + ], + "Projection": { + "ProjectionType": "ALL" + } + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/blob-service-reverse-index-test b/services/aws_backup/dynamo_db_tables/blob-service-reverse-index-test new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/blob-service-reverse-index-test @@ -0,0 +1,34 @@ +{ + "TableName": "blob-service-reverse-index-test", + "KeySchema": [ + { + "AttributeName": "holder", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "blobHash", + "AttributeType": "S" + }, + { + "AttributeName": "holder", + "AttributeType": "S" + } + ], + "GlobalSecondaryIndexes": [ + { + "IndexName": "blobHash-index", + "KeySchema": [ + { + "AttributeName": "blobHash", + "KeyType": "HASH" + } + ], + "Projection": { + "ProjectionType": "ALL" + } + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/tunnelbroker-device-session b/services/aws_backup/dynamo_db_tables/tunnelbroker-device-session new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/tunnelbroker-device-session @@ -0,0 +1,16 @@ +{ + "TableName": "tunnelbroker-device-session", + "KeySchema": [ + { + "AttributeName": "SessionId", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "SessionId", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/tunnelbroker-message b/services/aws_backup/dynamo_db_tables/tunnelbroker-message new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/tunnelbroker-message @@ -0,0 +1,24 @@ +{ + "TableName": "tunnelbroker-message", + "KeySchema": [ + { + "AttributeName": "ToDeviceID", + "KeyType": "HASH" + }, + { + "AttributeName": "MessageID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "MessageID", + "AttributeType": "S" + }, + { + "AttributeName": "ToDeviceID", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/tunnelbroker-messages b/services/aws_backup/dynamo_db_tables/tunnelbroker-messages new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/tunnelbroker-messages @@ -0,0 +1,24 @@ +{ + "TableName": "tunnelbroker-messages", + "KeySchema": [ + { + "AttributeName": "MessageID", + "KeyType": "HASH" + }, + { + "AttributeName": "ToDeviceID", + "KeyType": "RANGE" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "MessageID", + "AttributeType": "S" + }, + { + "AttributeName": "ToDeviceID", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/tunnelbroker-public-key b/services/aws_backup/dynamo_db_tables/tunnelbroker-public-key new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/tunnelbroker-public-key @@ -0,0 +1,16 @@ +{ + "TableName": "tunnelbroker-public-key", + "KeySchema": [ + { + "AttributeName": "DeviceId", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "DeviceId", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/dynamo_db_tables/tunnelbroker-verification-message b/services/aws_backup/dynamo_db_tables/tunnelbroker-verification-message new file mode 100644 --- /dev/null +++ b/services/aws_backup/dynamo_db_tables/tunnelbroker-verification-message @@ -0,0 +1,16 @@ +{ + "TableName": "tunnelbroker-verification-message", + "KeySchema": [ + { + "AttributeName": "DeviceId", + "KeyType": "HASH" + } + ], + "AttributeDefinitions": [ + { + "AttributeName": "DeviceId", + "AttributeType": "S" + } + ], + "BillingMode": "PAY_PER_REQUEST" +} diff --git a/services/aws_backup/s3.dump b/services/aws_backup/s3.dump new file mode 100644 --- /dev/null +++ b/services/aws_backup/s3.dump @@ -0,0 +1,5 @@ +comm-landing-assets +commapp-backup +commapp-blob +commapp-test +karol-comm-test diff --git a/services/package.json b/services/package.json --- a/services/package.json +++ b/services/package.json @@ -21,6 +21,7 @@ "run-all-services": "./scripts/run_all_services.sh", "test-all-services": "./scripts/test_all_services.sh", "test-all-services-dev-mode": "export COMM_SERVICES_DEV_MODE=1 && ./scripts/test_all_services.sh", - "run-all-services-dev-mode": "export COMM_SERVICES_DEV_MODE=1 && ./scripts/run_all_services.sh" + "run-all-services-dev-mode": "export COMM_SERVICES_DEV_MODE=1 && ./scripts/run_all_services.sh", + "backup-aws": "./scripts/backup_aws.sh" } } diff --git a/services/scripts/backup_aws.sh b/services/scripts/backup_aws.sh new file mode 100755 --- /dev/null +++ b/services/scripts/backup_aws.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +AWS_BACKUP_FOLDER="./aws_backup" +S3_DUMP_FILE="${AWS_BACKUP_FOLDER}/s3.dump" +DYNAMO_DB_TABLES_FOLDER="$AWS_BACKUP_FOLDER/dynamo_db_tables" + +rm -rf $AWS_BACKUP_FOLDER +mkdir $AWS_BACKUP_FOLDER +mkdir $DYNAMO_DB_TABLES_FOLDER + +# S3 +touch $S3_DUMP_FILE +echo "backing up S3..." +BUCKETS=`aws s3 ls | cut -d ' ' -f 3` + +for BUCKET in $BUCKETS; do + echo $BUCKET >> $S3_DUMP_FILE +done + +echo "backing up S3 - DONE" + +# dynamoDB +echo "backing up database..." + +AWS_PAGER="" +TABLES=`aws dynamodb list-tables | jq '.TableNames[]' | tr -d '"'` + +TMP=`mktemp` +for TABLE in $TABLES; do + echo " backing up table $TABLE..." + aws dynamodb describe-table --table-name $TABLE > $TMP + # todo treat this as generated in phabricator + jq < $TMP '.Table | {TableName, KeySchema, AttributeDefinitions} + (try {LocalSecondaryIndexes: [ .LocalSecondaryIndexes[] | {IndexName, KeySchema, Projection} ]} // {}) + (try {GlobalSecondaryIndexes: [ .GlobalSecondaryIndexes[] | {IndexName, KeySchema, Projection} ]} // {}) + {BillingMode: "PAY_PER_REQUEST"}' > $DYNAMO_DB_TABLES_FOLDER/$TABLE + echo " backing up table $TABLE - DONE" +done + +rm $TMP + +echo "backing up database - DONE" diff --git a/services/scripts/list_services.sh b/services/scripts/list_services.sh --- a/services/scripts/list_services.sh +++ b/services/scripts/list_services.sh @@ -2,4 +2,4 @@ set -e -ls | grep -vE 'base-image|docker.compose.yml|package.json|scripts' +ls | grep -vE 'base-image|docker.compose.yml|package.json|scripts|aws_backup'