diff --git a/services/backup/Cargo.lock b/services/backup/Cargo.lock index 35dfa333a..c15cc448e 100644 --- a/services/backup/Cargo.lock +++ b/services/backup/Cargo.lock @@ -1,2362 +1,2196 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 3 [[package]] name = "aho-corasick" version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] [[package]] name = "android_system_properties" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] [[package]] name = "anyhow" version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" [[package]] name = "async-stream" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", "pin-project-lite", ] [[package]] name = "async-stream-impl" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", "syn 2.0.28", ] -[[package]] -name = "async-trait" -version = "0.1.59" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.105", -] - [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" dependencies = [ "aws-credential-types", "aws-http", "aws-sdk-sso", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", "http", "hyper", "ring", "time 0.3.17", "tokio", "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" dependencies = [ "aws-smithy-async", "aws-smithy-types", "fastrand", "tokio", "tracing", "zeroize", ] [[package]] name = "aws-endpoint" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" dependencies = [ "aws-smithy-http", "aws-smithy-types", "aws-types", "http", "regex", "tracing", ] [[package]] name = "aws-http" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" dependencies = [ "aws-credential-types", "aws-smithy-http", "aws-smithy-types", "aws-types", "bytes", "http", "http-body", "lazy_static", "percent-encoding", "pin-project-lite", "tracing", ] [[package]] name = "aws-sdk-dynamodb" version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67fb64867fe098cffee7e34352b01bbfa2beb3aa1b2ff0e0a7bf9ff293557852" dependencies = [ "aws-credential-types", "aws-endpoint", "aws-http", "aws-sig-auth", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http", "regex", "tokio-stream", "tower", "tracing", ] [[package]] name = "aws-sdk-sso" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" dependencies = [ "aws-credential-types", "aws-endpoint", "aws-http", "aws-sig-auth", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-types", "aws-types", "bytes", "http", "regex", "tokio-stream", "tower", "tracing", ] [[package]] name = "aws-sdk-sts" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" dependencies = [ "aws-credential-types", "aws-endpoint", "aws-http", "aws-sig-auth", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-query", "aws-smithy-types", "aws-smithy-xml", "aws-types", "bytes", "http", "regex", "tower", "tracing", ] [[package]] name = "aws-sig-auth" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-http", "aws-types", "http", "tracing", ] [[package]] name = "aws-sigv4" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" dependencies = [ "aws-smithy-http", "form_urlencoded", "hex", "hmac", "http", "once_cell", "percent-encoding", "regex", "sha2", "time 0.3.17", "tracing", ] [[package]] name = "aws-smithy-async" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" dependencies = [ "futures-util", "pin-project-lite", "tokio", "tokio-stream", ] [[package]] name = "aws-smithy-client" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-types", "bytes", "fastrand", "http", "http-body", "hyper", "hyper-rustls", "lazy_static", "pin-project-lite", "rustls", "tokio", "tower", "tracing", ] [[package]] name = "aws-smithy-http" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" dependencies = [ "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", "hyper", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", "tokio", "tokio-util", "tracing", ] [[package]] name = "aws-smithy-http-tower" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", "http", "http-body", "pin-project-lite", "tower", "tracing", ] [[package]] name = "aws-smithy-json" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" dependencies = [ "aws-smithy-types", "urlencoding", ] [[package]] name = "aws-smithy-types" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" dependencies = [ "base64-simd", "itoa", "num-integer", "ryu", "time 0.3.17", ] [[package]] name = "aws-smithy-xml" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", "aws-smithy-types", "http", "rustc_version", "tracing", ] -[[package]] -name = "axum" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-http", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - [[package]] name = "backup" version = "0.1.0" dependencies = [ "anyhow", "async-stream", "aws-config", "aws-sdk-dynamodb", "aws-types", "chrono", "clap", "comm-services-lib", "once_cell", - "prost", "rand", "tokio", "tokio-stream", - "tonic", "tonic-build", "tracing", "tracing-futures", "tracing-subscriber", "uuid", ] [[package]] name = "base64" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64-simd" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" dependencies = [ "outref", "vsimd", ] [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" [[package]] name = "bytes" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "bytes-utils" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" dependencies = [ "bytes", "either", ] [[package]] name = "cc" version = "1.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ "iana-time-zone", "js-sys", "num-integer", "num-traits", "time 0.1.45", "wasm-bindgen", "winapi", ] [[package]] name = "clap" version = "4.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ "bitflags", "clap_derive", "clap_lex", "is-terminal", "once_cell", "strsim", "termcolor", ] [[package]] name = "clap_derive" version = "4.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ "heck", "proc-macro-error", "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "clap_lex" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" dependencies = [ "os_str_bytes", ] [[package]] name = "codespan-reporting" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" dependencies = [ "termcolor", "unicode-width", ] [[package]] name = "comm-services-lib" version = "0.1.0" dependencies = [ "aws-config", "aws-sdk-dynamodb", "aws-types", "chrono", "derive_more", "tracing", ] [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", ] [[package]] name = "core-foundation-sys" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" dependencies = [ "libc", ] [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", ] [[package]] name = "cxx" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf" dependencies = [ "cc", "cxxbridge-flags", "cxxbridge-macro", "link-cplusplus", ] [[package]] name = "cxx-build" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39" dependencies = [ "cc", "codespan-reporting", "once_cell", "proc-macro2", "quote", "scratch", "syn 1.0.105", ] [[package]] name = "cxxbridge-flags" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12" [[package]] name = "cxxbridge-macro" version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "derive_more" version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version", "syn 1.0.105", ] [[package]] name = "digest" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ "block-buffer", "crypto-common", "subtle", ] [[package]] name = "either" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "errno" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" dependencies = [ "errno-dragonfly", "libc", "winapi", ] [[package]] name = "errno-dragonfly" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" dependencies = [ "cc", "libc", ] [[package]] name = "fastrand" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" dependencies = [ "instant", ] [[package]] name = "fixedbitset" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ "percent-encoding", ] [[package]] name = "futures" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", "futures-executor", "futures-io", "futures-sink", "futures-task", "futures-util", ] [[package]] name = "futures-channel" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", "futures-sink", ] [[package]] name = "futures-core" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" [[package]] name = "futures-executor" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" dependencies = [ "futures-core", "futures-task", "futures-util", ] [[package]] name = "futures-io" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" [[package]] name = "futures-macro" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "futures-sink" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" [[package]] name = "futures-task" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" [[package]] name = "futures-util" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-channel", "futures-core", "futures-io", "futures-macro", "futures-sink", "futures-task", "memchr", "pin-project-lite", "pin-utils", "slab", ] [[package]] name = "generic-array" version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", ] [[package]] name = "getrandom" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "h2" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", "http", "indexmap", "slab", "tokio", "tokio-util", "tracing", ] [[package]] name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "heck" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] [[package]] name = "hermit-abi" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ "digest", ] [[package]] name = "http" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", "itoa", ] [[package]] name = "http-body" version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - [[package]] name = "httparse" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", "h2", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", "socket2", "tokio", "tower-service", "tracing", "want", ] [[package]] name = "hyper-rustls" version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" dependencies = [ "http", "hyper", "log", "rustls", "rustls-native-certs", "tokio", "tokio-rustls", ] -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - [[package]] name = "iana-time-zone" version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", "winapi", ] [[package]] name = "iana-time-zone-haiku" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" dependencies = [ "cxx", "cxx-build", ] [[package]] name = "indexmap" version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", ] [[package]] name = "instant" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", ] [[package]] name = "io-lifetimes" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" dependencies = [ "libc", "windows-sys 0.42.0", ] [[package]] name = "is-terminal" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" dependencies = [ "hermit-abi 0.2.6", "io-lifetimes", "rustix", "windows-sys 0.42.0", ] [[package]] name = "itertools" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" [[package]] name = "js-sys" version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" version = "0.2.138" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" [[package]] name = "link-cplusplus" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369" dependencies = [ "cc", ] [[package]] name = "linux-raw-sys" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "log" version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] [[package]] name = "matchers" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - [[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - [[package]] name = "mio" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.42.0", ] [[package]] name = "multimap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "nu-ansi-term" version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", "winapi", ] [[package]] name = "num-integer" version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", "num-traits", ] [[package]] name = "num-traits" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ "hermit-abi 0.1.19", "libc", ] [[package]] name = "once_cell" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "os_str_bytes" version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "outref" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" [[package]] name = "overload" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "percent-encoding" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "petgraph" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" dependencies = [ "fixedbitset", "indexmap", ] [[package]] name = "pin-project" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "pin-project-lite" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "ppv-lite86" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" dependencies = [ "proc-macro2", "syn 1.0.105", ] [[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", "syn 1.0.105", "version_check", ] [[package]] name = "proc-macro-error-attr" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", "version_check", ] [[package]] name = "proc-macro2" version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] [[package]] name = "prost" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b18e655c21ff5ac2084a5ad0611e827b3f92badf79f4910b5a5c58f4d87ff0" dependencies = [ "bytes", "prost-derive", ] [[package]] name = "prost-build" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e330bf1316db56b12c2bcfa399e8edddd4821965ea25ddb2c134b610b1c1c604" dependencies = [ "bytes", "heck", "itertools", "lazy_static", "log", "multimap", "petgraph", "prettyplease", "prost", "prost-types", "regex", "syn 1.0.105", "tempfile", "which", ] [[package]] name = "prost-derive" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "prost-types" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" dependencies = [ "bytes", "prost", ] [[package]] name = "quote" version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", ] [[package]] name = "rand_chacha" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", ] [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] [[package]] name = "redox_syscall" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] [[package]] name = "regex" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ "winapi", ] [[package]] name = "ring" version = "0.16.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", "once_cell", "spin", "untrusted", "web-sys", "winapi", ] [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ "semver", ] [[package]] name = "rustix" version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", "windows-sys 0.42.0", ] [[package]] name = "rustls" version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ "log", "ring", "sct", "webpki", ] [[package]] name = "rustls-native-certs" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ "openssl-probe", "rustls-pemfile", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55" dependencies = [ "base64", ] -[[package]] -name = "rustversion" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" - [[package]] name = "ryu" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" [[package]] name = "schannel" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" dependencies = [ "lazy_static", "windows-sys 0.36.1", ] [[package]] name = "scratch" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] name = "sct" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ "ring", "untrusted", ] [[package]] name = "security-framework" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bc1bb97804af6631813c55739f771071e0f2ed33ee20b68c86ec505d906356c" dependencies = [ "bitflags", "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", ] [[package]] name = "security-framework-sys" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" dependencies = [ "core-foundation-sys", "libc", ] [[package]] name = "semver" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" [[package]] name = "sha2" version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", "digest", ] [[package]] name = "sharded-slab" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" dependencies = [ "lazy_static", ] [[package]] name = "slab" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" dependencies = [ "autocfg", ] [[package]] name = "smallvec" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi", ] [[package]] name = "spin" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "syn" version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" - [[package]] name = "tempfile" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ "cfg-if", "fastrand", "libc", "redox_syscall", "remove_dir_all", "winapi", ] [[package]] name = "termcolor" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] [[package]] name = "thread_local" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" dependencies = [ "once_cell", ] [[package]] name = "time" version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", "winapi", ] [[package]] name = "time" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ "serde", "time-core", "time-macros", ] [[package]] name = "time-core" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" dependencies = [ "time-core", ] [[package]] name = "tokio" version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg", "bytes", "libc", "memchr", "mio", "num_cpus", "pin-project-lite", "socket2", "tokio-macros", "windows-sys 0.42.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", "tokio", "webpki", ] [[package]] name = "tokio-stream" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", "tokio", ] [[package]] name = "tokio-util" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", "tracing", ] -[[package]] -name = "tonic" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - [[package]] name = "tonic-build" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ "prettyplease", "proc-macro2", "prost-build", "quote", "syn 1.0.105", ] [[package]] name = "tower" version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", "pin-project", "pin-project-lite", - "rand", - "slab", "tokio", - "tokio-util", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "tower-http" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" [[package]] name = "tower-service" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", "pin-project-lite", "tracing-attributes", "tracing-core", ] [[package]] name = "tracing-attributes" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", ] [[package]] name = "tracing-core" version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", ] [[package]] name = "tracing-futures" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ "futures", "futures-task", "pin-project", "tracing", ] [[package]] name = "tracing-log" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", "tracing-core", ] [[package]] name = "tracing-subscriber" version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ "matchers", "nu-ansi-term", "once_cell", "regex", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", ] [[package]] name = "try-lock" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-ident" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" [[package]] name = "unicode-width" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "untrusted" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "urlencoding" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" [[package]] name = "uuid" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" dependencies = [ "getrandom", ] [[package]] name = "valuable" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vsimd" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "want" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" dependencies = [ "log", "try-lock", ] [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", "syn 1.0.105", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", ] [[package]] name = "wasm-bindgen-macro-support" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", "syn 1.0.105", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" dependencies = [ "js-sys", "wasm-bindgen", ] [[package]] name = "webpki" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ "ring", "untrusted", ] [[package]] name = "which" version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" dependencies = [ "either", "libc", "once_cell", ] [[package]] name = "winapi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", ] [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ "winapi", ] [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc 0.36.1", "windows_i686_gnu 0.36.1", "windows_i686_msvc 0.36.1", "windows_x86_64_gnu 0.36.1", "windows_x86_64_msvc 0.36.1", ] [[package]] name = "windows-sys" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc 0.42.0", "windows_i686_gnu 0.42.0", "windows_i686_msvc 0.42.0", "windows_x86_64_gnu 0.42.0", "windows_x86_64_gnullvm", "windows_x86_64_msvc 0.42.0", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" [[package]] name = "windows_aarch64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" [[package]] name = "windows_i686_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" [[package]] name = "windows_i686_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" [[package]] name = "windows_x86_64_gnu" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" [[package]] name = "windows_x86_64_gnullvm" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" [[package]] name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" [[package]] name = "xmlparser" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" [[package]] name = "zeroize" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" diff --git a/services/backup/Cargo.toml b/services/backup/Cargo.toml index b7ac9f662..389889deb 100644 --- a/services/backup/Cargo.toml +++ b/services/backup/Cargo.toml @@ -1,30 +1,28 @@ [package] name = "backup" version = "0.1.0" description = "Backup service" homepage = "https://comm.app" license = "BSD-3-Clause" edition = "2021" [dependencies] anyhow = "1.0" async-stream = "0.3" aws-config = "0.55" aws-sdk-dynamodb = "0.27" aws-types = "0.55" chrono = "0.4" clap = { version = "4.0", features = ["derive", "env"] } comm-services-lib = { path = "../comm-services-lib" } once_cell = "1.17" -prost = "0.11" rand = "0.8.5" -tokio = { version = "1.24", features = ["rt-multi-thread"]} +tokio = { version = "1.24", features = ["rt-multi-thread", "macros"] } tokio-stream = "0.1" -tonic = "0.8" tracing = "0.1" -tracing-futures = { version = "0.2", features = ["futures-03"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"]} +tracing-futures = { version = "0.2", features = ["futures-03"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = { version = "1.2", features = ["v4"] } [build-dependencies] tonic-build = "0.8" diff --git a/services/backup/Dockerfile b/services/backup/Dockerfile index 922dd8d4f..9820c5dc2 100644 --- a/services/backup/Dockerfile +++ b/services/backup/Dockerfile @@ -1,53 +1,51 @@ FROM commapp/services-base:1.3.2 as builder ENV PATH=/root/.cargo/bin:$PATH # Install Curl RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y curl \ && rm -rf /var/lib/apt/lists/* # Install Rust RUN curl https://sh.rustup.rs -sSf | sh -s -- -y RUN mkdir -p /home/comm/app/backup WORKDIR /home/comm/app/backup RUN cargo init --bin # Cache build dependencies in a new layer COPY services/backup/Cargo.toml services/backup/Cargo.lock ./ COPY services/comm-services-lib ../comm-services-lib RUN cargo build --release && rm src/*.rs # Copy actual application sources COPY services/backup . -COPY shared/protos/backup.proto ../../shared/protos/ -COPY shared/protos/blob.proto ../../shared/protos/ # Remove the previously-built binary so that only the application itself is # rebuilt RUN rm ./target/release/deps/backup* RUN cargo build --release # Runner stage FROM commapp/services-base:1.3.2 as runner # Create a new user comm and use it to run subsequent commands RUN useradd -m comm USER comm # Only copy built binary from builder stage WORKDIR /home/comm/app/backup COPY --from=builder /home/comm/app/backup/target/release/backup . ARG COMM_SERVICES_SANDBOX ENV COMM_SERVICES_SANDBOX=${COMM_SERVICES_SANDBOX} ARG LOCALSTACK_URL ENV LOCALSTACK_URL=${LOCALSTACK_URL} ARG BLOB_SERVICE_URL ENV BLOB_SERVICE_URL=${BLOB_SERVICE_URL} CMD ./backup \ --localstack-url ${LOCALSTACK_URL} \ --blob-service-url ${BLOB_SERVICE_URL} # The COMM_SERVICES_SANDBOX is processed by the executable diff --git a/services/backup/build.rs b/services/backup/build.rs deleted file mode 100644 index 4dcab23fd..000000000 --- a/services/backup/build.rs +++ /dev/null @@ -1,10 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=src/main.rs"); - - println!("cargo:rerun-if-changed=../../shared/protos/backup.proto"); - println!("cargo:rerun-if-changed=../../shared/protos/blob.proto"); - tonic_build::compile_protos("../../shared/protos/backup.proto") - .expect("Failed to compile Backup protobuf file"); - tonic_build::compile_protos("../../shared/protos/blob.proto") - .expect("Failed to compile Blob protobuf file"); -} diff --git a/services/backup/src/blob/downloader.rs b/services/backup/src/blob/downloader.rs deleted file mode 100644 index c6e9b7868..000000000 --- a/services/backup/src/blob/downloader.rs +++ /dev/null @@ -1,82 +0,0 @@ -use anyhow::{bail, Result}; -use tokio::{ - sync::mpsc::{self, Receiver}, - task::JoinHandle, -}; -use tracing::{error, instrument, Instrument}; - -use super::{proto, BlobClient}; -use crate::constants::MPSC_CHANNEL_BUFFER_CAPACITY; - -pub use proto::put_request::Data as PutRequestData; -pub use proto::{PutRequest, PutResponse}; - -/// The BlobDownloader instance is a handle holder of a Tokio task running the -/// actual blob client instance. The communication is done via a MPSC channel -/// and is one-sided - the data is transmitted from the client task to the -/// caller. Blob chunks received in response stream are waiting -/// for the channel to have capacity, so it is recommended to read them quickly -/// to make room for more. -/// The client task can be stopped and awaited for result via the `terminate()` -/// method. -pub struct BlobDownloader { - rx: Receiver>, - handle: JoinHandle>, -} - -impl BlobDownloader { - /// Connects to the Blob service and keeps the client connection open - /// in a separate Tokio task. - #[instrument(name = "blob_downloader")] - pub fn start(holder: String, mut blob_client: BlobClient) -> Self { - let (blob_res_tx, blob_res_rx) = - mpsc::channel(MPSC_CHANNEL_BUFFER_CAPACITY); - let client_thread = async move { - let response = blob_client - .get(proto::GetRequest { holder }) - .await - .map_err(|err| { - error!("Get request failed: {:?}", err); - err - })?; - let mut inner_response = response.into_inner(); - loop { - match inner_response.message().await? { - Some(data) => { - let data: Vec = data.data_chunk; - if let Err(err) = blob_res_tx.send(data).await { - bail!(err); - } - } - // Response stream was closed - None => break, - } - } - Ok(()) - }; - let handle = tokio::spawn(client_thread.in_current_span()); - - BlobDownloader { - rx: blob_res_rx, - handle, - } - } - - /// Receives the next chunk of blob data if ready or sleeps - /// until the data is available. - /// - /// Returns `None` when the transmission is finished, but this doesn't - /// determine if it was successful. After receiving `None`, the client - /// should be consumed by calling [`GetClient::terminate`] to handle - /// possible errors. - pub async fn next_chunk(&mut self) -> Option> { - self.rx.recv().await - } - - /// Stops receiving messages and awaits the client thread to exit - /// and returns its status. - pub async fn terminate(mut self) -> Result<()> { - self.rx.close(); - self.handle.await? - } -} diff --git a/services/backup/src/blob/mod.rs b/services/backup/src/blob/mod.rs deleted file mode 100644 index b691359d1..000000000 --- a/services/backup/src/blob/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -mod proto { - tonic::include_proto!("blob"); -} -use proto::blob_service_client::BlobServiceClient; -pub use proto::put_request::Data as PutRequestData; -pub use proto::{PutRequest, PutResponse}; - -mod downloader; -mod uploader; -pub use downloader::*; -pub use uploader::*; - -pub type BlobClient = BlobServiceClient; - -/// Creates a new Blob service client instance. It does not attempt to connect -/// to the service until first use. -pub fn init_blob_client() -> BlobClient { - let service_url = &crate::CONFIG.blob_service_url; - let channel = - tonic::transport::Channel::from_static(service_url).connect_lazy(); - BlobServiceClient::new(channel) -} diff --git a/services/backup/src/blob/uploader.rs b/services/backup/src/blob/uploader.rs deleted file mode 100644 index 28a265543..000000000 --- a/services/backup/src/blob/uploader.rs +++ /dev/null @@ -1,168 +0,0 @@ -use anyhow::{anyhow, bail, Result}; -use tokio::{ - sync::mpsc::{self, Receiver, Sender}, - task::JoinHandle, -}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::Status; -use tracing::{error, instrument, Instrument}; - -use super::{proto, BlobClient}; -use crate::constants::MPSC_CHANNEL_BUFFER_CAPACITY; - -pub use proto::put_request::Data as PutRequestData; -pub use proto::{PutRequest, PutResponse}; - -pub struct BlobUploader { - req_tx: Sender, - res_rx: Receiver, - handle: JoinHandle>, -} - -/// The BlobUploader instance is a handle holder of a Tokio task running the -/// actual blob client instance. The communication is done via two MPSC -/// channels - one sending requests to the client task, and another for sending -/// responses back to the caller. These messages should go in pairs -/// - one request for one response. -/// The client task can be stopped and awaited for result via the `terminate()` -/// method. -impl BlobUploader { - /// Connects to the Blob service and keeps the client connection open - /// in a separate Tokio task. - #[instrument(name = "blob_uploader")] - pub fn start(mut blob_client: BlobClient) -> Self { - let (blob_req_tx, blob_req_rx) = - mpsc::channel(MPSC_CHANNEL_BUFFER_CAPACITY); - let (blob_res_tx, blob_res_rx) = - mpsc::channel(MPSC_CHANNEL_BUFFER_CAPACITY); - let client_thread = async move { - match blob_client - .put(tonic::Request::new(ReceiverStream::new(blob_req_rx))) - .await - { - Ok(response) => { - let mut response_stream = response.into_inner(); - loop { - match response_stream.message().await? { - Some(response_message) => { - // warning: this will produce an error if there's more unread - // responses than MPSC_CHANNEL_BUFFER_CAPACITY - // so you should always read the response MPSC channel - // right after sending a request to dequeue the responses - // and make room for more. - // The PutClient::put() function should take care of that - if let Err(err) = blob_res_tx.try_send(response_message) { - bail!(err); - } - } - // Response stream was closed - None => break, - } - } - } - Err(err) => { - error!("Put request failed: {:?}", err); - bail!(err.to_string()); - } - }; - Ok(()) - }; - let handle = tokio::spawn(client_thread.in_current_span()); - - BlobUploader { - req_tx: blob_req_tx, - res_rx: blob_res_rx, - handle, - } - } - - /// Sends a [`PutRequest`] to the stream and waits for blob service - /// to send a response. After all data is sent, the [`PutClient::terminate`] - /// should be called to end the transmission and handle possible errors. - pub async fn put(&mut self, req: PutRequest) -> Result { - self.req_tx.try_send(req)?; - self - .res_rx - .recv() - .await - .ok_or_else(|| anyhow!("Blob client channel closed")) - } - - /// Convenience wrapper for - /// ``` - /// BlobClient::put(PutRequest { - /// data: Some(PutRequestData::DataChunk(data)) - /// }) - /// ``` - pub async fn put_data(&mut self, data: Vec) -> Result { - self - .put(PutRequest { - data: Some(PutRequestData::DataChunk(data)), - }) - .await - } - - /// Closes the connection and awaits the blob client task to finish. - pub async fn terminate(self) -> Result<()> { - drop(self.req_tx); - let thread_result = self.handle.await?; - thread_result - } -} - -/// Starts a put client instance. Fulfills request with blob hash and holder. -/// -/// `None` is returned if given `holder` already exists. -/// -/// ## Example -/// ``` -/// if let Some(mut client) = -/// start_simple_uploader("my_holder", "my_hash").await? { -/// let my_data = vec![1,2,3,4]; -/// let _ = client.put_data(my_data).await; -/// -/// let status = client.terminate().await; -/// } -/// ``` -pub async fn start_simple_uploader( - holder: &str, - blob_hash: &str, - blob_client: BlobClient, -) -> Result, Status> { - // start upload request - let mut uploader = BlobUploader::start(blob_client); - - // send holder - uploader - .put(PutRequest { - data: Some(PutRequestData::Holder(holder.to_string())), - }) - .await - .map_err(|err| { - error!("Failed to set blob holder: {:?}", err); - Status::aborted("Internal error") - })?; - - // send hash - let PutResponse { data_exists } = uploader - .put(PutRequest { - data: Some(PutRequestData::BlobHash(blob_hash.to_string())), - }) - .await - .map_err(|err| { - error!("Failed to set blob hash: {:?}", err); - Status::aborted("Internal error") - })?; - - // Blob with given holder already exists, nothing to do - if data_exists { - // the connection is already terminated by server, - // but it's good to await it anyway - uploader.terminate().await.map_err(|err| { - error!("Put client task closed with error: {:?}", err); - Status::aborted("Internal error") - })?; - return Ok(None); - } - Ok(Some(uploader)) -} diff --git a/services/backup/src/config.rs b/services/backup/src/config.rs index 71beea109..d8850b0d3 100644 --- a/services/backup/src/config.rs +++ b/services/backup/src/config.rs @@ -1,54 +1,50 @@ use clap::{builder::FalseyValueParser, Parser}; use once_cell::sync::Lazy; use tracing::info; use crate::constants::{ - DEFAULT_BLOB_SERVICE_URL, DEFAULT_GRPC_SERVER_PORT, DEFAULT_LOCALSTACK_URL, - SANDBOX_ENV_VAR, + DEFAULT_BLOB_SERVICE_URL, DEFAULT_LOCALSTACK_URL, SANDBOX_ENV_VAR, }; #[derive(Parser)] #[command(version, about, long_about = None)] pub struct AppConfig { - /// gRPC server listening port - #[arg(long = "port", default_value_t = DEFAULT_GRPC_SERVER_PORT)] - pub listening_port: u64, /// Run the service in sandbox #[arg(long = "sandbox", default_value_t = false)] // support the env var for compatibility reasons #[arg(env = SANDBOX_ENV_VAR)] #[arg(value_parser = FalseyValueParser::new())] pub is_sandbox: bool, /// AWS Localstack service URL, applicable in sandbox mode #[arg(long, default_value_t = DEFAULT_LOCALSTACK_URL.to_string())] pub localstack_url: String, /// Blob service URL #[arg(long, default_value_t = DEFAULT_BLOB_SERVICE_URL.to_string())] pub blob_service_url: String, } /// Stores configuration parsed from command-line arguments /// and environment variables pub static CONFIG: Lazy = Lazy::new(|| AppConfig::parse()); /// Processes the command-line arguments and environment variables. /// Should be called at the beginning of the `main()` function. pub(super) fn parse_cmdline_args() { // force evaluation of the lazy initialized config Lazy::force(&CONFIG); } /// Provides region/credentials configuration for AWS SDKs pub async fn load_aws_config() -> aws_types::SdkConfig { let mut config_builder = aws_config::from_env(); if CONFIG.is_sandbox { info!( "Running in sandbox environment. Localstack URL: {}", &CONFIG.localstack_url ); config_builder = config_builder.endpoint_url(&CONFIG.localstack_url); } config_builder.load().await } diff --git a/services/backup/src/constants.rs b/services/backup/src/constants.rs index 04d94a438..308e4be53 100644 --- a/services/backup/src/constants.rs +++ b/services/backup/src/constants.rs @@ -1,62 +1,60 @@ // Assorted constants pub const MPSC_CHANNEL_BUFFER_CAPACITY: usize = 1; pub const ID_SEPARATOR: &str = ":"; pub const ATTACHMENT_HOLDER_SEPARATOR: &str = ";"; // 400KiB limit (in docs there is KB but they mean KiB) - // https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html // This includes both attribute names' and values' lengths // // This has to be smaller than GRPC_CHUNK_SIZE_LIMIT because we need to // recognize if we may receive multiple chunks or just one. If it was larger // than the chunk limit, once we get the amount of data of size equal to the // limit, we wouldn't know if we should put this in the database right away or // wait for more data. pub const LOG_DATA_SIZE_DATABASE_LIMIT: usize = 1024 * 400; // 4MB limit // WARNING: use keeping in mind that grpc adds its own headers to messages // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md // so the message that actually is being sent over the network looks like this // [Compressed-Flag] [Message-Length] [Message] // [Compressed-Flag] 1 byte - added by grpc // [Message-Length] 4 bytes - added by grpc // [Message] N bytes - actual data // so for every message we get 5 additional bytes of data // as mentioned here // https://github.com/grpc/grpc/issues/15734#issuecomment-396962671 // grpc stream may contain more than one message pub const GRPC_CHUNK_SIZE_LIMIT: usize = 4 * 1024 * 1024; pub const GRPC_METADATA_SIZE_PER_MESSAGE: usize = 5; // Configuration defaults - -pub const DEFAULT_GRPC_SERVER_PORT: u64 = 50051; pub const DEFAULT_LOCALSTACK_URL: &str = "http://localhost:4566"; pub const DEFAULT_BLOB_SERVICE_URL: &str = "http://localhost:50053"; // Environment variable names pub const SANDBOX_ENV_VAR: &str = "COMM_SERVICES_SANDBOX"; pub const LOG_LEVEL_ENV_VAR: &str = tracing_subscriber::filter::EnvFilter::DEFAULT_ENV; // DynamoDB constants pub const BACKUP_TABLE_NAME: &str = "backup-service-backup"; pub const BACKUP_TABLE_FIELD_USER_ID: &str = "userID"; pub const BACKUP_TABLE_FIELD_BACKUP_ID: &str = "backupID"; pub const BACKUP_TABLE_FIELD_CREATED: &str = "created"; pub const BACKUP_TABLE_FIELD_RECOVERY_DATA: &str = "recoveryData"; pub const BACKUP_TABLE_FIELD_COMPACTION_HOLDER: &str = "compactionHolder"; pub const BACKUP_TABLE_FIELD_ATTACHMENT_HOLDERS: &str = "attachmentHolders"; pub const BACKUP_TABLE_INDEX_USERID_CREATED: &str = "userID-created-index"; pub const LOG_TABLE_NAME: &str = "backup-service-log"; pub const LOG_TABLE_FIELD_BACKUP_ID: &str = "backupID"; pub const LOG_TABLE_FIELD_LOG_ID: &str = "logID"; pub const LOG_TABLE_FIELD_PERSISTED_IN_BLOB: &str = "persistedInBlob"; pub const LOG_TABLE_FIELD_VALUE: &str = "value"; pub const LOG_TABLE_FIELD_ATTACHMENT_HOLDERS: &str = "attachmentHolders"; pub const LOG_TABLE_FIELD_DATA_HASH: &str = "dataHash"; diff --git a/services/backup/src/main.rs b/services/backup/src/main.rs index 17ceb67ab..2750b8dd1 100644 --- a/services/backup/src/main.rs +++ b/services/backup/src/main.rs @@ -1,57 +1,33 @@ use anyhow::Result; -use std::net::SocketAddr; -use tonic::transport::Server; -use tracing::{info, Level}; +use tracing::Level; use tracing_subscriber::EnvFilter; -use crate::blob::BlobClient; -use crate::service::{BackupServiceServer, MyBackupService}; - -pub mod blob; pub mod config; pub mod constants; pub mod database; -pub mod service; pub mod utils; // re-export this to be available as crate::CONFIG pub use config::CONFIG; fn configure_logging() -> Result<()> { let filter = EnvFilter::builder() .with_default_directive(Level::INFO.into()) .with_env_var(constants::LOG_LEVEL_ENV_VAR) .from_env_lossy(); let subscriber = tracing_subscriber::fmt().with_env_filter(filter).finish(); tracing::subscriber::set_global_default(subscriber)?; Ok(()) } -async fn run_grpc_server( - db: database::DatabaseClient, - blob_client: BlobClient, -) -> Result<()> { - let addr: SocketAddr = format!("[::]:{}", CONFIG.listening_port).parse()?; - let backup_service = MyBackupService::new(db, blob_client); - - info!("Starting gRPC server listening at {}", addr.to_string()); - Server::builder() - .add_service(BackupServiceServer::new(backup_service)) - .serve(addr) - .await?; - - Ok(()) -} - #[tokio::main] async fn main() -> Result<()> { config::parse_cmdline_args(); configure_logging()?; let aws_config = config::load_aws_config().await; - let db = database::DatabaseClient::new(&aws_config); - let blob_client = blob::init_blob_client(); + let _db = database::DatabaseClient::new(&aws_config); - run_grpc_server(db, blob_client).await + Ok(()) } diff --git a/services/backup/src/service/handlers/add_attachments.rs b/services/backup/src/service/handlers/add_attachments.rs deleted file mode 100644 index 1860e20af..000000000 --- a/services/backup/src/service/handlers/add_attachments.rs +++ /dev/null @@ -1,208 +0,0 @@ -use tonic::Status; -use tracing::debug; -use tracing::error; - -use super::handle_db_error; -use super::proto; -use crate::{ - blob::BlobClient, - constants::{ATTACHMENT_HOLDER_SEPARATOR, LOG_DATA_SIZE_DATABASE_LIMIT}, - database::{DatabaseClient, LogItem}, -}; - -pub async fn handle_add_attachments( - db: &DatabaseClient, - blob_client: &BlobClient, - request: proto::AddAttachmentsRequest, -) -> Result<(), Status> { - let proto::AddAttachmentsRequest { - user_id, - backup_id, - log_id, - holders, - } = request; - - if user_id.is_empty() { - return Err(Status::invalid_argument( - "user id required but not provided", - )); - } - if backup_id.is_empty() { - return Err(Status::invalid_argument( - "backup id required but not provided", - )); - } - if holders.is_empty() { - return Err(Status::invalid_argument( - "holders required but not provided", - )); - } - - if log_id.is_empty() { - let backup_item_result = db - .find_backup_item(&user_id, &backup_id) - .await - .map_err(handle_db_error)?; - let mut backup_item = backup_item_result.ok_or_else(|| { - debug!("Backup item not found"); - Status::not_found("Backup item not found") - })?; - - add_new_attachments(&mut backup_item.attachment_holders, &holders); - - db.put_backup_item(backup_item) - .await - .map_err(handle_db_error)?; - } else { - let log_item_result = db - .find_log_item(&backup_id, &log_id) - .await - .map_err(handle_db_error)?; - let mut log_item = log_item_result.ok_or_else(|| { - debug!("Log item not found"); - Status::not_found("Log item not found") - })?; - - add_new_attachments(&mut log_item.attachment_holders, &holders); - - // log item too large for database, move it to blob-service stroage - if !log_item.persisted_in_blob - && log_item.total_size() > LOG_DATA_SIZE_DATABASE_LIMIT - { - debug!("Log item too large. Persisting in blob service..."); - log_item = move_to_blob(log_item, blob_client).await?; - } - - db.put_log_item(log_item).await.map_err(handle_db_error)?; - } - - Ok(()) -} - -async fn move_to_blob( - log_item: LogItem, - blob_client: &BlobClient, -) -> Result { - let holder = crate::utils::generate_blob_holder( - &log_item.data_hash, - &log_item.backup_id, - Some(&log_item.log_id), - ); - - if let Some(mut uploader) = crate::blob::start_simple_uploader( - &holder, - &log_item.data_hash, - blob_client.clone(), - ) - .await? - { - let blob_chunk = log_item.value.into_bytes(); - uploader.put_data(blob_chunk).await.map_err(|err| { - error!("Failed to upload data chunk: {:?}", err); - Status::aborted("Internal error") - })?; - - uploader.terminate().await.map_err(|err| { - error!("Put client task closed with error: {:?}", err); - Status::aborted("Internal error") - })?; - } else { - debug!("Blob holder for log ID={} already exists", &log_item.log_id); - } - - Ok(LogItem { - persisted_in_blob: true, - value: holder, - ..log_item - }) -} - -/// Modifies the [`current_holders_str`] by appending attachment holders -/// contained in [`new_holders`]. Removes duplicates. Both arguments -/// are expected to be [`ATTACHMENT_HOLDER_SEPARATOR`] separated strings. -fn add_new_attachments(current_holders_str: &mut String, new_holders: &str) { - let mut current_holders = parse_attachment_holders(current_holders_str); - - let new_holders = parse_attachment_holders(new_holders); - current_holders.extend(new_holders); - - *current_holders_str = current_holders - .into_iter() - .collect::>() - .join(ATTACHMENT_HOLDER_SEPARATOR); -} - -/// Parses an [`ATTACHMENT_HOLDER_SEPARATOR`] separated string into a `HashSet` -/// of attachment holder string slices. -fn parse_attachment_holders( - holders_str: &str, -) -> std::collections::HashSet<&str> { - holders_str - .split(ATTACHMENT_HOLDER_SEPARATOR) - .filter(|holder| !holder.is_empty()) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashSet; - - #[test] - fn test_parse_attachments() { - let holders = "h1;h2;h3"; - let expected = HashSet::from(["h1", "h2", "h3"]); - assert_eq!(parse_attachment_holders(holders), expected); - } - - #[test] - fn test_empty_attachments() { - let actual = parse_attachment_holders(""); - let expected = HashSet::new(); - assert_eq!(actual, expected); - } - - #[test] - fn test_add_attachments() { - let mut current_holders = "holder1;holder2".to_string(); - let new_holders = "holder3;holder4"; - add_new_attachments(&mut current_holders, new_holders); - assert_eq!( - parse_attachment_holders(¤t_holders), - HashSet::from(["holder1", "holder2", "holder3", "holder4"]) - ); - } - - #[test] - fn test_add_to_empty() { - let mut current_holders = String::new(); - let new_holders = "holder3;holder4"; - add_new_attachments(&mut current_holders, new_holders); - assert_eq!( - parse_attachment_holders(¤t_holders), - HashSet::from(["holder3", "holder4"]) - ); - } - - #[test] - fn test_add_none() { - let mut current_holders = "holder1;holder2".to_string(); - let new_holders = ""; - add_new_attachments(&mut current_holders, new_holders); - assert_eq!( - parse_attachment_holders(¤t_holders), - HashSet::from(["holder1", "holder2"]) - ); - } - - #[test] - fn test_remove_duplicates() { - let mut current_holders = "holder1;holder2".to_string(); - let new_holders = "holder2;holder3"; - add_new_attachments(&mut current_holders, new_holders); - assert_eq!( - parse_attachment_holders(¤t_holders), - HashSet::from(["holder1", "holder2", "holder3"]) - ); - } -} diff --git a/services/backup/src/service/handlers/create_backup.rs b/services/backup/src/service/handlers/create_backup.rs deleted file mode 100644 index fb76aa57c..000000000 --- a/services/backup/src/service/handlers/create_backup.rs +++ /dev/null @@ -1,234 +0,0 @@ -use tonic::Status; -use tracing::{debug, error, trace, warn}; - -use crate::{ - blob::{start_simple_uploader, BlobClient, BlobUploader}, - database::{BackupItem, DatabaseClient}, - service::proto, -}; - -use super::handle_db_error; - -type CreateBackupResult = Result; - -enum HandlerState { - /// Initial state. Handler is receiving non-data inputs - ReceivingParams, - /// Handler is receiving data chunks - ReceivingData { uploader: BlobUploader }, - /// A special case when Blob service claims that a blob with given - /// [`CreateBackupHandler::data_hash`] already exists - DataAlreadyExists, -} - -pub struct CreateBackupHandler { - // flow control - pub should_close_stream: bool, - - // inputs - user_id: Option, - device_id: Option, - key_entropy: Option>, - data_hash: Option, - - // client instances - db: DatabaseClient, - blob_client: BlobClient, - - // internal state - state: HandlerState, - backup_id: String, - holder: Option, -} - -impl CreateBackupHandler { - pub fn new(db: DatabaseClient, blob_client: BlobClient) -> Self { - CreateBackupHandler { - should_close_stream: false, - user_id: None, - device_id: None, - key_entropy: None, - data_hash: None, - db, - blob_client, - state: HandlerState::ReceivingParams, - backup_id: String::new(), - holder: None, - } - } - - pub async fn handle_user_id( - &mut self, - user_id: String, - ) -> CreateBackupResult { - if self.user_id.is_some() { - warn!("user ID already provided"); - return Err(Status::invalid_argument("User ID already provided")); - } - self.user_id = Some(user_id); - self.handle_internal().await - } - pub async fn handle_device_id( - &mut self, - device_id: String, - ) -> CreateBackupResult { - if self.device_id.is_some() { - warn!("Device ID already provided"); - return Err(Status::invalid_argument("Device ID already provided")); - } - tracing::Span::current().record("device_id", &device_id); - self.device_id = Some(device_id); - self.handle_internal().await - } - pub async fn handle_key_entropy( - &mut self, - key_entropy: Vec, - ) -> CreateBackupResult { - if self.key_entropy.is_some() { - warn!("Key entropy already provided"); - return Err(Status::invalid_argument("Key entropy already provided")); - } - self.key_entropy = Some(key_entropy); - self.handle_internal().await - } - pub async fn handle_data_hash( - &mut self, - data_hash: Vec, - ) -> CreateBackupResult { - if self.data_hash.is_some() { - warn!("Data hash already provided"); - return Err(Status::invalid_argument("Data hash already provided")); - } - let hash_str = String::from_utf8(data_hash).map_err(|err| { - error!("Failed to convert data_hash into string: {:?}", err); - Status::aborted("Unexpected error") - })?; - debug!("Received data hash: {}", &hash_str); - self.data_hash = Some(hash_str); - self.handle_internal().await - } - - pub async fn handle_data_chunk( - &mut self, - data_chunk: Vec, - ) -> CreateBackupResult { - let HandlerState::ReceivingData { ref mut uploader } = self.state else { - self.should_close_stream = true; - error!("Data chunk sent before other inputs"); - return Err(Status::invalid_argument( - "Data chunk sent before other inputs", - )); - }; - - // empty chunk ends transmission - if data_chunk.is_empty() { - self.should_close_stream = true; - return Ok(proto::CreateNewBackupResponse { - backup_id: self.backup_id.clone(), - }); - } - - trace!("Received {} bytes of data", data_chunk.len()); - uploader.put_data(data_chunk).await.map_err(|err| { - error!("Failed to upload data chunk: {:?}", err); - Status::aborted("Internal error") - })?; - - Ok(proto::CreateNewBackupResponse { - // actual Backup ID should be sent only once, the time it is generated - // see handle_internal() - backup_id: String::new(), - }) - } - - /// This function should be called after the input stream is finished. - pub async fn finish(self) -> Result<(), Status> { - match self.state { - HandlerState::ReceivingParams => { - // client probably aborted early - trace!("Nothing to store in database. Finishing early"); - return Ok(()); - } - HandlerState::ReceivingData { uploader } => { - uploader.terminate().await.map_err(|err| { - error!("Uploader task closed with error: {:?}", err); - Status::aborted("Internal error") - })?; - } - HandlerState::DataAlreadyExists => (), - } - - let (Some(user_id), Some(holder)) = (self.user_id, self.holder) else { - error!("Holder / UserID absent in data mode. This should never happen!"); - return Err(Status::failed_precondition("Internal error")); - }; - if self.backup_id.is_empty() { - error!("Backup ID was not generated. This should never happen!"); - return Err(Status::failed_precondition("Internal error")); - } - let backup_item = BackupItem::new(user_id, self.backup_id, holder); - - self - .db - .put_backup_item(backup_item) - .await - .map_err(handle_db_error)?; - - Ok(()) - } - - // internal param handler helper - async fn handle_internal(&mut self) -> CreateBackupResult { - if !matches!(self.state, HandlerState::ReceivingParams) { - error!("CreateBackupHandler already received all non-data params."); - return Err(Status::failed_precondition("Backup data chunk expected")); - } - - // all non-data inputs must be set before receiving backup data chunks - let (Some(data_hash), Some(device_id), Some(_), Some(_)) = ( - self.data_hash.as_ref(), - self.device_id.as_ref(), - self.user_id.as_ref(), - self.key_entropy.as_ref(), - ) else { - // return empty backup ID when inputs are incomplete - return Ok(proto::CreateNewBackupResponse { - backup_id: "".to_string(), - }); - }; - - let backup_id = generate_backup_id(device_id); - let holder = - crate::utils::generate_blob_holder(data_hash, &backup_id, None); - self.backup_id = backup_id.clone(); - self.holder = Some(holder.clone()); - tracing::Span::current().record("backup_id", &backup_id); - tracing::Span::current().record("blob_holder", &holder); - - match start_simple_uploader(&holder, data_hash, self.blob_client.clone()) - .await? - { - Some(uploader) => { - self.state = HandlerState::ReceivingData { uploader }; - trace!("Everything prepared, waiting for data..."); - } - None => { - // Blob with given data_hash already exists - debug!("Blob already exists, finishing"); - self.should_close_stream = true; - self.state = HandlerState::DataAlreadyExists; - } - }; - - Ok(proto::CreateNewBackupResponse { backup_id }) - } -} - -/// Generates ID for a new backup -fn generate_backup_id(device_id: &str) -> String { - format!( - "{device_id}_{timestamp}", - device_id = device_id, - timestamp = chrono::Utc::now().timestamp_millis() - ) -} diff --git a/services/backup/src/service/handlers/pull_backup.rs b/services/backup/src/service/handlers/pull_backup.rs deleted file mode 100644 index 67835cce9..000000000 --- a/services/backup/src/service/handlers/pull_backup.rs +++ /dev/null @@ -1,330 +0,0 @@ -use async_stream::try_stream; -use tokio_stream::{Stream, StreamExt}; -use tonic::Status; -use tracing::{debug, error, trace, warn}; -use tracing_futures::Instrument; - -use super::handle_db_error; -use super::proto::{self, PullBackupResponse}; -use crate::{ - blob::{BlobClient, BlobDownloader}, - constants::{ - BACKUP_TABLE_FIELD_ATTACHMENT_HOLDERS, BACKUP_TABLE_FIELD_BACKUP_ID, - GRPC_CHUNK_SIZE_LIMIT, GRPC_METADATA_SIZE_PER_MESSAGE, - LOG_TABLE_FIELD_ATTACHMENT_HOLDERS, LOG_TABLE_FIELD_LOG_ID, - }, - database::{BackupItem, DatabaseClient, LogItem}, -}; - -pub struct PullBackupHandler { - blob_client: BlobClient, - backup_item: BackupItem, - logs: Vec, -} - -impl PullBackupHandler { - pub async fn new( - db: &DatabaseClient, - blob_client: &BlobClient, - request: proto::PullBackupRequest, - ) -> Result { - let proto::PullBackupRequest { user_id, backup_id } = request; - let backup_item = db - .find_backup_item(&user_id, &backup_id) - .await - .map_err(handle_db_error)? - .ok_or_else(|| { - debug!("Backup item not found"); - Status::not_found("Backup item not found") - })?; - - let backup_id = backup_item.backup_id.as_str(); - let logs = db - .find_log_items_for_backup(backup_id) - .await - .map_err(handle_db_error)?; - - Ok(PullBackupHandler { - backup_item, - logs, - blob_client: blob_client.clone(), - }) - } - - /// Consumes the handler and provides a response `Stream`. The stream will - /// produce the following in order: - /// - Backup compaction data chunks - /// - Backup logs - /// - Whole log, if stored in db - /// - Log chunks, if stored in blob - pub fn into_response_stream( - self, - ) -> impl Stream> { - use proto::pull_backup_response::*; - - try_stream! { - debug!("Pulling backup..."); - { - let compaction_stream = data_stream(&self.backup_item, self.blob_client.clone()); - tokio::pin!(compaction_stream); - while let Some(response) = compaction_stream.try_next().await? { - yield response; - } - } - trace!("Backup data pull complete."); - - if self.logs.is_empty() { - debug!("No logs to pull. Finishing"); - return; - } - - debug!("Pulling logs..."); - for log in self.logs { - trace!("Pulling log ID={}", &log.log_id); - let span = tracing::trace_span!("log", log_id = &log.log_id); - - if log.persisted_in_blob { - trace!(parent: &span, "Log persisted in blob"); - let log_data_stream = data_stream(&log, self.blob_client.clone()).instrument(span); - tokio::pin!(log_data_stream); - while let Some(response) = log_data_stream.try_next().await? { - yield response; - } - } else { - trace!(parent: &span, "Log persisted in database"); - yield proto::PullBackupResponse { - attachment_holders: Some(log.attachment_holders), - id: Some(Id::LogId(log.log_id)), - data: Some(Data::LogChunk(log.value.into_bytes())), - }; - } - } - trace!("Pulled all logs, done"); - } - } -} - -/// Downloads a blob-stored [`BlobStoredItem`] and streams its content into -/// stream of [`PullBackupResponse`] objects, handles gRPC message size details. -fn data_stream( - item: &Item, - blob_client: BlobClient, -) -> impl Stream> + '_ -where - Item: BlobStoredItem, -{ - try_stream! { - let mut buffer = ResponseBuffer::default(); - let mut downloader = - BlobDownloader::start(item.get_holder().to_string(), blob_client); - - let mut is_first_chunk = true; - loop { - if !buffer.is_saturated() { - if let Some(data) = downloader.next_chunk().await { - buffer.put(data); - } - } - if buffer.is_empty() { - break; - } - - // get data chunk, shortened by length of metadata - let padding = item.metadata_size(is_first_chunk); - let chunk = buffer.get_chunk(padding); - - trace!( - with_attachments = is_first_chunk, - data_size = chunk.len(), - "Sending data chunk" - ); - yield item.to_response(chunk, is_first_chunk); - is_first_chunk = false; - } - - downloader.terminate().await.map_err(|err| { - error!("Blob downloader failed: {:?}", err); - Status::aborted("Internal error") - })?; - } -} - -/// Represents downloadable item stored in Blob service -trait BlobStoredItem { - // Blob holder representing this item - fn get_holder(&self) -> &str; - - /// Generates a gRPC response for given `data_chunk`. - /// The response may be in extended version, with `include_extra_info`, - /// ususally sent with first chunk - fn to_response( - &self, - data_chunk: Vec, - include_extra_info: bool, - ) -> proto::PullBackupResponse; - - /// Size in bytes of non-data fields contained in response message. - fn metadata_size(&self, include_extra_info: bool) -> usize; -} - -impl BlobStoredItem for BackupItem { - fn get_holder(&self) -> &str { - &self.compaction_holder - } - - fn to_response( - &self, - data_chunk: Vec, - include_extra_info: bool, - ) -> proto::PullBackupResponse { - use proto::pull_backup_response::*; - let attachment_holders = if include_extra_info { - Some(self.attachment_holders.clone()) - } else { - None - }; - proto::PullBackupResponse { - id: Some(Id::BackupId(self.backup_id.clone())), - data: Some(Data::CompactionChunk(data_chunk)), - attachment_holders, - } - } - - fn metadata_size(&self, include_extra_info: bool) -> usize { - let mut extra_bytes: usize = 0; - extra_bytes += BACKUP_TABLE_FIELD_BACKUP_ID.as_bytes().len(); - extra_bytes += self.backup_id.as_bytes().len(); - if include_extra_info { - extra_bytes += BACKUP_TABLE_FIELD_ATTACHMENT_HOLDERS.as_bytes().len(); - extra_bytes += self.attachment_holders.as_bytes().len(); - } - extra_bytes - } -} - -impl BlobStoredItem for LogItem { - fn get_holder(&self) -> &str { - &self.value - } - - fn to_response( - &self, - data_chunk: Vec, - include_extra_info: bool, - ) -> proto::PullBackupResponse { - use proto::pull_backup_response::*; - let attachment_holders = if include_extra_info { - Some(self.attachment_holders.clone()) - } else { - None - }; - proto::PullBackupResponse { - id: Some(Id::LogId(self.log_id.clone())), - data: Some(Data::LogChunk(data_chunk)), - attachment_holders, - } - } - - fn metadata_size(&self, include_extra_info: bool) -> usize { - let mut extra_bytes: usize = 0; - extra_bytes += LOG_TABLE_FIELD_LOG_ID.as_bytes().len(); - extra_bytes += self.log_id.as_bytes().len(); - if include_extra_info { - extra_bytes += LOG_TABLE_FIELD_ATTACHMENT_HOLDERS.as_bytes().len(); - extra_bytes += self.attachment_holders.as_bytes().len(); - } - extra_bytes - } -} - -/// A utility structure that buffers downloaded data and allows to retrieve it -/// as chunks of arbitrary size, not greater than provided `limit`. -struct ResponseBuffer { - buf: Vec, - limit: usize, -} - -impl Default for ResponseBuffer { - /// Buffer size defaults to max usable gRPC message size - fn default() -> Self { - ResponseBuffer::new(GRPC_CHUNK_SIZE_LIMIT - GRPC_METADATA_SIZE_PER_MESSAGE) - } -} - -impl ResponseBuffer { - pub fn new(limit: usize) -> Self { - ResponseBuffer { - buf: Vec::new(), - limit, - } - } - - pub fn put(&mut self, data: Vec) { - if data.len() > self.limit { - warn!("Data saved to buffer is larger than chunk limit."); - } - - self.buf.extend(data); - } - - /// Gets chunk of size `limit - padding` and leaves remainder in buffer - pub fn get_chunk(&mut self, padding: usize) -> Vec { - let mut chunk = std::mem::take(&mut self.buf); - - let target_size = self.limit - padding; - if chunk.len() > target_size { - // after this operation, chunk=0..target_size, self.buf=target_size..end - self.buf = chunk.split_off(target_size); - } - return chunk; - } - - /// Does buffer length exceed given limit - pub fn is_saturated(&self) -> bool { - self.buf.len() >= self.limit - } - - pub fn is_empty(&self) -> bool { - self.buf.is_empty() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const LIMIT: usize = 100; - - #[test] - fn test_response_buffer() { - let mut buffer = ResponseBuffer::new(LIMIT); - assert_eq!(buffer.is_empty(), true); - - // put 80 bytes of data - buffer.put(vec![0u8; 80]); - assert_eq!(buffer.is_empty(), false); - assert_eq!(buffer.is_saturated(), false); - - // put next 80 bytes, should be saturated as 160 > 100 - buffer.put(vec![0u8; 80]); - let buf_size = buffer.buf.len(); - assert_eq!(buffer.is_saturated(), true); - assert_eq!(buf_size, 160); - - // get one chunk - let padding: usize = 10; - let expected_chunk_size = LIMIT - padding; - let chunk = buffer.get_chunk(padding); - assert_eq!(chunk.len(), expected_chunk_size); // 90 - - // buffer should not be saturated now (160 - 90 < 100) - let remaining_buf_size = buffer.buf.len(); - assert_eq!(remaining_buf_size, buf_size - expected_chunk_size); - assert_eq!(buffer.is_saturated(), false); - - // get last chunk - let chunk = buffer.get_chunk(padding); - assert_eq!(chunk.len(), remaining_buf_size); - assert_eq!(buffer.is_empty(), true); - } -} diff --git a/services/backup/src/service/handlers/send_log.rs b/services/backup/src/service/handlers/send_log.rs deleted file mode 100644 index bc116891d..000000000 --- a/services/backup/src/service/handlers/send_log.rs +++ /dev/null @@ -1,272 +0,0 @@ -use tonic::Status; -use tracing::{debug, error, trace, warn}; -use uuid::Uuid; - -use super::handle_db_error; -use crate::{ - blob::{BlobClient, BlobUploader}, - constants::{ID_SEPARATOR, LOG_DATA_SIZE_DATABASE_LIMIT}, - database::{DatabaseClient, LogItem}, - service::proto::SendLogResponse, -}; - -enum LogPersistence { - /// Log entirely stored in DynamoDB database - DB, - /// Log contents stored with Blob service - BLOB { holder: String }, -} - -pub struct SendLogHandler { - // flow control - pub should_close_stream: bool, - - // inputs - user_id: Option, - backup_id: Option, - log_hash: Option, - - // internal state - log_id: Option, - log_buffer: Vec, - persistence_method: LogPersistence, - should_receive_data: bool, - - // client instances - db: DatabaseClient, - blob_client: BlobClient, - uploader: Option, -} - -impl SendLogHandler { - pub fn new(db: &DatabaseClient, blob_client: &BlobClient) -> Self { - SendLogHandler { - db: db.clone(), - blob_client: blob_client.clone(), - uploader: None, - user_id: None, - backup_id: None, - log_hash: None, - log_id: None, - log_buffer: Vec::new(), - persistence_method: LogPersistence::DB, - should_receive_data: false, - should_close_stream: false, - } - } - - pub async fn handle_user_id( - &mut self, - user_id: String, - ) -> Result<(), Status> { - if self.user_id.is_some() { - warn!("user ID already provided"); - return Err(Status::invalid_argument("User ID already provided")); - } - self.user_id = Some(user_id); - self.handle_internal().await - } - pub async fn handle_backup_id( - &mut self, - backup_id: String, - ) -> Result<(), Status> { - if self.backup_id.is_some() { - warn!("backup ID already provided"); - return Err(Status::invalid_argument("Backup ID already provided")); - } - tracing::Span::current().record("backup_id", &backup_id); - self.backup_id = Some(backup_id); - self.handle_internal().await - } - pub async fn handle_log_hash( - &mut self, - log_hash: Vec, - ) -> Result<(), Status> { - if self.log_hash.is_some() { - warn!("Log hash already provided"); - return Err(Status::invalid_argument("Log hash already provided")); - } - let hash_str = String::from_utf8(log_hash).map_err(|err| { - error!("Failed to convert data_hash into string: {:?}", err); - Status::aborted("Unexpected error") - })?; - debug!("Received log hash: {}", &hash_str); - self.log_hash = Some(hash_str); - self.handle_internal().await - } - pub async fn handle_log_data( - &mut self, - data_chunk: Vec, - ) -> Result<(), Status> { - if !self.should_receive_data || self.log_id.is_none() { - self.should_close_stream = true; - error!("Data chunk sent before other inputs"); - return Err(Status::invalid_argument( - "Data chunk sent before other inputs", - )); - } - - // empty chunk ends transmission - if data_chunk.is_empty() { - self.should_close_stream = true; - return Ok(()); - } - - match self.persistence_method { - LogPersistence::DB => { - self.log_buffer.extend(data_chunk); - self.ensure_size_constraints().await?; - } - LogPersistence::BLOB { .. } => { - let Some(client) = self.uploader.as_mut() else { - self.should_close_stream = true; - error!("Put client uninitialized. This should never happen!"); - return Err(Status::failed_precondition("Internal error")); - }; - client.put_data(data_chunk).await.map_err(|err| { - error!("Failed to upload data chunk: {:?}", err); - Status::aborted("Internal error") - })?; - } - } - Ok(()) - } - - pub async fn finish(self) -> Result { - if let Some(client) = self.uploader { - client.terminate().await.map_err(|err| { - error!("Put client task closed with error: {:?}", err); - Status::aborted("Internal error") - })?; - } else { - trace!("No uploader initialized. Skipping termination"); - } - - if !self.should_receive_data { - // client probably aborted early - trace!("Nothing to store in database. Finishing early"); - return Ok(SendLogResponse { - log_checkpoint: "".to_string(), - }); - } - - let (Some(backup_id), Some(log_id), Some(data_hash)) = ( - self.backup_id, - self.log_id, - self.log_hash - ) else { - error!("Log info absent in data mode. This should never happen!"); - return Err(Status::failed_precondition("Internal error")); - }; - - let (log_value, persisted_in_blob) = match self.persistence_method { - LogPersistence::BLOB { holder } => (holder, true), - LogPersistence::DB => { - let contents = String::from_utf8(self.log_buffer).map_err(|err| { - error!("Failed to convert log contents data into string: {:?}", err); - Status::aborted("Unexpected error") - })?; - (contents, false) - } - }; - - let log_item = LogItem { - backup_id, - log_id: log_id.clone(), - persisted_in_blob, - value: log_value, - attachment_holders: String::new(), - data_hash, - }; - - self - .db - .put_log_item(log_item) - .await - .map_err(handle_db_error)?; - - Ok(SendLogResponse { - log_checkpoint: log_id, - }) - } - - // internal param handler helper - async fn handle_internal(&mut self) -> Result<(), Status> { - if self.should_receive_data { - error!("SendLogHandler is already expecting data chunks"); - return Err(Status::failed_precondition("Log data chunk expected")); - } - - // all non-data inputs must be set before receiving log contents - let (Some(backup_id), Some(_), Some(_)) = ( - self.backup_id.as_ref(), - self.user_id.as_ref(), - self.log_hash.as_ref() - ) else { return Ok(()); }; - - let log_id = generate_log_id(backup_id); - tracing::Span::current().record("log_id", &log_id); - self.log_id = Some(log_id); - - trace!("Everything prepared, waiting for data..."); - self.should_receive_data = true; - Ok(()) - } - - /// Ensures log fits db size constraints. If not, it is moved to blob - /// persistence - async fn ensure_size_constraints(&mut self) -> Result<(), Status> { - let (Some(backup_id), Some(log_id), Some(log_hash)) = ( - self.backup_id.as_ref(), - self.log_id.as_ref(), - self.log_hash.as_ref() - ) else { - self.should_close_stream = true; - error!("Log info absent in data mode. This should never happen!"); - return Err(Status::failed_precondition("Internal error")); - }; - - let log_size = LogItem::size_from_components( - backup_id, - log_id, - log_hash, - &self.log_buffer, - ); - if log_size > LOG_DATA_SIZE_DATABASE_LIMIT { - debug!("Log too large, switching persistence to Blob"); - let holder = - crate::utils::generate_blob_holder(log_hash, backup_id, Some(log_id)); - match crate::blob::start_simple_uploader( - &holder, - &log_hash, - self.blob_client.clone(), - ) - .await? - { - Some(mut uploader) => { - let blob_chunk = std::mem::take(&mut self.log_buffer); - uploader.put_data(blob_chunk).await.map_err(|err| { - error!("Failed to upload data chunk: {:?}", err); - Status::aborted("Internal error") - })?; - self.uploader = Some(uploader); - } - None => { - debug!("Log hash already exists"); - self.should_close_stream = true; - } - } - self.persistence_method = LogPersistence::BLOB { holder }; - } - Ok(()) - } -} - -fn generate_log_id(backup_id: &str) -> String { - format!( - "{backup_id}{sep}{uuid}", - backup_id = backup_id, - sep = ID_SEPARATOR, - uuid = Uuid::new_v4() - ) -} diff --git a/services/backup/src/service/mod.rs b/services/backup/src/service/mod.rs deleted file mode 100644 index 5e385c1f7..000000000 --- a/services/backup/src/service/mod.rs +++ /dev/null @@ -1,241 +0,0 @@ -use aws_sdk_dynamodb::Error as DynamoDBError; -use comm_services_lib::database::Error as DBError; -use proto::backup_service_server::BackupService; -use std::pin::Pin; -use tokio::sync::mpsc; -use tokio_stream::{wrappers::ReceiverStream, Stream, StreamExt}; -use tonic::{Request, Response, Status}; -use tracing::{debug, error, info, instrument, trace, warn}; -use tracing_futures::Instrument; - -use crate::{ - blob::BlobClient, constants::MPSC_CHANNEL_BUFFER_CAPACITY, - database::DatabaseClient, -}; - -mod proto { - tonic::include_proto!("backup"); -} -pub use proto::backup_service_server::BackupServiceServer; - -/// submodule containing gRPC endpoint handler implementations -mod handlers { - pub(super) mod add_attachments; - pub(super) mod create_backup; - pub(super) mod pull_backup; - pub(super) mod send_log; - - // re-exports for convenient usage in handlers - pub(self) use super::handle_db_error; - pub(self) use super::proto; -} -use self::handlers::create_backup::CreateBackupHandler; -use self::handlers::pull_backup::PullBackupHandler; -use self::handlers::send_log::SendLogHandler; - -pub struct MyBackupService { - db: DatabaseClient, - blob_client: BlobClient, -} - -impl MyBackupService { - pub fn new(db_client: DatabaseClient, blob_client: BlobClient) -> Self { - MyBackupService { - db: db_client, - blob_client, - } - } -} - -// gRPC implementation -#[tonic::async_trait] -impl BackupService for MyBackupService { - type CreateNewBackupStream = Pin< - Box< - dyn Stream> + Send, - >, - >; - - #[instrument(skip_all, fields(device_id, backup_id, blob_holder))] - async fn create_new_backup( - &self, - request: Request>, - ) -> Result, Status> { - use proto::create_new_backup_request::Data::*; - - info!("CreateNewBackup request: {:?}", request); - let mut in_stream = request.into_inner(); - let (tx, rx) = mpsc::channel(MPSC_CHANNEL_BUFFER_CAPACITY); - let db = self.db.clone(); - let blob_client = self.blob_client.clone(); - let worker = async move { - let mut handler = CreateBackupHandler::new(db, blob_client); - while let Some(message) = in_stream.next().await { - let response = match message { - Ok(proto::CreateNewBackupRequest { - data: Some(UserId(user_id)), - }) => handler.handle_user_id(user_id).await, - Ok(proto::CreateNewBackupRequest { - data: Some(DeviceId(device_id)), - }) => handler.handle_device_id(device_id).await, - Ok(proto::CreateNewBackupRequest { - data: Some(KeyEntropy(key_entropy)), - }) => handler.handle_key_entropy(key_entropy).await, - Ok(proto::CreateNewBackupRequest { - data: Some(NewCompactionHash(hash)), - }) => handler.handle_data_hash(hash).await, - Ok(proto::CreateNewBackupRequest { - data: Some(NewCompactionChunk(chunk)), - }) => handler.handle_data_chunk(chunk).await, - unexpected => { - error!("Received an unexpected request: {:?}", unexpected); - Err(Status::unknown("unknown error")) - } - }; - - trace!("Sending response: {:?}", response); - if let Err(e) = tx.send(response).await { - error!("Response was dropped: {}", e); - break; - } - if handler.should_close_stream { - trace!("Handler requested to close stream"); - break; - } - } - if let Err(status) = handler.finish().await { - trace!("Sending error response: {:?}", status); - let _ = tx.send(Err(status)).await; - } - debug!("Request finished processing"); - }; - tokio::spawn(worker.in_current_span()); - - let out_stream = ReceiverStream::new(rx); - Ok(Response::new( - Box::pin(out_stream) as Self::CreateNewBackupStream - )) - } - - #[instrument(skip_all, fields(backup_id, log_id))] - async fn send_log( - &self, - request: Request>, - ) -> Result, Status> { - use proto::send_log_request::Data::*; - - info!("SendLog request: {:?}", request); - let mut handler = SendLogHandler::new(&self.db, &self.blob_client); - - let mut in_stream = request.into_inner(); - while let Some(message) = in_stream.next().await { - let result = match message { - Ok(proto::SendLogRequest { - data: Some(UserId(user_id)), - }) => handler.handle_user_id(user_id).await, - Ok(proto::SendLogRequest { - data: Some(BackupId(backup_id)), - }) => handler.handle_backup_id(backup_id).await, - Ok(proto::SendLogRequest { - data: Some(LogHash(log_hash)), - }) => handler.handle_log_hash(log_hash).await, - Ok(proto::SendLogRequest { - data: Some(LogData(chunk)), - }) => handler.handle_log_data(chunk).await, - unexpected => { - error!("Received an unexpected request: {:?}", unexpected); - Err(Status::unknown("unknown error")) - } - }; - - if let Err(err) = result { - error!("An error occurred when processing request: {:?}", err); - return Err(err); - } - if handler.should_close_stream { - trace!("Handler requested to close request stream"); - break; - } - } - - let response = handler.finish().await; - debug!("Finished. Sending response: {:?}", response); - response.map(|response_body| Response::new(response_body)) - } - - type RecoverBackupKeyStream = Pin< - Box< - dyn Stream> + Send, - >, - >; - - #[instrument(skip(self))] - async fn recover_backup_key( - &self, - _request: Request>, - ) -> Result, Status> { - Err(Status::unimplemented("unimplemented")) - } - - type PullBackupStream = Pin< - Box> + Send>, - >; - - #[instrument(skip_all, fields(backup_id = &request.get_ref().backup_id))] - async fn pull_backup( - &self, - request: Request, - ) -> Result, Status> { - info!("PullBackup request: {:?}", request); - - let handler = - PullBackupHandler::new(&self.db, &self.blob_client, request.into_inner()) - .await?; - - let stream = handler.into_response_stream().in_current_span(); - Ok(Response::new(Box::pin(stream) as Self::PullBackupStream)) - } - - #[instrument(skip_all, - fields( - backup_id = &request.get_ref().backup_id, - log_id = &request.get_ref().log_id) - )] - async fn add_attachments( - &self, - request: Request, - ) -> Result, Status> { - info!( - "AddAttachment request. New holders: {}", - &request.get_ref().holders - ); - - handlers::add_attachments::handle_add_attachments( - &self.db, - &self.blob_client, - request.into_inner(), - ) - .await?; - - info!("Request processed successfully"); - Ok(Response::new(())) - } -} - -/// A helper converting our Database errors into gRPC responses -fn handle_db_error(db_error: DBError) -> Status { - match db_error { - DBError::AwsSdk(DynamoDBError::InternalServerError(_)) - | DBError::AwsSdk(DynamoDBError::ProvisionedThroughputExceededException( - _, - )) - | DBError::AwsSdk(DynamoDBError::RequestLimitExceeded(_)) => { - warn!("AWS transient error occurred"); - Status::unavailable("please retry") - } - e => { - error!("Encountered an unexpected error: {}", e); - Status::failed_precondition("unexpected error") - } - } -} diff --git a/shared/protos/backup.proto b/shared/protos/backup.proto deleted file mode 100644 index e9ba24453..000000000 --- a/shared/protos/backup.proto +++ /dev/null @@ -1,96 +0,0 @@ -syntax = "proto3"; - -package backup; - -import "google/protobuf/empty.proto"; - -/** - * API - description - * CreateNewBackup - This method is called when we want to create a new backup. - * We send a new backup key encrypted with the user's password and also the - * new compaction. New logs that will be sent from now on will be assigned to - * this backup. - * SendLog - User sends a new log to the backup service. The log is being - * assigned to the latest(or desired) backup's compaction item. - * RecoverBackupKey - Pulls data necessary for regenerating the backup key - * on the client-side for the latest(or desired) backup - * PullBackup - Fetches compaction + all logs assigned to it for the - * specified backup(default is the last backup) - */ - -service BackupService { - rpc CreateNewBackup(stream CreateNewBackupRequest) returns (stream CreateNewBackupResponse) {} - rpc SendLog(stream SendLogRequest) returns (SendLogResponse) {} - rpc RecoverBackupKey(stream RecoverBackupKeyRequest) returns (stream RecoverBackupKeyResponse) {} - rpc PullBackup(PullBackupRequest) returns (stream PullBackupResponse) {} - rpc AddAttachments(AddAttachmentsRequest) returns (google.protobuf.Empty) {} -} - -// CreateNewBackup - -message CreateNewBackupRequest { - oneof data { - string userID = 1; - string deviceID = 2; - bytes keyEntropy = 3; - bytes newCompactionHash = 4; - bytes newCompactionChunk = 5; - } -} - -message CreateNewBackupResponse { - string backupID = 1; -} - -// SendLog - -message SendLogRequest { - oneof data { - string userID = 1; - string backupID = 2; - bytes logHash = 3; - bytes logData = 4; - } -} - -message SendLogResponse { - string logCheckpoint = 1; -} - -// RecoverBackupKey - -message RecoverBackupKeyRequest { - string userID = 1; -} - -message RecoverBackupKeyResponse { - string backupID = 4; -} - -// PullBackup - -message PullBackupRequest { - string userID = 1; - string backupID = 2; -} - -message PullBackupResponse { - oneof id { - string backupID = 1; - string logID = 2; - } - oneof data { - bytes compactionChunk = 3; - bytes logChunk = 4; - } - optional string attachmentHolders = 5; -} - -// AddAttachment - -message AddAttachmentsRequest { - string userID = 1; - string backupID = 2; - string logID = 3; - string holders = 4; -}