diff --git a/services/commtest/Cargo.lock b/services/commtest/Cargo.lock --- a/services/commtest/Cargo.lock +++ b/services/commtest/Cargo.lock @@ -2,11 +2,50 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "argon2" @@ -57,6 +96,324 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "hyper", + "ring", + "time 0.3.26", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "fastrand", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fb64867fe098cffee7e34352b01bbfa2beb3aa1b2ff0e0a7bf9ff293557852" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "regex", + "tower", + "tracing", +] + +[[package]] +name = "aws-sig-auth" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +dependencies = [ + "aws-smithy-http", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time 0.3.26", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-client" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "rustls", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +dependencies = [ + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time 0.3.26", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.6.1" @@ -103,6 +460,21 @@ "tower-service", ] +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base16ct" version = "0.1.1" @@ -127,6 +499,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -175,6 +557,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +[[package]] +name = "bytes-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.1.0" @@ -193,6 +585,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "time 0.1.45", + "wasm-bindgen", + "winapi", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "comm-opaque2" version = "0.2.0" @@ -205,6 +622,24 @@ "wasm-bindgen", ] +[[package]] +name = "comm-services-lib" +version = "0.1.0" +dependencies = [ + "anyhow", + "aws-config", + "aws-sdk-dynamodb", + "aws-types", + "base64 0.21.2", + "chrono", + "derive_more", + "rand", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "commtest" version = "0.1.0" @@ -214,6 +649,7 @@ "base64 0.20.0", "bytesize", "comm-opaque2", + "comm-services-lib", "derive_more", "futures", "futures-util", @@ -307,6 +743,50 @@ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.15", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + [[package]] name = "der" version = "0.6.1" @@ -316,6 +796,12 @@ "const-oid", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" + [[package]] name = "derive-where" version = "1.0.0-rc.3" @@ -481,6 +967,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + [[package]] name = "futures-macro" version = "0.3.28" @@ -511,9 +1003,11 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -538,9 +1032,15 @@ dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "group" version = "0.12.1" @@ -618,9 +1118,9 @@ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -658,9 +1158,9 @@ [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -673,13 +1173,28 @@ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.7", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -705,6 +1220,30 @@ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "idna" version = "0.3.0" @@ -776,6 +1315,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + [[package]] name = "log" version = "0.4.17" @@ -813,16 +1361,24 @@ "unicase", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", - "wasi", - "windows-sys", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", ] [[package]] @@ -849,6 +1405,25 @@ "tempfile", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.14.0" @@ -859,6 +1434,15 @@ "libc", ] +[[package]] +name = "object" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.16.0" @@ -931,6 +1515,12 @@ "vcpkg", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "password-hash" version = "0.4.2" @@ -980,9 +1570,9 @@ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1130,6 +1720,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", ] @@ -1178,14 +1770,37 @@ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "winreg", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1195,6 +1810,39 @@ "semver", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +dependencies = [ + "base64 0.21.2", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -1213,7 +1861,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", +] + +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -1342,6 +2006,22 @@ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "subtle" version = "2.5.0" @@ -1390,6 +2070,15 @@ "winapi", ] +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.40" @@ -1410,6 +2099,44 @@ "syn 2.0.15", ] +[[package]] +name = "time" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a79d09ac6b08c1ab3906a2f7cc2e81a0e27c7ae89c63812df75e52bef0751e07" +dependencies = [ + "deranged", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c65469ed6b3a4809d987a41eb1dc918e9bc1d92211cbad7ae82931846f7451" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1427,20 +2154,19 @@ [[package]] name = "tokio" -version = "1.24.2" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", - "memchr", "mio", "num_cpus", "pin-project-lite", - "socket2", + "socket2 0.5.3", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1455,13 +2181,13 @@ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.15", ] [[package]] @@ -1474,6 +2200,17 @@ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-stream" version = "0.1.11" @@ -1739,6 +2476,18 @@ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -1750,6 +2499,12 @@ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -1787,6 +2542,12 @@ "zeroize", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.0" @@ -1797,6 +2558,12 @@ "try-lock", ] +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1869,6 +2636,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -1879,6 +2659,16 @@ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "which" version = "4.3.0" @@ -1906,25 +2696,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.0", + "windows_aarch64_msvc 0.42.0", + "windows_i686_gnu 0.42.0", + "windows_i686_msvc 0.42.0", + "windows_x86_64_gnu 0.42.0", + "windows_x86_64_gnullvm 0.42.0", + "windows_x86_64_msvc 0.42.0", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -1933,42 +2765,84 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_i686_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_x86_64_gnu" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "winreg" version = "0.10.1" @@ -1978,6 +2852,12 @@ "winapi", ] +[[package]] +name = "xmlparser" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" + [[package]] name = "zeroize" version = "1.6.0" diff --git a/services/commtest/Cargo.toml b/services/commtest/Cargo.toml --- a/services/commtest/Cargo.toml +++ b/services/commtest/Cargo.toml @@ -26,8 +26,9 @@ futures-util = "0.3.28" serde_json = "1.0.96" rand = "0.8.5" -reqwest = { version = "0.11", features = ["json", "multipart"] } +reqwest = { version = "0.11", features = ["json", "multipart", "stream"] } serde = "1.0" +comm-services-lib = { path = "../comm-services-lib" } [build-dependencies] tonic-build = "0.8" diff --git a/services/commtest/src/backup/add_attachments.rs b/services/commtest/src/backup/add_attachments.rs deleted file mode 100644 --- a/services/commtest/src/backup/add_attachments.rs +++ /dev/null @@ -1,48 +0,0 @@ -use crate::backup::backup_utils::{ - proto::AddAttachmentsRequest, BackupData, BackupServiceClient, -}; -use crate::constants::ATTACHMENT_DELIMITER; -use crate::tools::Error; -use tonic::Request; - -// log_index = None means that we add attachments to the backup -// log_index = Some(x) means that we add attachments to a specific log -pub async fn run( - client: &mut BackupServiceClient, - backup_data: &BackupData, - log_index: Option, -) -> Result<(), Error> { - let cloned_user_id = backup_data.user_id.clone(); - let cloned_backup_id = backup_data.backup_item.id.clone(); - let log_id: String = match log_index { - Some(index) => { - let log_id = backup_data.log_items[index].id.clone(); - println!("add attachments for log {}/{}", index, log_id); - log_id - } - None => { - println!("add attachments for backup"); - String::new() - } - }; - - let holders: String = match log_index { - Some(log_index) => backup_data.log_items[log_index] - .attachments_holders - .join(ATTACHMENT_DELIMITER), - None => backup_data - .backup_item - .attachments_holders - .join(ATTACHMENT_DELIMITER), - }; - - client - .add_attachments(Request::new(AddAttachmentsRequest { - user_id: cloned_user_id, - backup_id: cloned_backup_id, - log_id, - holders, - })) - .await?; - Ok(()) -} diff --git a/services/commtest/src/backup/backup_utils.rs b/services/commtest/src/backup/backup_utils.rs --- a/services/commtest/src/backup/backup_utils.rs +++ b/services/commtest/src/backup/backup_utils.rs @@ -1,101 +1,9 @@ -pub mod proto { - tonic::include_proto!("backup"); -} -pub use proto::backup_service_client::BackupServiceClient; -use std::collections::HashMap; - -// stands for both, backup and log items -#[derive(Clone)] -pub struct Item { - pub id: String, - pub chunks_sizes: Vec, - pub attachments_holders: Vec, -} - -impl Item { - pub fn new( - id: String, - chunks_sizes: Vec, - attachments_holders: Vec, - ) -> Item { - Item { - id, - chunks_sizes, - attachments_holders, - } - } -} - -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct BackupData { - pub user_id: String, - pub device_id: String, - pub backup_item: Item, - pub log_items: Vec, -} - -pub fn compare_backups(backup_data: &BackupData, result: &BackupData) { - // check backup size - let expected: usize = backup_data.backup_item.chunks_sizes.iter().sum(); - let from_result: usize = result.backup_item.chunks_sizes.iter().sum(); - assert_eq!( - from_result, expected, - "backup sizes do not match, expected {}, got {}", - expected, from_result - ); - - // check backup attachments - let expected: usize = backup_data.backup_item.attachments_holders.len(); - let from_result: usize = result.backup_item.attachments_holders.len(); - assert_eq!( - from_result, expected, - "backup: number of attachments holders do not match, expected {}, got {}", - expected, from_result - ); - - // check number of logs - let expected: usize = backup_data.log_items.len(); - let from_result: usize = result.log_items.len(); - assert_eq!( - expected, from_result, - "backup id {} number of logs do not match, expected {}, got {}", - backup_data.backup_item.id, expected, from_result - ); - - // check log sizes - // map - let mut expected_log_map: HashMap = HashMap::new(); - let mut result_log_map: HashMap = HashMap::new(); - for i in 0..backup_data.log_items.len() { - let expected: usize = backup_data.log_items[i].chunks_sizes.iter().sum(); - let insert_result = - expected_log_map.insert(backup_data.log_items[i].id.clone(), expected); - assert_eq!( - insert_result, None, - "expected collection contained duplicated log id: {}", - backup_data.log_items[i].id - ); - let from_result: usize = result.log_items[i].chunks_sizes.iter().sum(); - let insert_result = - result_log_map.insert(result.log_items[i].id.clone(), from_result); - assert_eq!( - insert_result, None, - "expected collection contained duplicated log id: {}", - result.log_items[i].id - ); - } - - for (expected_id, expected_size) in &expected_log_map { - let result_size = result_log_map.get(expected_id).expect(&format!( - "comparing logs: expected id found in result: {}", - expected_id - )); - assert_eq!( - expected_size, result_size, - "comparing logs, sizes don't match, backup {}", - backup_data.backup_item.id - ); - } - - // todo: check logs attachment holders + pub backup_id: String, + pub user_keys_hash: String, + pub user_keys: Vec, + pub user_data_hash: String, + pub user_data: Vec, + pub attachments: Vec, } diff --git a/services/commtest/src/backup/create_new_backup.rs b/services/commtest/src/backup/create_new_backup.rs --- a/services/commtest/src/backup/create_new_backup.rs +++ b/services/commtest/src/backup/create_new_backup.rs @@ -1,69 +1,60 @@ -use crate::backup::backup_utils::{ - proto::create_new_backup_request::Data::*, proto::CreateNewBackupRequest, - BackupData, BackupServiceClient, +use std::convert::Infallible; + +use crate::tools::Error; +use async_stream::stream; +use comm_services_lib::auth::UserIdentity; +use reqwest::{ + multipart::{Form, Part}, + Body, }; -use crate::tools::{generate_stable_nbytes, DataHasher, Error}; -use tonic::Request; + +use super::backup_utils::BackupData; pub async fn run( - client: &mut BackupServiceClient, + url: reqwest::Url, + user_identity: &UserIdentity, backup_data: &BackupData, -) -> Result { - println!("create new backup"); - let cloned_user_id = backup_data.user_id.clone(); - let cloned_device_id = backup_data.device_id.clone(); - let cloned_backup_chunk_sizes = backup_data.backup_item.chunks_sizes.clone(); - let predefined_byte_value = None; - let outbound = async_stream::stream! { - println!(" - sending user id"); - let request = CreateNewBackupRequest { - data: Some(UserId(cloned_user_id)), - }; - yield request; - println!(" - sending device id"); - let request = CreateNewBackupRequest { - data: Some(DeviceId(cloned_device_id)), - }; - yield request; - println!(" - sending key entropy"); - let request = CreateNewBackupRequest { - data: Some(KeyEntropy(vec![65,66,67,68])), - }; - yield request; - println!(" - sending data hash"); - let mut hasher = DataHasher::new(); - for chunk_size in &cloned_backup_chunk_sizes { - DataHasher::update(&mut hasher, generate_stable_nbytes(*chunk_size, predefined_byte_value)); - } +) -> Result<(), Error> { + println!("Creating new backup"); + + let BackupData { + backup_id, + user_keys_hash, + user_keys, + user_data_hash, + user_data, + attachments, + } = backup_data.clone(); - let request = CreateNewBackupRequest { - data: Some(NewCompactionHash(hasher.get_hash().as_bytes().to_vec())), - }; - yield request; - for chunk_size in &cloned_backup_chunk_sizes { - println!(" - sending data chunk {}", chunk_size); - let request = CreateNewBackupRequest { - data: Some(NewCompactionChunk(generate_stable_nbytes(*chunk_size, predefined_byte_value))), - }; - yield request; - } - }; + let client = reqwest::Client::new(); + let form = Form::new() + .text("backup_id", backup_id) + .text("user_keys_hash", user_keys_hash) + .part( + "user_keys", + Part::stream(Body::wrap_stream( + stream! { yield Ok::, Infallible>(user_keys); }, + )), + ) + .text("user_data_hash", user_data_hash) + .part( + "user_data", + Part::stream(Body::wrap_stream( + stream! { yield Ok::, Infallible>(user_data); }, + )), + ) + .text("attachments", attachments.join("\n")); - let mut backup_id: String = String::new(); - let response = client.create_new_backup(Request::new(outbound)).await?; - let mut inbound = response.into_inner(); - while let Some(response) = inbound.message().await? { - if !response.backup_id.is_empty() { - assert!( - backup_id.is_empty(), - "backup id should be returned only once" - ); - backup_id = response.backup_id; - } + let response = client + .post(url.join("backups")?) + .bearer_auth(user_identity.as_authorization_token()?) + .multipart(form) + .send() + .await?; + + if !response.status().is_success() { + return Err(Error::HttpStatus(response.status())); } - assert!( - !backup_id.is_empty(), - "could not get a backup id from the server" - ); - Ok(backup_id) + + Ok(()) } diff --git a/services/commtest/src/backup/mod.rs b/services/commtest/src/backup/mod.rs --- a/services/commtest/src/backup/mod.rs +++ b/services/commtest/src/backup/mod.rs @@ -1,5 +1,3 @@ -pub mod add_attachments; pub mod backup_utils; pub mod create_new_backup; pub mod pull_backup; -pub mod send_log; diff --git a/services/commtest/src/backup/pull_backup.rs b/services/commtest/src/backup/pull_backup.rs --- a/services/commtest/src/backup/pull_backup.rs +++ b/services/commtest/src/backup/pull_backup.rs @@ -1,130 +1,80 @@ -use crate::backup::backup_utils::{ - proto::pull_backup_response::Data, proto::pull_backup_response::Data::*, - proto::pull_backup_response::Id, proto::pull_backup_response::Id::*, - proto::PullBackupRequest, BackupData, BackupServiceClient, Item, -}; -use crate::constants::ATTACHMENT_DELIMITER; +use std::fmt::Display; + use crate::tools::Error; -use std::io::{Error as IOError, ErrorKind}; -use tonic::Request; +use comm_services_lib::auth::UserIdentity; -#[derive(PartialEq, Debug)] -enum State { - Compaction, - Log, +#[derive(Debug, Clone)] +pub enum BackupDescriptor { + BackupID { + backup_id: String, + user_identity: UserIdentity, + }, + Latest { + username: String, + }, } -pub async fn run( - client: &mut BackupServiceClient, - backup_data: &BackupData, -) -> Result { - println!("pull backup"); - let cloned_user_id = backup_data.user_id.clone(); - let cloned_backup_id = backup_data.backup_item.id.clone(); +impl Display for BackupDescriptor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BackupDescriptor::BackupID { + backup_id, + user_identity, + } => write!( + f, + "backup '{backup_id}' for user '{}'", + user_identity.user_id + ), + BackupDescriptor::Latest { username } => { + write!(f, "latest backup for user '{username}'") + } + } + } +} - let mut result = BackupData { - user_id: String::new(), - device_id: String::new(), - backup_item: Item::new(String::new(), Vec::new(), Vec::new()), - log_items: Vec::new(), - }; +#[derive(Debug, Clone, Copy)] +pub enum RequestedData { + BackupID, + UserKeys, + UserData, +} - let response = client - .pull_backup(Request::new(PullBackupRequest { - user_id: cloned_user_id, - backup_id: cloned_backup_id, - })) - .await?; - let mut inbound = response.into_inner(); - let mut state: State = State::Compaction; - let mut current_id: String = String::new(); - while let Some(response) = inbound.message().await? { - let response_data: Option = response.data; - let id: Option = response.id; - let mut backup_id: Option = None; - let mut log_id: Option = None; - match id { - Some(BackupId(id)) => backup_id = Some(id), - Some(LogId(id)) => log_id = Some(id), - None => {} - }; - match response_data { - Some(CompactionChunk(chunk)) => { - assert_eq!( - state, - State::Compaction, - "invalid state, expected compaction, got {:?}", - state - ); - current_id = backup_id.ok_or(IOError::new( - ErrorKind::Other, - "backup id expected but not received", - ))?; - println!( - "compaction (id {}), pulling chunk (size: {})", - current_id, - chunk.len() - ); - result.backup_item.chunks_sizes.push(chunk.len()) - } - Some(LogChunk(chunk)) => { - if state == State::Compaction { - state = State::Log; - } - assert_eq!( - state, - State::Log, - "invalid state, expected log, got {:?}", - state - ); - let log_id = log_id.ok_or(IOError::new( - ErrorKind::Other, - "log id expected but not received", - ))?; - if log_id != current_id { - result.log_items.push(Item::new( - log_id.clone(), - Vec::new(), - Vec::new(), - )); - current_id = log_id; - } - let log_items_size = result.log_items.len() - 1; - result.log_items[log_items_size] - .chunks_sizes - .push(chunk.len()); +pub async fn run( + url: reqwest::Url, + backup_descriptor: BackupDescriptor, + requested_data: RequestedData, +) -> Result, Error> { + println!("Pulling data: {requested_data:?}, from {backup_descriptor}"); - println!("log (id {}) chunk size {}", current_id, chunk.len()); - } - None => {} + let client = reqwest::Client::new(); + let url = url.join("backups/")?; + let url = match &backup_descriptor { + BackupDescriptor::BackupID { backup_id, .. } => { + url.join(&format!("{backup_id}/"))? } - if let Some(holders) = response.attachment_holders { - let holders_split: Vec<&str> = - holders.split(ATTACHMENT_DELIMITER).collect(); - if state == State::Compaction { - for holder in holders_split { - if holder.len() == 0 { - continue; - } - println!("attachments for the backup: {}", holder); - result - .backup_item - .attachments_holders - .push(holder.to_string()); - } - } else if state == State::Log { - println!("attachments for the log {:?}: {}", current_id, holders); - for holder in holders_split { - if holder.len() == 0 { - continue; - } - let log_items_size = result.log_items.len() - 1; - result.log_items[log_items_size] - .attachments_holders - .push(holder.to_string()) - } - } + BackupDescriptor::Latest { username } => { + url.join(&format!("latest/{username}/"))? } + }; + let url = match &requested_data { + RequestedData::BackupID => url.join("backup_id")?, + RequestedData::UserKeys => url.join("user_keys")?, + RequestedData::UserData => url.join("user_data")?, + }; + + let mut request = client.get(url); + + if let BackupDescriptor::BackupID { user_identity, .. } = backup_descriptor { + request = request.bearer_auth(user_identity.as_authorization_token()?) } + + let response = request.send().await?; + + if !response.status().is_success() { + return Err(Error::HttpStatus(response.status())); + } + + let result = response.bytes().await?.to_vec(); + Ok(result) } diff --git a/services/commtest/src/backup/send_log.rs b/services/commtest/src/backup/send_log.rs deleted file mode 100644 --- a/services/commtest/src/backup/send_log.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::backup::backup_utils::{ - proto::{send_log_request::Data::*, SendLogRequest}, - BackupData, BackupServiceClient, -}; -use crate::tools::{generate_stable_nbytes, DataHasher, Error}; -use tonic::Request; - -pub async fn run( - client: &mut BackupServiceClient, - backup_data: &BackupData, - log_index: usize, -) -> Result { - println!("send log"); - let cloned_user_id = backup_data.user_id.clone(); - let cloned_backup_id = backup_data.backup_item.id.clone(); - let cloned_log_sizes = backup_data.log_items[log_index].chunks_sizes.clone(); - let predefined_byte_value = None; - let outbound = async_stream::stream! { - println!(" - sending user id"); - let request = SendLogRequest { - data: Some(UserId(cloned_user_id)), - }; - yield request; - println!(" - sending backup id"); - let request = SendLogRequest { - data: Some(BackupId(cloned_backup_id)), - }; - yield request; - println!(" - sending log hash"); - let mut hasher = DataHasher::new(); - for chunk_size in &cloned_log_sizes { - DataHasher::update(&mut hasher, generate_stable_nbytes(*chunk_size, predefined_byte_value)); - } - - let request = SendLogRequest { - data: Some(LogHash(hasher.get_hash().as_bytes().to_vec())), - }; - yield request; - println!(" - sending log data"); - for log_size in &cloned_log_sizes { - println!(" - sending log data {}", *log_size); - let request = SendLogRequest { - data: Some(LogData(generate_stable_nbytes(*log_size, predefined_byte_value))), - }; - yield request; - } - }; - - let response = client.send_log(Request::new(outbound)).await?; - let inbound = response.into_inner(); - println!("send log response: {:?}", inbound.log_checkpoint); - Ok(inbound.log_checkpoint) -} diff --git a/services/commtest/src/tools.rs b/services/commtest/src/tools.rs --- a/services/commtest/src/tools.rs +++ b/services/commtest/src/tools.rs @@ -28,6 +28,10 @@ HttpStatus(#[error(ignore)] reqwest::StatusCode), #[display(...)] ParseError(ParseError), + #[display(...)] + JsonError(serde_json::error::Error), + #[display(...)] + FromUtf8Error(std::string::FromUtf8Error), } pub fn obtain_number_of_threads() -> usize { diff --git a/services/commtest/tests/backup_integration_test.rs b/services/commtest/tests/backup_integration_test.rs --- a/services/commtest/tests/backup_integration_test.rs +++ b/services/commtest/tests/backup_integration_test.rs @@ -1,126 +1,110 @@ use bytesize::ByteSize; -use commtest::backup::{ - add_attachments, - backup_utils::{self, BackupData, BackupServiceClient, Item}, - create_new_backup, pull_backup, send_log, +use comm_services_lib::{auth::UserIdentity, backup::LatestBackupIDResponse}; +use commtest::{ + backup::{ + backup_utils::BackupData, + create_new_backup, + pull_backup::{self, BackupDescriptor, RequestedData}, + }, + tools::{generate_stable_nbytes, Error}, }; -use commtest::constants; -use commtest::tools::Error; -use std::collections::HashMap; use std::env; #[tokio::test] async fn backup_integration_test() -> Result<(), Error> { let port = env::var("COMM_SERVICES_PORT_BACKUP") - .expect("port env var expected but not received"); - let mut client = - BackupServiceClient::connect(format!("http://localhost:{}", port)).await?; + .expect("port env var expected but not received") + .parse() + .expect("port env var should be a number"); - let attachments_fill_size: u64 = 500; + let mut url = reqwest::Url::parse("http://localhost")?; + url.set_port(Some(port)).expect("failed to set port"); - let mut backup_data = BackupData { - user_id: "user0000".to_string(), - device_id: "device0000".to_string(), - backup_item: Item::new( - String::new(), - vec![ByteSize::mib(1).as_u64() as usize; 6], - vec![ - "holder0".to_string(), - "holder1".to_string(), - "holder2".to_string(), - ], - ), - log_items: vec![ - // the item that almost hits the DB limit, we're going to later add a long - // list of attachments, so that causes it to exceed the limit. - // In this case its data should be moved to the S3 - Item::new( - String::new(), - vec![ - *constants::DYNAMO_DB_ITEM_SIZE_LIMIT - - ByteSize::b(attachments_fill_size / 2).as_u64() as usize, - ], - vec!["holder0".to_string(), "holder1".to_string()], + let backup_datas = [ + BackupData { + backup_id: "b1".to_string(), + user_keys_hash: "kh1".to_string(), + user_keys: generate_stable_nbytes( + ByteSize::kib(4).as_u64() as usize, + Some(b'a'), ), - // just a small item - Item::new( - String::new(), - vec![ByteSize::b(100).as_u64() as usize], - vec!["holder0".to_string()], + user_data_hash: "dh1".to_string(), + user_data: generate_stable_nbytes( + ByteSize::mib(4).as_u64() as usize, + Some(b'A'), ), - // a big item that should be placed in the S3 right away - Item::new( - String::new(), - vec![ - *constants::GRPC_CHUNK_SIZE_LIMIT, - *constants::GRPC_CHUNK_SIZE_LIMIT, - ], - vec![ - "holder0".to_string(), - "holder1".to_string(), - "holder2".to_string(), - ], + attachments: vec![], + }, + BackupData { + backup_id: "b2".to_string(), + user_keys_hash: "kh2".to_string(), + user_keys: generate_stable_nbytes( + ByteSize::kib(4).as_u64() as usize, + Some(b'b'), ), - ], + user_data_hash: "dh2".to_string(), + user_data: generate_stable_nbytes( + ByteSize::mib(4).as_u64() as usize, + Some(b'B'), + ), + attachments: vec![], + }, + ]; + + let user_identity = UserIdentity { + user_id: "1".to_string(), + access_token: "dummy access token".to_string(), + device_id: "dummy device_id".to_string(), }; - backup_data.backup_item.id = - create_new_backup::run(&mut client, &backup_data).await?; - println!("backup id in main: {}", backup_data.backup_item.id); + create_new_backup::run(url.clone(), &user_identity, &backup_datas[0]).await?; + create_new_backup::run(url.clone(), &user_identity, &backup_datas[1]).await?; - add_attachments::run(&mut client, &backup_data, None).await?; + // Test direct lookup + let second_backup_descriptor = BackupDescriptor::BackupID { + backup_id: backup_datas[1].backup_id.clone(), + user_identity: user_identity.clone(), + }; - for log_index in 0..backup_data.log_items.len() { - backup_data.log_items[log_index].id = - send_log::run(&mut client, &backup_data, log_index).await?; - add_attachments::run(&mut client, &backup_data, Some(log_index)).await?; - } + let user_keys = pull_backup::run( + url.clone(), + second_backup_descriptor.clone(), + RequestedData::UserKeys, + ) + .await?; + assert_eq!(user_keys, backup_datas[1].user_keys); - let result: BackupData = pull_backup::run(&mut client, &backup_data).await?; + let user_data = pull_backup::run( + url.clone(), + second_backup_descriptor.clone(), + RequestedData::UserData, + ) + .await?; + assert_eq!(user_data, backup_datas[1].user_data); - backup_utils::compare_backups(&backup_data, &result); + // Test latest backup lookup + let latest_backup_descriptor = BackupDescriptor::Latest { + // Initial version of the backup service uses `user_id` in place of a username + username: "1".to_string(), + }; - // push so many attachments that the log item's data will have to be moved - // from the db to the s3 - let mut attachments_size = 0; - let mut i = backup_data.log_items[0].attachments_holders.len(); - let mut new_attachments: Vec = Vec::new(); - while attachments_size < (attachments_fill_size as usize) { - let att = format!("holder{}", i); - attachments_size += att.len(); - new_attachments.push(att); - i += 1; - } + let backup_id_response = pull_backup::run( + url.clone(), + latest_backup_descriptor.clone(), + RequestedData::BackupID, + ) + .await?; + let response: LatestBackupIDResponse = + serde_json::from_slice(&backup_id_response)?; + assert_eq!(response.backup_id, backup_datas[1].backup_id); - let mut old_attachments = - backup_data.log_items[0].attachments_holders.clone(); - backup_data.log_items[0].attachments_holders = new_attachments; - add_attachments::run(&mut client, &backup_data, Some(0)).await?; - backup_data.log_items[0] - .attachments_holders - .append(&mut old_attachments); - let result = pull_backup::run(&mut client, &backup_data).await?; - // check logs attachments - // map - let mut expected_log_map: HashMap = HashMap::new(); - let mut result_log_map: HashMap = HashMap::new(); - for i in 0..backup_data.log_items.len() { - let expected: usize = backup_data.log_items[i].attachments_holders.len(); - expected_log_map.insert(backup_data.log_items[i].id.clone(), expected); - let from_result: usize = result.log_items[i].attachments_holders.len(); - result_log_map.insert(result.log_items[i].id.clone(), from_result); - } - for (expected_id, expected_size) in &expected_log_map { - let result_size = result_log_map.get(expected_id).expect(&format!( - "comparing logs attachments: expected id found in result: {}", - expected_id - )); - assert_eq!( - expected_size, result_size, - "comparing logs attachments, sizes don't match, backup {}", - backup_data.backup_item.id - ); - } + let user_keys = pull_backup::run( + url.clone(), + latest_backup_descriptor.clone(), + RequestedData::UserKeys, + ) + .await?; + assert_eq!(user_keys, backup_datas[1].user_keys); Ok(()) } diff --git a/services/commtest/tests/backup_performance_test.rs b/services/commtest/tests/backup_performance_test.rs --- a/services/commtest/tests/backup_performance_test.rs +++ b/services/commtest/tests/backup_performance_test.rs @@ -1,228 +1,187 @@ use bytesize::ByteSize; -use commtest::backup::{ - add_attachments, - backup_utils::{self, BackupData, BackupServiceClient, Item}, - create_new_backup, pull_backup, send_log, +use comm_services_lib::{auth::UserIdentity, backup::LatestBackupIDResponse}; +use commtest::{ + backup::{ + backup_utils::BackupData, + create_new_backup, + pull_backup::{self, BackupDescriptor}, + }, + tools::{generate_stable_nbytes, obtain_number_of_threads, Error}, }; -use commtest::tools::{obtain_number_of_threads, Error}; use std::env; -use std::sync::mpsc::channel; -use tokio::runtime::Runtime; +use tokio::{runtime::Runtime, task::JoinSet}; #[tokio::test] async fn backup_performance_test() -> Result<(), Error> { let port = env::var("COMM_SERVICES_PORT_BACKUP") - .expect("port env var expected but not received"); - let client = - BackupServiceClient::connect(format!("http://localhost:{}", port)).await?; + .expect("port env var expected but not received") + .parse() + .expect("port env var should be a number"); + + let mut url = reqwest::Url::parse("http://localhost")?; + url.set_port(Some(port)).expect("failed to set port"); let number_of_threads = obtain_number_of_threads(); + let rt = Runtime::new().unwrap(); + println!( "Running performance tests for backup, number of threads: {}", number_of_threads ); let mut backup_data = vec![]; - for i in 0..number_of_threads { backup_data.push(BackupData { - user_id: format!("user{}", i), - device_id: format!("device{}", i), - backup_item: Item::new( - String::new(), - vec![ByteSize::mib(1).as_u64() as usize; 3 + (i % 5)], - (0..(i % 5)).map(|x| format!("holder{}", x)).collect(), + backup_id: format!("b{i}"), + user_keys_hash: format!("kh{i}"), + user_keys: generate_stable_nbytes( + ByteSize::kib(4).as_u64() as usize, + Some(i as u8), ), - log_items: (0..(i % 4)) - .map(|x| { - Item::new( - String::new(), - vec![ByteSize::mib(1).as_u64() as usize; 2 + (x % 2)], - (0..(i % 5)).map(|x| format!("holder{}-{}", i, x)).collect(), - ) - }) - .collect(), + user_data_hash: format!("dh{i}"), + user_data: generate_stable_nbytes( + ByteSize::mib(4).as_u64() as usize, + Some(i as u8), + ), + attachments: vec![], }); } - let rt = Runtime::new().unwrap(); + let user_identities = [ + UserIdentity { + user_id: "1".to_string(), + access_token: "dummy access token".to_string(), + device_id: "dummy device_id".to_string(), + }, + UserIdentity { + user_id: "2".to_string(), + access_token: "dummy access token".to_string(), + device_id: "dummy device_id".to_string(), + }, + ]; + tokio::task::spawn_blocking(move || { - // CREATE NEW BACKUP + println!("Creating new backups"); rt.block_on(async { - println!("performing CREATE NEW BACKUP operations"); - let mut handlers = vec![]; - let (sender, receiver) = channel::<(usize, String)>(); + let mut set = JoinSet::new(); for (i, item) in backup_data.iter().enumerate() { - let item_cloned = item.clone(); - let mut client_cloned = client.clone(); - let sender_cloned = sender.clone(); - handlers.push(tokio::spawn(async move { - let id = create_new_backup::run(&mut client_cloned, &item_cloned) - .await - .unwrap(); - assert!( - !id.is_empty(), - "backup id should not be empty after creating a new backup" - ); - sender_cloned.send((i, id)).unwrap(); - })); + let url = url.clone(); + let user = user_identities[i % user_identities.len()].clone(); + let item = item.clone(); + set.spawn(async move { + create_new_backup::run(url, &user, &item).await.unwrap(); + }); } - // https://docs.rs/tokio/1.1.0/tokio/sync/mpsc/struct.Receiver.html#method.recv - // The channel is closed when all senders have been dropped, or when close - // is called. The best option here is to clone the sender for every - // thread, drop the original one and let all the clones be dropped when - // going out of scope which is equal to the parent thread's termination. - drop(sender); - for handler in handlers { - handler.await.unwrap(); - } - for data in receiver { - println!("received: {:?}", data); - let (index, id) = data; - backup_data[index].backup_item.id = id; + while let Some(result) = set.join_next().await { + result.unwrap(); } }); - // check if backup IDs are properly set - for (i, item) in backup_data.iter().enumerate() { - assert!( - !item.backup_item.id.is_empty(), - "missing backup id for index {}", - i - ); - } - - // ADD ATTACHMENTS - BACKUPS + let mut latest_ids_for_user = vec![]; + println!("Reading latest ids"); rt.block_on(async { - println!("performing ADD ATTACHMENTS - BACKUPS operations"); let mut handlers = vec![]; - for item in &backup_data { - let item_cloned = item.clone(); - let mut client_cloned = client.clone(); + for user in &user_identities { + let url = url.clone(); + let descriptor = BackupDescriptor::Latest { + username: user.user_id.clone(), + }; handlers.push(tokio::spawn(async move { - if !item_cloned.backup_item.attachments_holders.is_empty() { - add_attachments::run(&mut client_cloned, &item_cloned, None) - .await - .unwrap(); - } + let response = pull_backup::run( + url, + descriptor, + pull_backup::RequestedData::BackupID, + ) + .await + .unwrap(); + + serde_json::from_slice::(&response).unwrap() })); } for handler in handlers { - handler.await.unwrap(); + latest_ids_for_user.push(handler.await.unwrap().backup_id); } }); - // SEND LOG + assert_eq!(latest_ids_for_user.len(), user_identities.len()); + + let mut latest_user_keys_for_user = vec![]; + println!("Reading latest user keys"); rt.block_on(async { - println!("performing SEND LOG operations"); let mut handlers = vec![]; - let (sender, receiver) = channel::<(usize, usize, String)>(); - for (backup_index, backup_item) in backup_data.iter().enumerate() { - let backup_item_cloned = backup_item.clone(); - for log_index in 0..backup_item_cloned.log_items.len() { - let backup_item_recloned = backup_item_cloned.clone(); - let mut client_cloned = client.clone(); - let sender_cloned = sender.clone(); - handlers.push(tokio::spawn(async move { - println!( - "sending log, backup index: [{}] log index: [{}]", - backup_index, log_index - ); - let id = send_log::run( - &mut client_cloned, - &backup_item_recloned, - log_index, - ) - .await - .unwrap(); - assert!(!id.is_empty(), "log id should not be empty after sending"); - sender_cloned.send((backup_index, log_index, id)).unwrap(); - })); - } + for user in &user_identities { + let url = url.clone(); + let descriptor = BackupDescriptor::Latest { + username: user.user_id.clone(), + }; + handlers.push(tokio::spawn(async move { + pull_backup::run( + url, + descriptor, + pull_backup::RequestedData::UserKeys, + ) + .await + .unwrap() + })); } - // https://docs.rs/tokio/1.1.0/tokio/sync/mpsc/struct.Receiver.html#method.recv - // The channel is closed when all senders have been dropped, or when close - // is called. The best option here is to clone the sender for every - // thread, drop the original one and let all the clones be dropped when - // going out of scope which is equal to the parent thread's termination. - drop(sender); for handler in handlers { - handler.await.unwrap(); - } - for data in receiver { - println!("received: {:?}", data); - let (backup_index, log_index, id) = data; - backup_data[backup_index].log_items[log_index].id = id; + latest_user_keys_for_user.push(handler.await.unwrap()); } }); - // check if log IDs are properly set - for (backup_index, backup_item) in backup_data.iter().enumerate() { - for (log_index, log_item) in backup_item.log_items.iter().enumerate() { - assert!( - !log_item.id.is_empty(), - "missing log id for backup index {} and log index {}", - backup_index, - log_index - ); - } - } - - // ADD ATTACHMENTS - LOGS - rt.block_on(async { - println!("performing ADD ATTACHMENTS - LOGS operations"); - let mut handlers = vec![]; - for backup_item in &backup_data { - let backup_item_cloned = backup_item.clone(); - for log_index in 0..backup_item_cloned.log_items.len() { - let backup_item_recloned = backup_item_cloned.clone(); - let mut client_cloned = client.clone(); - handlers.push(tokio::spawn(async move { - if !backup_item_recloned - .backup_item - .attachments_holders - .is_empty() - { - add_attachments::run( - &mut client_cloned, - &backup_item_recloned, - Some(log_index), - ) - .await - .unwrap(); - } - })); - } - } + assert_eq!(latest_user_keys_for_user.len(), user_identities.len()); + for (backup_id, user_keys) in + latest_ids_for_user.iter().zip(latest_user_keys_for_user) + { + let backup = backup_data + .iter() + .find(|data| data.backup_id == *backup_id) + .expect("Request should return existing backup data"); - for handler in handlers { - handler.await.unwrap(); - } - }); + assert_eq!(backup.user_keys, user_keys); + } - // PULL BACKUP + let mut latest_user_data_for_user = vec![]; + println!("Reading latest user data"); rt.block_on(async { - println!("performing PULL BACKUP operations"); let mut handlers = vec![]; - for item in backup_data { - let item_cloned = item.clone(); - let mut client_cloned = client.clone(); + for (i, backup_id) in latest_ids_for_user.iter().enumerate() { + let url = url.clone(); + let descriptor = BackupDescriptor::BackupID { + backup_id: backup_id.clone(), + user_identity: user_identities[i % user_identities.len()].clone(), + }; handlers.push(tokio::spawn(async move { - let result = pull_backup::run(&mut client_cloned, &item_cloned) - .await - .unwrap(); - backup_utils::compare_backups(&item_cloned, &result); + pull_backup::run( + url, + descriptor, + pull_backup::RequestedData::UserData, + ) + .await + .unwrap() })); } for handler in handlers { - handler.await.unwrap(); + latest_user_data_for_user.push(handler.await.unwrap()); } }); + + assert_eq!(latest_user_data_for_user.len(), user_identities.len()); + for (backup_id, user_data) in + latest_ids_for_user.iter().zip(latest_user_data_for_user) + { + let backup = backup_data + .iter() + .find(|data| data.backup_id == *backup_id) + .expect("Request should return existing backup data"); + + assert_eq!(backup.user_data, user_data); + } }) .await .expect("Task panicked");