diff --git a/native/cpp/CommonCpp/DatabaseManagers/DatabaseQueryExecutor.h b/native/cpp/CommonCpp/DatabaseManagers/DatabaseQueryExecutor.h --- a/native/cpp/CommonCpp/DatabaseManagers/DatabaseQueryExecutor.h +++ b/native/cpp/CommonCpp/DatabaseManagers/DatabaseQueryExecutor.h @@ -91,6 +91,8 @@ virtual void restoreFromMainCompaction( std::string mainCompactionPath, std::string mainCompactionEncryptionKey) const = 0; + virtual void + restoreFromBackupLog(const std::vector<std::uint8_t> &backupLog) const = 0; #ifdef EMSCRIPTEN virtual std::vector<WebThread> getAllThreadsWeb() const = 0; diff --git a/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.h b/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.h --- a/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.h +++ b/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.h @@ -21,6 +21,7 @@ public: NativeSQLiteConnectionManager(); void setLogsMonitoring(bool enabled); + bool getLogsMonitoring(); void initializeConnection( std::string sqliteFilePath, std::function<void(sqlite3 *)> on_db_open_callback) override; @@ -30,5 +31,7 @@ std::string backupID, std::string logID, std::string encryptionKey); + void + restoreFromBackupLog(const std::vector<std::uint8_t> &backupLog) override; }; } // namespace comm diff --git a/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.cpp b/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.cpp --- a/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.cpp +++ b/native/cpp/CommonCpp/DatabaseManagers/NativeSQLiteConnectionManager.cpp @@ -184,6 +184,13 @@ sqlite3session_enable(backupLogsSession, enabled); } +bool NativeSQLiteConnectionManager::getLogsMonitoring() { + if (!backupLogsSession) { + return false; + } + return sqlite3session_enable(backupLogsSession, -1); +} + void NativeSQLiteConnectionManager::initializeConnection( std::string sqliteFilePath, std::function<void(sqlite3 *)> on_db_open_callback) { @@ -232,4 +239,12 @@ attachSession(); return true; } + +void NativeSQLiteConnectionManager::restoreFromBackupLog( + const std::vector<std::uint8_t> &backupLog) { + bool initialEnabledValue = getLogsMonitoring(); + setLogsMonitoring(false); + SQLiteConnectionManager::restoreFromBackupLog(backupLog); + setLogsMonitoring(initialEnabledValue); +} } // namespace comm diff --git a/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.h b/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.h --- a/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.h +++ b/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.h @@ -26,5 +26,6 @@ std::function<void(sqlite3 *)> on_db_open_callback); virtual void closeConnection(); virtual ~SQLiteConnectionManager(); + virtual void restoreFromBackupLog(const std::vector<std::uint8_t> &backupLog); }; } // namespace comm diff --git a/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.cpp b/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.cpp --- a/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.cpp +++ b/native/cpp/CommonCpp/DatabaseManagers/SQLiteConnectionManager.cpp @@ -1,5 +1,6 @@ #include "SQLiteConnectionManager.h" +#include "Logger.h" #include <sstream> #include <stdexcept> #include <string> @@ -53,4 +54,49 @@ SQLiteConnectionManager::~SQLiteConnectionManager() { closeConnectionInternal(); } + +void SQLiteConnectionManager::restoreFromBackupLog( + const std::vector<std::uint8_t> &backupLog) { + if (!dbConnection) { + throw std::runtime_error( + "Programmer error: attempt to restore from backup log but database " + "connection is not initialized."); + } + + static auto backupLogRestoreConflictHandler = + [](void *, int conflictReason, sqlite3_changeset_iter *changesetIter) { + const char *tableName; + int columnsNumber; + int operationType; + + int getOperationResult = sqlite3changeset_op( + changesetIter, &tableName, &columnsNumber, &operationType, nullptr); + handleSQLiteError( + getOperationResult, + "Failed to extract operation from log iterator."); + + std::stringstream conflictMessage; + conflictMessage << "Conflict of type " << conflictReason + << " occurred for operation of type " << operationType + << " for table " << tableName + << " during backup log application"; + Logger::log(conflictMessage.str()); + + if (operationType == SQLITE_INSERT && + conflictReason == SQLITE_CHANGESET_CONFLICT) { + return SQLITE_CHANGESET_REPLACE; + } + + return SQLITE_CHANGESET_OMIT; + }; + + int applyChangesetResult = sqlite3changeset_apply( + dbConnection, + backupLog.size(), + (void *)backupLog.data(), + nullptr, + backupLogRestoreConflictHandler, + nullptr); + handleSQLiteError(applyChangesetResult, "Failed to apply backup log."); +} } // namespace comm diff --git a/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.h b/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.h --- a/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.h +++ b/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.h @@ -104,6 +104,8 @@ void restoreFromMainCompaction( std::string mainCompactionPath, std::string mainCompactionEncryptionKey) const override; + void restoreFromBackupLog( + const std::vector<std::uint8_t> &backupLog) const override; #ifdef EMSCRIPTEN std::vector<WebThread> getAllThreadsWeb() const override; diff --git a/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.cpp b/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.cpp --- a/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.cpp +++ b/native/cpp/CommonCpp/DatabaseManagers/SQLiteQueryExecutor.cpp @@ -1899,4 +1899,9 @@ "Failed to delete main compaction file after successful restore."); } +void SQLiteQueryExecutor::restoreFromBackupLog( + const std::vector<std::uint8_t> &backupLog) const { + SQLiteQueryExecutor::connectionManager.restoreFromBackupLog(backupLog); +} + } // namespace comm diff --git a/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.h b/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.h --- a/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.h +++ b/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.h @@ -1,6 +1,7 @@ #pragma once #include <string> +#include <vector> namespace comm { class BackupOperationsExecutor { @@ -9,5 +10,6 @@ static void restoreFromMainCompaction( std::string mainCompactionPath, std::string mainCompactionEncryptionKey); + static void restoreFromBackupLog(const std::vector<std::uint8_t> &backupLog); }; } // namespace comm diff --git a/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.cpp b/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.cpp --- a/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.cpp +++ b/native/cpp/CommonCpp/NativeModules/PersistentStorageUtilities/BackupOperationsUtilities/BackupOperationsExecutor.cpp @@ -36,4 +36,19 @@ }; GlobalDBSingleton::instance.scheduleOrRunCancellable(job); } + +void BackupOperationsExecutor::restoreFromBackupLog( + const std::vector<std::uint8_t> &backupLog) { + taskType job = [backupLog]() { + try { + DatabaseManager::getQueryExecutor().restoreFromBackupLog(backupLog); + } catch (const std::exception &e) { + // TODO: Inform Rust networking about failure + // of restoration from backup log. + Logger::log( + "Restore from backup log failed. Details: " + std::string(e.what())); + } + }; + GlobalDBSingleton::instance.scheduleOrRunCancellable(job); +} } // namespace comm diff --git a/native/native_rust_library/RustBackupExecutor.h b/native/native_rust_library/RustBackupExecutor.h --- a/native/native_rust_library/RustBackupExecutor.h +++ b/native/native_rust_library/RustBackupExecutor.h @@ -13,5 +13,6 @@ void restoreFromMainCompaction( rust::String mainCompactionPath, rust::String mainCompactionEncryptionKey); +void restoreFromBackupLog(rust::Vec<std::uint8_t> backupLog); } // namespace comm diff --git a/native/native_rust_library/RustBackupExecutor.cpp b/native/native_rust_library/RustBackupExecutor.cpp --- a/native/native_rust_library/RustBackupExecutor.cpp +++ b/native/native_rust_library/RustBackupExecutor.cpp @@ -37,4 +37,9 @@ std::string(mainCompactionPath), std::string(mainCompactionEncryptionKey)); } + +void restoreFromBackupLog(rust::Vec<std::uint8_t> backupLog) { + BackupOperationsExecutor::restoreFromBackupLog( + std::move(std::vector<std::uint8_t>(backupLog.begin(), backupLog.end()))); +} } // namespace comm diff --git a/native/native_rust_library/src/lib.rs b/native/native_rust_library/src/lib.rs --- a/native/native_rust_library/src/lib.rs +++ b/native/native_rust_library/src/lib.rs @@ -311,6 +311,10 @@ main_compaction_path: String, main_compaction_encryption_key: String, ) -> Result<()>; + + #[allow(unused)] + #[cxx_name = "restoreFromBackupLog"] + fn restore_from_backup_log(backup_log: Vec<u8>) -> Result<()>; } } diff --git a/web/cpp/SQLiteQueryExecutorBindings.cpp b/web/cpp/SQLiteQueryExecutorBindings.cpp --- a/web/cpp/SQLiteQueryExecutorBindings.cpp +++ b/web/cpp/SQLiteQueryExecutorBindings.cpp @@ -176,7 +176,9 @@ "rollbackTransaction", &SQLiteQueryExecutor::rollbackTransaction) .function( "restoreFromMainCompaction", - &SQLiteQueryExecutor::restoreFromMainCompaction); + &SQLiteQueryExecutor::restoreFromMainCompaction) + .function( + "restoreFromBackupLog", &SQLiteQueryExecutor::restoreFromBackupLog); } } // namespace comm diff --git a/web/database/_generated/comm-query-executor.js b/web/database/_generated/comm-query-executor.js --- a/web/database/_generated/comm-query-executor.js +++ b/web/database/_generated/comm-query-executor.js @@ -136,11 +136,11 @@ S(b);var c="std::string"===b;R(a,{name:b,fromWireType:function(d){var f=F[d>>2],g=d+4;if(c)for(var k=g,h=0;h<=f;++h){var n=g+h;if(h==f||0==z[n]){k=y(k,n-k);if(void 0===q)var q=k;else q+=String.fromCharCode(0),q+=k;k=n+1}}else{q=Array(f);for(h=0;h<f;++h)q[h]=String.fromCharCode(z[g+h]);q=q.join("")}X(d);return q},toWireType:function(d,f){f instanceof ArrayBuffer&&(f=new Uint8Array(f));var g="string"==typeof f;g||f instanceof Uint8Array||f instanceof Uint8ClampedArray||f instanceof Int8Array||T("Cannot pass non-string to std::string"); var k=c&&g?ta(f):f.length;var h=wc(4+k+1),n=h+4;F[h>>2]=k;if(c&&g)A(f,z,n,k+1);else if(g)for(g=0;g<k;++g){var q=f.charCodeAt(g);255<q&&(X(n),T("String has UTF-16 code units that do not fit in 8 bits"));z[n+g]=q}else for(g=0;g<k;++g)z[n+g]=f[g];null!==d&&d.push(X,h);return h},argPackAdvance:8,readValueFromPointer:lb,ib:function(d){X(d)}})},p:function(a,b,c){c=S(c);if(2===b){var d=ic;var f=jc;var g=kc;var k=()=>va;var h=1}else 4===b&&(d=lc,f=mc,g=nc,k=()=>F,h=2);R(a,{name:c,fromWireType:function(n){for(var q= F[n>>2],p=k(),t,x=n+4,l=0;l<=q;++l){var u=n+4+l*b;if(l==q||0==p[u>>h])x=d(x,u-x),void 0===t?t=x:(t+=String.fromCharCode(0),t+=x),x=u+b}X(n);return t},toWireType:function(n,q){"string"!=typeof q&&T("Cannot pass non-string to C++ string type "+c);var p=g(q),t=wc(4+p+b);F[t>>2]=p>>h;f(q,t+4,p+b);null!==n&&n.push(X,t);return t},argPackAdvance:8,readValueFromPointer:lb,ib:function(n){X(n)}})},g:function(a,b,c,d,f,g){jb[a]={name:S(b),rc:W(c,d),pb:W(f,g),Lc:[]}},d:function(a,b,c,d,f,g,k,h,n,q){jb[a].Lc.push({rd:S(b), -zd:c,Sb:W(d,f),yd:g,Sd:k,Rd:W(h,n),Td:q})},ga:function(a,b){b=S(b);R(a,{Dd:!0,name:b,argPackAdvance:0,fromWireType:function(){},toWireType:function(){}})},o:function(){return Date.now()},aa:function(){return!0},m:function(a,b,c){a=ec(a);b=oc(b,"emval::as");var d=[],f=Sb(d);F[c>>2]=f;return b.toWireType(d,a)},A:function(a,b,c,d){a=rc[a];b=ec(b);c=qc(c);a(b,c,null,d)},na:dc,oa:function(a,b){var c=tc(a,b),d=c[0];b=d.name+"_$"+c.slice(1).map(function(p){return p.name}).join("_")+"$";var f=uc[b];if(void 0!== +zd:c,Sb:W(d,f),yd:g,Sd:k,Rd:W(h,n),Td:q})},ga:function(a,b){b=S(b);R(a,{Dd:!0,name:b,argPackAdvance:0,fromWireType:function(){},toWireType:function(){}})},o:function(){return Date.now()},aa:function(){return!0},l:function(a,b,c){a=ec(a);b=oc(b,"emval::as");var d=[],f=Sb(d);F[c>>2]=f;return b.toWireType(d,a)},A:function(a,b,c,d){a=rc[a];b=ec(b);c=qc(c);a(b,c,null,d)},na:dc,oa:function(a,b){var c=tc(a,b),d=c[0];b=d.name+"_$"+c.slice(1).map(function(p){return p.name}).join("_")+"$";var f=uc[b];if(void 0!== f)return f;f=["retType"];for(var g=[d],k="",h=0;h<a-1;++h)k+=(0!==h?", ":"")+"arg"+h,f.push("argType"+h),g.push(c[1+h]);var n="return function "+pb("methodCaller_"+b)+"(handle, name, destructors, args) {\n",q=0;for(h=0;h<a-1;++h)n+=" var arg"+h+" = argType"+h+".readValueFromPointer(args"+(q?"+"+q:"")+");\n",q+=c[h+1].argPackAdvance;n+=" var rv = handle[name]("+k+");\n";for(h=0;h<a-1;++h)c[h+1].deleteObject&&(n+=" argType"+h+".deleteObject(arg"+h+");\n");d.Dd||(n+=" return retType.toWireType(destructors, rv);\n"); f.push(n+"};\n");a=ac(f).apply(null,g);f=sc(a);return uc[b]=f},z:function(a,b){a=ec(a);b=ec(b);return Sb(a[b])},C:function(a){4<a&&(Y[a].uc+=1)},B:function(){return Sb([])},ma:function(a){return Sb(qc(a))},la:function(a){var b=ec(a);kb(b);dc(a)},j:function(a,b){a=oc(a,"_emval_take_value");a=a.readValueFromPointer(b);return Sb(a)},ba:function(a,b){a=new Date(1E3*ib(a));E[b>>2]=a.getSeconds();E[b+4>>2]=a.getMinutes();E[b+8>>2]=a.getHours();E[b+12>>2]=a.getDate();E[b+16>>2]=a.getMonth();E[b+20>>2]=a.getFullYear()- 1900;E[b+24>>2]=a.getDay();var c=new Date(a.getFullYear(),0,1);E[b+28>>2]=(a.getTime()-c.getTime())/864E5|0;E[b+36>>2]=-(60*a.getTimezoneOffset());var d=(new Date(a.getFullYear(),6,1)).getTimezoneOffset();c=c.getTimezoneOffset();E[b+32>>2]=(d!=c&&a.getTimezoneOffset()==Math.min(c,d))|0},N:function(a,b,c,d,f,g){try{var k=N.rb(d);if(!k)return-8;var h=N.yb(k,a,f,b,c),n=h.Sa;E[g>>2]=h.zc;return n}catch(q){if("undefined"==typeof N||!(q instanceof N.Ma))throw q;return-q.Ra}},O:function(a,b,c,d,f,g){try{var k= -N.rb(f);if(k&&c&2){var h=z.slice(a,a+b);N.Fb(k,h,g,b,d)}}catch(n){if("undefined"==typeof N||!(n instanceof N.Ma))throw n;return-n.Ra}},ca:yc,l:function(){v("")},J:function(){return 2147483648},w:zc,n:function(a){var b=z.length;a>>>=0;if(2147483648<a)return!1;for(var c=1;4>=c;c*=2){var d=b*(1+.2/c);d=Math.min(d,a+100663296);var f=Math;d=Math.max(a,d);f=f.min.call(f,2147483648,d+(65536-d%65536)%65536);a:{try{pa.grow(f-ua.byteLength+65535>>>16);ya();var g=1;break a}catch(k){}g=void 0}if(g)return!0}return!1}, +N.rb(f);if(k&&c&2){var h=z.slice(a,a+b);N.Fb(k,h,g,b,d)}}catch(n){if("undefined"==typeof N||!(n instanceof N.Ma))throw n;return-n.Ra}},ca:yc,m:function(){v("")},J:function(){return 2147483648},w:zc,n:function(a){var b=z.length;a>>>=0;if(2147483648<a)return!1;for(var c=1;4>=c;c*=2){var d=b*(1+.2/c);d=Math.min(d,a+100663296);var f=Math;d=Math.max(a,d);f=f.min.call(f,2147483648,d+(65536-d%65536)%65536);a:{try{pa.grow(f-ua.byteLength+65535>>>16);ya();var g=1;break a}catch(k){}g=void 0}if(g)return!0}return!1}, Q:function(a,b){var c=0;Bc().forEach(function(d,f){var g=b+c;f=F[a+4*f>>2]=g;for(g=0;g<d.length;++g)B[f++>>0]=d.charCodeAt(g);B[f>>0]=0;c+=d.length+1});return 0},R:function(a,b){var c=Bc();F[a>>2]=c.length;var d=0;c.forEach(function(f){d+=f.length+1});F[b>>2]=d;return 0},k:function(a){try{var b=Q(a);N.close(b);return 0}catch(c){if("undefined"==typeof N||!(c instanceof N.Ma))throw c;return c.Ra}},H:function(a,b){try{var c=Q(a);B[b>>0]=c.tty?2:N.Ya(c.mode)?3:N.vb(c.mode)?7:4;return 0}catch(d){if("undefined"== typeof N||!(d instanceof N.Ma))throw d;return d.Ra}},u:function(a,b,c,d){try{a:{var f=Q(a);a=b;for(var g=b=0;g<c;g++){var k=F[a>>2],h=F[a+4>>2];a+=8;var n=N.read(f,B,k,h,void 0);if(0>n){var q=-1;break a}b+=n;if(n<h)break}q=b}E[d>>2]=q;return 0}catch(p){if("undefined"==typeof N||!(p instanceof N.Ma))throw p;return p.Ra}},D:function(a,b,c,d,f){try{b=c+2097152>>>0<4194305-!!b?(b>>>0)+4294967296*c:NaN;if(isNaN(b))return 61;var g=Q(a);N.bb(g,b,d);J=[g.position>>>0,(G=g.position,1<=+Math.abs(G)?0<G?(Math.min(+Math.floor(G/ 4294967296),4294967295)|0)>>>0:~~+Math.ceil((G-+(~~G>>>0))/4294967296)>>>0:0)];E[f>>2]=J[0];E[f+4>>2]=J[1];g.ub&&0===b&&0===d&&(g.ub=null);return 0}catch(k){if("undefined"==typeof N||!(k instanceof N.Ma))throw k;return k.Ra}},T:function(a){try{var b=Q(a);return b.Qa&&b.Qa.fsync?-b.Qa.fsync(b):0}catch(c){if("undefined"==typeof N||!(c instanceof N.Ma))throw c;return c.Ra}},t:function(a,b,c,d){try{a:{var f=Q(a);a=b;for(var g=b=0;g<c;g++){var k=F[a>>2],h=F[a+4>>2];a+=8;var n=N.write(f,B,k,h,void 0);if(0> diff --git a/web/database/_generated/comm_query_executor.wasm b/web/database/_generated/comm_query_executor.wasm index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@<O00001 literal 0 Hc$@<O00001 diff --git a/web/database/types/sqlite-query-executor.js b/web/database/types/sqlite-query-executor.js --- a/web/database/types/sqlite-query-executor.js +++ b/web/database/types/sqlite-query-executor.js @@ -113,6 +113,8 @@ mainCompactionEncryptionKey: string, ): void; + restoreFromBackupLog(backupLog: Uint8Array): void; + // method is provided to manually signal that a C++ object // is no longer needed and can be deleted delete(): void; diff --git a/web/scripts/run_emscripten.sh b/web/scripts/run_emscripten.sh --- a/web/scripts/run_emscripten.sh +++ b/web/scripts/run_emscripten.sh @@ -31,7 +31,7 @@ OPENSSL_LIBCRYPTO="${OPENSSL_DIR}/libcrypto.a" # SQLCipher resources -SQLCIPHER_AMALGAMATION_VERSION="4.5.5b" +SQLCIPHER_AMALGAMATION_VERSION="4.5.5-d" SQLCIPHER_AMALGAMATION="sqlcipher-amalgamation-${SQLCIPHER_AMALGAMATION_VERSION}" SQLCIPHER_AMALGAMATION_URL="https://codeload.github.com/CommE2E/sqlcipher-amalgamation/zip/refs/tags/${SQLCIPHER_AMALGAMATION_VERSION}" SQLCIPHER_AMALGAMATION_FILE="${SQLITE_DIR}${SQLCIPHER_AMALGAMATION}" @@ -47,6 +47,8 @@ -DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2 -DSQLCIPHER_CRYPTO_OPENSSL + -DSQLITE_ENABLE_SESSION + -DSQLITE_ENABLE_PREUPDATE_HOOK ) download_openssl() {