diff --git a/README.md b/README.md index c3adb97..efaed10 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,54 @@ using var db = tx.OpenDatabase(configuration: config); Reverse variants are available for most comparers (e.g., `ReverseSignedIntegerComparer`). +## LMDB 1.0 + +Starting with LightningDB 0.23.0, the bundled native library is from the LMDB 1.0 line. + +> **⚠️ Data migration required:** the LMDB 1.0 on-disk format is incompatible with 0.9.x. +> Existing environments created by earlier LightningDB versions cannot be opened; migrate +> them once by dumping with the 0.9 `mdb_dump` tool and loading with the 1.0 `mdb_load` +> tool. Named-database (DBI) names are also stored differently (NUL-terminated) in 1.0. + +New capabilities exposed by LightningDB: + +### Encryption and checksums (opt-in) + +Every page can be encrypted and/or checksummed. LMDB itself ships no cipher — the +application supplies one; LightningDB includes hardware-accelerated AES-256-GCM and +SHA-256 implementations out of the box (custom `LightningCipher`/`LightningChecksum` +implementations are also supported, and required on netstandard2.0 or browser-wasm): + +```csharp +var config = new EnvironmentConfiguration +{ + Encryption = new EncryptionConfiguration(new AesGcmCipher(), key), // key: e.g. 32 random bytes + Checksum = new Sha256Checksum(), // independent of encryption; either can be used alone +}; +using var env = new LightningEnvironment("path_to_your_database", config); +env.Open(); +``` + +The same cipher and key must be configured every time the environment is opened. + +### Two-phase commit + +`tx.Prepare()` performs the first phase of a two-phase commit; follow with `Commit()` or +`Abort()`. If a remote participant fails after a local commit, the last committed +transaction can be undone with `env.RollbackLastTransaction(txId)`. + +### Incremental backup + +`env.CopyTo(path)` remains the full-backup primitive; `env.IncrementalCopyTo(file, sinceTxnId)` +dumps only pages newer than a previous backup's `env.Info.LastTransactionId`, and +`env.LoadIncrementalFromStream(stream)` applies the dump to a restored copy. + +### Other additions + +- `EnvironmentConfiguration.PageSize` — set the database page size (power of 2, 512–65536) at creation time. +- `EnvironmentOpenFlags.PreviousSnapshot` — open using the previous snapshot, recovering from a bad last transaction. +- `TransactionBeginFlags.NoSync`/`NoMetaSync` are now honored per-transaction. + ## Additional Resources For more detailed examples and advanced usage, refer to the unit tests in the [Lightning.NET](https://github.com/CoreyKaylor/Lightning.NET) repository. diff --git a/lmdb/compile-lmdb-macos.sh b/lmdb/compile-lmdb-macos.sh index 09a5000..dc69eb4 100755 --- a/lmdb/compile-lmdb-macos.sh +++ b/lmdb/compile-lmdb-macos.sh @@ -4,26 +4,34 @@ if [ ! -d "lmdb" ]; then git clone https://git.openldap.org/openldap/openldap.git lmdb fi cd ./lmdb/libraries/liblmdb || exit -git checkout LMDB_0.9.35 +# Pinned mdb.master3 (LMDB 1.0 dev line): the LMDB_1.0.0 tag doesn't compile for +# Windows and lacks post-release fixes to encryption/checksums/incremental backup +# (ITS#10515, ITS#10518-10523). Re-pin to the release tag once upstream cuts 1.0.1. +git fetch origin mdb.master3 +git checkout -f 7ed58053d01cd21bd5ba07ed888d3583d4561ab4 +# The new win32 incremental-backup/copy code doesn't compile under mingw-w64 +# (LARGE_INTEGER union misuse, pointer-type mismatches) even on mdb.master3; +# drop the patch once fixed upstream. +git apply ../../../mingw-win32-fixes.patch declare -A build_outputs declare -A supported_targets=( - [ios-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphoneos --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [iossimulator-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [iossimulator-x64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [osx-arm64/native/lmdb.dylib]="make LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [osx/native/lmdb.dylib]="make CC='clang -mmacosx-version-min=10.15 -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" + [ios-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphoneos --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" + [iossimulator-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" + [iossimulator-x64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" + [osx-arm64/native/lmdb.dylib]="make LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" + [osx/native/lmdb.dylib]="make CC='clang -mmacosx-version-min=10.15 -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'" [linux-arm/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/arm/7 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'" [linux-arm64/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/arm64 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'" [linux-x64/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/amd64 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [win-x64/native/lmdb.dll]="make CC='x86_64-w64-mingw32-gcc' AR='x86_64-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [win-x86/native/lmdb.dll]="make CC='i686-w64-mingw32-gcc' AR='i686-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG'" - [win-arm64/native/lmdb.dll]="docker run --mount type=bind,source='$(pwd)',target=/lmdb --rm -w /lmdb dockcross/windows-arm64 bash -c 'make CC=aarch64-w64-mingw32-gcc AR=aarch64-w64-mingw32-ar LDFLAGS=-s XCFLAGS=-DNDEBUG'" + [win-x64/native/lmdb.dll]="make CC='x86_64-w64-mingw32-gcc' AR='x86_64-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG' LDL= VERSION_OPT=" + [win-x86/native/lmdb.dll]="make CC='i686-w64-mingw32-gcc' AR='i686-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG' LDL= VERSION_OPT=" + [win-arm64/native/lmdb.dll]="docker run --mount type=bind,source='$(pwd)',target=/lmdb --rm -w /lmdb dockcross/windows-arm64 bash -c 'make CC=aarch64-w64-mingw32-gcc AR=aarch64-w64-mingw32-ar LDFLAGS=-s XCFLAGS=-DNDEBUG LDL= VERSION_OPT='" [android-arm64/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/aarch64-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" [android-arm/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" [android-x86/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/i686-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" [android-x64/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/x86_64-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'" - [browser-wasm/native/liblmdb.wasm]="emmake make LDFLAGS='-s' XCFLAGS='-DNDEBUG'" + [browser-wasm/native/liblmdb.wasm]="emcc -O2 -pthread -fPIC -DNDEBUG -sSIDE_MODULE=1 -o liblmdb.so mdb.c midl.c module.c" ) function compile_lib() { diff --git a/lmdb/mingw-win32-fixes.patch b/lmdb/mingw-win32-fixes.patch new file mode 100644 index 0000000..2112e01 --- /dev/null +++ b/lmdb/mingw-win32-fixes.patch @@ -0,0 +1,83 @@ +diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c +index 0e8f6212de..5c5ca124b7 100644 +--- a/libraries/liblmdb/mdb.c ++++ b/libraries/liblmdb/mdb.c +@@ -11614,10 +11614,15 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + #endif + #if MDB_RPAGE_CACHE + #ifdef _WIN32 +- LARGE_INTEGER off = 0; ++ LARGE_INTEGER off = {0}; ++ SIZE_T mlen; ++#define OFF_ADD(off,val) off.QuadPart += (val) + #else + off_t off = 0; ++ size_t mlen; ++#define OFF_ADD(off,val) off += (val) + #endif ++ void *mptr; + #endif + + /* Do the lock/unlock of the reader mutex before starting the +@@ -11678,7 +11683,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + } + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { +- off = wsize; ++ SET_OFF(off, wsize); + } + #endif + wsize = w3 - wsize; +@@ -11686,7 +11691,9 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize; + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { +- MAP(rc, env, ptr, w2, off); ++ mlen = w2; ++ MAP(rc, env, mptr, mlen, off); ++ ptr = mptr; + } + #endif + DO_WRITE(rc, fd, ptr, w2, len); +@@ -11704,7 +11711,7 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd) + wsize -= len; + #if MDB_RPAGE_CACHE + if (MDB_REMAPPING(env->me_flags)) { +- off += len; ++ OFF_ADD(off, len); + } + #endif + continue; +@@ -11946,7 +11953,7 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) + return ENOMEM; + + #ifdef _WIN32 +- SetFilePointerEx(env->me_fd, 0, NULL, FILE_BEGIN); ++ { LARGE_INTEGER li0; li0.QuadPart = 0; SetFilePointerEx(env->me_fd, li0, NULL, FILE_BEGIN); } + #else + lseek(env->me_fd, 0, SEEK_SET); + #endif +@@ -12011,15 +12018,19 @@ mdb_env_incr_loadfd(MDB_env *env, HANDLE fd) + ptr += rlen; + rsize -= rlen; + } +- off = (pg-prevpg-numprev) * env->me_psize; +- rsize = numpgs * env->me_psize; +- if (off) { + #ifdef _WIN32 ++ off.QuadPart = (pg-prevpg-numprev) * env->me_psize; ++ rsize = numpgs * env->me_psize; ++ if (off.QuadPart) { + SetFilePointerEx(env->me_fd, off, NULL, FILE_CURRENT); ++ } + #else ++ off = (pg-prevpg-numprev) * env->me_psize; ++ rsize = numpgs * env->me_psize; ++ if (off) { + lseek(env->me_fd, off, SEEK_CUR); +-#endif + } ++#endif + ptr = pbuf; + while (rsize > 0) { + w2 = (rsize > MAX_WRITE) ? MAX_WRITE: rsize; diff --git a/src/LightningDB.Tests/DatabaseTests.cs b/src/LightningDB.Tests/DatabaseTests.cs index 8ef2103..cf31c14 100644 --- a/src/LightningDB.Tests/DatabaseTests.cs +++ b/src/LightningDB.Tests/DatabaseTests.cs @@ -74,7 +74,8 @@ public void named_database_name_exists_in_master() using (var cursor = tx.CreateCursor(db)) { cursor.Next(); - UTF8.GetString(cursor.GetCurrent().key.CopyToNewArray()) + //LMDB 1.0 stores DBI names with their NUL terminator + UTF8.GetString(cursor.GetCurrent().key.CopyToNewArray()).TrimEnd('\0') .ShouldBe("customdb"); } } diff --git a/src/LightningDB.Tests/EncryptionTests.cs b/src/LightningDB.Tests/EncryptionTests.cs new file mode 100644 index 0000000..c5e2cee --- /dev/null +++ b/src/LightningDB.Tests/EncryptionTests.cs @@ -0,0 +1,146 @@ +using System; +using System.Text; +using Shouldly; + +namespace LightningDB.Tests; + +public class EncryptionTests : TestBase +{ + private static byte[] Key() => "0123456789abcdef0123456789abcdef"u8.ToArray(); + + private LightningEnvironment CreateEncryptedEnvironment(string path, byte[]? key = null) => + CreateEnvironment(path, new EnvironmentConfiguration + { + Encryption = new EncryptionConfiguration(new AesGcmCipher(), key ?? Key()) + }); + + public void can_write_and_read_in_encrypted_environment() + { + using var env = CreateEncryptedEnvironment(TempPath()); + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "key"u8.ToArray(), "value"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + using (var db = tx.OpenDatabase()) + { + var (resultCode, _, value) = tx.Get(db, "key"u8.ToArray()); + resultCode.ShouldBe(MDBResultCode.Success); + Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("value"); + } + } + + public void encrypted_environment_survives_reopen_with_same_key() + { + var path = TempPath(); + using (var env = CreateEncryptedEnvironment(path)) + { + env.Open(); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Put(db, "persisted"u8.ToArray(), "still here"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var env = CreateEncryptedEnvironment(path)) + { + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (resultCode, _, value) = tx.Get(db, "persisted"u8.ToArray()); + resultCode.ShouldBe(MDBResultCode.Success); + Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("still here"); + } + } + + public void opening_encrypted_environment_with_wrong_key_fails() + { + var path = TempPath(); + using (var env = CreateEncryptedEnvironment(path)) + { + env.Open(); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Put(db, "secret"u8.ToArray(), "data"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + var wrongKey = "ffffffffffffffffffffffffffffffff"u8.ToArray(); + using (var env = CreateEncryptedEnvironment(path, wrongKey)) + { + //data pages decrypt lazily, so the failure surfaces on first read + //rather than at open + try + { + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (resultCode, _, _) = tx.Get(db, "secret"u8.ToArray()); + resultCode.ShouldNotBe(MDBResultCode.Success); + } + catch (LightningException) + { + } + } + } + + public void opening_encrypted_environment_without_cipher_fails() + { + var path = TempPath(); + using (var env = CreateEncryptedEnvironment(path)) + { + env.Open(); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Put(db, "secret"u8.ToArray(), "data"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var env = CreateEnvironment(path)) + { + Should.Throw(() => + { + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + tx.Get(db, "secret"u8.ToArray()); + }); + } + } + + public void encrypted_environment_reports_encrypted_flag() + { + using var env = CreateEncryptedEnvironment(TempPath()); + env.Open(); + env.Flags.HasFlag(EnvironmentOpenFlags.Encrypted).ShouldBeTrue(); + } + + public void checksum_environment_round_trip() + { + var path = TempPath(); + var config = new EnvironmentConfiguration { Checksum = new Sha256Checksum() }; + using (var env = CreateEnvironment(path, config)) + { + env.Open(); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Put(db, "summed"u8.ToArray(), "verified"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var env = CreateEnvironment(path, new EnvironmentConfiguration { Checksum = new Sha256Checksum() })) + { + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + var (resultCode, _, value) = tx.Get(db, "summed"u8.ToArray()); + resultCode.ShouldBe(MDBResultCode.Success); + Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("verified"); + } + } +} diff --git a/src/LightningDB.Tests/EnvironmentTests.cs b/src/LightningDB.Tests/EnvironmentTests.cs index ff9b25a..eccaf71 100644 --- a/src/LightningDB.Tests/EnvironmentTests.cs +++ b/src/LightningDB.Tests/EnvironmentTests.cs @@ -11,7 +11,8 @@ public void can_get_environment_version() using var env = CreateEnvironment(); var version = env.Version; version.ShouldNotBeNull(); - version.Minor.ShouldBeGreaterThan(0); + //0.9.x and the 1.0 dev line (0.9.90) have Minor > 0; a 1.0 release has Major > 0 + (version.Major + version.Minor).ShouldBeGreaterThan(0); } public void environment_should_be_created_if_without_flags() @@ -215,7 +216,8 @@ public void should_get_version_info() // Verify version info properties are valid versionInfo.ShouldNotBeNull(); versionInfo.Major.ShouldBeGreaterThanOrEqualTo(0); - versionInfo.Minor.ShouldBeGreaterThan(0); + //0.9.x and the 1.0 dev line (0.9.90) have Minor > 0; a 1.0 release has Major > 0 + (versionInfo.Major + versionInfo.Minor).ShouldBeGreaterThan(0); versionInfo.Patch.ShouldBeGreaterThanOrEqualTo(0); } @@ -403,4 +405,136 @@ public void max_key_size_throws_when_environment_not_opened() // Attempting to get MaxKeySize should throw, as the environment must be opened Should.Throw(() => { _ = env.MaxKeySize; }); } + + public void can_set_page_size_before_open() + { + using var env = CreateEnvironment(config: new EnvironmentConfiguration { PageSize = 8192 }); + env.Open(); + env.EnvironmentStats.PageSize.ShouldBe(8192u); + } + + public void setting_page_size_after_open_throws() + { + using var env = CreateEnvironment(); + env.Open(); + Should.Throw(() => env.PageSize = 8192); + } + + public void invalid_page_size_throws() + { + using var env = CreateEnvironment(); + Should.Throw(() => env.PageSize = 1000); + Should.Throw(() => env.PageSize = 256); + Should.Throw(() => env.PageSize = 131072); + } + + public void can_open_previous_snapshot() + { + var path = TempPath(); + using (var env = CreateEnvironment(path)) + { + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "first"u8.ToArray(), "kept"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "second"u8.ToArray(), "lost"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + } + + using (var env = CreateEnvironment(path)) + { + env.Open(EnvironmentOpenFlags.PreviousSnapshot); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + tx.ContainsKey(db, "first"u8.ToArray()).ShouldBeTrue(); + tx.ContainsKey(db, "second"u8.ToArray()).ShouldBeFalse(); + } + } + + public void incremental_backup_and_restore_round_trip() + { + var sourcePath = TempPath(); + var backupPath = TempPath(); + var incrementalFile = Path.Combine(TempPath(), "incremental.dump"); + + using var env = CreateEnvironment(sourcePath); + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "before"u8.ToArray(), "full backup"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + env.CopyTo(backupPath).ThrowOnError(); + var baselineTxnId = (ulong)env.Info.LastTransactionId; + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "after"u8.ToArray(), "incremental"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + env.IncrementalCopyTo(incrementalFile, baselineTxnId).ThrowOnError(); + new FileInfo(incrementalFile).Length.ShouldBeGreaterThan(0); + + //apply the incremental dump, then reopen the environment to observe the result + using (var restored = CreateEnvironment(backupPath)) + { + restored.Open(); + using var dumpStream = File.OpenRead(incrementalFile); + restored.LoadIncrementalFromStream(dumpStream).ThrowOnError(); + } + + using (var restored = CreateEnvironment(backupPath)) + { + restored.Open(); + using var tx = restored.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + tx.ContainsKey(db, "before"u8.ToArray()).ShouldBeTrue(); + tx.ContainsKey(db, "after"u8.ToArray()).ShouldBeTrue(); + } + } + + public void incremental_copy_to_stream_works() + { + var incrementalFile = Path.Combine(TempPath(), "incremental.dump"); + + using var env = CreateEnvironment(); + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "base"u8.ToArray(), "line"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + var baselineTxnId = (ulong)env.Info.LastTransactionId; + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "more"u8.ToArray(), "data"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + using (var stream = File.Open(incrementalFile, FileMode.CreateNew, FileAccess.Write)) + { + env.IncrementalCopyToStream(stream, baselineTxnId).ThrowOnError(); + } + + new FileInfo(incrementalFile).Length.ShouldBeGreaterThan(0); + } } diff --git a/src/LightningDB.Tests/TransactionTests.cs b/src/LightningDB.Tests/TransactionTests.cs index 43bde68..397e61a 100644 --- a/src/LightningDB.Tests/TransactionTests.cs +++ b/src/LightningDB.Tests/TransactionTests.cs @@ -605,4 +605,123 @@ public void should_handle_reserve_space_option() tx.ContainsKey(db, key).ShouldBeTrue(); }); } + + public void can_prepare_and_commit_transaction() + { + using var env = CreateEnvironment(); + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "prepared"u8.ToArray(), "committed"u8.ToArray()); + tx.Prepare().ShouldBe(MDBResultCode.Success); + tx.State.ShouldBe(LightningTransactionState.Prepared); + tx.Commit().ShouldBe(MDBResultCode.Success); + } + + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + using (var db = tx.OpenDatabase()) + { + var (resultCode, _, value) = tx.Get(db, "prepared"u8.ToArray()); + resultCode.ShouldBe(MDBResultCode.Success); + value.CopyToNewArray().ShouldBe("committed"u8.ToArray()); + } + } + + public void can_prepare_and_abort_transaction() + { + using var env = CreateEnvironment(); + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "prepared"u8.ToArray(), "aborted"u8.ToArray()); + tx.Prepare().ShouldBe(MDBResultCode.Success); + tx.Abort(); + } + + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + using (var db = tx.OpenDatabase()) + { + var (resultCode, _, _) = tx.Get(db, "prepared"u8.ToArray()); + resultCode.ShouldBe(MDBResultCode.NotFound); + } + } + + public void prepare_on_readonly_transaction_throws() + { + using var env = CreateEnvironment(); + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + Should.Throw(() => tx.Prepare()); + } + + public void prepare_twice_throws() + { + using var env = CreateEnvironment(); + env.Open(); + using var tx = env.BeginTransaction(); + using var db = tx.OpenDatabase(); + tx.Put(db, "key"u8.ToArray(), "value"u8.ToArray()); + tx.Prepare().ShouldBe(MDBResultCode.Success); + Should.Throw(() => tx.Prepare()); + tx.Abort(); + } + + public void can_rollback_last_committed_transaction() + { + var path = TempPath(); + using (var env = CreateEnvironment(path)) + { + env.Open(); + + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "first"u8.ToArray(), "kept"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + + ulong lastId; + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "second"u8.ToArray(), "rolled back"u8.ToArray()); + lastId = tx.Id; + tx.Commit().ThrowOnError(); + } + + env.RollbackLastTransaction(lastId).ShouldBe(MDBResultCode.Success); + env.RollbackLastTransaction(lastId).ShouldBe(MDBResultCode.CantRollback); + + //the rollback takes effect immediately in the running environment + using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly)) + using (var db = tx.OpenDatabase()) + { + tx.ContainsKey(db, "first"u8.ToArray()).ShouldBeTrue(); + tx.ContainsKey(db, "second"u8.ToArray()).ShouldBeFalse(); + } + + //a write commit is required after a rollback before the environment + //can be closed and reopened (see RollbackLastTransaction remarks) + using (var tx = env.BeginTransaction()) + using (var db = tx.OpenDatabase()) + { + tx.Put(db, "third"u8.ToArray(), "after rollback"u8.ToArray()); + tx.Commit().ThrowOnError(); + } + } + + using (var env = CreateEnvironment(path)) + { + env.Open(); + using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly); + using var db = tx.OpenDatabase(); + tx.ContainsKey(db, "first"u8.ToArray()).ShouldBeTrue(); + tx.ContainsKey(db, "second"u8.ToArray()).ShouldBeFalse(); + tx.ContainsKey(db, "third"u8.ToArray()).ShouldBeTrue(); + } + } } diff --git a/src/LightningDB/AesGcmCipher.cs b/src/LightningDB/AesGcmCipher.cs new file mode 100644 index 0000000..2b2963d --- /dev/null +++ b/src/LightningDB/AesGcmCipher.cs @@ -0,0 +1,94 @@ +#if NET8_0_OR_GREATER +using System; +using System.Security.Cryptography; +using System.Threading; + +namespace LightningDB; + +/// +/// The built-in AES-256-GCM page cipher. AES-GCM is hardware-accelerated (AES-NI / ARMv8 +/// crypto extensions) on all supported platforms except browser-wasm, making it the +/// recommended default for encrypted environments. +/// +/// +/// The 12-byte GCM nonce is derived from LMDB's per-page IV (page number + transaction id, +/// unique per page write) and the 16-byte tag is stored as the page's authentication data. +/// Not available on platforms where is false (e.g. +/// browser-wasm); supply a custom there. +/// +public sealed class AesGcmCipher : LightningCipher, IDisposable +{ + private const int TagSize = 16; + private const int NonceSize = 12; + + //AesGcm instance methods aren't documented as thread-safe, and pages decrypt + //concurrently from reader threads, so each thread gets its own instance + private readonly ThreadLocal _aes; + + public AesGcmCipher() : base(TagSize) + { + if (!AesGcm.IsSupported) + throw new PlatformNotSupportedException( + "AES-GCM is not supported on this platform; supply a custom LightningCipher instead."); + _aes = new ThreadLocal(trackAllValues: true); + } + + public override bool Encrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, Span authData) + { + Span nonce = stackalloc byte[NonceSize]; + FillNonce(iv, nonce); + GetAes(key).Encrypt(nonce, source, destination, authData); + return true; + } + + public override bool Decrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, ReadOnlySpan authData) + { + Span nonce = stackalloc byte[NonceSize]; + FillNonce(iv, nonce); + try + { + GetAes(key).Decrypt(nonce, source, authData, destination); + return true; + } + catch (AuthenticationTagMismatchException) + { + return false; + } + } + + private AesGcm GetAes(ReadOnlySpan key) + { + var aes = _aes.Value; + if (aes == null) + { + aes = new AesGcm(key, TagSize); + _aes.Value = aes; + } + return aes; + } + + private static void FillNonce(ReadOnlySpan iv, Span nonce) + { + //the IV is 16 bytes on 64-bit platforms and 8 on 32-bit; take what fits + //and zero-pad the remainder + if (iv.Length >= nonce.Length) + { + iv[..nonce.Length].CopyTo(nonce); + } + else + { + nonce.Clear(); + iv.CopyTo(nonce); + } + } + + public void Dispose() + { + foreach (var aes in _aes.Values) + aes.Dispose(); + _aes.Dispose(); + } +} +#endif diff --git a/src/LightningDB/EncryptionConfiguration.cs b/src/LightningDB/EncryptionConfiguration.cs new file mode 100644 index 0000000..980d0c1 --- /dev/null +++ b/src/LightningDB/EncryptionConfiguration.cs @@ -0,0 +1,31 @@ +using System; + +namespace LightningDB; + +/// +/// Encryption settings for an environment: the page cipher and the encryption key. +/// Assign to before the environment is created. +/// +public sealed class EncryptionConfiguration +{ + /// + /// Creates encryption settings for an environment. + /// + /// The page cipher, e.g. AesGcmCipher + /// The encryption key; copied defensively. The same key must be + /// supplied every time the environment is opened. + public EncryptionConfiguration(LightningCipher cipher, ReadOnlySpan key) + { + Cipher = cipher ?? throw new ArgumentNullException(nameof(cipher)); + if (key.IsEmpty) + throw new ArgumentException("An encryption key is required", nameof(key)); + Key = key.ToArray(); + } + + /// + /// The page cipher. + /// + public LightningCipher Cipher { get; } + + internal byte[] Key { get; } +} diff --git a/src/LightningDB/EnvironmentConfiguration.cs b/src/LightningDB/EnvironmentConfiguration.cs index 46aead7..bd8b9f8 100644 --- a/src/LightningDB/EnvironmentConfiguration.cs +++ b/src/LightningDB/EnvironmentConfiguration.cs @@ -68,6 +68,32 @@ public class EnvironmentConfiguration /// public bool AutoReduceMapSizeIn32BitProcess { get; set; } + /// + /// Gets or sets the database page size in bytes, a power of 2 from 512 to 65536. + /// Zero (the default) uses the operating system page size. + /// + /// + /// The page size is stored persistently in the environment when it is first created; + /// it cannot be changed for an existing environment. Larger pages can improve + /// sequential-read throughput; smaller pages can reduce write amplification. + /// + public int PageSize { get; set; } + + /// + /// Gets or sets the encryption settings for the environment. Null (the default) + /// leaves the environment unencrypted. When set, every page is encrypted with the + /// configured cipher before being written and decrypted when read, and the same + /// cipher and key must be configured every time the environment is opened. + /// + public EncryptionConfiguration? Encryption { get; set; } + + /// + /// Gets or sets the page checksum for the environment. Null (the default) disables + /// checksums. When set, every page is checksummed when written and verified when + /// read; a mismatch surfaces as . + /// + public LightningChecksum? Checksum { get; set; } + internal void Configure(LightningEnvironment env) { if (MapSize > 0) @@ -78,5 +104,11 @@ internal void Configure(LightningEnvironment env) if (MaxReaders > 0) env.MaxReaders = MaxReaders; + + if (PageSize > 0) + env.PageSize = PageSize; + + if (Encryption != null || Checksum != null) + env.ConfigurePageCallbacks(Encryption, Checksum); } } \ No newline at end of file diff --git a/src/LightningDB/EnvironmentOpenFlags.cs b/src/LightningDB/EnvironmentOpenFlags.cs index fff272c..6fd893a 100644 --- a/src/LightningDB/EnvironmentOpenFlags.cs +++ b/src/LightningDB/EnvironmentOpenFlags.cs @@ -90,6 +90,26 @@ public enum EnvironmentOpenFlags /// /// MDB_NOMEMINIT. don't initialize malloc'd memory before writing to datafile /// - NoMemoryInitialization = 0x1000000 + NoMemoryInitialization = 0x1000000, + /// + /// MDB_ENCRYPT. Informational flag set internally by the library when encryption is + /// configured via ; visible when reading the + /// environment's flags. Do not pass this flag when opening an environment. + /// + Encrypted = 0x2000, + + /// + /// MDB_PREVSNAPSHOT. Open the environment using the previous snapshot (meta page), + /// losing the latest transaction. Useful for recovery after an application error. + /// If opened with write access this process must be the only one using the environment; + /// the flag is automatically reset after a successful write commit. + /// + PreviousSnapshot = 0x2000000, + + /// + /// MDB_REMAP_CHUNKS. Don't use a single memory map; remap individual chunks on demand. + /// Implied when encryption is configured. + /// + RemapChunks = 0x4000000 } \ No newline at end of file diff --git a/src/LightningDB/LightningChecksum.cs b/src/LightningDB/LightningChecksum.cs new file mode 100644 index 0000000..3c7d80f --- /dev/null +++ b/src/LightningDB/LightningChecksum.cs @@ -0,0 +1,42 @@ +using System; + +namespace LightningDB; + +/// +/// A page checksum for an environment. The configured checksum is computed for every page +/// written and verified for every page read; a mismatch surfaces as +/// . +/// +/// +/// Implementations must be thread-safe and should avoid per-call allocations; the +/// computation sits on the environment's page I/O hot path. +/// +public abstract class LightningChecksum +{ + /// + /// The size of computed checksum values in bytes. + /// + public abstract int Size { get; } + + /// + /// Computes the checksum of the source data. + /// + /// The data to checksum + /// Storage for the checksum, bytes + /// The encryption key when encryption is also configured on the + /// environment (for keyed hashes), otherwise empty + public abstract void Compute(ReadOnlySpan source, Span destination, ReadOnlySpan key); +} + +#if NET8_0_OR_GREATER +/// +/// The built-in SHA-256 page checksum. +/// +public sealed class Sha256Checksum : LightningChecksum +{ + public override int Size => 32; + + public override void Compute(ReadOnlySpan source, Span destination, ReadOnlySpan key) + => System.Security.Cryptography.SHA256.HashData(source, destination); +} +#endif diff --git a/src/LightningDB/LightningCipher.cs b/src/LightningDB/LightningCipher.cs new file mode 100644 index 0000000..bdfc9d5 --- /dev/null +++ b/src/LightningDB/LightningCipher.cs @@ -0,0 +1,60 @@ +using System; + +namespace LightningDB; + +/// +/// A page cipher for an encrypted environment. LMDB stores no cipher itself; the +/// configured cipher is invoked for every page written to or read from the data file. +/// +/// +/// Implementations must be thread-safe: pages may be decrypted concurrently from multiple +/// reader threads. Implementations should avoid per-call allocations; both methods sit on +/// the environment's page I/O hot path. The transform must produce exactly +/// src.Length output bytes; authentication data (e.g. an AEAD tag) goes in +/// authData, sized by . +/// +public abstract class LightningCipher +{ + /// + /// Initializes the cipher. + /// + /// Bytes of per-page authentication data (e.g. 16 for an + /// AES-GCM tag); zero for unauthenticated ciphers + protected LightningCipher(int authDataSize) + { + if (authDataSize < 0) + throw new ArgumentOutOfRangeException(nameof(authDataSize)); + AuthDataSize = authDataSize; + } + + /// + /// Bytes of per-page authentication data reserved at the end of every page. + /// + public int AuthDataSize { get; } + + /// + /// Encrypts a page. + /// + /// The plaintext page data + /// Storage for the ciphertext; exactly source.Length bytes + /// The encryption key configured on the environment + /// The per-page initialization vector supplied by LMDB + /// (the page number followed by the transaction id; unique per page write) + /// Storage for authentication data, bytes + /// True on success, false on failure + public abstract bool Encrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, Span authData); + + /// + /// Decrypts a page. + /// + /// The ciphertext page data + /// Storage for the plaintext; exactly source.Length bytes + /// The encryption key configured on the environment + /// The per-page initialization vector supplied by LMDB + /// The authentication data stored with the page, + /// bytes + /// True on success, false on failure (including authentication failure) + public abstract bool Decrypt(ReadOnlySpan source, Span destination, + ReadOnlySpan key, ReadOnlySpan iv, ReadOnlySpan authData); +} diff --git a/src/LightningDB/LightningDB.csproj b/src/LightningDB/LightningDB.csproj index a3475c8..7c3ebaa 100644 --- a/src/LightningDB/LightningDB.csproj +++ b/src/LightningDB/LightningDB.csproj @@ -2,7 +2,8 @@ LightningDB - 0.22.0 + 0.23.0 + alpha Ilya Lukyanov;Corey Kaylor netstandard2.0;net8.0;net9.0;net10.0 14 diff --git a/src/LightningDB/LightningEnvironment.cs b/src/LightningDB/LightningEnvironment.cs index 36b3b7a..5469d48 100644 --- a/src/LightningDB/LightningEnvironment.cs +++ b/src/LightningDB/LightningEnvironment.cs @@ -12,6 +12,7 @@ namespace LightningDB; public sealed class LightningEnvironment : IDisposable { private readonly EnvironmentConfiguration _config = new(); + private Native.PageCallbackKeepAlive? _pageCallbacks; private bool _disposed; internal nint _handle; @@ -148,6 +149,36 @@ public int MaxDatabases } } + /// + /// Gets or sets the database page size in bytes, a power of 2 from 512 to 65536. + /// Zero (the default) uses the operating system page size. + /// This may only be set before the environment is opened; the page size is stored + /// persistently in the environment when it is first created. + /// + public int PageSize + { + get => _config.PageSize; + set + { + if (IsOpened) + throw new InvalidOperationException("Can't change PageSize of opened environment"); + + if (value < 512 || value > 65536 || (value & (value - 1)) != 0) + throw new ArgumentOutOfRangeException(nameof(value), value, + "PageSize must be a power of 2 from 512 to 65536"); + + mdb_env_set_pagesize(_handle, value).ThrowOnError(); + + _config.PageSize = value; + } + } + + internal void ConfigurePageCallbacks(EncryptionConfiguration? encryption, LightningChecksum? checksum) + { + _pageCallbacks = new Native.PageCallbackKeepAlive(encryption, checksum); + _pageCallbacks.Install(_handle); + } + /// /// Get statistics about the LMDB environment. /// @@ -464,6 +495,105 @@ public MDBResultCode CopyToStream(FileStream fileStream, bool compact = false) : mdb_env_copyfd(_handle, fd); } + /// + /// Rolls back the last committed transaction in the environment, for example when a + /// remote participant of a two-phase commit (see ) + /// successfully prepared but failed to commit. + /// + /// The ID of the transaction to roll back, obtained from + /// of that transaction or + /// .LastTransactionId + /// on success; + /// when a rollback was already performed + /// (only the single last transaction can be rolled back) or when the ID doesn't + /// match the last committed transaction + /// + /// The rollback takes effect immediately in the running environment. As of LMDB 1.0, + /// at least one write transaction must be committed after a rollback before the + /// environment is closed, otherwise the environment cannot be reopened. + /// + public MDBResultCode RollbackLastTransaction(ulong transactionId) + { + EnsureOpened(); + return mdb_env_rollback(_handle, (nuint)transactionId); + } + + /// + /// Dumps pages newer than the given transaction ID to a new file at the specified path, + /// creating an incremental backup. Capture .LastTransactionId at the + /// time of the previous (full or incremental) backup and pass it here. + /// + /// The file in which the incremental dump will reside; must not already exist + /// Dump pages newer than this transaction ID; must be greater than zero + /// A result code indicating success or failure + public MDBResultCode IncrementalCopyTo(string path, ulong sinceTransactionId) + { + if (path == null) + throw new ArgumentNullException(nameof(path)); + if (sinceTransactionId == 0) + throw new ArgumentOutOfRangeException(nameof(sinceTransactionId), + "sinceTransactionId must be greater than zero; use CopyTo for a full backup"); + + EnsureOpened(); + + return mdb_env_incr_dump(_handle, path, (nuint)sinceTransactionId); + } + + /// + /// Dumps pages newer than the given transaction ID to the specified FileStream, + /// creating an incremental backup. Capture .LastTransactionId at the + /// time of the previous (full or incremental) backup and pass it here. + /// + /// The FileStream to write to + /// Dump pages newer than this transaction ID; must be greater than zero + /// A result code indicating success or failure + public MDBResultCode IncrementalCopyToStream(FileStream fileStream, ulong sinceTransactionId) + { + if (fileStream == null) + throw new ArgumentNullException(nameof(fileStream)); + if (!fileStream.CanWrite) + throw new ArgumentException("FileStream must be writable", nameof(fileStream)); + if (sinceTransactionId == 0) + throw new ArgumentOutOfRangeException(nameof(sinceTransactionId), + "sinceTransactionId must be greater than zero; use CopyToStream for a full backup"); + + EnsureOpened(); + + var safeHandle = fileStream.SafeFileHandle; + if (safeHandle == null || safeHandle.IsInvalid) + throw new ArgumentException("Invalid file handle from FileStream", nameof(fileStream)); + + return mdb_env_incr_dumpfd(_handle, safeHandle.DangerousGetHandle(), (nuint)sinceTransactionId); + } + + /// + /// Applies an incremental dump produced by or + /// to this environment. + /// + /// The FileStream to read the incremental dump from + /// A result code indicating success or failure + /// + /// No other operations may run on the environment while the load is in progress, and + /// the environment must be closed and reopened afterwards before the loaded data is + /// visible (mirroring the behavior of the mdb_load -i tool, which opens the + /// environment solely for the load). + /// + public MDBResultCode LoadIncrementalFromStream(FileStream fileStream) + { + if (fileStream == null) + throw new ArgumentNullException(nameof(fileStream)); + if (!fileStream.CanRead) + throw new ArgumentException("FileStream must be readable", nameof(fileStream)); + + EnsureOpened(); + + var safeHandle = fileStream.SafeFileHandle; + if (safeHandle == null || safeHandle.IsInvalid) + throw new ArgumentException("Invalid file handle from FileStream", nameof(fileStream)); + + return mdb_env_incr_loadfd(_handle, safeHandle.DangerousGetHandle()); + } + private void EnsureOpened() { if (!IsOpened) @@ -489,6 +619,11 @@ private void Dispose(bool disposing) IsOpened = false; } + //must outlive mdb_env_close: native code may invoke the page callbacks + //and read the key memory until the environment is closed + _pageCallbacks?.Dispose(); + _pageCallbacks = null; + _handle = 0; GC.SuppressFinalize(this); } diff --git a/src/LightningDB/LightningTransaction.cs b/src/LightningDB/LightningTransaction.cs index 09fb122..be5c5c9 100644 --- a/src/LightningDB/LightningTransaction.cs +++ b/src/LightningDB/LightningTransaction.cs @@ -345,12 +345,35 @@ public MDBResultCode Renew() return result; } + /// + /// Prepares all the operations of a transaction as the first phase of a two-phase commit. + /// + /// + /// After a successful prepare, the only legal operations on the transaction are + /// and ; the effect of any data operation on a + /// prepared transaction is undefined. A prepared transaction that was committed can + /// later be undone with , for + /// example when a remote participant of the two-phase commit failed to commit. + /// + public MDBResultCode Prepare() + { + if (IsReadOnly) + throw new InvalidOperationException("Can't prepare a read-only transaction"); + if (State != LightningTransactionState.Ready) + throw new InvalidOperationException("Transaction that is not ready cannot be prepared"); + + var result = mdb_txn_prepare(_handle); + if (result == MDBResultCode.Success) + State = LightningTransactionState.Prepared; + return result; + } + /// /// Commit all the operations of a transaction into the database. /// public MDBResultCode Commit() { - if(State != LightningTransactionState.Ready) + if(State != LightningTransactionState.Ready && State != LightningTransactionState.Prepared) throw new InvalidOperationException("Transaction that is not ready cannot be committed"); if (ParentTransaction != null && ParentTransaction.State != LightningTransactionState.Ready) { @@ -367,7 +390,7 @@ public MDBResultCode Commit() /// public void Abort() { - if(State != LightningTransactionState.Ready) + if(State != LightningTransactionState.Ready && State != LightningTransactionState.Prepared) throw new InvalidOperationException("Transaction that is not ready cannot be committed"); State = LightningTransactionState.Done; mdb_txn_abort(_handle); @@ -479,7 +502,7 @@ private void Dispose(bool disposing) _disposed = true; if (!Environment.IsOpened) throw new InvalidOperationException("A transaction must be disposed before closing the environment"); - if (State == LightningTransactionState.Ready && Environment.IsOpened) + if (State is LightningTransactionState.Ready or LightningTransactionState.Prepared && Environment.IsOpened) { Abort(); } diff --git a/src/LightningDB/LightningTransactionState.cs b/src/LightningDB/LightningTransactionState.cs index 8cd137e..cd84b4b 100644 --- a/src/LightningDB/LightningTransactionState.cs +++ b/src/LightningDB/LightningTransactionState.cs @@ -23,5 +23,11 @@ public enum LightningTransactionState /// /// Transaction is committed. /// - Released + Released, + + /// + /// Transaction has been prepared (first phase of a two-phase commit) and is awaiting + /// Commit or Abort. + /// + Prepared } \ No newline at end of file diff --git a/src/LightningDB/MDBResultCode.cs b/src/LightningDB/MDBResultCode.cs index f277dbd..1bd1c2d 100644 --- a/src/LightningDB/MDBResultCode.cs +++ b/src/LightningDB/MDBResultCode.cs @@ -95,6 +95,46 @@ public enum MDBResultCode /// Problem = -30779, /// + /// Page checksum incorrect + /// + BadChecksum = -30778, + /// + /// Encryption/decryption failed + /// + CryptoFail = -30777, + /// + /// Environment encryption mismatch + /// + EnvEncryption = -30776, + /// + /// Transaction was already prepared + /// + TxnPending = -30775, + /// + /// Environment can't rollback the last transaction + /// + CantRollback = -30774, + /// + /// Can't drop the main DBI while other DBIs are open + /// + DbisBusy = -30773, + /// + /// Write was incomplete + /// + ShortWrite = -30772, + /// + /// Environment is busy, can't use previous snapshot + /// + EnvBusy = -30771, + /// + /// Environment or transaction is read-only, can't write + /// + ReadOnly = -30770, + /// + /// The requested map address is unavailable + /// + AddressBusy = -30769, + /// /// ENOENT error from C-runtime /// FileNotFound = 2, diff --git a/src/LightningDB/Native/Lmdb.cs b/src/LightningDB/Native/Lmdb.cs index 50d5d5a..e141e03 100644 --- a/src/LightningDB/Native/Lmdb.cs +++ b/src/LightningDB/Native/Lmdb.cs @@ -338,6 +338,91 @@ public static partial class Lmdb [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial MDBResultCode mdb_env_copyfd2(nint env, nint fd, EnvironmentCopyFlags copyFlags); + /// + /// Sets the page size for the environment. Must be called before mdb_env_open. + /// + /// The environment handle + /// The page size in bytes, a power of 2 from 512 to 65536 + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_set_pagesize(nint env, int size); + + /// + /// Sets encryption on the environment. Must be called before mdb_env_open. + /// Implicitly sets MDB_REMAP_CHUNKS on the environment. + /// + /// The environment handle + /// The encryption callback + /// The encryption key + /// The size of per-page authentication data in bytes, zero for unauthenticated ciphers + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_set_encrypt(nint env, EncryptFunction func, ref MDBValue key, uint size); + + /// + /// Sets checksums on the environment. Must be called before mdb_env_open. + /// + /// The environment handle + /// The checksum callback + /// The size of computed checksum values in bytes + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_set_checksum(nint env, ChecksumFunction func, uint size); + + /// + /// Prepares (first phase of a two-phase commit) all the operations of a transaction. + /// + /// The transaction handle + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_txn_prepare(nint txn); + + /// + /// Rolls back the last committed transaction in the environment. + /// + /// The environment handle + /// The ID of the transaction to roll back + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_rollback(nint env, nuint txnid); + + /// + /// Dumps pages newer than the given transaction ID to a new file at the specified path. + /// + /// The environment handle + /// The file in which the incremental dump will reside; must not exist + /// Dump pages newer than this transaction ID + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME, StringMarshalling = StringMarshalling.Utf8)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_incr_dump(nint env, string path, nuint txnid); + + /// + /// Dumps pages newer than the given transaction ID to the specified file descriptor. + /// + /// The environment handle + /// The file descriptor to write to + /// Dump pages newer than this transaction ID + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_incr_dumpfd(nint env, nint fd, nuint txnid); + + /// + /// Applies an incremental dump read from the specified file descriptor to the environment. + /// + /// The environment handle + /// The file descriptor to read from + /// A result code indicating success or failure + [LibraryImport(MDB_DLL_NAME)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial MDBResultCode mdb_env_incr_loadfd(nint env, nint fd); + /// /// Returns information about the LMDB environment. /// @@ -995,6 +1080,96 @@ public static MDBResultCode mdb_env_copy2(nint env, string path, EnvironmentCopy return mdb_env_copy2(env, bytes, copyFlags); } + /// + /// Sets the page size for the environment. Must be called before mdb_env_open. + /// + /// The environment handle + /// The page size in bytes, a power of 2 from 512 to 65536 + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_set_pagesize(nint env, int size); + + /// + /// Sets encryption on the environment. Must be called before mdb_env_open. + /// Implicitly sets MDB_REMAP_CHUNKS on the environment. + /// + /// The environment handle + /// The encryption callback + /// The encryption key + /// The size of per-page authentication data in bytes, zero for unauthenticated ciphers + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_set_encrypt(nint env, EncryptFunction func, ref MDBValue key, uint size); + + /// + /// Sets checksums on the environment. Must be called before mdb_env_open. + /// + /// The environment handle + /// The checksum callback + /// The size of computed checksum values in bytes + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_set_checksum(nint env, ChecksumFunction func, uint size); + + /// + /// Prepares (first phase of a two-phase commit) all the operations of a transaction. + /// + /// The transaction handle + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_txn_prepare(nint txn); + + /// + /// Rolls back the last committed transaction in the environment. + /// + /// The environment handle + /// The ID of the transaction to roll back + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_rollback(nint env, nuint txnid); + + /// + /// Dumps pages newer than the given transaction ID to a new file at the specified path. + /// + /// The environment handle + /// The file in which the incremental dump will reside; must not exist + /// Dump pages newer than this transaction ID + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_incr_dump(nint env, byte[] path, nuint txnid); + + /// + /// Dumps pages newer than the given transaction ID to a new file at the specified path. + /// + /// The environment handle + /// The file in which the incremental dump will reside; must not exist + /// Dump pages newer than this transaction ID + /// A result code indicating success or failure + public static MDBResultCode mdb_env_incr_dump(nint env, string path, nuint txnid) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(path + "\0"); + return mdb_env_incr_dump(env, bytes, txnid); + } + + /// + /// Dumps pages newer than the given transaction ID to the specified file descriptor. + /// + /// The environment handle + /// The file descriptor to write to + /// Dump pages newer than this transaction ID + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_incr_dumpfd(nint env, nint fd, nuint txnid); + + /// + /// Applies an incremental dump read from the specified file descriptor to the environment. + /// + /// The environment handle + /// The file descriptor to read from + /// A result code indicating success or failure + [DllImport(MDB_DLL_NAME, CallingConvention = CallingConvention.Cdecl)] + public static extern MDBResultCode mdb_env_incr_loadfd(nint env, nint fd); + /// /// Returns information about the LMDB environment. /// diff --git a/src/LightningDB/Native/PageCallbackKeepAlive.cs b/src/LightningDB/Native/PageCallbackKeepAlive.cs new file mode 100644 index 0000000..67eafe0 --- /dev/null +++ b/src/LightningDB/Native/PageCallbackKeepAlive.cs @@ -0,0 +1,102 @@ +using System; +using System.Runtime.InteropServices; +using static LightningDB.Native.Lmdb; + +namespace LightningDB.Native; + +/// +/// Owns the native resources behind an environment's encryption/checksum configuration: +/// the callback delegates (which must outlive the environment because native code holds +/// their function pointers) and an unmanaged copy of the encryption key (which LMDB reads +/// on every page operation). +/// +internal sealed unsafe class PageCallbackKeepAlive : IDisposable +{ + private readonly LightningCipher? _cipher; + private readonly LightningChecksum? _checksum; + private readonly EncryptFunction? _encryptFunction; + private readonly ChecksumFunction? _checksumFunction; + private GCHandle _encryptHandle; + private GCHandle _checksumHandle; + private nint _keyPtr; + private readonly int _keyLength; + + public PageCallbackKeepAlive(EncryptionConfiguration? encryption, LightningChecksum? checksum) + { + _checksum = checksum; + if (checksum != null) + { + _checksumFunction = OnChecksum; + _checksumHandle = GCHandle.Alloc(_checksumFunction); + } + + if (encryption == null) + return; + _cipher = encryption.Cipher; + _encryptFunction = OnEncrypt; + _encryptHandle = GCHandle.Alloc(_encryptFunction); + _keyLength = encryption.Key.Length; + _keyPtr = Marshal.AllocHGlobal(_keyLength); + Marshal.Copy(encryption.Key, 0, _keyPtr, _keyLength); + } + + public void Install(nint envHandle) + { + if (_encryptFunction != null) + { + var key = new MDBValue(_keyLength, (byte*)_keyPtr); + mdb_env_set_encrypt(envHandle, _encryptFunction, ref key, (uint)_cipher!.AuthDataSize).ThrowOnError(); + } + if (_checksumFunction != null) + { + mdb_env_set_checksum(envHandle, _checksumFunction, (uint)_checksum!.Size).ThrowOnError(); + } + } + + private int OnEncrypt(MDBValue* src, MDBValue* dst, MDBValue* key, int encdec) + { + try + { + var ok = encdec != 0 + ? _cipher!.Encrypt(src->AsSpan(), dst->AsWritableSpan(), + key[0].AsSpan(), key[1].AsSpan(), key[2].AsWritableSpan()) + : _cipher!.Decrypt(src->AsSpan(), dst->AsWritableSpan(), + key[0].AsSpan(), key[1].AsSpan(), key[2].AsSpan()); + return ok ? 0 : -1; + } + catch + { + //exceptions must never cross the native boundary + return -1; + } + } + + private void OnChecksum(MDBValue* src, MDBValue* dst, MDBValue* key) + { + try + { + _checksum!.Compute(src->AsSpan(), dst->AsWritableSpan(), + key == null ? default : key->AsSpan()); + } + catch + { + //exceptions must never cross the native boundary; a wrong checksum + //surfaces as MDBResultCode.BadChecksum on the next read + } + } + + public void Dispose() + { + if (_keyPtr != 0) + { + new Span((void*)_keyPtr, _keyLength).Clear(); + Marshal.FreeHGlobal(_keyPtr); + _keyPtr = 0; + } + if (_encryptHandle.IsAllocated) + _encryptHandle.Free(); + if (_checksumHandle.IsAllocated) + _checksumHandle.Free(); + (_cipher as IDisposable)?.Dispose(); + } +} diff --git a/src/LightningDB/Native/PageFunctions.cs b/src/LightningDB/Native/PageFunctions.cs new file mode 100644 index 0000000..6c8e98d --- /dev/null +++ b/src/LightningDB/Native/PageFunctions.cs @@ -0,0 +1,28 @@ +using System.Runtime.InteropServices; + +namespace LightningDB.Native; + +/// +/// MDB_enc_func. A callback used to encrypt/decrypt pages in the environment. +/// Encrypt or decrypt the data in src and store the result in dst using the provided key. +/// The result must be the same number of bytes as the input. +/// +/// The input data to be transformed +/// Storage for the result +/// A pointer to an array of THREE MDB_vals: key[0] is the encryption key, +/// key[1] is the initialization vector, and key[2] is the authentication data (written on +/// encrypt, verified on decrypt), if any +/// 1 to encrypt, 0 to decrypt +/// Zero on success, non-zero on failure +[UnmanagedFunctionPointer(CallingConvention.Cdecl)] +public unsafe delegate int EncryptFunction(MDBValue* src, MDBValue* dst, MDBValue* key, int encdec); + +/// +/// MDB_sum_func. A callback used to checksum pages in the environment. +/// Compute the checksum of the data in src and store the result in dst. +/// +/// The input data to be transformed +/// Storage for the result +/// The encryption key if encryption is also configured, otherwise null +[UnmanagedFunctionPointer(CallingConvention.Cdecl)] +public unsafe delegate void ChecksumFunction(MDBValue* src, MDBValue* dst, MDBValue* key); diff --git a/src/LightningDB/TransactionBeginFlags.cs b/src/LightningDB/TransactionBeginFlags.cs index 8fdde99..a8fbd63 100644 --- a/src/LightningDB/TransactionBeginFlags.cs +++ b/src/LightningDB/TransactionBeginFlags.cs @@ -19,6 +19,7 @@ public enum TransactionBeginFlags /// Note that (MDB_NOSYNC | MDB_WRITEMAP) leaves the system with no hint for when to write transactions to disk, unless mdb_env_sync() is called. /// (MDB_MAPASYNC | MDB_WRITEMAP) may be preferable. /// This flag may be changed at any time using mdb_env_set_flags(). + /// As of LMDB 1.0 this flag is honored per-transaction when passed to BeginTransaction. /// NoSync = 0x10000, diff --git a/src/LightningDB/runtimes/android-arm/native/liblmdb.so b/src/LightningDB/runtimes/android-arm/native/liblmdb.so index dc81f2d..806d3c6 100755 Binary files a/src/LightningDB/runtimes/android-arm/native/liblmdb.so and b/src/LightningDB/runtimes/android-arm/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/android-arm64/native/liblmdb.so b/src/LightningDB/runtimes/android-arm64/native/liblmdb.so index fa2aa0e..11d0dfe 100755 Binary files a/src/LightningDB/runtimes/android-arm64/native/liblmdb.so and b/src/LightningDB/runtimes/android-arm64/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/android-x64/native/liblmdb.so b/src/LightningDB/runtimes/android-x64/native/liblmdb.so index 9627433..6bc7bc0 100755 Binary files a/src/LightningDB/runtimes/android-x64/native/liblmdb.so and b/src/LightningDB/runtimes/android-x64/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/android-x86/native/liblmdb.so b/src/LightningDB/runtimes/android-x86/native/liblmdb.so index 6e74965..543a0b8 100755 Binary files a/src/LightningDB/runtimes/android-x86/native/liblmdb.so and b/src/LightningDB/runtimes/android-x86/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/browser-wasm/native/liblmdb.wasm b/src/LightningDB/runtimes/browser-wasm/native/liblmdb.wasm index 2a33733..343fe46 100755 Binary files a/src/LightningDB/runtimes/browser-wasm/native/liblmdb.wasm and b/src/LightningDB/runtimes/browser-wasm/native/liblmdb.wasm differ diff --git a/src/LightningDB/runtimes/ios-arm64/native/liblmdb.dylib b/src/LightningDB/runtimes/ios-arm64/native/liblmdb.dylib index 9be988a..3651177 100755 Binary files a/src/LightningDB/runtimes/ios-arm64/native/liblmdb.dylib and b/src/LightningDB/runtimes/ios-arm64/native/liblmdb.dylib differ diff --git a/src/LightningDB/runtimes/iossimulator-arm64/native/liblmdb.dylib b/src/LightningDB/runtimes/iossimulator-arm64/native/liblmdb.dylib index 6a4b8d1..7694a1c 100755 Binary files a/src/LightningDB/runtimes/iossimulator-arm64/native/liblmdb.dylib and b/src/LightningDB/runtimes/iossimulator-arm64/native/liblmdb.dylib differ diff --git a/src/LightningDB/runtimes/iossimulator-x64/native/liblmdb.dylib b/src/LightningDB/runtimes/iossimulator-x64/native/liblmdb.dylib index 09597a8..04045fb 100755 Binary files a/src/LightningDB/runtimes/iossimulator-x64/native/liblmdb.dylib and b/src/LightningDB/runtimes/iossimulator-x64/native/liblmdb.dylib differ diff --git a/src/LightningDB/runtimes/linux-arm/native/liblmdb.so b/src/LightningDB/runtimes/linux-arm/native/liblmdb.so index 7c69206..ef0bfd0 100755 Binary files a/src/LightningDB/runtimes/linux-arm/native/liblmdb.so and b/src/LightningDB/runtimes/linux-arm/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/linux-arm64/native/liblmdb.so b/src/LightningDB/runtimes/linux-arm64/native/liblmdb.so index fd4e414..e43e237 100755 Binary files a/src/LightningDB/runtimes/linux-arm64/native/liblmdb.so and b/src/LightningDB/runtimes/linux-arm64/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/linux-x64/native/liblmdb.so b/src/LightningDB/runtimes/linux-x64/native/liblmdb.so index 6171b7d..eb50a28 100755 Binary files a/src/LightningDB/runtimes/linux-x64/native/liblmdb.so and b/src/LightningDB/runtimes/linux-x64/native/liblmdb.so differ diff --git a/src/LightningDB/runtimes/osx-arm64/native/lmdb.dylib b/src/LightningDB/runtimes/osx-arm64/native/lmdb.dylib index 2a33733..7aff571 100755 Binary files a/src/LightningDB/runtimes/osx-arm64/native/lmdb.dylib and b/src/LightningDB/runtimes/osx-arm64/native/lmdb.dylib differ diff --git a/src/LightningDB/runtimes/osx/native/lmdb.dylib b/src/LightningDB/runtimes/osx/native/lmdb.dylib index 704668a..32d0970 100755 Binary files a/src/LightningDB/runtimes/osx/native/lmdb.dylib and b/src/LightningDB/runtimes/osx/native/lmdb.dylib differ diff --git a/src/LightningDB/runtimes/win-arm64/native/lmdb.dll b/src/LightningDB/runtimes/win-arm64/native/lmdb.dll index 28a03d8..9a999cb 100755 Binary files a/src/LightningDB/runtimes/win-arm64/native/lmdb.dll and b/src/LightningDB/runtimes/win-arm64/native/lmdb.dll differ diff --git a/src/LightningDB/runtimes/win-x64/native/lmdb.dll b/src/LightningDB/runtimes/win-x64/native/lmdb.dll index 1e2c7bb..e8e645b 100755 Binary files a/src/LightningDB/runtimes/win-x64/native/lmdb.dll and b/src/LightningDB/runtimes/win-x64/native/lmdb.dll differ diff --git a/src/LightningDB/runtimes/win-x86/native/lmdb.dll b/src/LightningDB/runtimes/win-x86/native/lmdb.dll index cbaccce..5e9ab9a 100755 Binary files a/src/LightningDB/runtimes/win-x86/native/lmdb.dll and b/src/LightningDB/runtimes/win-x86/native/lmdb.dll differ