diff --git a/Tharga.Cache.Redis.Tests/RedisResiliencePolicyTests.cs b/Tharga.Cache.Redis.Tests/RedisResiliencePolicyTests.cs new file mode 100644 index 0000000..bd02e09 --- /dev/null +++ b/Tharga.Cache.Redis.Tests/RedisResiliencePolicyTests.cs @@ -0,0 +1,78 @@ +using FluentAssertions; +using Polly; +using Polly.CircuitBreaker; +using Xunit; + +namespace Tharga.Cache.Redis.Tests; + +public class RedisResiliencePolicyTests +{ + [Fact] + public async Task Circuit_opens_after_threshold_and_then_fails_fast_without_invoking_backend() + { + //Arrange — no retry so the breaker trips deterministically after exactly `threshold` failures. + var options = new RedisCacheOptions + { + RetryCount = 0, + CircuitBreakerFailureThreshold = 2, + CircuitBreakerDuration = TimeSpan.FromMinutes(1) + }; + var policy = RedisResiliencePolicy.Create(options, null); + + var invocations = 0; + Func failing = async () => + { + invocations++; + await Task.Yield(); + throw new TimeoutException("simulated outage"); + }; + + //Act — trip the breaker. + for (var i = 0; i < options.CircuitBreakerFailureThreshold; i++) + { + await policy.Invoking(p => p.ExecuteAsync(failing)).Should().ThrowAsync(); + } + + var invocationsBeforeOpen = invocations; + + //Assert — the circuit is now open: the next call fails fast without touching the backend. + await policy.Invoking(p => p.ExecuteAsync(failing)).Should().ThrowAsync(); + invocations.Should().Be(invocationsBeforeOpen, "an open circuit must short-circuit instead of invoking the backend"); + } + + [Fact] + public async Task Successful_call_passes_through() + { + //Arrange + var policy = RedisResiliencePolicy.Create(new RedisCacheOptions { RetryCount = 0, CircuitBreakerFailureThreshold = 5 }, null); + + //Act + var result = await policy.ExecuteAsync(() => Task.FromResult(42)); + + //Assert + result.Should().Be(42); + } + + [Fact] + public async Task Retries_transient_failures_then_succeeds() + { + //Arrange + var options = new RedisCacheOptions { RetryCount = 2, CircuitBreakerFailureThreshold = 5 }; + var policy = RedisResiliencePolicy.Create(options, null); + + var attempts = 0; + + //Act + var result = await policy.ExecuteAsync(async () => + { + attempts++; + await Task.Yield(); + if (attempts < 3) throw new TimeoutException("transient"); + return "ok"; + }); + + //Assert + result.Should().Be("ok"); + attempts.Should().Be(3, "the call should be retried twice before succeeding on the third attempt"); + } +} diff --git a/Tharga.Cache.Redis/README.md b/Tharga.Cache.Redis/README.md index 69f683d..6271379 100644 --- a/Tharga.Cache.Redis/README.md +++ b/Tharga.Cache.Redis/README.md @@ -31,6 +31,28 @@ Any type registered with `IRedis` is persisted to Redis. Unregistered types defa - **Survives restarts** — cached data is not lost on deploy - **High throughput** with low latency +## Resilience (fail-open) + +If Redis becomes unreachable, the cache **fails open**: a backend read error is treated as a miss so the +call falls through to your source loader, and a backend write error is swallowed. A cache outage therefore +never faults the caller as long as the source of truth is healthy. This is on by default and can be turned +off with `CacheOptions.FailOpenOnBackendError = false` (which restores the previous throwing behavior). + +A Polly **circuit breaker** sits in front of the Redis connection so a sustained outage short-circuits +immediately instead of paying retry latency on every call (which is what prevents thread-pool starvation). +The breaker recovers automatically once Redis is healthy again. + +```csharp +o.AddRedisDBOptions(r => +{ + r.ConnectionStringLoader = sp => "localhost:6379"; + r.RetryCount = 3; // transient-error retries before a call fails (default 3) + r.CircuitBreakerFailureThreshold = 5; // consecutive failures before the circuit opens (default 5) + r.CircuitBreakerDuration = TimeSpan.FromSeconds(30); // how long it stays open before probing again (default 30s) + r.CommandTimeout = TimeSpan.FromSeconds(1); // optional shorter per-command timeout for fast fail-open +}); +``` + ## Documentation Full documentation, configuration options, and samples are available on the [GitHub project page](https://github.com/Tharga/Cache). diff --git a/Tharga.Cache.Redis/Redis.cs b/Tharga.Cache.Redis/Redis.cs index 6c024ce..3423173 100644 --- a/Tharga.Cache.Redis/Redis.cs +++ b/Tharga.Cache.Redis/Redis.cs @@ -1,11 +1,10 @@ using System.Diagnostics; -using System.Net.Sockets; using System.Text.Json; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Polly; -using Polly.Retry; +using Polly.CircuitBreaker; using StackExchange.Redis; using Tharga.Cache.Core; @@ -17,7 +16,7 @@ internal class Redis : IRedis private readonly IHostEnvironment _hostEnvironment; private readonly RedisCacheOptions _options; private readonly ILogger _logger; - private readonly AsyncRetryPolicy _retryPolicy; + private readonly IAsyncPolicy _resiliencePolicy; private ConnectionMultiplexer _redisConnection; public Redis(IServiceProvider serviceProvider, IHostEnvironment hostEnvironment, IManagedCacheMonitor cacheMonitor, IOptions options, ILogger logger) @@ -26,14 +25,7 @@ public Redis(IServiceProvider serviceProvider, IHostEnvironment hostEnvironment, _hostEnvironment = hostEnvironment; _options = options.Value; _logger = logger; - _retryPolicy = Policy - .Handle() - .Or() - .Or() - .WaitAndRetryAsync( - 3, - attempt => TimeSpan.FromMilliseconds(200 * Math.Pow(2, attempt)), - (exception, timeSpan, retryCount, _) => { _logger.LogWarning($"Retry {retryCount} after {timeSpan.TotalMilliseconds}ms due to: {exception.Message}"); }); + _resiliencePolicy = RedisResiliencePolicy.Create(_options, logger); cacheMonitor.RequestEvictEvent += async (_, e) => { @@ -50,7 +42,7 @@ public Redis(IServiceProvider serviceProvider, IHostEnvironment hostEnvironment, public async Task> GetAsync(Key key) { - return await _retryPolicy.ExecuteAsync(async () => + return await _resiliencePolicy.ExecuteAsync(async () => { var redisConnection = await GetConnection(); if (redisConnection.Multiplexer == null) return null; @@ -74,7 +66,7 @@ public async Task> GetAsync(Key key) public async Task SetAsync(Key key, CacheItem cacheItem, bool staleWhileRevalidate) { - await _retryPolicy.ExecuteAsync(async () => + await _resiliencePolicy.ExecuteAsync(async () => { var item = JsonSerializer.Serialize(cacheItem); if (Debugger.IsAttached) @@ -101,17 +93,17 @@ await _retryPolicy.ExecuteAsync(async () => public async Task BuyMoreTime(Key key) { - return await _retryPolicy.ExecuteAsync(async () => await SetUpdateTime(key, DateTime.UtcNow)); + return await _resiliencePolicy.ExecuteAsync(async () => await SetUpdateTime(key, DateTime.UtcNow)); } public async Task Invalidate(Key key) { - return await _retryPolicy.ExecuteAsync(async () => await SetUpdateTime(key, DateTime.MinValue)); + return await _resiliencePolicy.ExecuteAsync(async () => await SetUpdateTime(key, DateTime.MinValue)); } public async Task DropAsync(Key key) { - return await _retryPolicy.ExecuteAsync(async () => + return await _resiliencePolicy.ExecuteAsync(async () => { var redisConnection = await GetConnection(); if (redisConnection.Multiplexer == null) return false; @@ -124,13 +116,20 @@ public async Task DropAsync(Key key) public async Task<(bool Success, string Message)> CanConnectAsync() { - return await _retryPolicy.ExecuteAsync(async () => + try { - var redisConnection = await GetConnection(); - if (redisConnection.Multiplexer == null) return (false, redisConnection.Message); + return await _resiliencePolicy.ExecuteAsync(async () => + { + var redisConnection = await GetConnection(); + if (redisConnection.Multiplexer == null) return (false, redisConnection.Message); - return (redisConnection.Multiplexer.IsConnected, redisConnection.Message); - }); + return (redisConnection.Multiplexer.IsConnected, redisConnection.Message); + }); + } + catch (BrokenCircuitException e) + { + return (false, $"Redis circuit is open: {e.Message}"); + } } private async Task SetUpdateTime(Key key, DateTime updateTime) @@ -174,7 +173,20 @@ private async Task SetUpdateTime(Key key, DateTime updateTime) try { - _redisConnection = await ConnectionMultiplexer.ConnectAsync(connectionString); + if (_options.CommandTimeout is { } commandTimeout) + { + var config = ConfigurationOptions.Parse(connectionString); + var milliseconds = (int)commandTimeout.TotalMilliseconds; + config.AsyncTimeout = milliseconds; + config.SyncTimeout = milliseconds; + config.ConnectTimeout = milliseconds; + _redisConnection = await ConnectionMultiplexer.ConnectAsync(config); + } + else + { + _redisConnection = await ConnectionMultiplexer.ConnectAsync(connectionString); + } + return (_redisConnection, "Connected to Redis."); } catch (Exception e) diff --git a/Tharga.Cache.Redis/RedisCacheOptions.cs b/Tharga.Cache.Redis/RedisCacheOptions.cs index 514fe33..073bd8c 100644 --- a/Tharga.Cache.Redis/RedisCacheOptions.cs +++ b/Tharga.Cache.Redis/RedisCacheOptions.cs @@ -1,6 +1,33 @@ -namespace Tharga.Cache.Redis; +namespace Tharga.Cache.Redis; public record RedisCacheOptions { public Func ConnectionStringLoader; -} \ No newline at end of file + + /// + /// Number of retry attempts on a transient Redis error before a call is considered failed. Default 3. + /// Set to 0 to disable retries (each call is attempted once). + /// + public int RetryCount { get; set; } = 3; + + /// + /// Number of consecutive failed calls before the circuit opens and subsequent calls fail fast + /// (throwing ) instead of paying retry latency. + /// Combined with , an open circuit means the + /// cache falls straight through to the source loader. Default 5. + /// + public int CircuitBreakerFailureThreshold { get; set; } = 5; + + /// + /// How long the circuit stays open before it transitions to half-open and probes the backend with a + /// single trial call. Default 30 seconds. + /// + public TimeSpan CircuitBreakerDuration { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Optional per-command timeout applied to the StackExchange.Redis connection + /// (Async / Sync / Connect timeout). A shorter timeout makes the fail-open path fast even before the + /// circuit breaker opens. When null, the connection-string / library defaults apply. + /// + public TimeSpan? CommandTimeout { get; set; } +} diff --git a/Tharga.Cache.Redis/RedisResiliencePolicy.cs b/Tharga.Cache.Redis/RedisResiliencePolicy.cs new file mode 100644 index 0000000..3d1cc51 --- /dev/null +++ b/Tharga.Cache.Redis/RedisResiliencePolicy.cs @@ -0,0 +1,42 @@ +using System.Net.Sockets; +using Microsoft.Extensions.Logging; +using Polly; +using StackExchange.Redis; + +namespace Tharga.Cache.Redis; + +/// +/// Builds the resilience pipeline used by the Redis persist backend: a retry policy wrapped by a circuit breaker. +/// The breaker is the outer policy, so once it is open calls fail fast () +/// without paying any retry latency — which is what prevents a sustained outage from starving the thread pool. +/// +internal static class RedisResiliencePolicy +{ + internal static IAsyncPolicy Create(RedisCacheOptions options, ILogger logger) + { + var retryCount = Math.Max(0, options.RetryCount); + var retryPolicy = Policy + .Handle() + .Or() + .Or() + .WaitAndRetryAsync( + retryCount, + attempt => TimeSpan.FromMilliseconds(200 * Math.Pow(2, attempt)), + (exception, timeSpan, retryCount, _) => logger?.LogWarning("Redis retry {RetryCount} after {Delay}ms due to: {Message}", retryCount, timeSpan.TotalMilliseconds, exception.Message)); + + var threshold = Math.Max(1, options.CircuitBreakerFailureThreshold); + var circuitBreakerPolicy = Policy + .Handle() + .Or() + .Or() + .CircuitBreakerAsync( + threshold, + options.CircuitBreakerDuration, + onBreak: (exception, breakDelay) => logger?.LogWarning("Redis circuit opened for {Delay}ms after {Threshold} consecutive failures: {Message}", breakDelay.TotalMilliseconds, threshold, exception.Message), + onReset: () => logger?.LogInformation("Redis circuit reset; calls are flowing to the backend again."), + onHalfOpen: () => logger?.LogInformation("Redis circuit half-open; probing the backend with a trial call.")); + + // Breaker (outer) wraps retry (inner): when the breaker is open, the retry never runs. + return Policy.WrapAsync(circuitBreakerPolicy, retryPolicy); + } +} diff --git a/Tharga.Cache.Tests/FailOpenTests.cs b/Tharga.Cache.Tests/FailOpenTests.cs new file mode 100644 index 0000000..c0e8263 --- /dev/null +++ b/Tharga.Cache.Tests/FailOpenTests.cs @@ -0,0 +1,129 @@ +using FluentAssertions; +using Moq; +using Tharga.Cache.Core; +using Tharga.Cache.Persist; +using Xunit; + +namespace Tharga.Cache.Tests; + +public class FailOpenTests +{ + [Fact] + public async Task GetAsync_returns_source_loader_result_when_backend_read_throws() + { + //Arrange + var (sut, persist) = BuildCache(failOpen: true); + + //Act + var result = await sut.GetAsync("Key", () => Task.FromResult("from-source")); + + //Assert + result.Should().Be("from-source", "a backend read failure should fail open to the source loader"); + persist.GetCalls.Should().BeGreaterThan(0, "the backend should have been attempted before failing open"); + } + + [Fact] + public async Task GetAsync_does_not_fault_when_backend_write_throws() + { + //Arrange — read fails open to the loader, then the write-back to the backend also throws. + var (sut, persist) = BuildCache(failOpen: true); + + //Act + var result = await sut.GetAsync("Key", () => Task.FromResult("from-source")); + + //Assert + result.Should().Be("from-source", "a failed cache write must not fault the caller"); + persist.SetCalls.Should().BeGreaterThan(0, "the write-back should have been attempted"); + } + + [Fact] + public async Task SetAsync_does_not_fault_when_backend_write_throws() + { + //Arrange + var (sut, persist) = BuildCache(failOpen: true); + + //Act + var act = async () => await sut.SetAsync("Key", "value"); + + //Assert + await act.Should().NotThrowAsync("an explicit SetAsync must not fault when the backend write fails"); + persist.SetCalls.Should().BeGreaterThan(0); + } + + [Fact] + public async Task GetAsync_rethrows_when_FailOpenOnBackendError_is_disabled() + { + //Arrange + var (sut, _) = BuildCache(failOpen: false); + + //Act + var act = async () => await sut.GetAsync("Key", () => Task.FromResult("from-source")); + + //Assert + await act.Should().ThrowAsync("fail-open is opt-out; disabling it preserves the throwing behavior"); + } + + private static (ICache Cache, ThrowingPersist Persist) BuildCache(bool failOpen) + { + var options = new CacheOptions + { + FailOpenOnBackendError = failOpen, + Default = new CacheTypeOptions { DefaultFreshSpan = TimeSpan.FromSeconds(10) } + }; + options.RegisterType(s => s.DefaultFreshSpan = TimeSpan.FromSeconds(10)); + + var persist = new ThrowingPersist(); + var persistLoader = new Mock(MockBehavior.Strict); + persistLoader.Setup(x => x.GetPersist(It.IsAny())).Returns(persist); + var cacheMonitor = new CacheMonitor(persistLoader.Object, options); + var fetchQueue = new FetchQueue(cacheMonitor, options, null); + var cache = new TimeToLiveCache(cacheMonitor, persistLoader.Object, fetchQueue, options, null); + return (cache, persist); + } + + /// An IPersist whose reads and writes always throw, simulating a backend that times out commands. + private sealed class ThrowingPersist : IPersist + { + private int _getCalls; + private int _setCalls; + + public int GetCalls => _getCalls; + public int SetCalls => _setCalls; + + public async Task> GetAsync(Key key) + { + Interlocked.Increment(ref _getCalls); + await Task.Yield(); + throw new TimeoutException("Simulated backend read timeout."); + } + + public IAsyncEnumerable<(Key Key, CacheItem CacheItem)> FindAsync(Key key) => throw new NotImplementedException(); + + public async Task SetAsync(Key key, CacheItem cacheItem, bool staleWhileRevalidate) + { + Interlocked.Increment(ref _setCalls); + await Task.Yield(); + throw new TimeoutException("Simulated backend write timeout."); + } + + public async Task BuyMoreTime(Key key) + { + await Task.Yield(); + throw new TimeoutException("Simulated backend timeout."); + } + + public async Task Invalidate(Key key) + { + await Task.Yield(); + throw new TimeoutException("Simulated backend timeout."); + } + + public async Task DropAsync(Key key) + { + await Task.Yield(); + throw new TimeoutException("Simulated backend timeout."); + } + + public Task<(bool Success, string Message)> CanConnectAsync() => Task.FromResult((false, "Simulated backend outage.")); + } +} diff --git a/Tharga.Cache/CacheOptions.cs b/Tharga.Cache/CacheOptions.cs index 09a6bef..d52bc2c 100644 --- a/Tharga.Cache/CacheOptions.cs +++ b/Tharga.Cache/CacheOptions.cs @@ -16,6 +16,13 @@ public record CacheOptions /// public TimeSpan WatchDogInterval { get; set; } = TimeSpan.FromSeconds(60); + /// + /// When true (default), an exception thrown by the persist backend is logged and treated as a cache miss + /// for reads (control flows to the source loader) and is swallowed for writes — so a backend outage never + /// faults the caller. Set to false to restore the previous behavior where backend exceptions propagate. + /// + public bool FailOpenOnBackendError { get; set; } = true; + public void RegisterType(Action options = null) where TPersist : IPersist { var typeOptions = (Default ?? BuildDefault()) with { }; diff --git a/Tharga.Cache/CacheRegistrationExtensions.cs b/Tharga.Cache/CacheRegistrationExtensions.cs index 3b35c5b..97a242e 100644 --- a/Tharga.Cache/CacheRegistrationExtensions.cs +++ b/Tharga.Cache/CacheRegistrationExtensions.cs @@ -65,7 +65,7 @@ public static void AddCache(this IServiceCollection serviceCollection, Action(); var persistLoader = s.GetService(); var fetchQueue = s.GetService(); - return new EternalCache(cacheMonitor, persistLoader, fetchQueue, opts); + return new EternalCache(cacheMonitor, persistLoader, fetchQueue, opts, CreateCacheLogger(s)); }); serviceCollection.TryAddSingleton(s => { @@ -73,7 +73,7 @@ public static void AddCache(this IServiceCollection serviceCollection, Action(); var persistLoader = s.GetService(); var fetchQueue = s.GetService(); - return new TimeToLiveCache(cacheMonitor, persistLoader, fetchQueue, opts); + return new TimeToLiveCache(cacheMonitor, persistLoader, fetchQueue, opts, CreateCacheLogger(s)); }); serviceCollection.TryAddSingleton(s => { @@ -81,7 +81,7 @@ public static void AddCache(this IServiceCollection serviceCollection, Action(); var persistLoader = s.GetService(); var fetchQueue = s.GetService(); - return new TimeToIdleCache(cacheMonitor, persistLoader, fetchQueue, opts); + return new TimeToIdleCache(cacheMonitor, persistLoader, fetchQueue, opts, CreateCacheLogger(s)); }); serviceCollection.TryAddScoped(s => { @@ -89,7 +89,7 @@ public static void AddCache(this IServiceCollection serviceCollection, Action(); var persistLoader = s.GetService(); var fetchQueue = s.GetService(); - return new EternalCache(cacheMonitor, persistLoader, fetchQueue, opts); + return new EternalCache(cacheMonitor, persistLoader, fetchQueue, opts, CreateCacheLogger(s)); }); serviceCollection.TryAddSingleton(); @@ -120,6 +120,11 @@ private static void AppendPreviousRegistrations(CacheOptions o) } } + private static ILogger CreateCacheLogger(IServiceProvider serviceProvider) + { + return serviceProvider.GetService()?.CreateLogger("Tharga.Cache.Cache"); + } + private static void RegisterPersist(IServiceCollection serviceCollection) { serviceCollection.TryAddTransient(); diff --git a/Tharga.Cache/Core/CacheBase.cs b/Tharga.Cache/Core/CacheBase.cs index b4e6633..1973121 100644 --- a/Tharga.Cache/Core/CacheBase.cs +++ b/Tharga.Cache/Core/CacheBase.cs @@ -1,4 +1,5 @@ -using Tharga.Cache.Persist; +using Microsoft.Extensions.Logging; +using Tharga.Cache.Persist; namespace Tharga.Cache.Core; @@ -7,9 +8,10 @@ internal abstract class CacheBase : ICache private readonly IManagedCacheMonitor _cacheMonitor; private readonly IPersistLoader _persistLoader; private readonly IFetchQueue _fetchQueue; + private readonly ILogger _logger; protected readonly CacheOptions _options; - protected CacheBase(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) + protected CacheBase(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) { if (options.MaxConcurrentFetchCount <= 0) throw new InvalidOperationException($"Min value for {nameof(options.MaxConcurrentFetchCount)} is 1."); @@ -17,6 +19,7 @@ protected CacheBase(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoa _persistLoader = persistLoader; _fetchQueue = fetchQueue; _options = options; + _logger = logger; } public event EventHandler DataSetEvent; @@ -36,7 +39,7 @@ public virtual async Task GetAsync(Key key, Func> fetch) key = key.SetTypeKey(); - var result = await GetPersist().GetAsync(key); + var result = await TryGetPersistAsync(key); if (result.IsValid()) { @@ -79,8 +82,10 @@ private void BackgroundLoad(Key key, Func> fetch, Func callb private async Task FetchCallback(Key key, CacheItem item, bool staleWhileRevalidate) { - await GetPersist().SetAsync(key, item, staleWhileRevalidate); - await OnSetAsync(key, item, staleWhileRevalidate); + if (await TrySetPersistAsync(key, item, staleWhileRevalidate)) + { + await OnSetAsync(key, item, staleWhileRevalidate); + } } protected CacheTypeOptions GetTypeOptions() @@ -93,7 +98,7 @@ public virtual async Task PeekAsync(Key key) { key = key.SetTypeKey(); - var result = await GetPersist().GetAsync(key); + var result = await TryGetPersistAsync(key); if (result.IsValid()) { var response = result.GetData(); @@ -124,8 +129,10 @@ protected async Task SetCoreAsync(Key key, T data, TimeSpan freshSpan) var staleWhileRevalidate = GetTypeOptions().StaleWhileRevalidate; var item = CacheItemBuilder.BuildCacheItem(key.KeyParts, data, freshSpan); - await GetPersist().SetAsync(key, item, staleWhileRevalidate); - await OnSetAsync(key, item, staleWhileRevalidate); + if (await TrySetPersistAsync(key, item, staleWhileRevalidate)) + { + await OnSetAsync(key, item, staleWhileRevalidate); + } } public virtual async Task DropAsync(Key key) @@ -204,7 +211,14 @@ protected async Task OnGetCoreAsync(Key key, bool buyMoreTime) var moreTimeBought = false; if (buyMoreTime) { - moreTimeBought = await GetPersist().BuyMoreTime(key); + try + { + moreTimeBought = await GetPersist().BuyMoreTime(key); + } + catch (Exception ex) when (_options.FailOpenOnBackendError) + { + LogFailOpen(ex, "buy-more-time", key, typeof(T)); + } } _cacheMonitor.Accessed(typeof(T), key, moreTimeBought); @@ -228,6 +242,46 @@ private IPersist GetPersist() return persist; } + /// + /// Reads from the persist backend. When is enabled, a + /// backend exception is logged and treated as a miss (returns null) so the caller falls back to the source loader. + /// + private async Task> TryGetPersistAsync(Key key) + { + try + { + return await GetPersist().GetAsync(key); + } + catch (Exception ex) when (_options.FailOpenOnBackendError) + { + LogFailOpen(ex, "read", key, typeof(T)); + return null; + } + } + + /// + /// Writes to the persist backend. When is enabled, a backend + /// exception is logged and swallowed (returns false) so a failed cache write never faults the caller. + /// + private async Task TrySetPersistAsync(Key key, CacheItem item, bool staleWhileRevalidate) + { + try + { + await GetPersist().SetAsync(key, item, staleWhileRevalidate); + return true; + } + catch (Exception ex) when (_options.FailOpenOnBackendError) + { + LogFailOpen(ex, "write", key, typeof(T)); + return false; + } + } + + private void LogFailOpen(Exception ex, string operation, Key key, Type type) + { + _logger?.LogWarning(ex, "Cache persist {Operation} failed for key '{Key}' on type '{Type}'; failing open ({Message}).", operation, key.Value, type.Name, ex.Message); + } + private async Task EvictItems(T data) { var maxCount = GetTypeOptions().MaxCount; diff --git a/Tharga.Cache/Core/EternalCache.cs b/Tharga.Cache/Core/EternalCache.cs index cb079a1..640c3dc 100644 --- a/Tharga.Cache/Core/EternalCache.cs +++ b/Tharga.Cache/Core/EternalCache.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal class EternalCache : CacheBase, IEternalCache, IScopeCache { - public EternalCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + public EternalCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } diff --git a/Tharga.Cache/Core/GenericCache.cs b/Tharga.Cache/Core/GenericCache.cs index ea96a05..ba5ad12 100644 --- a/Tharga.Cache/Core/GenericCache.cs +++ b/Tharga.Cache/Core/GenericCache.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal class GenericCache : CacheBase { - public GenericCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + public GenericCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } diff --git a/Tharga.Cache/Core/GenericTimeCache.cs b/Tharga.Cache/Core/GenericTimeCache.cs index 7688fce..2b85a41 100644 --- a/Tharga.Cache/Core/GenericTimeCache.cs +++ b/Tharga.Cache/Core/GenericTimeCache.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal class GenericTimeCache : TimeCacheBase { - public GenericTimeCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + public GenericTimeCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } } \ No newline at end of file diff --git a/Tharga.Cache/Core/TimeCacheBase.cs b/Tharga.Cache/Core/TimeCacheBase.cs index dc62d9d..ae560dd 100644 --- a/Tharga.Cache/Core/TimeCacheBase.cs +++ b/Tharga.Cache/Core/TimeCacheBase.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal abstract class TimeCacheBase : CacheBase, ITimeCache { - protected TimeCacheBase(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + protected TimeCacheBase(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } diff --git a/Tharga.Cache/Core/TimeToIdleCache.cs b/Tharga.Cache/Core/TimeToIdleCache.cs index e83534a..29701ad 100644 --- a/Tharga.Cache/Core/TimeToIdleCache.cs +++ b/Tharga.Cache/Core/TimeToIdleCache.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal class TimeToIdleCache : TimeCacheBase, ITimeToIdleCache { - public TimeToIdleCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + public TimeToIdleCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } diff --git a/Tharga.Cache/Core/TimeToLiveCache.cs b/Tharga.Cache/Core/TimeToLiveCache.cs index dce0876..bb5f6d2 100644 --- a/Tharga.Cache/Core/TimeToLiveCache.cs +++ b/Tharga.Cache/Core/TimeToLiveCache.cs @@ -1,9 +1,11 @@ -namespace Tharga.Cache.Core; +using Microsoft.Extensions.Logging; + +namespace Tharga.Cache.Core; internal class TimeToLiveCache : TimeCacheBase, ITimeToLiveCache { - public TimeToLiveCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options) - : base(cacheMonitor, persistLoader, fetchQueue, options) + public TimeToLiveCache(IManagedCacheMonitor cacheMonitor, IPersistLoader persistLoader, IFetchQueue fetchQueue, CacheOptions options, ILogger logger = null) + : base(cacheMonitor, persistLoader, fetchQueue, options, logger) { } } \ No newline at end of file diff --git a/Tharga.Cache/README.md b/Tharga.Cache/README.md index 953ebf6..bd2f0f0 100644 --- a/Tharga.Cache/README.md +++ b/Tharga.Cache/README.md @@ -12,6 +12,7 @@ A flexible .NET caching library with multiple cache strategies, configurable evi - **Eviction policies** — LRU, FIFO, or Random when size or count limits are reached - **Stale-while-revalidate** — return cached data instantly while refreshing in the background - **Pluggable persistence** — in-memory by default, with optional Redis, MongoDB, and file backends +- **Fail-open** — when a persist backend is unreachable, reads fall through to the source loader and writes are skipped, so a cache outage never faults the caller (`CacheOptions.FailOpenOnBackendError`, on by default) - **Composite keys** — build cache keys from multiple parts with `KeyBuilder` - **Monitoring** — inspect cache state, item counts, and fetch queue depth via `ICacheMonitor` - **Background cleanup** — built-in WatchDog service removes stale entries automatically