dotnet-cqrs/Svrnty.CQRS.Events/Storage/InMemoryEventStreamStore.cs

645 lines
22 KiB
C#

using System;
using Svrnty.CQRS.Events.Abstractions.Delivery;
using Svrnty.CQRS.Events.Abstractions.EventStore;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using Svrnty.CQRS.Events.Abstractions;
using Svrnty.CQRS.Events.Abstractions.Models;
namespace Svrnty.CQRS.Events.Storage;
/// <summary>
/// In-memory implementation of <see cref="IEventStreamStore"/> for ephemeral streams.
/// Uses concurrent collections for thread-safe message queue operations.
/// </summary>
/// <remarks>
/// <para>
/// <strong>Phase 1 Implementation:</strong>
/// Supports ephemeral streams with visibility tracking, acknowledgment, and NACK.
/// Data is lost on application restart (in-memory only).
/// </para>
/// <para>
/// <strong>Thread Safety:</strong>
/// All operations are thread-safe using <see cref="ConcurrentQueue{T}"/> and <see cref="ConcurrentDictionary{TKey,TValue}"/>.
/// </para>
/// </remarks>
public class InMemoryEventStreamStore : IEventStreamStore
{
// EPHEMERAL STREAMS
// Stream name -> Queue of events
private readonly ConcurrentDictionary<string, ConcurrentQueue<ICorrelatedEvent>> _streams = new();
// Track in-flight events: (streamName, eventId) -> InFlightEvent
private readonly ConcurrentDictionary<string, InFlightEvent> _inFlightEvents = new();
// Dead letter queue: streamName -> Queue of events
private readonly ConcurrentDictionary<string, ConcurrentQueue<ICorrelatedEvent>> _deadLetterQueues = new();
// PERSISTENT STREAMS (Phase 2)
// Stream name -> Persistent stream storage
private readonly ConcurrentDictionary<string, PersistentStream> _persistentStreams = new();
// Timer for checking visibility timeouts
private readonly Timer _visibilityTimer;
// Event delivery providers (Phase 1.7+)
private readonly IEnumerable<IEventDeliveryProvider> _deliveryProviders;
private readonly ILogger<InMemoryEventStreamStore> _logger;
/// <summary>
/// Initializes a new instance of the <see cref="InMemoryEventStreamStore"/> class.
/// </summary>
public InMemoryEventStreamStore(
IEnumerable<IEventDeliveryProvider> deliveryProviders,
ILogger<InMemoryEventStreamStore> logger)
{
_deliveryProviders = deliveryProviders ?? Enumerable.Empty<IEventDeliveryProvider>();
_logger = logger;
// Check for expired visibility timeouts every 1 second
_visibilityTimer = new Timer(
CheckVisibilityTimeouts,
null,
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(1));
}
// ========================================================================
// EPHEMERAL STREAM OPERATIONS
// ========================================================================
/// <inheritdoc />
public async Task EnqueueAsync(
string streamName,
ICorrelatedEvent @event,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (@event == null)
throw new ArgumentNullException(nameof(@event));
var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue<ICorrelatedEvent>());
queue.Enqueue(@event);
// Notify all delivery providers that a new event is available
await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken);
}
/// <inheritdoc />
public async Task EnqueueBatchAsync(
string streamName,
IEnumerable<ICorrelatedEvent> events,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (events == null)
throw new ArgumentNullException(nameof(events));
var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue<ICorrelatedEvent>());
var eventList = events.Where(e => e != null).ToList();
foreach (var @event in eventList)
{
queue.Enqueue(@event);
}
// Notify delivery providers about all enqueued events
foreach (var @event in eventList)
{
await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken);
}
}
/// <inheritdoc />
public Task<ICorrelatedEvent?> DequeueAsync(
string streamName,
string consumerId,
TimeSpan visibilityTimeout,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (string.IsNullOrWhiteSpace(consumerId))
throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId));
if (visibilityTimeout <= TimeSpan.Zero)
throw new ArgumentException("Visibility timeout must be positive.", nameof(visibilityTimeout));
if (!_streams.TryGetValue(streamName, out var queue))
{
return Task.FromResult<ICorrelatedEvent?>(null);
}
// Try to dequeue an event
if (queue.TryDequeue(out var @event))
{
// Track as in-flight
var inFlightKey = GetInFlightKey(streamName, @event.EventId);
var inFlight = new InFlightEvent
{
Event = @event,
ConsumerId = consumerId,
StreamName = streamName,
VisibleAfter = DateTimeOffset.UtcNow.Add(visibilityTimeout)
};
_inFlightEvents[inFlightKey] = inFlight;
return Task.FromResult<ICorrelatedEvent?>(@event);
}
return Task.FromResult<ICorrelatedEvent?>(null);
}
/// <inheritdoc />
public Task<bool> AcknowledgeAsync(
string streamName,
string eventId,
string consumerId,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (string.IsNullOrWhiteSpace(eventId))
throw new ArgumentException("Event ID cannot be null or whitespace.", nameof(eventId));
if (string.IsNullOrWhiteSpace(consumerId))
throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId));
var inFlightKey = GetInFlightKey(streamName, eventId);
// Remove from in-flight tracking (event is now permanently deleted)
if (_inFlightEvents.TryRemove(inFlightKey, out var inFlight))
{
// Verify the consumer ID matches
if (inFlight.ConsumerId != consumerId)
{
// Put it back if wrong consumer
_inFlightEvents.TryAdd(inFlightKey, inFlight);
return Task.FromResult(false);
}
// Event is permanently removed (ephemeral stream semantics)
return Task.FromResult(true);
}
return Task.FromResult(false);
}
/// <inheritdoc />
public Task<bool> NackAsync(
string streamName,
string eventId,
string consumerId,
bool requeue = true,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (string.IsNullOrWhiteSpace(eventId))
throw new ArgumentException("Event ID cannot be null or whitespace.", nameof(eventId));
if (string.IsNullOrWhiteSpace(consumerId))
throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId));
var inFlightKey = GetInFlightKey(streamName, eventId);
if (_inFlightEvents.TryRemove(inFlightKey, out var inFlight))
{
// Verify the consumer ID matches
if (inFlight.ConsumerId != consumerId)
{
// Put it back if wrong consumer
_inFlightEvents.TryAdd(inFlightKey, inFlight);
return Task.FromResult(false);
}
if (requeue)
{
// Put the event back in the queue for reprocessing
var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue<ICorrelatedEvent>());
queue.Enqueue(inFlight.Event);
}
else
{
// Move to dead letter queue
var dlq = _deadLetterQueues.GetOrAdd(streamName, _ => new ConcurrentQueue<ICorrelatedEvent>());
dlq.Enqueue(inFlight.Event);
}
return Task.FromResult(true);
}
return Task.FromResult(false);
}
/// <inheritdoc />
public Task<int> GetPendingCountAsync(
string streamName,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (_streams.TryGetValue(streamName, out var queue))
{
return Task.FromResult(queue.Count);
}
return Task.FromResult(0);
}
// ========================================================================
// PERSISTENT STREAM OPERATIONS (Phase 2)
// ========================================================================
/// <inheritdoc />
public async Task<long> AppendAsync(
string streamName,
ICorrelatedEvent @event,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (@event == null)
throw new ArgumentNullException(nameof(@event));
var stream = _persistentStreams.GetOrAdd(streamName, _ => new PersistentStream(streamName));
var offset = stream.Append(@event);
// Notify delivery providers
await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken);
return offset;
}
/// <inheritdoc />
public Task<List<ICorrelatedEvent>> ReadStreamAsync(
string streamName,
long fromOffset,
int maxCount,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (fromOffset < 0)
throw new ArgumentException("Offset cannot be negative.", nameof(fromOffset));
if (maxCount <= 0)
throw new ArgumentException("Max count must be positive.", nameof(maxCount));
if (!_persistentStreams.TryGetValue(streamName, out var stream))
{
// Stream doesn't exist, return empty list
return Task.FromResult(new List<ICorrelatedEvent>());
}
var events = stream.Read(fromOffset, maxCount);
return Task.FromResult(events);
}
/// <inheritdoc />
public Task<long> GetStreamLengthAsync(
string streamName,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (!_persistentStreams.TryGetValue(streamName, out var stream))
{
// Stream doesn't exist, length is 0
return Task.FromResult(0L);
}
return Task.FromResult(stream.Length);
}
/// <inheritdoc />
public Task<StreamMetadata> GetStreamMetadataAsync(
string streamName,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(streamName))
throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName));
if (!_persistentStreams.TryGetValue(streamName, out var stream))
{
// Stream doesn't exist, return empty metadata
return Task.FromResult(new StreamMetadata
{
StreamName = streamName,
Length = 0,
OldestEventOffset = 0,
OldestEventTimestamp = null,
NewestEventTimestamp = null,
RetentionPolicy = null,
DeletedEventCount = 0
});
}
var metadata = stream.GetMetadata();
return Task.FromResult(metadata);
}
// ========================================================================
// INTERNAL HELPERS
// ========================================================================
private static string GetInFlightKey(string streamName, string eventId)
{
return $"{streamName}:{eventId}";
}
/// <summary>
/// Notify all registered delivery providers that a new event is available.
/// </summary>
private async Task NotifyDeliveryProvidersAsync(
string streamName,
ICorrelatedEvent @event,
CancellationToken cancellationToken)
{
foreach (var provider in _deliveryProviders)
{
try
{
await provider.NotifyEventAvailableAsync(streamName, @event, cancellationToken);
}
catch (Exception ex)
{
// Log and continue - don't let provider failures break event enqueueing
_logger.LogError(
ex,
"Delivery provider {ProviderName} failed to process event notification for stream {StreamName}, event {EventId}",
provider.ProviderName,
streamName,
@event.EventId);
}
}
}
/// <summary>
/// Background task that checks for events with expired visibility timeouts.
/// When an event's visibility timeout expires, it's automatically requeued.
/// </summary>
private void CheckVisibilityTimeouts(object? state)
{
var now = DateTimeOffset.UtcNow;
var expiredEvents = _inFlightEvents
.Where(kvp => kvp.Value.VisibleAfter <= now)
.ToList();
foreach (var kvp in expiredEvents)
{
if (_inFlightEvents.TryRemove(kvp.Key, out var inFlight))
{
// Requeue the event (visibility timeout expired without ack/nack)
var queue = _streams.GetOrAdd(inFlight.StreamName, _ => new ConcurrentQueue<ICorrelatedEvent>());
queue.Enqueue(inFlight.Event);
}
}
}
/// <summary>
/// Get dead letter queue events for a stream (for monitoring/debugging).
/// </summary>
public IEnumerable<ICorrelatedEvent> GetDeadLetterQueue(string streamName)
{
if (_deadLetterQueues.TryGetValue(streamName, out var dlq))
{
return dlq.ToList();
}
return Enumerable.Empty<ICorrelatedEvent>();
}
// ========================================================================
// CONSUMER OFFSET TRACKING - Phase 6 (Monitoring & Health Checks)
// ========================================================================
/// <inheritdoc />
public Task<long> GetConsumerOffsetAsync(
string streamName,
string consumerId,
CancellationToken cancellationToken = default)
{
if (_persistentStreams.TryGetValue(streamName, out var stream))
{
return Task.FromResult(stream.GetConsumerOffset(consumerId));
}
return Task.FromResult(0L);
}
/// <inheritdoc />
public Task<DateTimeOffset> GetConsumerLastUpdateTimeAsync(
string streamName,
string consumerId,
CancellationToken cancellationToken = default)
{
if (_persistentStreams.TryGetValue(streamName, out var stream))
{
return Task.FromResult(stream.GetConsumerLastUpdateTime(consumerId));
}
return Task.FromResult(DateTimeOffset.MinValue);
}
/// <inheritdoc />
public Task UpdateConsumerOffsetAsync(
string streamName,
string consumerId,
long newOffset,
CancellationToken cancellationToken = default)
{
if (_persistentStreams.TryGetValue(streamName, out var stream))
{
stream.UpdateConsumerOffset(consumerId, newOffset);
_logger?.LogInformation(
"Consumer offset updated: Stream={StreamName}, Consumer={ConsumerId}, NewOffset={NewOffset}",
streamName, consumerId, newOffset);
}
return Task.CompletedTask;
}
/// <summary>
/// Dispose resources (timer).
/// </summary>
public void Dispose()
{
_visibilityTimer?.Dispose();
}
// ========================================================================
// INTERNAL MODELS
// ========================================================================
/// <summary>
/// Represents an event that's currently being processed by a consumer.
/// </summary>
private class InFlightEvent
{
public required ICorrelatedEvent Event { get; init; }
public required string ConsumerId { get; init; }
public required string StreamName { get; init; }
public required DateTimeOffset VisibleAfter { get; init; }
}
/// <summary>
/// Represents a persistent event stream with append-only semantics.
/// Events are stored with sequential offsets and never deleted (in Phase 2.1).
/// </summary>
private class PersistentStream
{
private readonly string _streamName;
private readonly List<PersistedEvent> _events = new();
private readonly Dictionary<string, ConsumerOffsetInfo> _consumerOffsets = new();
private readonly object _lock = new();
private long _nextOffset = 0;
public PersistentStream(string streamName)
{
_streamName = streamName;
}
public long Length
{
get
{
lock (_lock)
{
return _events.Count;
}
}
}
/// <summary>
/// Append an event to the stream and return its assigned offset.
/// </summary>
public long Append(ICorrelatedEvent @event)
{
lock (_lock)
{
var offset = _nextOffset++;
var persistedEvent = new PersistedEvent
{
Offset = offset,
Event = @event,
Timestamp = DateTimeOffset.UtcNow
};
_events.Add(persistedEvent);
return offset;
}
}
/// <summary>
/// Read events starting from a specific offset.
/// </summary>
public List<ICorrelatedEvent> Read(long fromOffset, int maxCount)
{
lock (_lock)
{
return _events
.Where(e => e.Offset >= fromOffset)
.Take(maxCount)
.Select(e => e.Event)
.ToList();
}
}
/// <summary>
/// Get metadata about this stream.
/// </summary>
public StreamMetadata GetMetadata()
{
lock (_lock)
{
if (_events.Count == 0)
{
return new StreamMetadata
{
StreamName = _streamName,
Length = 0,
OldestEventOffset = 0,
OldestEventTimestamp = null,
NewestEventTimestamp = null,
RetentionPolicy = null,
DeletedEventCount = 0
};
}
var oldest = _events.First();
var newest = _events.Last();
return new StreamMetadata
{
StreamName = _streamName,
Length = _events.Count,
OldestEventOffset = oldest.Offset,
OldestEventTimestamp = oldest.Timestamp,
NewestEventTimestamp = newest.Timestamp,
RetentionPolicy = null, // Phase 2.4 will add retention policies
DeletedEventCount = 0 // Phase 2.4 will track deleted events
};
}
}
/// <summary>
/// Get consumer offset for health checks (Phase 6).
/// </summary>
public long GetConsumerOffset(string consumerId)
{
lock (_lock)
{
return _consumerOffsets.TryGetValue(consumerId, out var info) ? info.Offset : 0L;
}
}
/// <summary>
/// Get consumer last update time for health checks (Phase 6).
/// </summary>
public DateTimeOffset GetConsumerLastUpdateTime(string consumerId)
{
lock (_lock)
{
return _consumerOffsets.TryGetValue(consumerId, out var info)
? info.LastUpdated
: DateTimeOffset.MinValue;
}
}
/// <summary>
/// Update consumer offset (used by subscription clients during event processing).
/// </summary>
public void UpdateConsumerOffset(string consumerId, long offset)
{
lock (_lock)
{
_consumerOffsets[consumerId] = new ConsumerOffsetInfo
{
Offset = offset,
LastUpdated = DateTimeOffset.UtcNow
};
}
}
/// <summary>
/// Represents an event stored in a persistent stream.
/// </summary>
private class PersistedEvent
{
public required long Offset { get; init; }
public required ICorrelatedEvent Event { get; init; }
public required DateTimeOffset Timestamp { get; init; }
}
}
/// <summary>
/// Tracks consumer offset and last update time for health monitoring (Phase 6).
/// </summary>
private class ConsumerOffsetInfo
{
public required long Offset { get; init; }
public required DateTimeOffset LastUpdated { get; init; }
}
}