using System; using Svrnty.CQRS.Events.Abstractions.Delivery; using Svrnty.CQRS.Events.Abstractions.EventStore; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Svrnty.CQRS.Events.Abstractions; using Svrnty.CQRS.Events.Abstractions.Models; namespace Svrnty.CQRS.Events.Storage; /// /// In-memory implementation of for ephemeral streams. /// Uses concurrent collections for thread-safe message queue operations. /// /// /// /// Phase 1 Implementation: /// Supports ephemeral streams with visibility tracking, acknowledgment, and NACK. /// Data is lost on application restart (in-memory only). /// /// /// Thread Safety: /// All operations are thread-safe using and . /// /// public class InMemoryEventStreamStore : IEventStreamStore { // EPHEMERAL STREAMS // Stream name -> Queue of events private readonly ConcurrentDictionary> _streams = new(); // Track in-flight events: (streamName, eventId) -> InFlightEvent private readonly ConcurrentDictionary _inFlightEvents = new(); // Dead letter queue: streamName -> Queue of events private readonly ConcurrentDictionary> _deadLetterQueues = new(); // PERSISTENT STREAMS (Phase 2) // Stream name -> Persistent stream storage private readonly ConcurrentDictionary _persistentStreams = new(); // Timer for checking visibility timeouts private readonly Timer _visibilityTimer; // Event delivery providers (Phase 1.7+) private readonly IEnumerable _deliveryProviders; private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// public InMemoryEventStreamStore( IEnumerable deliveryProviders, ILogger logger) { _deliveryProviders = deliveryProviders ?? Enumerable.Empty(); _logger = logger; // Check for expired visibility timeouts every 1 second _visibilityTimer = new Timer( CheckVisibilityTimeouts, null, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); } // ======================================================================== // EPHEMERAL STREAM OPERATIONS // ======================================================================== /// public async Task EnqueueAsync( string streamName, ICorrelatedEvent @event, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (@event == null) throw new ArgumentNullException(nameof(@event)); var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue()); queue.Enqueue(@event); // Notify all delivery providers that a new event is available await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken); } /// public async Task EnqueueBatchAsync( string streamName, IEnumerable events, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (events == null) throw new ArgumentNullException(nameof(events)); var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue()); var eventList = events.Where(e => e != null).ToList(); foreach (var @event in eventList) { queue.Enqueue(@event); } // Notify delivery providers about all enqueued events foreach (var @event in eventList) { await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken); } } /// public Task DequeueAsync( string streamName, string consumerId, TimeSpan visibilityTimeout, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (string.IsNullOrWhiteSpace(consumerId)) throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId)); if (visibilityTimeout <= TimeSpan.Zero) throw new ArgumentException("Visibility timeout must be positive.", nameof(visibilityTimeout)); if (!_streams.TryGetValue(streamName, out var queue)) { return Task.FromResult(null); } // Try to dequeue an event if (queue.TryDequeue(out var @event)) { // Track as in-flight var inFlightKey = GetInFlightKey(streamName, @event.EventId); var inFlight = new InFlightEvent { Event = @event, ConsumerId = consumerId, StreamName = streamName, VisibleAfter = DateTimeOffset.UtcNow.Add(visibilityTimeout) }; _inFlightEvents[inFlightKey] = inFlight; return Task.FromResult(@event); } return Task.FromResult(null); } /// public Task AcknowledgeAsync( string streamName, string eventId, string consumerId, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (string.IsNullOrWhiteSpace(eventId)) throw new ArgumentException("Event ID cannot be null or whitespace.", nameof(eventId)); if (string.IsNullOrWhiteSpace(consumerId)) throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId)); var inFlightKey = GetInFlightKey(streamName, eventId); // Remove from in-flight tracking (event is now permanently deleted) if (_inFlightEvents.TryRemove(inFlightKey, out var inFlight)) { // Verify the consumer ID matches if (inFlight.ConsumerId != consumerId) { // Put it back if wrong consumer _inFlightEvents.TryAdd(inFlightKey, inFlight); return Task.FromResult(false); } // Event is permanently removed (ephemeral stream semantics) return Task.FromResult(true); } return Task.FromResult(false); } /// public Task NackAsync( string streamName, string eventId, string consumerId, bool requeue = true, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (string.IsNullOrWhiteSpace(eventId)) throw new ArgumentException("Event ID cannot be null or whitespace.", nameof(eventId)); if (string.IsNullOrWhiteSpace(consumerId)) throw new ArgumentException("Consumer ID cannot be null or whitespace.", nameof(consumerId)); var inFlightKey = GetInFlightKey(streamName, eventId); if (_inFlightEvents.TryRemove(inFlightKey, out var inFlight)) { // Verify the consumer ID matches if (inFlight.ConsumerId != consumerId) { // Put it back if wrong consumer _inFlightEvents.TryAdd(inFlightKey, inFlight); return Task.FromResult(false); } if (requeue) { // Put the event back in the queue for reprocessing var queue = _streams.GetOrAdd(streamName, _ => new ConcurrentQueue()); queue.Enqueue(inFlight.Event); } else { // Move to dead letter queue var dlq = _deadLetterQueues.GetOrAdd(streamName, _ => new ConcurrentQueue()); dlq.Enqueue(inFlight.Event); } return Task.FromResult(true); } return Task.FromResult(false); } /// public Task GetPendingCountAsync( string streamName, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (_streams.TryGetValue(streamName, out var queue)) { return Task.FromResult(queue.Count); } return Task.FromResult(0); } // ======================================================================== // PERSISTENT STREAM OPERATIONS (Phase 2) // ======================================================================== /// public async Task AppendAsync( string streamName, ICorrelatedEvent @event, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (@event == null) throw new ArgumentNullException(nameof(@event)); var stream = _persistentStreams.GetOrAdd(streamName, _ => new PersistentStream(streamName)); var offset = stream.Append(@event); // Notify delivery providers await NotifyDeliveryProvidersAsync(streamName, @event, cancellationToken); return offset; } /// public Task> ReadStreamAsync( string streamName, long fromOffset, int maxCount, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (fromOffset < 0) throw new ArgumentException("Offset cannot be negative.", nameof(fromOffset)); if (maxCount <= 0) throw new ArgumentException("Max count must be positive.", nameof(maxCount)); if (!_persistentStreams.TryGetValue(streamName, out var stream)) { // Stream doesn't exist, return empty list return Task.FromResult(new List()); } var events = stream.Read(fromOffset, maxCount); return Task.FromResult(events); } /// public Task GetStreamLengthAsync( string streamName, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (!_persistentStreams.TryGetValue(streamName, out var stream)) { // Stream doesn't exist, length is 0 return Task.FromResult(0L); } return Task.FromResult(stream.Length); } /// public Task GetStreamMetadataAsync( string streamName, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(streamName)) throw new ArgumentException("Stream name cannot be null or whitespace.", nameof(streamName)); if (!_persistentStreams.TryGetValue(streamName, out var stream)) { // Stream doesn't exist, return empty metadata return Task.FromResult(new StreamMetadata { StreamName = streamName, Length = 0, OldestEventOffset = 0, OldestEventTimestamp = null, NewestEventTimestamp = null, RetentionPolicy = null, DeletedEventCount = 0 }); } var metadata = stream.GetMetadata(); return Task.FromResult(metadata); } // ======================================================================== // INTERNAL HELPERS // ======================================================================== private static string GetInFlightKey(string streamName, string eventId) { return $"{streamName}:{eventId}"; } /// /// Notify all registered delivery providers that a new event is available. /// private async Task NotifyDeliveryProvidersAsync( string streamName, ICorrelatedEvent @event, CancellationToken cancellationToken) { foreach (var provider in _deliveryProviders) { try { await provider.NotifyEventAvailableAsync(streamName, @event, cancellationToken); } catch (Exception ex) { // Log and continue - don't let provider failures break event enqueueing _logger.LogError( ex, "Delivery provider {ProviderName} failed to process event notification for stream {StreamName}, event {EventId}", provider.ProviderName, streamName, @event.EventId); } } } /// /// Background task that checks for events with expired visibility timeouts. /// When an event's visibility timeout expires, it's automatically requeued. /// private void CheckVisibilityTimeouts(object? state) { var now = DateTimeOffset.UtcNow; var expiredEvents = _inFlightEvents .Where(kvp => kvp.Value.VisibleAfter <= now) .ToList(); foreach (var kvp in expiredEvents) { if (_inFlightEvents.TryRemove(kvp.Key, out var inFlight)) { // Requeue the event (visibility timeout expired without ack/nack) var queue = _streams.GetOrAdd(inFlight.StreamName, _ => new ConcurrentQueue()); queue.Enqueue(inFlight.Event); } } } /// /// Get dead letter queue events for a stream (for monitoring/debugging). /// public IEnumerable GetDeadLetterQueue(string streamName) { if (_deadLetterQueues.TryGetValue(streamName, out var dlq)) { return dlq.ToList(); } return Enumerable.Empty(); } // ======================================================================== // CONSUMER OFFSET TRACKING - Phase 6 (Monitoring & Health Checks) // ======================================================================== /// public Task GetConsumerOffsetAsync( string streamName, string consumerId, CancellationToken cancellationToken = default) { if (_persistentStreams.TryGetValue(streamName, out var stream)) { return Task.FromResult(stream.GetConsumerOffset(consumerId)); } return Task.FromResult(0L); } /// public Task GetConsumerLastUpdateTimeAsync( string streamName, string consumerId, CancellationToken cancellationToken = default) { if (_persistentStreams.TryGetValue(streamName, out var stream)) { return Task.FromResult(stream.GetConsumerLastUpdateTime(consumerId)); } return Task.FromResult(DateTimeOffset.MinValue); } /// public Task UpdateConsumerOffsetAsync( string streamName, string consumerId, long newOffset, CancellationToken cancellationToken = default) { if (_persistentStreams.TryGetValue(streamName, out var stream)) { stream.UpdateConsumerOffset(consumerId, newOffset); _logger?.LogInformation( "Consumer offset updated: Stream={StreamName}, Consumer={ConsumerId}, NewOffset={NewOffset}", streamName, consumerId, newOffset); } return Task.CompletedTask; } /// /// Dispose resources (timer). /// public void Dispose() { _visibilityTimer?.Dispose(); } // ======================================================================== // INTERNAL MODELS // ======================================================================== /// /// Represents an event that's currently being processed by a consumer. /// private class InFlightEvent { public required ICorrelatedEvent Event { get; init; } public required string ConsumerId { get; init; } public required string StreamName { get; init; } public required DateTimeOffset VisibleAfter { get; init; } } /// /// Represents a persistent event stream with append-only semantics. /// Events are stored with sequential offsets and never deleted (in Phase 2.1). /// private class PersistentStream { private readonly string _streamName; private readonly List _events = new(); private readonly Dictionary _consumerOffsets = new(); private readonly object _lock = new(); private long _nextOffset = 0; public PersistentStream(string streamName) { _streamName = streamName; } public long Length { get { lock (_lock) { return _events.Count; } } } /// /// Append an event to the stream and return its assigned offset. /// public long Append(ICorrelatedEvent @event) { lock (_lock) { var offset = _nextOffset++; var persistedEvent = new PersistedEvent { Offset = offset, Event = @event, Timestamp = DateTimeOffset.UtcNow }; _events.Add(persistedEvent); return offset; } } /// /// Read events starting from a specific offset. /// public List Read(long fromOffset, int maxCount) { lock (_lock) { return _events .Where(e => e.Offset >= fromOffset) .Take(maxCount) .Select(e => e.Event) .ToList(); } } /// /// Get metadata about this stream. /// public StreamMetadata GetMetadata() { lock (_lock) { if (_events.Count == 0) { return new StreamMetadata { StreamName = _streamName, Length = 0, OldestEventOffset = 0, OldestEventTimestamp = null, NewestEventTimestamp = null, RetentionPolicy = null, DeletedEventCount = 0 }; } var oldest = _events.First(); var newest = _events.Last(); return new StreamMetadata { StreamName = _streamName, Length = _events.Count, OldestEventOffset = oldest.Offset, OldestEventTimestamp = oldest.Timestamp, NewestEventTimestamp = newest.Timestamp, RetentionPolicy = null, // Phase 2.4 will add retention policies DeletedEventCount = 0 // Phase 2.4 will track deleted events }; } } /// /// Get consumer offset for health checks (Phase 6). /// public long GetConsumerOffset(string consumerId) { lock (_lock) { return _consumerOffsets.TryGetValue(consumerId, out var info) ? info.Offset : 0L; } } /// /// Get consumer last update time for health checks (Phase 6). /// public DateTimeOffset GetConsumerLastUpdateTime(string consumerId) { lock (_lock) { return _consumerOffsets.TryGetValue(consumerId, out var info) ? info.LastUpdated : DateTimeOffset.MinValue; } } /// /// Update consumer offset (used by subscription clients during event processing). /// public void UpdateConsumerOffset(string consumerId, long offset) { lock (_lock) { _consumerOffsets[consumerId] = new ConsumerOffsetInfo { Offset = offset, LastUpdated = DateTimeOffset.UtcNow }; } } /// /// Represents an event stored in a persistent stream. /// private class PersistedEvent { public required long Offset { get; init; } public required ICorrelatedEvent Event { get; init; } public required DateTimeOffset Timestamp { get; init; } } } /// /// Tracks consumer offset and last update time for health monitoring (Phase 6). /// private class ConsumerOffsetInfo { public required long Offset { get; init; } public required DateTimeOffset LastUpdated { get; init; } } }