dotnet-cqrs/docs/event-streaming/fundamentals/subscriptions.md

Subscriptions

Subscription modes and patterns for consuming event streams.

Overview

Subscriptions define how consumers receive events from streams. The framework supports two primary subscription modes: Broadcast (all consumers receive all events) and Queue (load-balanced delivery to consumer groups).

Key Features:

• ✅ Broadcast Mode - All subscribers receive all events
• ✅ Queue Mode - Events load-balanced across consumer group
• ✅ Offset Tracking - Resume from last processed position
• ✅ At-Least-Once - Guaranteed delivery with retries
• ✅ Exactly-Once per Group - No duplicate processing within group

Subscription Modes

Broadcast Subscriptions

All consumers receive all events independently:

┌───────────┐
│   Stream  │
│  [Events] │
└─────┬─────┘
      │
      ├─────────▶ Consumer A (all events)
      ├─────────▶ Consumer B (all events)
      └─────────▶ Consumer C (all events)

Use Cases:

• Analytics and reporting
• Audit logging
• Multiple independent projections
• Notifications to different channels

Queue Subscriptions

Events distributed across consumer group (load balanced):

┌───────────┐
│   Stream  │
│  [Events] │
└─────┬─────┘
      │
      ├─────────▶ Consumer A (event 1, 4, 7...)
      ├─────────▶ Consumer B (event 2, 5, 8...)
      └─────────▶ Consumer C (event 3, 6, 9...)

Use Cases:

• Parallel processing for scalability
• Background job processing
• Horizontal scaling of event handlers

Broadcast Mode

Creating Broadcast Subscription

public class AnalyticsProjection : BackgroundService
{
    private readonly IEventStreamStore _eventStore;
    private readonly ILogger<AnalyticsProjection> _logger;
    private long _lastProcessedOffset;

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        // Load checkpoint
        _lastProcessedOffset = await LoadCheckpointAsync();

        _logger.LogInformation(
            "Starting analytics projection from offset {Offset}",
            _lastProcessedOffset);

        // Subscribe to stream
        await foreach (var @event in _eventStore.ReadStreamAsync(
            streamName: "orders",
            fromOffset: _lastProcessedOffset + 1,
            cancellationToken: stoppingToken))
        {
            try
            {
                // Process event
                await ProcessEventAsync(@event);

                // Update checkpoint
                _lastProcessedOffset = @event.Offset;
                await SaveCheckpointAsync(_lastProcessedOffset);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error processing event {Offset}", @event.Offset);
                // Continue processing or implement retry logic
            }
        }
    }

    private async Task ProcessEventAsync(StoredEvent @event)
    {
        var eventData = JsonSerializer.Deserialize(
            @event.Data,
            Type.GetType(@event.EventType));

        switch (eventData)
        {
            case OrderPlacedEvent placed:
                await _analytics.RecordOrderAsync(placed);
                break;

            case OrderShippedEvent shipped:
                await _analytics.RecordShipmentAsync(shipped);
                break;
        }
    }
}

Multiple Independent Projections

Each projection maintains its own offset:

// Projection 1: Order summary
public class OrderSummaryProjection : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken ct)
    {
        var offset = await LoadCheckpointAsync("order-summary");

        await foreach (var evt in _eventStore.ReadStreamAsync("orders", offset + 1, ct))
        {
            await UpdateOrderSummaryAsync(evt);
            await SaveCheckpointAsync("order-summary", evt.Offset);
        }
    }
}

// Projection 2: Customer analytics (independent)
public class CustomerAnalyticsProjection : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken ct)
    {
        var offset = await LoadCheckpointAsync("customer-analytics");

        await foreach (var evt in _eventStore.ReadStreamAsync("orders", offset + 1, ct))
        {
            await UpdateCustomerAnalyticsAsync(evt);
            await SaveCheckpointAsync("customer-analytics", evt.Offset);
        }
    }
}

Queue Mode (Consumer Groups)

Creating Queue Subscription

public class OrderProcessingWorker : BackgroundService
{
    private readonly IConsumerGroupReader _consumerGroup;
    private readonly string _consumerId;

    public OrderProcessingWorker(IConsumerGroupReader consumerGroup)
    {
        _consumerGroup = consumerGroup;
        _consumerId = $"worker-{Environment.MachineName}-{Guid.NewGuid():N}";
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Worker {ConsumerId} started", _consumerId);

        // Join consumer group
        await foreach (var @event in _consumerGroup.ConsumeAsync(
            streamName: "orders",
            groupId: "order-processing",
            consumerId: _consumerId,
            options: new ConsumerGroupOptions
            {
                BatchSize = 100,
                CommitStrategy = OffsetCommitStrategy.AfterBatch,
                HeartbeatInterval = TimeSpan.FromSeconds(10),
                SessionTimeout = TimeSpan.FromSeconds(30)
            },
            cancellationToken: stoppingToken))
        {
            try
            {
                await ProcessOrderEventAsync(@event);
            }
            catch (Exception ex)
            {
                _logger.LogError(ex, "Error processing event");
                // Offset will not be committed, event will be reprocessed
            }
        }

        _logger.LogInformation("Worker {ConsumerId} stopped", _consumerId);
    }
}

Scaling with Multiple Workers

Run multiple instances of the same worker:

# Start 3 workers in same consumer group
dotnet run --WorkerId=1 &
dotnet run --WorkerId=2 &
dotnet run --WorkerId=3 &

# Events automatically load balanced across workers
# - Worker 1 processes events 1, 4, 7, 10...
# - Worker 2 processes events 2, 5, 8, 11...
# - Worker 3 processes events 3, 6, 9, 12...

Offset Management

Checkpoint Strategies

Manual Checkpoint:

await foreach (var @event in _eventStore.ReadStreamAsync("orders", offset))
{
    await ProcessEventAsync(@event);

    // Manual checkpoint after each event
    offset = @event.Offset;
    await SaveCheckpointAsync(offset);
}

Batch Checkpoint:

const int batchSize = 100;
var batch = new List<StoredEvent>();

await foreach (var @event in _eventStore.ReadStreamAsync("orders", offset))
{
    batch.Add(@event);

    if (batch.Count >= batchSize)
    {
        // Process batch
        await ProcessBatchAsync(batch);

        // Checkpoint after batch
        await SaveCheckpointAsync(batch.Max(e => e.Offset));
        batch.Clear();
    }
}

Periodic Checkpoint:

var lastCheckpoint = DateTimeOffset.UtcNow;
var checkpointInterval = TimeSpan.FromSeconds(30);

await foreach (var @event in _eventStore.ReadStreamAsync("orders", offset))
{
    await ProcessEventAsync(@event);
    offset = @event.Offset;

    // Checkpoint every 30 seconds
    if (DateTimeOffset.UtcNow - lastCheckpoint > checkpointInterval)
    {
        await SaveCheckpointAsync(offset);
        lastCheckpoint = DateTimeOffset.UtcNow;
    }
}

Consumer Group Offset Tracking

With consumer groups, offsets are tracked automatically:

await foreach (var @event in _consumerGroup.ConsumeAsync(
    streamName: "orders",
    groupId: "order-processing",
    consumerId: "worker-1",
    options: new ConsumerGroupOptions
    {
        // Offset committed after each event
        CommitStrategy = OffsetCommitStrategy.AfterEach
    }))
{
    await ProcessEventAsync(@event);
    // Offset committed automatically
}

Subscription Lifecycle

Starting Subscription

public class EventSubscriptionService : IHostedService
{
    private Task? _subscriptionTask;
    private CancellationTokenSource? _cts;

    public Task StartAsync(CancellationToken cancellationToken)
    {
        _cts = new CancellationTokenSource();

        _subscriptionTask = Task.Run(async () =>
        {
            await SubscribeAsync(_cts.Token);
        }, cancellationToken);

        return Task.CompletedTask;
    }

    public async Task StopAsync(CancellationToken cancellationToken)
    {
        if (_subscriptionTask == null)
            return;

        // Signal cancellation
        _cts?.Cancel();

        // Wait for graceful shutdown
        await Task.WhenAny(
            _subscriptionTask,
            Task.Delay(Timeout.Infinite, cancellationToken));
    }

    private async Task SubscribeAsync(CancellationToken ct)
    {
        var offset = await LoadCheckpointAsync();

        await foreach (var @event in _eventStore.ReadStreamAsync("orders", offset, ct))
        {
            await ProcessEventAsync(@event);
            await SaveCheckpointAsync(@event.Offset);
        }
    }
}

Graceful Shutdown

public class GracefulShutdownWorker : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        try
        {
            await foreach (var @event in _consumerGroup.ConsumeAsync(
                "orders",
                "workers",
                "worker-1",
                cancellationToken: stoppingToken))
            {
                // Process event
                await ProcessEventAsync(@event);
            }
        }
        catch (OperationCanceledException)
        {
            _logger.LogInformation("Shutdown requested, finishing current batch...");

            // Finish processing current batch before exit
            // Offset will be committed by consumer group
        }
        finally
        {
            _logger.LogInformation("Worker stopped gracefully");
        }
    }
}

Error Handling

Retry on Failure

public async Task ProcessWithRetryAsync(StoredEvent @event)
{
    const int maxRetries = 3;
    int attempt = 0;

    while (attempt < maxRetries)
    {
        try
        {
            await ProcessEventAsync(@event);
            return;  // Success
        }
        catch (Exception ex)
        {
            attempt++;

            if (attempt >= maxRetries)
            {
                _logger.LogError(ex,
                    "Failed to process event {EventId} after {Attempts} attempts",
                    @event.EventId,
                    attempt);

                // Move to dead letter queue
                await MoveToDLQAsync(@event);
                throw;
            }

            // Exponential backoff
            var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
            _logger.LogWarning(ex,
                "Retry {Attempt}/{MaxRetries} after {Delay}",
                attempt,
                maxRetries,
                delay);

            await Task.Delay(delay);
        }
    }
}

Dead Letter Queue

public async Task ProcessWithDLQAsync(StoredEvent @event)
{
    try
    {
        await ProcessEventAsync(@event);
    }
    catch (Exception ex)
    {
        _logger.LogError(ex, "Processing failed, moving to DLQ");

        // Move to dead letter queue for manual investigation
        await _eventStore.EnqueueAsync("dlq-orders", new DeadLetterMessage
        {
            OriginalEventId = @event.EventId,
            OriginalStreamName = @event.StreamName,
            OriginalOffset = @event.Offset,
            ErrorMessage = ex.Message,
            ErrorStackTrace = ex.StackTrace,
            FailedAt = DateTimeOffset.UtcNow
        });
    }
}

Monitoring Subscriptions

Lag Monitoring

public class SubscriptionMonitor : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        while (!stoppingToken.IsCancellationRequested)
        {
            var streamLength = await GetStreamLengthAsync("orders");
            var lastProcessedOffset = await LoadCheckpointAsync("order-processing");

            var lag = streamLength - lastProcessedOffset;

            if (lag > 1000)
            {
                _logger.LogWarning(
                    "Subscription lagging: {Lag} events behind",
                    lag);
            }

            _metrics.RecordConsumerLag("order-processing", lag);

            await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
        }
    }
}

Health Checks

public class SubscriptionHealthCheck : IHealthCheck
{
    private readonly IConsumerOffsetStore _offsetStore;

    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context,
        CancellationToken ct)
    {
        var consumers = await _offsetStore.GetConsumersAsync("orders", "order-processing");

        var staleConsumers = consumers.Where(c =>
            DateTimeOffset.UtcNow - c.LastHeartbeat > TimeSpan.FromMinutes(1));

        if (staleConsumers.Any())
        {
            return HealthCheckResult.Degraded(
                $"Stale consumers: {string.Join(", ", staleConsumers.Select(c => c.ConsumerId))}");
        }

        return HealthCheckResult.Healthy("All consumers active");
    }
}

Best Practices

✅ DO

• Use broadcast for independent projections
• Use queue mode for scalable processing
• Track offsets reliably
• Implement idempotent handlers
• Monitor consumer lag
• Handle errors gracefully
• Implement graceful shutdown

❌ DON'T

• Don't lose checkpoint data
• Don't process events without idempotency
• Don't ignore consumer lag
• Don't skip error handling
• Don't block event processing
• Don't commit offsets before processing

See Also

• Getting Started
• Consumer Groups
• Projections
• Health Checks
• Metrics