svrnty/dotnet-cqrs

Fork 0

Mathias Beaulieu-Duncan 5c497f8923 this is a mess

2025-12-11 01:18:24 -05:00

11 KiB

Raw Blame History

Progress Tracking

Monitor event replay progress with detailed metrics and estimated completion.

Overview

Progress tracking provides visibility into long-running replay operations:

Real-time event processing metrics
Throughput and performance monitoring
Estimated time to completion
Error tracking and reporting

Quick Start

using Svrnty.CQRS.Events.Abstractions;

var replayService = serviceProvider.GetRequiredService<IEventReplayService>();

await foreach (var @event in replayService.ReplayFromOffsetAsync(
    streamName: "orders",
    startOffset: 0,
    options: new ReplayOptions
    {
        ProgressCallback = progress =>
        {
            Console.WriteLine($"Processed: {progress.EventsProcessed}");
            Console.WriteLine($"Rate: {progress.EventsPerSecond:F0} events/sec");
        },
        ProgressInterval = 1000  // Callback every 1000 events
    }))
{
    await ProcessEventAsync(@event);
}

Progress Metrics

The ReplayProgress object provides comprehensive metrics:

public class ReplayProgress
{
    public long EventsProcessed { get; set; }      // Total events processed
    public long TotalEvents { get; set; }          // Total events to process
    public double PercentComplete { get; set; }    // 0.0 to 100.0
    public double EventsPerSecond { get; set; }    // Current throughput
    public TimeSpan Elapsed { get; set; }          // Time since replay started
    public TimeSpan? EstimatedRemaining { get; set; }  // ETA
    public DateTimeOffset? EstimatedCompletion { get; set; }  // Estimated finish time
    public long ErrorCount { get; set; }           // Failed events
}

Detailed Progress Callback

var options = new ReplayOptions
{
    ProgressCallback = progress =>
    {
        Console.Clear();
        Console.WriteLine("=== Event Replay Progress ===");
        Console.WriteLine($"Events Processed: {progress.EventsProcessed:N0}");

        if (progress.TotalEvents > 0)
        {
            Console.WriteLine($"Total Events: {progress.TotalEvents:N0}");
            Console.WriteLine($"Progress: {progress.PercentComplete:F2}%");

            // Progress bar
            var barWidth = 50;
            var filled = (int)(barWidth * progress.PercentComplete / 100);
            var bar = new string('█', filled) + new string('░', barWidth - filled);
            Console.WriteLine($"[{bar}]");
        }

        Console.WriteLine($"Throughput: {progress.EventsPerSecond:F0} events/sec");
        Console.WriteLine($"Elapsed: {progress.Elapsed:hh\\:mm\\:ss}");

        if (progress.EstimatedRemaining.HasValue)
        {
            Console.WriteLine($"ETA: {progress.EstimatedRemaining.Value:hh\\:mm\\:ss}");
            Console.WriteLine($"Completion: {progress.EstimatedCompletion:yyyy-MM-dd HH:mm:ss}");
        }

        if (progress.ErrorCount > 0)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine($"Errors: {progress.ErrorCount}");
            Console.ResetColor();
        }
    },
    ProgressInterval = 1000
};

await foreach (var @event in replayService.ReplayFromOffsetAsync(
    "orders",
    startOffset: 0,
    options))
{
    await ProcessEventAsync(@event);
}

Progress Interval Configuration

Control how often progress callbacks are invoked:

// Frequent updates - every 100 events
var options = new ReplayOptions
{
    ProgressInterval = 100,
    ProgressCallback = progress => { /* ... */ }
};

// Moderate updates - every 1000 events (default)
var options = new ReplayOptions
{
    ProgressInterval = 1000,
    ProgressCallback = progress => { /* ... */ }
};

// Infrequent updates - every 10000 events
var options = new ReplayOptions
{
    ProgressInterval = 10000,
    ProgressCallback = progress => { /* ... */ }
};

Logging Progress

using Microsoft.Extensions.Logging;

var options = new ReplayOptions
{
    ProgressCallback = progress =>
    {
        _logger.LogInformation(
            "Replay progress: {EventsProcessed}/{TotalEvents} ({Percent:F1}%) at {Rate:F0} events/sec, ETA: {ETA}",
            progress.EventsProcessed,
            progress.TotalEvents,
            progress.PercentComplete,
            progress.EventsPerSecond,
            progress.EstimatedRemaining?.ToString(@"hh\:mm\:ss") ?? "unknown");
    },
    ProgressInterval = 5000
};

Monitoring Dashboard Integration

Prometheus Metrics

using Prometheus;

var eventsProcessedCounter = Metrics.CreateCounter(
    "replay_events_processed_total",
    "Total events processed during replay");

var replayProgressGauge = Metrics.CreateGauge(
    "replay_progress_percent",
    "Replay progress percentage");

var options = new ReplayOptions
{
    ProgressCallback = progress =>
    {
        eventsProcessedCounter.Inc(progress.EventsProcessed - _lastCount);
        _lastCount = progress.EventsProcessed;

        replayProgressGauge.Set(progress.PercentComplete);
    },
    ProgressInterval = 1000
};

Application Insights

using Microsoft.ApplicationInsights;
using Microsoft.ApplicationInsights.DataContracts;

var telemetryClient = new TelemetryClient();

var options = new ReplayOptions
{
    ProgressCallback = progress =>
    {
        telemetryClient.TrackMetric(new MetricTelemetry
        {
            Name = "ReplayProgress",
            Sum = progress.PercentComplete,
            Properties =
            {
                ["StreamName"] = "orders",
                ["EventsProcessed"] = progress.EventsProcessed.ToString(),
                ["EventsPerSecond"] = progress.EventsPerSecond.ToString("F0")
            }
        });
    },
    ProgressInterval = 1000
};

Cancellation and Progress

Track progress during cancellable replays:

var cts = new CancellationTokenSource();

// Cancel after 5 minutes
cts.CancelAfter(TimeSpan.FromMinutes(5));

long lastProcessedCount = 0;

var options = new ReplayOptions
{
    ProgressCallback = progress =>
    {
        lastProcessedCount = progress.EventsProcessed;

        Console.WriteLine($"Processed {progress.EventsProcessed} events");

        if (progress.EventsProcessed >= 100000)
        {
            Console.WriteLine("Target reached, cancelling...");
            cts.Cancel();
        }
    },
    ProgressInterval = 1000
};

try
{
    await foreach (var @event in replayService.ReplayFromOffsetAsync(
        "orders",
        startOffset: 0,
        options,
        cts.Token))
    {
        await ProcessEventAsync(@event);
    }
}
catch (OperationCanceledException)
{
    Console.WriteLine($"Replay cancelled after processing {lastProcessedCount} events");
}

Background Replay with Progress Reporting

public class ReplayBackgroundService : BackgroundService
{
    private readonly IEventReplayService _replayService;
    private readonly ILogger<ReplayBackgroundService> _logger;
    private ReplayProgress? _currentProgress;

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        var options = new ReplayOptions
        {
            ProgressCallback = progress =>
            {
                _currentProgress = progress;

                _logger.LogInformation(
                    "Replay: {Processed}/{Total} ({Percent:F1}%) at {Rate:F0} events/sec",
                    progress.EventsProcessed,
                    progress.TotalEvents,
                    progress.PercentComplete,
                    progress.EventsPerSecond);
            },
            ProgressInterval = 5000
        };

        await foreach (var @event in _replayService.ReplayFromOffsetAsync(
            "orders",
            startOffset: 0,
            options,
            stoppingToken))
        {
            await ProcessEventAsync(@event, stoppingToken);
        }
    }

    public ReplayProgress? GetCurrentProgress() => _currentProgress;
}

// API endpoint to query progress
app.MapGet("/api/replay/progress", (ReplayBackgroundService service) =>
{
    var progress = service.GetCurrentProgress();
    return progress == null
        ? Results.NotFound("No replay in progress")
        : Results.Ok(progress);
});

Progress State Machine

Track replay lifecycle states:

public enum ReplayState
{
    NotStarted,
    Running,
    Paused,
    Completed,
    Failed,
    Cancelled
}

public class ReplayProgressTracker
{
    private ReplayState _state = ReplayState.NotStarted;
    private ReplayProgress? _progress;
    private readonly object _lock = new object();

    public void Start()
    {
        lock (_lock)
        {
            _state = ReplayState.Running;
        }
    }

    public void UpdateProgress(ReplayProgress progress)
    {
        lock (_lock)
        {
            _progress = progress;

            if (progress.PercentComplete >= 100)
            {
                _state = ReplayState.Completed;
            }
        }
    }

    public void Fail(Exception ex)
    {
        lock (_lock)
        {
            _state = ReplayState.Failed;
        }
    }

    public void Cancel()
    {
        lock (_lock)
        {
            _state = ReplayState.Cancelled;
        }
    }

    public (ReplayState State, ReplayProgress? Progress) GetStatus()
    {
        lock (_lock)
        {
            return (_state, _progress);
        }
    }
}

Performance Impact

Progress callbacks can impact throughput:

// High frequency - may impact performance
var options = new ReplayOptions
{
    ProgressInterval = 10,  // Every 10 events
    ProgressCallback = progress => { /* Heavy work */ }
};

// Recommended - balance between visibility and performance
var options = new ReplayOptions
{
    ProgressInterval = 1000,  // Every 1000 events
    ProgressCallback = progress => { /* Light work */ }
};

// Low frequency - minimal impact
var options = new ReplayOptions
{
    ProgressInterval = 10000,  // Every 10000 events
    ProgressCallback = progress => { /* Any work */ }
};

Best Practices

✅ DO

Use appropriate progress intervals (1000-5000 for most cases)
Keep progress callbacks lightweight
Log progress at INFO level
Track errors in progress metrics
Provide estimated completion time
Use cancellation tokens
Store final progress for audit

❌ DON'T

Don't use very frequent intervals (< 100)
Don't perform heavy computation in callbacks
Don't block in progress callbacks
Don't ignore error counts
Don't forget to handle cancellation
Don't log at DEBUG level for production

11 KiB Raw Blame History