15 KiB
15 KiB
Replaying and Rebuilding Projections
Learn how to replay events and rebuild projections from scratch with Svrnty.CQRS.
Why Replay Events?
Event replay allows you to:
✅ Rebuild Projections - Recreate read models from scratch ✅ Fix Bugs - Correct projection logic and reprocess events ✅ Add New Projections - Create new views from historical data ✅ Time Travel - Analyze data at specific points in time ✅ Audit - Investigate what happened and when
Simple Replay
Replay all events from the beginning:
public class ProjectionRebuilder
{
private readonly IEventStreamStore _eventStore;
private readonly ICheckpointStore _checkpointStore;
private readonly IDynamicProjection _projection;
public async Task RebuildAsync(CancellationToken ct = default)
{
Console.WriteLine($"Rebuilding projection: {_projection.ProjectionName}");
// Reset checkpoint to start from beginning
await _checkpointStore.SaveCheckpointAsync(_projection.ProjectionName, 0, ct);
// Clear existing projection data
if (_projection is IResettableProjection resettable)
{
await resettable.ResetAsync(ct);
}
// Replay all events
await _projection.RunAsync(ct);
Console.WriteLine("Rebuild complete!");
}
}
// Usage
var rebuilder = new ProjectionRebuilder(eventStore, checkpointStore, projection);
await rebuilder.RebuildAsync();
Replay from Specific Offset
Replay events from a specific point:
public class OffsetReplayService
{
private readonly IEventStreamStore _eventStore;
private readonly IDynamicProjection _projection;
public async Task ReplayFromOffsetAsync(
string streamName,
long startOffset,
CancellationToken ct = default)
{
Console.WriteLine($"Replaying from offset {startOffset}...");
var eventsProcessed = 0;
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
streamName,
fromOffset: startOffset,
cancellationToken: ct))
{
await ProcessEventAsync(storedEvent.Data, ct);
eventsProcessed++;
if (eventsProcessed % 100 == 0)
{
Console.WriteLine($"Processed {eventsProcessed} events...");
}
}
Console.WriteLine($"Replay complete. Processed {eventsProcessed} events.");
}
private async Task ProcessEventAsync(object @event, CancellationToken ct)
{
// Process event with projection logic
// ...
}
}
// Usage
var replayService = new OffsetReplayService(eventStore, projection);
await replayService.ReplayFromOffsetAsync("orders", startOffset: 1000);
Time-Based Replay
Replay events from a specific time:
public class TimeBasedReplayService
{
private readonly IEventReplayService _replayService;
private readonly IDynamicProjection _projection;
public async Task ReplayFromTimeAsync(
string streamName,
DateTimeOffset startTime,
CancellationToken ct = default)
{
Console.WriteLine($"Replaying events from {startTime}...");
var options = new ReplayOptions
{
BatchSize = 100,
ProgressInterval = 1000,
ProgressCallback = progress =>
{
Console.WriteLine(
$"Progress: {progress.EventsProcessed} events " +
$"@ {progress.EventsPerSecond:F0} events/sec " +
$"(ETA: {progress.EstimatedTimeRemaining})");
}
};
await foreach (var @event in _replayService.ReplayFromTimeAsync(
streamName,
startTime,
options,
ct))
{
await ProcessEventAsync(@event.Data, ct);
}
Console.WriteLine("Replay complete!");
}
}
// Usage
var replayService = new TimeBasedReplayService(eventReplayService, projection);
await replayService.ReplayFromTimeAsync("orders", DateTimeOffset.UtcNow.AddDays(-7));
Rate-Limited Replay
Replay with rate limiting to avoid overwhelming the system:
public class RateLimitedReplayService
{
private readonly IEventReplayService _replayService;
public async Task ReplayWithRateLimitAsync(
string streamName,
int maxEventsPerSecond,
CancellationToken ct = default)
{
var options = new ReplayOptions
{
BatchSize = 100,
MaxEventsPerSecond = maxEventsPerSecond, // Rate limit
ProgressInterval = 1000,
ProgressCallback = progress =>
{
Console.WriteLine(
$"Replaying: {progress.EventsProcessed} events " +
$"@ {progress.EventsPerSecond:F0} events/sec (limited to {maxEventsPerSecond})");
}
};
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
streamName,
startOffset: 0,
options,
ct))
{
await ProcessEventAsync(@event.Data, ct);
}
}
}
// Usage: Replay at 1000 events/sec to avoid overload
await replayService.ReplayWithRateLimitAsync("orders", maxEventsPerSecond: 1000);
Filtered Replay
Replay only specific event types:
public class FilteredReplayService
{
private readonly IEventReplayService _replayService;
public async Task ReplayEventTypesAsync(
string streamName,
string[] eventTypes,
CancellationToken ct = default)
{
var options = new ReplayOptions
{
EventTypeFilter = eventTypes,
BatchSize = 100,
ProgressInterval = 1000,
ProgressCallback = progress =>
{
Console.WriteLine($"Processed {progress.EventsProcessed} events...");
}
};
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
streamName,
startOffset: 0,
options,
ct))
{
await ProcessEventAsync(@event.Data, ct);
}
}
}
// Usage: Replay only order-related events
await replayService.ReplayEventTypesAsync(
"orders",
new[] { "OrderPlaced", "OrderShipped", "OrderDelivered" });
Parallel Projection Rebuild
Rebuild multiple projections in parallel:
public class ParallelProjectionRebuilder
{
private readonly IEventStreamStore _eventStore;
private readonly IEnumerable<IDynamicProjection> _projections;
public async Task RebuildAllAsync(CancellationToken ct = default)
{
Console.WriteLine($"Rebuilding {_projections.Count()} projections in parallel...");
var tasks = _projections.Select(async projection =>
{
Console.WriteLine($"Starting rebuild: {projection.ProjectionName}");
if (projection is IResettableProjection resettable)
{
await resettable.ResetAsync(ct);
}
await projection.RunAsync(ct);
Console.WriteLine($"Completed rebuild: {projection.ProjectionName}");
});
await Task.WhenAll(tasks);
Console.WriteLine("All projections rebuilt!");
}
}
// Usage
var rebuilder = new ParallelProjectionRebuilder(eventStore, projections);
await rebuilder.RebuildAllAsync();
Incremental Replay
Replay only events that haven't been processed yet:
public class IncrementalReplayService
{
private readonly IEventStreamStore _eventStore;
private readonly ICheckpointStore _checkpointStore;
private readonly IDynamicProjection _projection;
public async Task CatchUpAsync(CancellationToken ct = default)
{
var checkpoint = await _checkpointStore.GetCheckpointAsync(_projection.ProjectionName, ct);
Console.WriteLine($"Catching up from offset {checkpoint}...");
var eventsProcessed = 0;
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
"orders",
fromOffset: checkpoint + 1,
cancellationToken: ct))
{
await ProcessEventAsync(storedEvent.Data, ct);
await _checkpointStore.SaveCheckpointAsync(_projection.ProjectionName, storedEvent.Offset, ct);
eventsProcessed++;
if (eventsProcessed % 100 == 0)
{
Console.WriteLine($"Caught up {eventsProcessed} events...");
}
}
Console.WriteLine($"Catch-up complete. Processed {eventsProcessed} new events.");
}
}
// Usage: Catch up a projection that fell behind
var catchUpService = new IncrementalReplayService(eventStore, checkpointStore, projection);
await catchUpService.CatchUpAsync();
Projection Versioning
Rebuild projections when logic changes:
public interface IVersionedProjection : IDynamicProjection
{
int Version { get; }
}
public class UserSummaryProjectionV2 : IVersionedProjection, IResettableProjection
{
public string ProjectionName => "user-summary";
public int Version => 2; // Incremented when logic changes
private readonly IProjectionVersionStore _versionStore;
public async Task RunAsync(CancellationToken ct)
{
var storedVersion = await _versionStore.GetVersionAsync(ProjectionName, ct);
if (storedVersion < Version)
{
Console.WriteLine($"Projection version mismatch. Rebuilding from scratch...");
await ResetAsync(ct);
await _versionStore.SetVersionAsync(ProjectionName, Version, ct);
}
// Run projection normally
var checkpoint = await _checkpointStore.GetCheckpointAsync(ProjectionName, ct);
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
"users",
fromOffset: checkpoint + 1,
cancellationToken: ct))
{
await HandleEventAsync(storedEvent.Data, ct);
await _checkpointStore.SaveCheckpointAsync(ProjectionName, storedEvent.Offset, ct);
}
}
public async Task ResetAsync(CancellationToken ct)
{
await _repository.DeleteAllAsync(ct);
await _checkpointStore.SaveCheckpointAsync(ProjectionName, 0, ct);
}
}
Complete Rebuild Example
Here's a complete example with rebuild CLI:
// RebuildProjectionCommand.cs
public class RebuildProjectionCommand
{
public async Task ExecuteAsync(string projectionName)
{
var serviceProvider = BuildServiceProvider();
var projections = serviceProvider.GetServices<IDynamicProjection>();
var projection = projections.FirstOrDefault(p => p.ProjectionName == projectionName);
if (projection == null)
{
Console.WriteLine($"Projection '{projectionName}' not found.");
return;
}
Console.WriteLine($"Rebuilding projection: {projectionName}");
// Reset
if (projection is IResettableProjection resettable)
{
Console.WriteLine("Resetting projection...");
await resettable.ResetAsync();
}
// Replay
Console.WriteLine("Replaying events...");
var startTime = DateTimeOffset.UtcNow;
await projection.RunAsync(CancellationToken.None);
var elapsed = DateTimeOffset.UtcNow - startTime;
Console.WriteLine($"Rebuild complete in {elapsed.TotalSeconds:F2} seconds!");
}
private IServiceProvider BuildServiceProvider()
{
var services = new ServiceCollection();
// Register all services
services.AddEventStreaming()
.AddPostgresEventStore(connectionString);
services.AddSingleton<ICheckpointStore, PostgresCheckpointStore>();
services.AddSingleton<IDynamicProjection, UserSummaryProjection>();
services.AddSingleton<IDynamicProjection, OrderAnalyticsProjection>();
return services.BuildServiceProvider();
}
}
// Usage from CLI
// dotnet run -- rebuild user-summary
Best Practices
✅ DO:
- Reset projections before rebuilding
- Use rate limiting to avoid overload
- Track progress during long replays
- Version projections when logic changes
- Test replay logic before production rebuild
- Take database backups before rebuilding
❌ DON'T:
- Replay without resetting projection data
- Rebuild during peak traffic hours
- Forget to track checkpoint progress
- Change event schemas without versioning
- Rebuild without testing first
- Run multiple rebuilds simultaneously
Monitoring Replay Progress
Monitor replay progress with metrics:
public class MonitoredReplayService
{
private readonly IEventReplayService _replayService;
private readonly ILogger<MonitoredReplayService> _logger;
public async Task ReplayWithMonitoringAsync(
string streamName,
CancellationToken ct = default)
{
var startTime = DateTimeOffset.UtcNow;
var totalEvents = 0L;
var options = new ReplayOptions
{
BatchSize = 100,
ProgressInterval = 1000,
ProgressCallback = progress =>
{
_logger.LogInformation(
"Replay progress: {EventsProcessed} events " +
"@ {EventsPerSecond:F0} events/sec, " +
"ETA: {EstimatedTimeRemaining}",
progress.EventsProcessed,
progress.EventsPerSecond,
progress.EstimatedTimeRemaining);
totalEvents = progress.EventsProcessed;
}
};
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
streamName,
startOffset: 0,
options,
ct))
{
await ProcessEventAsync(@event.Data, ct);
}
var elapsed = DateTimeOffset.UtcNow - startTime;
_logger.LogInformation(
"Replay complete: {TotalEvents} events in {ElapsedSeconds:F2} seconds " +
"({AverageEventsPerSecond:F0} events/sec)",
totalEvents,
elapsed.TotalSeconds,
totalEvents / elapsed.TotalSeconds);
}
}
Next Steps
- E-Commerce Example Tutorial - Complete real-world example
- Event Replay API - Replay API documentation
- Projections - Projection documentation