509 lines
15 KiB
Markdown
509 lines
15 KiB
Markdown
# Replaying and Rebuilding Projections
|
|
|
|
Learn how to replay events and rebuild projections from scratch with Svrnty.CQRS.
|
|
|
|
## Why Replay Events?
|
|
|
|
Event replay allows you to:
|
|
|
|
✅ **Rebuild Projections** - Recreate read models from scratch
|
|
✅ **Fix Bugs** - Correct projection logic and reprocess events
|
|
✅ **Add New Projections** - Create new views from historical data
|
|
✅ **Time Travel** - Analyze data at specific points in time
|
|
✅ **Audit** - Investigate what happened and when
|
|
|
|
## Simple Replay
|
|
|
|
Replay all events from the beginning:
|
|
|
|
```csharp
|
|
public class ProjectionRebuilder
|
|
{
|
|
private readonly IEventStreamStore _eventStore;
|
|
private readonly ICheckpointStore _checkpointStore;
|
|
private readonly IDynamicProjection _projection;
|
|
|
|
public async Task RebuildAsync(CancellationToken ct = default)
|
|
{
|
|
Console.WriteLine($"Rebuilding projection: {_projection.ProjectionName}");
|
|
|
|
// Reset checkpoint to start from beginning
|
|
await _checkpointStore.SaveCheckpointAsync(_projection.ProjectionName, 0, ct);
|
|
|
|
// Clear existing projection data
|
|
if (_projection is IResettableProjection resettable)
|
|
{
|
|
await resettable.ResetAsync(ct);
|
|
}
|
|
|
|
// Replay all events
|
|
await _projection.RunAsync(ct);
|
|
|
|
Console.WriteLine("Rebuild complete!");
|
|
}
|
|
}
|
|
|
|
// Usage
|
|
var rebuilder = new ProjectionRebuilder(eventStore, checkpointStore, projection);
|
|
await rebuilder.RebuildAsync();
|
|
```
|
|
|
|
## Replay from Specific Offset
|
|
|
|
Replay events from a specific point:
|
|
|
|
```csharp
|
|
public class OffsetReplayService
|
|
{
|
|
private readonly IEventStreamStore _eventStore;
|
|
private readonly IDynamicProjection _projection;
|
|
|
|
public async Task ReplayFromOffsetAsync(
|
|
string streamName,
|
|
long startOffset,
|
|
CancellationToken ct = default)
|
|
{
|
|
Console.WriteLine($"Replaying from offset {startOffset}...");
|
|
|
|
var eventsProcessed = 0;
|
|
|
|
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
|
|
streamName,
|
|
fromOffset: startOffset,
|
|
cancellationToken: ct))
|
|
{
|
|
await ProcessEventAsync(storedEvent.Data, ct);
|
|
eventsProcessed++;
|
|
|
|
if (eventsProcessed % 100 == 0)
|
|
{
|
|
Console.WriteLine($"Processed {eventsProcessed} events...");
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"Replay complete. Processed {eventsProcessed} events.");
|
|
}
|
|
|
|
private async Task ProcessEventAsync(object @event, CancellationToken ct)
|
|
{
|
|
// Process event with projection logic
|
|
// ...
|
|
}
|
|
}
|
|
|
|
// Usage
|
|
var replayService = new OffsetReplayService(eventStore, projection);
|
|
await replayService.ReplayFromOffsetAsync("orders", startOffset: 1000);
|
|
```
|
|
|
|
## Time-Based Replay
|
|
|
|
Replay events from a specific time:
|
|
|
|
```csharp
|
|
public class TimeBasedReplayService
|
|
{
|
|
private readonly IEventReplayService _replayService;
|
|
private readonly IDynamicProjection _projection;
|
|
|
|
public async Task ReplayFromTimeAsync(
|
|
string streamName,
|
|
DateTimeOffset startTime,
|
|
CancellationToken ct = default)
|
|
{
|
|
Console.WriteLine($"Replaying events from {startTime}...");
|
|
|
|
var options = new ReplayOptions
|
|
{
|
|
BatchSize = 100,
|
|
ProgressInterval = 1000,
|
|
ProgressCallback = progress =>
|
|
{
|
|
Console.WriteLine(
|
|
$"Progress: {progress.EventsProcessed} events " +
|
|
$"@ {progress.EventsPerSecond:F0} events/sec " +
|
|
$"(ETA: {progress.EstimatedTimeRemaining})");
|
|
}
|
|
};
|
|
|
|
await foreach (var @event in _replayService.ReplayFromTimeAsync(
|
|
streamName,
|
|
startTime,
|
|
options,
|
|
ct))
|
|
{
|
|
await ProcessEventAsync(@event.Data, ct);
|
|
}
|
|
|
|
Console.WriteLine("Replay complete!");
|
|
}
|
|
}
|
|
|
|
// Usage
|
|
var replayService = new TimeBasedReplayService(eventReplayService, projection);
|
|
await replayService.ReplayFromTimeAsync("orders", DateTimeOffset.UtcNow.AddDays(-7));
|
|
```
|
|
|
|
## Rate-Limited Replay
|
|
|
|
Replay with rate limiting to avoid overwhelming the system:
|
|
|
|
```csharp
|
|
public class RateLimitedReplayService
|
|
{
|
|
private readonly IEventReplayService _replayService;
|
|
|
|
public async Task ReplayWithRateLimitAsync(
|
|
string streamName,
|
|
int maxEventsPerSecond,
|
|
CancellationToken ct = default)
|
|
{
|
|
var options = new ReplayOptions
|
|
{
|
|
BatchSize = 100,
|
|
MaxEventsPerSecond = maxEventsPerSecond, // Rate limit
|
|
ProgressInterval = 1000,
|
|
ProgressCallback = progress =>
|
|
{
|
|
Console.WriteLine(
|
|
$"Replaying: {progress.EventsProcessed} events " +
|
|
$"@ {progress.EventsPerSecond:F0} events/sec (limited to {maxEventsPerSecond})");
|
|
}
|
|
};
|
|
|
|
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
|
|
streamName,
|
|
startOffset: 0,
|
|
options,
|
|
ct))
|
|
{
|
|
await ProcessEventAsync(@event.Data, ct);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Usage: Replay at 1000 events/sec to avoid overload
|
|
await replayService.ReplayWithRateLimitAsync("orders", maxEventsPerSecond: 1000);
|
|
```
|
|
|
|
## Filtered Replay
|
|
|
|
Replay only specific event types:
|
|
|
|
```csharp
|
|
public class FilteredReplayService
|
|
{
|
|
private readonly IEventReplayService _replayService;
|
|
|
|
public async Task ReplayEventTypesAsync(
|
|
string streamName,
|
|
string[] eventTypes,
|
|
CancellationToken ct = default)
|
|
{
|
|
var options = new ReplayOptions
|
|
{
|
|
EventTypeFilter = eventTypes,
|
|
BatchSize = 100,
|
|
ProgressInterval = 1000,
|
|
ProgressCallback = progress =>
|
|
{
|
|
Console.WriteLine($"Processed {progress.EventsProcessed} events...");
|
|
}
|
|
};
|
|
|
|
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
|
|
streamName,
|
|
startOffset: 0,
|
|
options,
|
|
ct))
|
|
{
|
|
await ProcessEventAsync(@event.Data, ct);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Usage: Replay only order-related events
|
|
await replayService.ReplayEventTypesAsync(
|
|
"orders",
|
|
new[] { "OrderPlaced", "OrderShipped", "OrderDelivered" });
|
|
```
|
|
|
|
## Parallel Projection Rebuild
|
|
|
|
Rebuild multiple projections in parallel:
|
|
|
|
```csharp
|
|
public class ParallelProjectionRebuilder
|
|
{
|
|
private readonly IEventStreamStore _eventStore;
|
|
private readonly IEnumerable<IDynamicProjection> _projections;
|
|
|
|
public async Task RebuildAllAsync(CancellationToken ct = default)
|
|
{
|
|
Console.WriteLine($"Rebuilding {_projections.Count()} projections in parallel...");
|
|
|
|
var tasks = _projections.Select(async projection =>
|
|
{
|
|
Console.WriteLine($"Starting rebuild: {projection.ProjectionName}");
|
|
|
|
if (projection is IResettableProjection resettable)
|
|
{
|
|
await resettable.ResetAsync(ct);
|
|
}
|
|
|
|
await projection.RunAsync(ct);
|
|
|
|
Console.WriteLine($"Completed rebuild: {projection.ProjectionName}");
|
|
});
|
|
|
|
await Task.WhenAll(tasks);
|
|
|
|
Console.WriteLine("All projections rebuilt!");
|
|
}
|
|
}
|
|
|
|
// Usage
|
|
var rebuilder = new ParallelProjectionRebuilder(eventStore, projections);
|
|
await rebuilder.RebuildAllAsync();
|
|
```
|
|
|
|
## Incremental Replay
|
|
|
|
Replay only events that haven't been processed yet:
|
|
|
|
```csharp
|
|
public class IncrementalReplayService
|
|
{
|
|
private readonly IEventStreamStore _eventStore;
|
|
private readonly ICheckpointStore _checkpointStore;
|
|
private readonly IDynamicProjection _projection;
|
|
|
|
public async Task CatchUpAsync(CancellationToken ct = default)
|
|
{
|
|
var checkpoint = await _checkpointStore.GetCheckpointAsync(_projection.ProjectionName, ct);
|
|
|
|
Console.WriteLine($"Catching up from offset {checkpoint}...");
|
|
|
|
var eventsProcessed = 0;
|
|
|
|
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
|
|
"orders",
|
|
fromOffset: checkpoint + 1,
|
|
cancellationToken: ct))
|
|
{
|
|
await ProcessEventAsync(storedEvent.Data, ct);
|
|
await _checkpointStore.SaveCheckpointAsync(_projection.ProjectionName, storedEvent.Offset, ct);
|
|
|
|
eventsProcessed++;
|
|
|
|
if (eventsProcessed % 100 == 0)
|
|
{
|
|
Console.WriteLine($"Caught up {eventsProcessed} events...");
|
|
}
|
|
}
|
|
|
|
Console.WriteLine($"Catch-up complete. Processed {eventsProcessed} new events.");
|
|
}
|
|
}
|
|
|
|
// Usage: Catch up a projection that fell behind
|
|
var catchUpService = new IncrementalReplayService(eventStore, checkpointStore, projection);
|
|
await catchUpService.CatchUpAsync();
|
|
```
|
|
|
|
## Projection Versioning
|
|
|
|
Rebuild projections when logic changes:
|
|
|
|
```csharp
|
|
public interface IVersionedProjection : IDynamicProjection
|
|
{
|
|
int Version { get; }
|
|
}
|
|
|
|
public class UserSummaryProjectionV2 : IVersionedProjection, IResettableProjection
|
|
{
|
|
public string ProjectionName => "user-summary";
|
|
public int Version => 2; // Incremented when logic changes
|
|
|
|
private readonly IProjectionVersionStore _versionStore;
|
|
|
|
public async Task RunAsync(CancellationToken ct)
|
|
{
|
|
var storedVersion = await _versionStore.GetVersionAsync(ProjectionName, ct);
|
|
|
|
if (storedVersion < Version)
|
|
{
|
|
Console.WriteLine($"Projection version mismatch. Rebuilding from scratch...");
|
|
await ResetAsync(ct);
|
|
await _versionStore.SetVersionAsync(ProjectionName, Version, ct);
|
|
}
|
|
|
|
// Run projection normally
|
|
var checkpoint = await _checkpointStore.GetCheckpointAsync(ProjectionName, ct);
|
|
|
|
await foreach (var storedEvent in _eventStore.ReadStreamAsync(
|
|
"users",
|
|
fromOffset: checkpoint + 1,
|
|
cancellationToken: ct))
|
|
{
|
|
await HandleEventAsync(storedEvent.Data, ct);
|
|
await _checkpointStore.SaveCheckpointAsync(ProjectionName, storedEvent.Offset, ct);
|
|
}
|
|
}
|
|
|
|
public async Task ResetAsync(CancellationToken ct)
|
|
{
|
|
await _repository.DeleteAllAsync(ct);
|
|
await _checkpointStore.SaveCheckpointAsync(ProjectionName, 0, ct);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Complete Rebuild Example
|
|
|
|
Here's a complete example with rebuild CLI:
|
|
|
|
```csharp
|
|
// RebuildProjectionCommand.cs
|
|
public class RebuildProjectionCommand
|
|
{
|
|
public async Task ExecuteAsync(string projectionName)
|
|
{
|
|
var serviceProvider = BuildServiceProvider();
|
|
|
|
var projections = serviceProvider.GetServices<IDynamicProjection>();
|
|
var projection = projections.FirstOrDefault(p => p.ProjectionName == projectionName);
|
|
|
|
if (projection == null)
|
|
{
|
|
Console.WriteLine($"Projection '{projectionName}' not found.");
|
|
return;
|
|
}
|
|
|
|
Console.WriteLine($"Rebuilding projection: {projectionName}");
|
|
|
|
// Reset
|
|
if (projection is IResettableProjection resettable)
|
|
{
|
|
Console.WriteLine("Resetting projection...");
|
|
await resettable.ResetAsync();
|
|
}
|
|
|
|
// Replay
|
|
Console.WriteLine("Replaying events...");
|
|
|
|
var startTime = DateTimeOffset.UtcNow;
|
|
await projection.RunAsync(CancellationToken.None);
|
|
var elapsed = DateTimeOffset.UtcNow - startTime;
|
|
|
|
Console.WriteLine($"Rebuild complete in {elapsed.TotalSeconds:F2} seconds!");
|
|
}
|
|
|
|
private IServiceProvider BuildServiceProvider()
|
|
{
|
|
var services = new ServiceCollection();
|
|
|
|
// Register all services
|
|
services.AddEventStreaming()
|
|
.AddPostgresEventStore(connectionString);
|
|
|
|
services.AddSingleton<ICheckpointStore, PostgresCheckpointStore>();
|
|
services.AddSingleton<IDynamicProjection, UserSummaryProjection>();
|
|
services.AddSingleton<IDynamicProjection, OrderAnalyticsProjection>();
|
|
|
|
return services.BuildServiceProvider();
|
|
}
|
|
}
|
|
|
|
// Usage from CLI
|
|
// dotnet run -- rebuild user-summary
|
|
```
|
|
|
|
## Best Practices
|
|
|
|
✅ **DO:**
|
|
- Reset projections before rebuilding
|
|
- Use rate limiting to avoid overload
|
|
- Track progress during long replays
|
|
- Version projections when logic changes
|
|
- Test replay logic before production rebuild
|
|
- Take database backups before rebuilding
|
|
|
|
❌ **DON'T:**
|
|
- Replay without resetting projection data
|
|
- Rebuild during peak traffic hours
|
|
- Forget to track checkpoint progress
|
|
- Change event schemas without versioning
|
|
- Rebuild without testing first
|
|
- Run multiple rebuilds simultaneously
|
|
|
|
## Monitoring Replay Progress
|
|
|
|
Monitor replay progress with metrics:
|
|
|
|
```csharp
|
|
public class MonitoredReplayService
|
|
{
|
|
private readonly IEventReplayService _replayService;
|
|
private readonly ILogger<MonitoredReplayService> _logger;
|
|
|
|
public async Task ReplayWithMonitoringAsync(
|
|
string streamName,
|
|
CancellationToken ct = default)
|
|
{
|
|
var startTime = DateTimeOffset.UtcNow;
|
|
var totalEvents = 0L;
|
|
|
|
var options = new ReplayOptions
|
|
{
|
|
BatchSize = 100,
|
|
ProgressInterval = 1000,
|
|
ProgressCallback = progress =>
|
|
{
|
|
_logger.LogInformation(
|
|
"Replay progress: {EventsProcessed} events " +
|
|
"@ {EventsPerSecond:F0} events/sec, " +
|
|
"ETA: {EstimatedTimeRemaining}",
|
|
progress.EventsProcessed,
|
|
progress.EventsPerSecond,
|
|
progress.EstimatedTimeRemaining);
|
|
|
|
totalEvents = progress.EventsProcessed;
|
|
}
|
|
};
|
|
|
|
await foreach (var @event in _replayService.ReplayFromOffsetAsync(
|
|
streamName,
|
|
startOffset: 0,
|
|
options,
|
|
ct))
|
|
{
|
|
await ProcessEventAsync(@event.Data, ct);
|
|
}
|
|
|
|
var elapsed = DateTimeOffset.UtcNow - startTime;
|
|
|
|
_logger.LogInformation(
|
|
"Replay complete: {TotalEvents} events in {ElapsedSeconds:F2} seconds " +
|
|
"({AverageEventsPerSecond:F0} events/sec)",
|
|
totalEvents,
|
|
elapsed.TotalSeconds,
|
|
totalEvents / elapsed.TotalSeconds);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Next Steps
|
|
|
|
- [E-Commerce Example Tutorial](../ecommerce-example/README.md) - Complete real-world example
|
|
- [Event Replay API](../../event-streaming/event-replay/README.md) - Replay API documentation
|
|
- [Projections](../../event-streaming/projections/README.md) - Projection documentation
|
|
|
|
## See Also
|
|
|
|
- [Event Replay from Offset](../../event-streaming/event-replay/replay-from-offset.md)
|
|
- [Event Replay from Time](../../event-streaming/event-replay/replay-from-time.md)
|
|
- [Rate Limiting](../../event-streaming/event-replay/rate-limiting.md)
|
|
- [Progress Tracking](../../event-streaming/event-replay/progress-tracking.md)
|