504 lines
13 KiB
Markdown
504 lines
13 KiB
Markdown
# Compensation
|
|
|
|
Rollback completed saga steps when failures occur.
|
|
|
|
## Overview
|
|
|
|
Compensation undoes completed saga steps to maintain consistency when later steps fail:
|
|
- **Backward Recovery** - Undo completed operations
|
|
- **Idempotency** - Safe to retry compensations
|
|
- **Ordering** - Compensate in reverse order
|
|
- **Partial Compensation** - Only undo completed steps
|
|
|
|
## Quick Start
|
|
|
|
```csharp
|
|
private async Task CompensateAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
_logger.LogWarning("Starting compensation for saga {SagaId}", state.SagaId);
|
|
|
|
state.CurrentStep = SagaStep.Compensating;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
|
|
// Compensate in reverse order of execution
|
|
if (state.Shipped && !state.ShipmentCancelled)
|
|
{
|
|
await CompensateShipmentAsync(state, ct);
|
|
}
|
|
|
|
if (state.PaymentProcessed && !state.PaymentRefunded)
|
|
{
|
|
await CompensatePaymentAsync(state, ct);
|
|
}
|
|
|
|
if (state.InventoryReserved && !state.InventoryReleased)
|
|
{
|
|
await CompensateInventoryAsync(state, ct);
|
|
}
|
|
|
|
state.CurrentStep = SagaStep.Failed;
|
|
state.CompletedAt = DateTimeOffset.UtcNow;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
|
|
_logger.LogInformation("Compensation complete for saga {SagaId}", state.SagaId);
|
|
}
|
|
```
|
|
|
|
## Compensation Principles
|
|
|
|
### Reverse Order
|
|
|
|
Compensate steps in reverse order of execution:
|
|
|
|
```csharp
|
|
// Execution order:
|
|
// 1. Reserve Inventory
|
|
// 2. Process Payment
|
|
// 3. Ship Order → FAILS
|
|
|
|
// Compensation order:
|
|
// 1. (Ship order didn't complete, skip)
|
|
// 2. Refund Payment
|
|
// 3. Release Inventory
|
|
```
|
|
|
|
### Idempotency
|
|
|
|
Make compensations idempotent (safe to retry):
|
|
|
|
```csharp
|
|
private async Task CompensatePaymentAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
// Check if already refunded (idempotency)
|
|
if (state.PaymentRefunded)
|
|
{
|
|
_logger.LogInformation("Payment already refunded for order {OrderId}", state.OrderId);
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
await _paymentService.RefundAsync(state.OrderId, ct);
|
|
state.PaymentRefunded = true;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
|
|
_logger.LogInformation("Payment refunded for order {OrderId}", state.OrderId);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to refund payment for order {OrderId}", state.OrderId);
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
### Partial Compensation
|
|
|
|
Only compensate completed steps:
|
|
|
|
```csharp
|
|
private async Task CompensateAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
// Only compensate what was actually completed
|
|
var completedSteps = new List<string>();
|
|
|
|
if (state.Shipped)
|
|
completedSteps.Add("Shipment");
|
|
if (state.PaymentProcessed)
|
|
completedSteps.Add("Payment");
|
|
if (state.InventoryReserved)
|
|
completedSteps.Add("Inventory");
|
|
|
|
_logger.LogWarning(
|
|
"Compensating {Count} completed steps for {SagaId}: {Steps}",
|
|
completedSteps.Count,
|
|
state.SagaId,
|
|
string.Join(", ", completedSteps));
|
|
|
|
// Compensate only completed steps
|
|
if (state.Shipped && !state.ShipmentCancelled)
|
|
{
|
|
await CompensateShipmentAsync(state, ct);
|
|
}
|
|
|
|
if (state.PaymentProcessed && !state.PaymentRefunded)
|
|
{
|
|
await CompensatePaymentAsync(state, ct);
|
|
}
|
|
|
|
if (state.InventoryReserved && !state.InventoryReleased)
|
|
{
|
|
await CompensateInventoryAsync(state, ct);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Compensation Strategies
|
|
|
|
### Immediate Compensation
|
|
|
|
Undo immediately when step fails:
|
|
|
|
```csharp
|
|
public async Task HandleAsync(OrderPlacedEvent @event, CancellationToken ct)
|
|
{
|
|
var state = new OrderFulfillmentState { OrderId = @event.OrderId };
|
|
|
|
try
|
|
{
|
|
// Step 1
|
|
await _inventoryService.ReserveAsync(@event.OrderId, @event.Items, ct);
|
|
state.InventoryReserved = true;
|
|
|
|
// Step 2
|
|
await _paymentService.ChargeAsync(@event.OrderId, @event.TotalAmount, ct);
|
|
state.PaymentProcessed = true;
|
|
|
|
// Step 3 - fails
|
|
await _shippingService.ShipAsync(@event.OrderId, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Immediately compensate
|
|
await CompensateAsync(state, ct);
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
### Delayed Compensation
|
|
|
|
Queue compensation for later execution:
|
|
|
|
```csharp
|
|
public async Task HandleAsync(OrderPlacedEvent @event, CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
await ExecuteStepsAsync(@event, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Queue compensation for background processing
|
|
await _compensationQueue.EnqueueAsync(new CompensationTask
|
|
{
|
|
SagaId = state.SagaId,
|
|
State = state,
|
|
Reason = ex.Message,
|
|
ScheduledFor = DateTimeOffset.UtcNow.AddSeconds(30)
|
|
});
|
|
|
|
throw;
|
|
}
|
|
}
|
|
|
|
// Background service processes compensation queue
|
|
public class CompensationProcessor : BackgroundService
|
|
{
|
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
|
{
|
|
await foreach (var task in _compensationQueue.DequeueAsync(stoppingToken))
|
|
{
|
|
await CompensateAsync(task.State, stoppingToken);
|
|
}
|
|
}
|
|
}
|
|
```
|
|
|
|
### Retry with Backoff
|
|
|
|
Retry failed compensations with exponential backoff:
|
|
|
|
```csharp
|
|
private async Task CompensateWithRetryAsync(
|
|
OrderFulfillmentState state,
|
|
CancellationToken ct)
|
|
{
|
|
var maxAttempts = 5;
|
|
|
|
for (int attempt = 1; attempt <= maxAttempts; attempt++)
|
|
{
|
|
try
|
|
{
|
|
await CompensateAsync(state, ct);
|
|
return; // Success
|
|
}
|
|
catch (Exception ex) when (attempt < maxAttempts)
|
|
{
|
|
var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt));
|
|
|
|
_logger.LogWarning(ex,
|
|
"Compensation attempt {Attempt}/{Max} failed for {SagaId}, retrying in {Delay}",
|
|
attempt,
|
|
maxAttempts,
|
|
state.SagaId,
|
|
delay);
|
|
|
|
await Task.Delay(delay, ct);
|
|
}
|
|
}
|
|
|
|
// All retries failed - manual intervention required
|
|
await NotifyOpsTeamAsync(state, "Compensation failed after all retries");
|
|
}
|
|
```
|
|
|
|
## Compensation Examples
|
|
|
|
### Inventory Compensation
|
|
|
|
```csharp
|
|
private async Task CompensateInventoryAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
if (state.InventoryReleased)
|
|
{
|
|
_logger.LogInformation("Inventory already released for order {OrderId}", state.OrderId);
|
|
return;
|
|
}
|
|
|
|
_logger.LogInformation("Releasing inventory for order {OrderId}", state.OrderId);
|
|
|
|
try
|
|
{
|
|
await _inventoryService.ReleaseReservationAsync(state.OrderId, ct);
|
|
|
|
state.InventoryReleased = true;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
}
|
|
catch (ReservationNotFoundException)
|
|
{
|
|
// Already released or never existed - treat as success
|
|
state.InventoryReleased = true;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to release inventory for order {OrderId}", state.OrderId);
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
### Payment Compensation
|
|
|
|
```csharp
|
|
private async Task CompensatePaymentAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
if (state.PaymentRefunded)
|
|
{
|
|
_logger.LogInformation("Payment already refunded for order {OrderId}", state.OrderId);
|
|
return;
|
|
}
|
|
|
|
_logger.LogInformation("Refunding payment for order {OrderId}", state.OrderId);
|
|
|
|
try
|
|
{
|
|
var refundId = await _paymentService.RefundAsync(
|
|
state.OrderId,
|
|
state.PaymentTransactionId,
|
|
reason: "Order fulfillment failed",
|
|
ct);
|
|
|
|
state.PaymentRefunded = true;
|
|
state.RefundTransactionId = refundId;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
|
|
// Publish refund event
|
|
await _eventPublisher.PublishAsync(new PaymentRefundedEvent
|
|
{
|
|
OrderId = state.OrderId,
|
|
RefundId = refundId,
|
|
Reason = "Order fulfillment failed"
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to refund payment for order {OrderId}", state.OrderId);
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
### Shipment Compensation
|
|
|
|
```csharp
|
|
private async Task CompensateShipmentAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
if (state.ShipmentCancelled)
|
|
{
|
|
_logger.LogInformation("Shipment already cancelled for order {OrderId}", state.OrderId);
|
|
return;
|
|
}
|
|
|
|
_logger.LogWarning("Cancelling shipment for order {OrderId}", state.OrderId);
|
|
|
|
try
|
|
{
|
|
await _shippingService.CancelShipmentAsync(
|
|
state.OrderId,
|
|
state.ShipmentTrackingNumber,
|
|
ct);
|
|
|
|
state.ShipmentCancelled = true;
|
|
await _stateStore.SaveStateAsync(state.SagaId, state, ct);
|
|
|
|
// Notify customer
|
|
await _notificationService.SendAsync(new ShipmentCancelledNotification
|
|
{
|
|
OrderId = state.OrderId,
|
|
Reason = "Order fulfillment failed"
|
|
});
|
|
}
|
|
catch (ShipmentAlreadyDeliveredException)
|
|
{
|
|
_logger.LogError("Cannot cancel shipment for order {OrderId} - already delivered", state.OrderId);
|
|
|
|
// Can't undo delivery - create return label instead
|
|
await CreateReturnLabelAsync(state.OrderId, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to cancel shipment for order {OrderId}", state.OrderId);
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
## Compensation Limitations
|
|
|
|
### Non-Compensatable Actions
|
|
|
|
Some actions cannot be undone:
|
|
|
|
```csharp
|
|
// ✅ Compensatable
|
|
await _inventoryService.ReserveAsync(orderId, items); // Can release
|
|
await _paymentService.ChargeAsync(orderId, amount); // Can refund
|
|
|
|
// ❌ Non-Compensatable
|
|
await _emailService.SendOrderConfirmationAsync(orderId); // Cannot unsend
|
|
await _externalApiService.NotifyPartnerAsync(orderId); // May not support undo
|
|
```
|
|
|
|
### Handling Non-Compensatable Actions
|
|
|
|
```csharp
|
|
private async Task CompensateAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
// Compensate what we can
|
|
if (state.PaymentProcessed)
|
|
{
|
|
await _paymentService.RefundAsync(state.OrderId, ct);
|
|
}
|
|
|
|
if (state.InventoryReserved)
|
|
{
|
|
await _inventoryService.ReleaseAsync(state.OrderId, ct);
|
|
}
|
|
|
|
// Handle non-compensatable actions
|
|
if (state.ConfirmationEmailSent)
|
|
{
|
|
// Send cancellation email instead
|
|
await _emailService.SendOrderCancellationAsync(state.OrderId, ct);
|
|
}
|
|
|
|
if (state.PartnerNotified)
|
|
{
|
|
// Notify partner of cancellation
|
|
await _externalApiService.NotifyCancellationAsync(state.OrderId, ct);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Monitoring Compensation
|
|
|
|
### Compensation Metrics
|
|
|
|
```csharp
|
|
public class CompensationMetrics
|
|
{
|
|
public int TotalCompensations { get; set; }
|
|
public int SuccessfulCompensations { get; set; }
|
|
public int FailedCompensations { get; set; }
|
|
public Dictionary<string, int> CompensationsByStep { get; set; }
|
|
|
|
public double SuccessRate =>
|
|
TotalCompensations > 0
|
|
? (double)SuccessfulCompensations / TotalCompensations * 100
|
|
: 100;
|
|
}
|
|
|
|
// Track metrics
|
|
_metrics.RecordCompensation(state.SagaId, success: true);
|
|
|
|
if (metrics.FailedCompensations > 10)
|
|
{
|
|
_logger.LogWarning("High compensation failure rate: {Rate:F1}%",
|
|
100 - metrics.SuccessRate);
|
|
}
|
|
```
|
|
|
|
### Alerting
|
|
|
|
```csharp
|
|
private async Task CompensateAsync(OrderFulfillmentState state, CancellationToken ct)
|
|
{
|
|
try
|
|
{
|
|
await ExecuteCompensationStepsAsync(state, ct);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogCritical(ex, "Compensation failed for saga {SagaId}", state.SagaId);
|
|
|
|
// Alert operations team
|
|
await _alertService.SendAsync(new Alert
|
|
{
|
|
Severity = AlertSeverity.Critical,
|
|
Title = $"Saga compensation failed: {state.SagaId}",
|
|
Description = $"Order {state.OrderId} requires manual intervention",
|
|
SagaId = state.SagaId,
|
|
OrderId = state.OrderId,
|
|
FailureReason = ex.Message
|
|
});
|
|
|
|
throw;
|
|
}
|
|
}
|
|
```
|
|
|
|
## Best Practices
|
|
|
|
### ✅ DO
|
|
|
|
- Compensate in reverse order
|
|
- Make compensations idempotent
|
|
- Log all compensation attempts
|
|
- Track compensation state
|
|
- Retry with backoff
|
|
- Handle non-compensatable actions
|
|
- Alert on compensation failures
|
|
- Test compensation thoroughly
|
|
- Document compensation logic
|
|
|
|
### ❌ DON'T
|
|
|
|
- Don't assume compensation always succeeds
|
|
- Don't forget partial compensation
|
|
- Don't skip logging
|
|
- Don't ignore non-compensatable actions
|
|
- Don't retry indefinitely
|
|
- Don't leave system in inconsistent state
|
|
- Don't forget to update state after compensation
|
|
- Don't compensate non-completed steps
|
|
|
|
## See Also
|
|
|
|
- [Sagas Overview](README.md)
|
|
- [Saga Pattern](saga-pattern.md)
|
|
- [Creating Sagas](creating-sagas.md)
|
|
- [Saga Context](saga-context.md)
|
|
- [Error Handling Best Practices](../../best-practices/error-handling.md)
|