svrnty/dotnet-cqrs

Fork 0

Mathias Beaulieu-Duncan 5c497f8923 this is a mess

2025-12-11 01:18:24 -05:00

12 KiB

Raw Blame History

Lifecycle Configuration

Automate stream lifecycle management with automatic creation, archival, and deletion.

Overview

Lifecycle configuration automates stream management:

Auto-Create - Create streams on first append
Auto-Archive - Move old events to cold storage
Auto-Delete - Delete archived or expired events
Custom Archive Locations - Specify S3, Azure Blob, or file storage

Quick Start

using Svrnty.CQRS.Events.Abstractions;

var configStore = serviceProvider.GetRequiredService<IStreamConfigurationStore>();

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "orders",
    Lifecycle = new LifecycleConfiguration
    {
        AutoCreate = true,
        AutoArchive = true,
        ArchiveAfter = TimeSpan.FromDays(90),
        ArchiveLocation = "s3://archive/orders"
    }
});

Lifecycle Properties

public class LifecycleConfiguration
{
    public bool AutoCreate { get; set; }                   // Create on first append
    public bool AutoArchive { get; set; }                  // Enable archival
    public TimeSpan ArchiveAfter { get; set; }             // Archive age threshold
    public string? ArchiveLocation { get; set; }           // Archive storage URI
    public bool AutoDelete { get; set; }                   // Delete after archive
    public TimeSpan DeleteAfter { get; set; }              // Delete age threshold
    public bool CompressOnArchive { get; set; }            // Compress archived events
    public string? ArchiveFormat { get; set; }             // Parquet, JSON, Avro
}

Auto-Create

Automatically create streams on first append:

// Enable auto-create (default: false)
var lifecycle = new LifecycleConfiguration
{
    AutoCreate = true
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "user-activity",
    Lifecycle = lifecycle
});

// Now you can append without explicitly creating stream
await eventStore.AppendAsync("user-activity", new UserLoginEvent());
// Stream created automatically

Manual vs Auto-Create

// ❌ Manual - Requires explicit creation
await eventStore.CreateStreamAsync("orders");
await eventStore.AppendAsync("orders", @event);

// ✅ Auto-Create - Stream created on first append
var lifecycle = new LifecycleConfiguration { AutoCreate = true };
await eventStore.AppendAsync("orders", @event);  // Creates if not exists

Auto-Archive

Move old events to cold storage:

// Archive after 90 days to S3
var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),
    ArchiveLocation = "s3://my-bucket/archives/orders",
    CompressOnArchive = true,
    ArchiveFormat = "parquet"  // Efficient columnar format
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "orders",
    Lifecycle = lifecycle
});

Archive Locations

S3

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(365),
    ArchiveLocation = "s3://prod-archives/orders/{year}/{month}",
    CompressOnArchive = true
};

// Results in: s3://prod-archives/orders/2025/12/events-12345.parquet.gz

Azure Blob Storage

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(365),
    ArchiveLocation = "azure://archivescontainer/orders/{year}/{month}",
    CompressOnArchive = true
};

Local/Network File System

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(30),
    ArchiveLocation = "file:///mnt/archives/orders/{year}/{month}",
    CompressOnArchive = true
};

Auto-Delete

Automatically delete old or archived events:

// Delete after archiving
var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),
    ArchiveLocation = "s3://archives/orders",
    AutoDelete = true,
    DeleteAfter = TimeSpan.FromDays(100)  // Delete 10 days after archive
};

// Delete without archiving (data loss!)
var lifecycle = new LifecycleConfiguration
{
    AutoArchive = false,
    AutoDelete = true,
    DeleteAfter = TimeSpan.FromDays(7)  // Delete after 7 days
};

Archive Formats

Parquet (Recommended)

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),
    ArchiveLocation = "s3://archives/orders",
    ArchiveFormat = "parquet",  // Columnar, efficient for analytics
    CompressOnArchive = true
};

// Best for:
// - Analytics queries
// - Large datasets
// - Efficient storage

JSON

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),
    ArchiveLocation = "s3://archives/orders",
    ArchiveFormat = "json",  // Human-readable
    CompressOnArchive = true  // GZIP compression
};

// Best for:
// - Human inspection
// - Simple tooling
// - Debugging

Avro

var lifecycle = new LifecycleConfiguration
{
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),
    ArchiveLocation = "s3://archives/orders",
    ArchiveFormat = "avro",  // Schema evolution support
    CompressOnArchive = true
};

// Best for:
// - Schema evolution
// - Cross-language compatibility
// - Event versioning

Domain-Specific Examples

Audit Logs - Long-term Archival

var auditLifecycle = new LifecycleConfiguration
{
    AutoCreate = true,
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(365),  // Archive after 1 year
    ArchiveLocation = "s3://compliance-archives/audit-logs/{year}",
    CompressOnArchive = true,
    ArchiveFormat = "parquet",
    AutoDelete = false  // Keep in database AND archive for compliance
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "audit-logs",
    Lifecycle = auditLifecycle,
    Tags = new List<string> { "compliance", "audit" }
});

Analytics Events - Archive and Delete

var analyticsLifecycle = new LifecycleConfiguration
{
    AutoCreate = true,
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(90),  // Archive after 90 days
    ArchiveLocation = "s3://analytics-archives/events/{year}/{month}",
    CompressOnArchive = true,
    ArchiveFormat = "parquet",  // Efficient for analytics
    AutoDelete = true,
    DeleteAfter = TimeSpan.FromDays(100)  // Delete from DB after archive
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "analytics",
    Lifecycle = analyticsLifecycle
});

Temporary Sessions - Delete Only

var sessionLifecycle = new LifecycleConfiguration
{
    AutoCreate = true,
    AutoArchive = false,  // No archival needed
    AutoDelete = true,
    DeleteAfter = TimeSpan.FromHours(24)  // Delete after 24 hours
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "user-sessions",
    Lifecycle = sessionLifecycle,
    Tags = new List<string> { "temporary" }
});

Financial Transactions - Permanent Archive

var financialLifecycle = new LifecycleConfiguration
{
    AutoCreate = true,
    AutoArchive = true,
    ArchiveAfter = TimeSpan.FromDays(180),  // Archive after 6 months
    ArchiveLocation = "s3://financial-archives/transactions/{year}/{month}",
    CompressOnArchive = true,
    ArchiveFormat = "parquet",
    AutoDelete = false  // Never delete, keep both DB and archive
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "financial-transactions",
    Lifecycle = financialLifecycle,
    Tags = new List<string> { "financial", "compliance", "permanent" }
});

Archive Process

Automatic Archival

// Background service handles archival automatically
public class ArchivalBackgroundService : BackgroundService
{
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        using var timer = new PeriodicTimer(TimeSpan.FromHours(1));

        while (await timer.WaitForNextTickAsync(stoppingToken))
        {
            await ArchiveEligibleEventsAsync(stoppingToken);
        }
    }

    private async Task ArchiveEligibleEventsAsync(CancellationToken ct)
    {
        var configs = await _configStore.GetAllConfigurationsAsync();

        foreach (var config in configs.Where(c => c.Lifecycle.AutoArchive))
        {
            var eligibleEvents = await GetEventsEligibleForArchivalAsync(
                config.StreamName,
                config.Lifecycle.ArchiveAfter);

            await ArchiveEventsAsync(
                eligibleEvents,
                config.Lifecycle.ArchiveLocation,
                config.Lifecycle.ArchiveFormat,
                config.Lifecycle.CompressOnArchive);

            if (config.Lifecycle.AutoDelete)
            {
                await DeleteArchivedEventsAsync(eligibleEvents);
            }
        }
    }
}

Manual Archive Trigger

// Trigger manual archival
var archivalService = serviceProvider.GetRequiredService<IArchivalService>();

await archivalService.ArchiveStreamAsync(
    streamName: "orders",
    fromDate: DateTimeOffset.UtcNow.AddDays(-365),
    toDate: DateTimeOffset.UtcNow.AddDays(-90));

_logger.LogInformation("Manual archival completed");

Restoring from Archive

public class ArchiveRestoreService
{
    public async Task RestoreFromArchiveAsync(
        string streamName,
        DateTimeOffset fromDate,
        DateTimeOffset toDate,
        CancellationToken ct)
    {
        var config = await _configStore.GetConfigurationAsync(streamName);
        var archiveLocation = config.Lifecycle.ArchiveLocation;

        // Download from S3/Azure/File
        var archivedEvents = await DownloadArchivedEventsAsync(
            archiveLocation,
            fromDate,
            toDate,
            ct);

        // Restore to database
        foreach (var @event in archivedEvents)
        {
            await _eventStore.AppendAsync(streamName, @event);
        }

        _logger.LogInformation(
            "Restored {Count} events from archive for {Stream}",
            archivedEvents.Count,
            streamName);
    }
}

Monitoring Lifecycle

// Monitor archival status
var archivalStatus = new
{
    StreamName = "orders",
    TotalEvents = await GetEventCountAsync("orders"),
    ArchivedEvents = await GetArchivedEventCountAsync("orders"),
    EligibleForArchival = await GetArchivalEligibleCountAsync("orders"),
    NextArchivalRun = _archivalService.GetNextRunTime()
};

if (archivalStatus.EligibleForArchival > 10000)
{
    _logger.LogWarning(
        "{Count} events eligible for archival in {Stream}",
        archivalStatus.EligibleForArchival,
        archivalStatus.StreamName);
}

Best Practices

✅ DO

Enable auto-create for dynamic stream names
Archive to durable storage (S3, Azure Blob)
Use Parquet format for analytics
Compress archived events
Test restore process regularly
Document archive locations
Monitor archival success/failures
Implement archive verification
Use appropriate archive timing
Keep financial/audit data indefinitely

❌ DON'T

Don't delete without archiving critical data
Don't use auto-delete for compliance data
Don't forget to test restore procedures
Don't archive too frequently (adds overhead)
Don't use local file system for production archives
Don't forget archive access credentials
Don't skip compression for large datasets
Don't auto-delete before verifying archive

12 KiB Raw Blame History