svrnty/dotnet-cqrs

Fork 0

Mathias Beaulieu-Duncan 5c497f8923 this is a mess

2025-12-11 01:18:24 -05:00

10 KiB

Raw Blame History

Retention Configuration

Configure per-stream retention policies for time, size, and count-based event cleanup.

Overview

Retention configuration controls how long events are kept in a stream:

Time-based: Delete events older than specified age
Size-based: Limit total stream size in bytes
Count-based: Keep only last N events
Partitioning: Organize events for efficient cleanup

Quick Start

using Svrnty.CQRS.Events.Abstractions;

var configStore = serviceProvider.GetRequiredService<IStreamConfigurationStore>();

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "orders",
    Retention = new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(90),      // Keep 90 days
        MaxEventCount = 1000000              // Keep last 1M events
    }
});

Retention Properties

public class RetentionConfiguration
{
    public TimeSpan? MaxAge { get; set; }              // Maximum event age
    public long? MaxSizeBytes { get; set; }            // Maximum stream size
    public long? MaxEventCount { get; set; }           // Maximum event count
    public bool EnablePartitioning { get; set; }       // Enable time partitioning
    public PartitionInterval PartitionInterval { get; set; }  // Partition granularity
}

public enum PartitionInterval
{
    Hourly,
    Daily,
    Weekly,
    Monthly
}

Time-Based Retention

Delete events older than specified age:

// Keep 30 days
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(30)
};

// Keep 7 days
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(7)
};

// Keep 1 year
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(365)
};

Common Retention Periods

// Audit logs - long retention
var auditRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(2555)  // 7 years for compliance
};

// Analytics - medium retention
var analyticsRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(90)  // 3 months
};

// Temporary data - short retention
var tempRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(7)  // 1 week
};

// Session data - very short retention
var sessionRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromHours(24)  // 24 hours
};

Size-Based Retention

Limit total stream size:

// 10 GB limit
var retention = new RetentionConfiguration
{
    MaxSizeBytes = 10L * 1024 * 1024 * 1024
};

// 100 MB limit
var retention = new RetentionConfiguration
{
    MaxSizeBytes = 100L * 1024 * 1024
};

// 1 TB limit
var retention = new RetentionConfiguration
{
    MaxSizeBytes = 1L * 1024 * 1024 * 1024 * 1024
};

Count-Based Retention

Keep only last N events:

// Keep last 1 million events
var retention = new RetentionConfiguration
{
    MaxEventCount = 1000000
};

// Keep last 10,000 events
var retention = new RetentionConfiguration
{
    MaxEventCount = 10000
};

// Keep last 100 events
var retention = new RetentionConfiguration
{
    MaxEventCount = 100
};

Combined Retention

Use multiple retention criteria (first to trigger wins):

// Keep 90 days OR 10 million events OR 100 GB (whichever reached first)
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(90),
    MaxEventCount = 10000000,
    MaxSizeBytes = 100L * 1024 * 1024 * 1024
};

Partitioning

Enable partitioning for efficient cleanup:

// Daily partitioning
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(30),
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Daily
};

// Monthly partitioning for long retention
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(365),
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Monthly
};

// Hourly partitioning for high-volume streams
var retention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(7),
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Hourly
};

Partition Benefits

Faster Cleanup: Drop entire partitions instead of deleting rows
Better Performance: Query only relevant partitions
Easier Archival: Archive partitions independently
Predictable I/O: Cleanup doesn't impact live writes

Domain-Specific Examples

E-Commerce Orders

var orderRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(365 * 2),  // 2 years for tax compliance
    MaxSizeBytes = 100L * 1024 * 1024 * 1024,  // 100 GB
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Monthly
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "orders",
    Retention = orderRetention,
    Tags = new List<string> { "production", "compliance" }
});

Application Logs

var logRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(30),  // 30 days
    MaxSizeBytes = 50L * 1024 * 1024 * 1024,  // 50 GB
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Daily
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "application-logs",
    Retention = logRetention
});

User Sessions

var sessionRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromHours(24),  // 24 hours
    MaxEventCount = 100000,  // Last 100k sessions
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Hourly
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "user-sessions",
    Retention = sessionRetention
});

Analytics Events

var analyticsRetention = new RetentionConfiguration
{
    MaxAge = TimeSpan.FromDays(90),  // 90 days
    MaxEventCount = 50000000,  // 50M events
    MaxSizeBytes = 500L * 1024 * 1024 * 1024,  // 500 GB
    EnablePartitioning = true,
    PartitionInterval = PartitionInterval.Daily
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "analytics",
    Retention = analyticsRetention
});

Environment-Specific Retention

var environment = builder.Environment.EnvironmentName;

var retention = environment switch
{
    "Production" => new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(90),
        EnablePartitioning = true,
        PartitionInterval = PartitionInterval.Daily
    },
    "Staging" => new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(14),
        EnablePartitioning = true,
        PartitionInterval = PartitionInterval.Daily
    },
    "Development" => new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(3),
        EnablePartitioning = false
    },
    _ => new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(7)
    }
};

await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "orders",
    Retention = retention
});

Multi-Tenant Retention

// Premium tenant - long retention
await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "tenant-premium-events",
    Retention = new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(365),
        MaxSizeBytes = 100L * 1024 * 1024 * 1024
    },
    Tags = new List<string> { "tenant-premium", "premium-tier" }
});

// Standard tenant - standard retention
await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "tenant-standard-events",
    Retention = new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(90),
        MaxSizeBytes = 10L * 1024 * 1024 * 1024
    },
    Tags = new List<string> { "tenant-standard", "standard-tier" }
});

// Free tenant - short retention
await configStore.SetConfigurationAsync(new StreamConfiguration
{
    StreamName = "tenant-free-events",
    Retention = new RetentionConfiguration
    {
        MaxAge = TimeSpan.FromDays(7),
        MaxEventCount = 10000
    },
    Tags = new List<string> { "tenant-free", "free-tier" }
});

Monitoring Retention

Query current retention status:

var config = await configStore.GetConfigurationAsync("orders");

if (config?.Retention != null)
{
    var retention = config.Retention;

    Console.WriteLine($"Stream: {config.StreamName}");
    Console.WriteLine($"Max Age: {retention.MaxAge}");
    Console.WriteLine($"Max Size: {retention.MaxSizeBytes?.ToString() ?? "unlimited"}");
    Console.WriteLine($"Max Count: {retention.MaxEventCount?.ToString() ?? "unlimited"}");
    Console.WriteLine($"Partitioning: {retention.EnablePartitioning}");

    if (retention.EnablePartitioning)
    {
        Console.WriteLine($"Partition Interval: {retention.PartitionInterval}");
    }
}

Database Schema Impact

Without Partitioning

-- Single table for all events
CREATE TABLE events (
    event_id BIGSERIAL PRIMARY KEY,
    stream_name TEXT NOT NULL,
    timestamp TIMESTAMPTZ NOT NULL,
    event_data JSONB NOT NULL
);

-- Cleanup requires DELETE (slow for large tables)
DELETE FROM events
WHERE stream_name = 'orders'
  AND timestamp < NOW() - INTERVAL '90 days';

With Partitioning

-- Parent table
CREATE TABLE events (
    event_id BIGSERIAL,
    stream_name TEXT NOT NULL,
    timestamp TIMESTAMPTZ NOT NULL,
    event_data JSONB NOT NULL
) PARTITION BY RANGE (timestamp);

-- Monthly partitions
CREATE TABLE events_2025_01 PARTITION OF events
    FOR VALUES FROM ('2025-01-01') TO ('2025-02-01');

CREATE TABLE events_2025_02 PARTITION OF events
    FOR VALUES FROM ('2025-02-01') TO ('2025-03-01');

-- Cleanup just drops partition (instant)
DROP TABLE events_2024_12;

Best Practices

✅ DO

Set retention based on compliance requirements
Enable partitioning for large streams
Use daily partitions for high-volume streams
Use monthly partitions for long retention
Combine multiple retention criteria
Monitor stream size regularly
Test retention in non-production first
Document retention policies

❌ DON'T

Don't set retention too short for audit logs
Don't disable partitioning for large streams
Don't use hourly partitions unless necessary
Don't forget about compliance requirements
Don't mix incompatible retention settings
Don't change retention without approval
Don't forget to archive before deleting

10 KiB Raw Blame History