using System.Text; using System.Text.RegularExpressions; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; using OpenHarbor.Storage.Abstractions; namespace OpenHarbor.Storage.S3; public partial class S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) : IStorageProvider { private S3UsEast1RegionalEndpointValue? _s3UsEast1RegionalEndpointValue = null; private bool _forcePathStyle = false; public void SetForcePathStyle(bool forcePathStyle) { _forcePathStyle = forcePathStyle; } public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value) { _s3UsEast1RegionalEndpointValue = value; } protected virtual IAmazonS3 GetClient() { var config = new AmazonS3Config { USEast1RegionalEndpointValue = _s3UsEast1RegionalEndpointValue, ServiceURL = endpoint, ForcePathStyle = _forcePathStyle }; var client = new AmazonS3Client(accessKey, secret, config); return client; } public Task CreateDirectoryAsync(string path, CancellationToken cancellationToken) { return Task.FromResult(new S3NotExistingDirectoryInfo(path)); } /// /// Can only delete 1000 at a time. /// /// /// /// /// public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default) { using var client = GetClient(); var files = await GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken); var next = files.AsQueryable(); while(next.Any()) { var next1000 = next.Take(1000); var keys = next1000.Select(s3Object => new KeyVersion { Key = s3Object.Key }) .ToList(); await client.DeleteObjectsAsync(new DeleteObjectsRequest { BucketName = bucketName, Objects = keys }, cancellationToken); next = next.Skip(1000); } } public async Task DeleteFileAsync(string path, CancellationToken cancellationToken) { using var client = GetClient(); var response = await client.DeleteObjectAsync(new DeleteObjectRequest { BucketName = bucketName, Key = path }, cancellationToken); } public async Task FileExistsAsync(string path, CancellationToken cancellationToken = default) { var s3Object = await GetS3FileByPathAsync(path, cancellationToken); return s3Object != null; } public Task> GetDirectoriesAsync(string path, CancellationToken cancellationToken) { return Task.FromResult(new List()); } public async Task GetFileBytesAsync(string path, CancellationToken cancellationToken = default) { await using var fileStream = await GetFileStreamAsync(path, cancellationToken); using var memoryStream = new MemoryStream(); await fileStream.CopyToAsync(memoryStream, cancellationToken); return memoryStream.ToArray(); } public async Task GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken = default) { await using var fileStream = await this.GetFileStreamAsync(path, cancellationToken); using var streamReader = new StreamReader(fileStream, encoding); return await streamReader.ReadToEndAsync(cancellationToken); } public async Task> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default) { if (pattern != null) throw new NotSupportedException(); var finalPath = SanitizeDirectoryRequest(path); var s3Files = await GetS3FilesAsync(prefix: finalPath, delimiter: "/", cancellationToken: cancellationToken); var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable().ToList(); return ret; } private static string SanitizeDirectoryRequest(string path) { string finalPath; if (path == "/") finalPath = ""; else finalPath = $"{path?.TrimEnd('/')}/"; return finalPath; } public async Task GetFileStreamAsync(string path, CancellationToken cancellationToken) { using var client = GetClient(); return await client.GetObjectStreamAsync(bucketName, path, null, cancellationToken); } protected virtual async Task> GetS3FilesAsync(string? prefix = null, string? delimiter = null, CancellationToken cancellationToken = default) { using var client = GetClient(); var items = new List(); string? nextKey = null; do { var response = await client.ListObjectsV2Async(new ListObjectsV2Request { BucketName = bucketName, Prefix = prefix, Delimiter = delimiter, MaxKeys = 1000, ContinuationToken = nextKey }, cancellationToken); items.AddRange(response.S3Objects); nextKey = response.NextContinuationToken; } while (nextKey != null); return items; } public async Task> GetListAsync(string path, CancellationToken cancellationToken) { var files = await GetFilesAsync(path, cancellationToken: cancellationToken); return files.Cast().ToList(); } public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken) { await using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read); return await WriteFileAsync(fileStream, path, options, cancellationToken); } public Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken) { return WriteFileAsync(new MemoryStream(bytes), path, options, cancellationToken); } public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(options); if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) throw new FileAlreadyExistsException(path); using var client = GetClient(); var request = new PutObjectRequest { BucketName = bucketName, InputStream = stream, Key = path }; if (options is IS3FileWriteOptions s3FileWriteOptions) request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl); // todo: unhandled response var result = await client.PutObjectAsync(request, cancellationToken); var file = await GetFileInfoByPathAsync(path, cancellationToken); return file; } private async Task GetS3FileByPathAsync(string path, CancellationToken cancellationToken) { var files = await GetS3FilesAsync(path, cancellationToken: cancellationToken); return files.FirstOrDefault(); } private async Task GetFileInfoByPathAsync(string path, CancellationToken cancellationToken) { var s3Object = await GetS3FileByPathAsync(path, cancellationToken); if (s3Object == null) throw new FileDoesNotExistException(path); var ret = new S3FileInfo(s3Object); return ret; } public string SanitizeFileName(string key, string replacement) { var regex = S3FileNameRegex(); var result = regex.Replace(key, replacement); return result; } public bool IsFileNameAllowed(string fileName) { var regex = S3FileNameRegex(); var hasMatches = regex.IsMatch(fileName); return false == hasMatches; } public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) { return WriteFileAsync(sourcePath, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }, cancellationToken); } public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) { return WriteFileAsync(bytes, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }, cancellationToken); } public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) { return WriteFileAsync(stream, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }, cancellationToken); } [GeneratedRegex(@"[^a-zA-Z0-9.!\-_*'()]", RegexOptions.Multiline)] private static partial Regex S3FileNameRegex(); }