using Amazon.S3; using Amazon.S3.Model; using PoweredSoft.Storage.Core; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; namespace PoweredSoft.Storage.S3 { public class S3StorageProvider : IStorageProvider { protected readonly string endpoint; protected readonly string bucketName; protected readonly string accessKey; protected readonly string secret; public S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) { this.endpoint = endpoint; this.bucketName = bucketName; this.accessKey = accessKey; this.secret = secret; } protected virtual IAmazonS3 GetClient() { var config = new AmazonS3Config { ServiceURL = endpoint }; var client = new AmazonS3Client(this.accessKey, this.secret, config); return client; } public Task CreateDirectoryAsync(string path) { return Task.FromResult(new S3NotExistingDirectoryInfo(path)); } /// /// Can only delete 1000 at a time. /// /// /// /// public async Task DeleteDirectoryAsync(string path, bool force = false) { using var client = GetClient(); var files = await this.GetS3FilesAsync(prefix: path, delimiter: null); var next = files.AsQueryable(); while(next.Any()) { var next1000 = next.Take(1000); var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList(); await client.DeleteObjectsAsync(new DeleteObjectsRequest { BucketName = this.bucketName, Objects = keys }); next = next.Skip(1000); } } public async Task DeleteFileAsync(string path) { using var client = GetClient(); var response = await client.DeleteObjectAsync(new DeleteObjectRequest { BucketName = this.bucketName, Key = path }); } public async Task FileExistsAsync(string path) { var item = await GetS3FileByPath(path); return item != null; } public Task> GetDirectories(string path) { return Task.FromResult(new List()); } public async Task GetFileBytesAsync(string path) { using var fileStream = await this.GetFileStreamAsync(path); using var memoryStream = new MemoryStream(); await fileStream.CopyToAsync(memoryStream); return memoryStream.ToArray(); } public async Task GetFileContentAsync(string path, Encoding encoding) { using var fileStream = await this.GetFileStreamAsync(path); using var streamReader = new StreamReader(fileStream, encoding); return await streamReader.ReadToEndAsync(); } public async Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly) { if (pattern != null) throw new NotSupportedException(); string finalPath = SantizeDirectoryRequest(path); var s3Files = await this.GetS3FilesAsync(prefix: finalPath, delimiter: "/"); var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable().ToList(); return ret; } private static string SantizeDirectoryRequest(string path) { string finalPath; if (path == "/") finalPath = ""; else finalPath = $"{path?.TrimEnd('/')}/"; return finalPath; } public Task GetFileStreamAsync(string path) { using var client = GetClient(); return client.GetObjectStreamAsync(this.bucketName, path, null); } protected virtual async Task> GetS3FilesAsync(string prefix = null, string delimiter = null) { using var client = GetClient(); var items = new List(); string nextKey = null; do { var response = await client.ListObjectsV2Async(new ListObjectsV2Request { BucketName = this.bucketName, Prefix = prefix, Delimiter = delimiter, MaxKeys = 1000, ContinuationToken = nextKey }); items.AddRange(response.S3Objects); nextKey = response.NextContinuationToken; } while (nextKey != null); return items; } public async Task> GetListAsync(string path) { var files = await this.GetFilesAsync(path); return files.Cast().ToList(); } public async Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true) { using var client = GetClient(); await client.UploadObjectFromFilePathAsync(this.bucketName, path, sourcePath, null); var file = await GetFileInfoByPath(path); return file; } public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true) { return WriteFileAsync(new MemoryStream(bytes), path, overrideIfExists: overrideIfExists); } public async Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true) { if (!overrideIfExists && await FileExistsAsync(path)) throw new FileAlreadyExistsException(path); using var client = GetClient(); var request = new PutObjectRequest { BucketName = this.bucketName, InputStream = stream, Key = path }; var result = await client.PutObjectAsync(request); var file = await GetFileInfoByPath(path); return file; } private async Task GetS3FileByPath(string path) { var files = await this.GetS3FilesAsync(path); var s3o = files.FirstOrDefault(); return s3o; } private async Task GetFileInfoByPath(string path) { var s3o = await GetS3FileByPath(path); if (s3o == null) throw new FileDoesNotExistException(path); var ret = new S3FileInfo(s3o); return ret; } } }