using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; using PoweredSoft.Storage.Core; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace PoweredSoft.Storage.S3 { public class S3StorageProvider : IStorageProvider { protected readonly string endpoint; protected readonly string bucketName; protected readonly string accessKey; protected readonly string secret; protected S3UsEast1RegionalEndpointValue? s3UsEast1RegionalEndpointValue = null; protected bool forcePathStyle = false; public S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) { this.endpoint = endpoint; this.bucketName = bucketName; this.accessKey = accessKey; this.secret = secret; } public void SetForcePathStyle(bool forcePathStyle) { this.forcePathStyle = forcePathStyle; } public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value) { this.s3UsEast1RegionalEndpointValue = value; } protected virtual IAmazonS3 GetClient() { var config = new AmazonS3Config { USEast1RegionalEndpointValue = s3UsEast1RegionalEndpointValue, ServiceURL = endpoint, ForcePathStyle = forcePathStyle }; var client = new AmazonS3Client(this.accessKey, this.secret, config); return client; } public Task CreateDirectoryAsync(string path) { return Task.FromResult(new S3NotExistingDirectoryInfo(path)); } /// /// Can only delete 1000 at a time. /// /// /// /// public async Task DeleteDirectoryAsync(string path, bool force = false) { using var client = GetClient(); var files = await this.GetS3FilesAsync(prefix: path, delimiter: null); var next = files.AsQueryable(); while(next.Any()) { var next1000 = next.Take(1000); var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList(); await client.DeleteObjectsAsync(new DeleteObjectsRequest { BucketName = this.bucketName, Objects = keys }); next = next.Skip(1000); } } public async Task DeleteFileAsync(string path) { using var client = GetClient(); var response = await client.DeleteObjectAsync(new DeleteObjectRequest { BucketName = this.bucketName, Key = path }); } public async Task FileExistsAsync(string path) { var item = await GetS3FileByPath(path); return item != null; } public Task> GetDirectories(string path) { return Task.FromResult(new List()); } public async Task GetFileBytesAsync(string path) { using var fileStream = await this.GetFileStreamAsync(path); using var memoryStream = new MemoryStream(); await fileStream.CopyToAsync(memoryStream); return memoryStream.ToArray(); } public async Task GetFileContentAsync(string path, Encoding encoding) { using var fileStream = await this.GetFileStreamAsync(path); using var streamReader = new StreamReader(fileStream, encoding); return await streamReader.ReadToEndAsync(); } public async Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly) { if (pattern != null) throw new NotSupportedException(); string finalPath = SantizeDirectoryRequest(path); var s3Files = await this.GetS3FilesAsync(prefix: finalPath, delimiter: "/"); var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable().ToList(); return ret; } private static string SantizeDirectoryRequest(string path) { string finalPath; if (path == "/") finalPath = ""; else finalPath = $"{path?.TrimEnd('/')}/"; return finalPath; } public Task GetFileStreamAsync(string path) { using var client = GetClient(); return client.GetObjectStreamAsync(this.bucketName, path, null); } protected virtual async Task> GetS3FilesAsync(string prefix = null, string delimiter = null) { using var client = GetClient(); var items = new List(); string nextKey = null; do { var response = await client.ListObjectsV2Async(new ListObjectsV2Request { BucketName = this.bucketName, Prefix = prefix, Delimiter = delimiter, MaxKeys = 1000, ContinuationToken = nextKey }); items.AddRange(response.S3Objects); nextKey = response.NextContinuationToken; } while (nextKey != null); return items; } public async Task> GetListAsync(string path) { var files = await this.GetFilesAsync(path); return files.Cast().ToList(); } public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options) { using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read); return await WriteFileAsync(fileStream, path, options); } public Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options) { return WriteFileAsync(new MemoryStream(bytes), path, options); } public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options) { if (options is null) { throw new ArgumentNullException(nameof(options)); } if (!options.OverrideIfExists && await FileExistsAsync(path)) throw new FileAlreadyExistsException(path); using var client = GetClient(); var request = new PutObjectRequest { BucketName = this.bucketName, InputStream = stream, Key = path }; if (options is IS3FileWriteOptions s3FileWriteOptions) { if (s3FileWriteOptions.Acl != null) request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl); } var result = await client.PutObjectAsync(request); var file = await GetFileInfoByPath(path); return file; } private async Task GetS3FileByPath(string path) { var files = await this.GetS3FilesAsync(path); var s3o = files.FirstOrDefault(); return s3o; } private async Task GetFileInfoByPath(string path) { var s3o = await GetS3FileByPath(path); if (s3o == null) throw new FileDoesNotExistException(path); var ret = new S3FileInfo(s3o); return ret; } public string SanitizeFileName(string key, string replacement) { string pattern = @"[^a-zA-Z0-9.!\-_*'()]"; string substitution = replacement; string input = key; RegexOptions options = RegexOptions.Multiline; Regex regex = new Regex(pattern, options); string result = regex.Replace(input, substitution); return result; } public bool IsFileNameAllowed(string fileName) { string pattern = @"[^a-zA-Z0-9.!\-_*'()]"; RegexOptions options = RegexOptions.Multiline; Regex regex = new Regex(pattern, options); var hasMatches = regex.IsMatch(fileName); return false == hasMatches; } public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true) { return WriteFileAsync(sourcePath, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }); } public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true) { return WriteFileAsync(bytes, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }); } public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true) { return WriteFileAsync(stream, path, new DefaultWriteOptions { OverrideIfExists = overrideIfExists }); } } }