From c0b8b92c210fb22aad64b184ab42adf5ac4e26e1 Mon Sep 17 00:00:00 2001 From: Mathias Beaulieu-Duncan Date: Sun, 10 Nov 2024 16:57:24 -0500 Subject: [PATCH 1/2] Refractor a bunch of code, change projects names, update icon, upgrade to .net 8 --- .../FileAlreadyExistsException.cs | 3 + .../FileDoesNotExistException.cs | 3 + OpenHarbor.Abstractions/IDirectory.cs | 6 + OpenHarbor.Abstractions/IDirectoryOrFile.cs | 7 + OpenHarbor.Abstractions/IFileInfo.cs | 14 + OpenHarbor.Abstractions/IStorageProvider.cs | 26 ++ OpenHarbor.Abstractions/IWriteFileOptions.cs | 11 + .../OpenHarbor.Abstractions.csproj | 9 + .../Blob/AzureBlobDirectoryInfo.cs | 16 + .../Blob/AzureBlobFileInfo.cs | 19 ++ .../Blob/AzureBlobStorageProvider.cs | 251 ++++++++++++++++ .../OpenHarbor.Storage.Azure.csproj | 17 ++ .../OpenHarbor.Storage.Physical.csproj | 13 + .../PhysicalDirectory.cs | 9 + .../PhysicalFileInfo.cs | 4 +- .../PhysicalStorageProvider.cs | 189 ++++++++++++ .../IS3FileWriteOptions.cs | 4 +- .../OpenHarbor.Storage.S3.csproj | 17 ++ .../S3FileInfo.cs | 4 +- .../S3NotExistingDirectoryInfo.cs | 4 +- OpenHarbor.Storage.S3/S3StorageProvider.cs | 260 ++++++++++++++++ ...Soft.Storage.sln => OpenHarbor.Storage.sln | 14 +- .../Blob/AzureBlobDirectoryInfo.cs | 29 -- .../Blob/AzureBlobFileInfo.cs | 30 -- .../Blob/AzureBlobStorageProvider.cs | 265 ---------------- .../PoweredSoft.Storage.Azure.csproj | 18 -- .../FileAlreadyExistsException.cs | 12 - .../FileDoesNotExistException.cs | 12 - PoweredSoft.Storage.Core/IDirectory.cs | 7 - PoweredSoft.Storage.Core/IDirectoryOrFile.cs | 8 - PoweredSoft.Storage.Core/IFileInfo.cs | 17 -- PoweredSoft.Storage.Core/IStorageProvider.cs | 31 -- PoweredSoft.Storage.Core/IWriteFileOptions.cs | 12 - .../PoweredSoft.Storage.Core.csproj | 10 - .../PhysicalDirectory.cs | 15 - .../PhysicalStorageProvider.cs | 199 ------------ .../PoweredSoft.Storage.Physical.csproj | 14 - .../PoweredSoft.Storage.S3.csproj | 21 -- PoweredSoft.Storage.S3/S3StorageProvider.cs | 283 ------------------ .../PoweredSoft.Storage.Test.csproj | 20 -- PoweredSoft.Storage.Test/S3Tests.cs | 55 ---- 41 files changed, 882 insertions(+), 1076 deletions(-) create mode 100644 OpenHarbor.Abstractions/FileAlreadyExistsException.cs create mode 100644 OpenHarbor.Abstractions/FileDoesNotExistException.cs create mode 100644 OpenHarbor.Abstractions/IDirectory.cs create mode 100644 OpenHarbor.Abstractions/IDirectoryOrFile.cs create mode 100644 OpenHarbor.Abstractions/IFileInfo.cs create mode 100644 OpenHarbor.Abstractions/IStorageProvider.cs create mode 100644 OpenHarbor.Abstractions/IWriteFileOptions.cs create mode 100644 OpenHarbor.Abstractions/OpenHarbor.Abstractions.csproj create mode 100644 OpenHarbor.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs create mode 100644 OpenHarbor.Storage.Azure/Blob/AzureBlobFileInfo.cs create mode 100644 OpenHarbor.Storage.Azure/Blob/AzureBlobStorageProvider.cs create mode 100644 OpenHarbor.Storage.Azure/OpenHarbor.Storage.Azure.csproj create mode 100644 OpenHarbor.Storage.Physical/OpenHarbor.Storage.Physical.csproj create mode 100644 OpenHarbor.Storage.Physical/PhysicalDirectory.cs rename {PoweredSoft.Storage.Physical => OpenHarbor.Storage.Physical}/PhysicalFileInfo.cs (92%) create mode 100644 OpenHarbor.Storage.Physical/PhysicalStorageProvider.cs rename {PoweredSoft.Storage.S3 => OpenHarbor.Storage.S3}/IS3FileWriteOptions.cs (81%) create mode 100644 OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj rename {PoweredSoft.Storage.S3 => OpenHarbor.Storage.S3}/S3FileInfo.cs (94%) rename {PoweredSoft.Storage.S3 => OpenHarbor.Storage.S3}/S3NotExistingDirectoryInfo.cs (78%) create mode 100644 OpenHarbor.Storage.S3/S3StorageProvider.cs rename PoweredSoft.Storage.sln => OpenHarbor.Storage.sln (61%) delete mode 100644 PoweredSoft.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs delete mode 100644 PoweredSoft.Storage.Azure/Blob/AzureBlobFileInfo.cs delete mode 100644 PoweredSoft.Storage.Azure/Blob/AzureBlobStorageProvider.cs delete mode 100644 PoweredSoft.Storage.Azure/PoweredSoft.Storage.Azure.csproj delete mode 100644 PoweredSoft.Storage.Core/FileAlreadyExistsException.cs delete mode 100644 PoweredSoft.Storage.Core/FileDoesNotExistException.cs delete mode 100644 PoweredSoft.Storage.Core/IDirectory.cs delete mode 100644 PoweredSoft.Storage.Core/IDirectoryOrFile.cs delete mode 100644 PoweredSoft.Storage.Core/IFileInfo.cs delete mode 100644 PoweredSoft.Storage.Core/IStorageProvider.cs delete mode 100644 PoweredSoft.Storage.Core/IWriteFileOptions.cs delete mode 100644 PoweredSoft.Storage.Core/PoweredSoft.Storage.Core.csproj delete mode 100644 PoweredSoft.Storage.Physical/PhysicalDirectory.cs delete mode 100644 PoweredSoft.Storage.Physical/PhysicalStorageProvider.cs delete mode 100644 PoweredSoft.Storage.Physical/PoweredSoft.Storage.Physical.csproj delete mode 100644 PoweredSoft.Storage.S3/PoweredSoft.Storage.S3.csproj delete mode 100644 PoweredSoft.Storage.S3/S3StorageProvider.cs delete mode 100644 PoweredSoft.Storage.Test/PoweredSoft.Storage.Test.csproj delete mode 100644 PoweredSoft.Storage.Test/S3Tests.cs diff --git a/OpenHarbor.Abstractions/FileAlreadyExistsException.cs b/OpenHarbor.Abstractions/FileAlreadyExistsException.cs new file mode 100644 index 0000000..f3ab6ed --- /dev/null +++ b/OpenHarbor.Abstractions/FileAlreadyExistsException.cs @@ -0,0 +1,3 @@ +namespace OpenHarbor.Abstractions; + +public class FileAlreadyExistsException(string path) : Exception($"{path} already exists.."); diff --git a/OpenHarbor.Abstractions/FileDoesNotExistException.cs b/OpenHarbor.Abstractions/FileDoesNotExistException.cs new file mode 100644 index 0000000..45fa174 --- /dev/null +++ b/OpenHarbor.Abstractions/FileDoesNotExistException.cs @@ -0,0 +1,3 @@ +namespace OpenHarbor.Abstractions; + +public class FileDoesNotExistException(string path) : Exception($"{path} does not exist."); \ No newline at end of file diff --git a/OpenHarbor.Abstractions/IDirectory.cs b/OpenHarbor.Abstractions/IDirectory.cs new file mode 100644 index 0000000..d3125a8 --- /dev/null +++ b/OpenHarbor.Abstractions/IDirectory.cs @@ -0,0 +1,6 @@ +namespace OpenHarbor.Abstractions; + +public interface IDirectoryInfo: IDirectoryOrFile +{ + +} diff --git a/OpenHarbor.Abstractions/IDirectoryOrFile.cs b/OpenHarbor.Abstractions/IDirectoryOrFile.cs new file mode 100644 index 0000000..65134dc --- /dev/null +++ b/OpenHarbor.Abstractions/IDirectoryOrFile.cs @@ -0,0 +1,7 @@ +namespace OpenHarbor.Abstractions; + +public interface IDirectoryOrFile +{ + string Path { get; } + bool IsDirectory { get; } +} \ No newline at end of file diff --git a/OpenHarbor.Abstractions/IFileInfo.cs b/OpenHarbor.Abstractions/IFileInfo.cs new file mode 100644 index 0000000..94f6155 --- /dev/null +++ b/OpenHarbor.Abstractions/IFileInfo.cs @@ -0,0 +1,14 @@ +namespace OpenHarbor.Abstractions; + +public interface IFileInfo : IDirectoryOrFile +{ + string FileName { get; } + string Extension { get; } + long FileSize { get; } + DateTimeOffset? CreatedTime { get; } + DateTimeOffset? LastModifiedTime { get; } + DateTimeOffset? LastAccessTime { get; } + DateTime? CreatedTimeUtc { get; } + DateTime? LastModifiedTimeUtc { get; } + DateTime? LastAccessTimeUtc { get; } +} \ No newline at end of file diff --git a/OpenHarbor.Abstractions/IStorageProvider.cs b/OpenHarbor.Abstractions/IStorageProvider.cs new file mode 100644 index 0000000..ed3affb --- /dev/null +++ b/OpenHarbor.Abstractions/IStorageProvider.cs @@ -0,0 +1,26 @@ +using System.Text; + +namespace OpenHarbor.Abstractions; + +public interface IStorageProvider +{ + Task> GetListAsync(string path, CancellationToken cancellationToken); + Task> GetDirectoriesAsync(string path, CancellationToken cancellationToken); + Task> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default); + Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default); + Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default); + Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default); + Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken); + Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken); + Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken); + Task GetFileStreamAsync(string path, CancellationToken cancellationToken); + Task GetFileBytesAsync(string path, CancellationToken cancellationToken); + Task GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken); + Task FileExistsAsync(string path, CancellationToken cancellationToken); + Task DeleteFileAsync(string path, CancellationToken cancellationToken); + Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default); + Task CreateDirectoryAsync(string path, CancellationToken cancellationToken); + + bool IsFileNameAllowed(string fileName); + string SanitizeFileName(string key, string replacement); +} \ No newline at end of file diff --git a/OpenHarbor.Abstractions/IWriteFileOptions.cs b/OpenHarbor.Abstractions/IWriteFileOptions.cs new file mode 100644 index 0000000..35cc594 --- /dev/null +++ b/OpenHarbor.Abstractions/IWriteFileOptions.cs @@ -0,0 +1,11 @@ +namespace OpenHarbor.Abstractions; + +public interface IWriteFileOptions +{ + bool OverrideIfExists { get; } +} + +public class DefaultWriteOptions : IWriteFileOptions +{ + public bool OverrideIfExists { get; set; } +} \ No newline at end of file diff --git a/OpenHarbor.Abstractions/OpenHarbor.Abstractions.csproj b/OpenHarbor.Abstractions/OpenHarbor.Abstractions.csproj new file mode 100644 index 0000000..5f2d77b --- /dev/null +++ b/OpenHarbor.Abstractions/OpenHarbor.Abstractions.csproj @@ -0,0 +1,9 @@ + + + net8.0 + enable + enable + Open Harbor + https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d + + diff --git a/OpenHarbor.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs b/OpenHarbor.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs new file mode 100644 index 0000000..dee90d7 --- /dev/null +++ b/OpenHarbor.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs @@ -0,0 +1,16 @@ +using Microsoft.WindowsAzure.Storage.Blob; +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.Azure.Blob; + +public class AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory) : IDirectoryInfo +{ + public string Path => blobDirectory.Prefix.TrimEnd('/'); + public bool IsDirectory => true; +} + +public class AzureBlobNotExistingDirectoryInfo(string path) : IDirectoryInfo +{ + public string Path { get; } = path; + public bool IsDirectory => true; +} \ No newline at end of file diff --git a/OpenHarbor.Storage.Azure/Blob/AzureBlobFileInfo.cs b/OpenHarbor.Storage.Azure/Blob/AzureBlobFileInfo.cs new file mode 100644 index 0000000..17bddaa --- /dev/null +++ b/OpenHarbor.Storage.Azure/Blob/AzureBlobFileInfo.cs @@ -0,0 +1,19 @@ +using Microsoft.WindowsAzure.Storage.Blob; +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.Azure.Blob; + +public class AzureBlobFileInfo(CloudBlockBlob fileBlock) : IFileInfo +{ + public string FileName => System.IO.Path.GetFileName(fileBlock.Name); + public string Extension => System.IO.Path.GetExtension(fileBlock.Name); + public long FileSize => fileBlock.Properties.Length; + public DateTimeOffset? CreatedTime => fileBlock.Properties.Created; + public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified; + public DateTimeOffset? LastAccessTime => null; + public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime; + public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime; + public DateTime? LastAccessTimeUtc => null; + public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", ""); + public bool IsDirectory => false; +} \ No newline at end of file diff --git a/OpenHarbor.Storage.Azure/Blob/AzureBlobStorageProvider.cs b/OpenHarbor.Storage.Azure/Blob/AzureBlobStorageProvider.cs new file mode 100644 index 0000000..f6f2048 --- /dev/null +++ b/OpenHarbor.Storage.Azure/Blob/AzureBlobStorageProvider.cs @@ -0,0 +1,251 @@ +using System.Text; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.Azure.Blob; + +public class AzureBlobStorageProvider : IStorageProvider +{ + private string? _connectionString = null; + private string? _containerName = null; + + public AzureBlobStorageProvider(string connectionString, string containerName) + { + SetConnectionString(connectionString); + SetContainerName(containerName); + } + + public void SetContainerName(string name) + { + _containerName = name; + } + + public void SetConnectionString(string connectionString) + { + _connectionString = connectionString; + } + + public Task CreateDirectoryAsync(string path, CancellationToken cancellationToken) + { + var ret = new AzureBlobNotExistingDirectoryInfo(path); + return Task.FromResult(ret); + } + + public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default) + { + var ret = new List(); + var container = GetContainer(); + var finalPath = CleanDirectoryPath(path); + + BlobContinuationToken? continuationToken = null; + List results = new List(); + do + { + BlobResultSegment response; + if (continuationToken == null) + response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null, cancellationToken); + else + response = await container.ListBlobsSegmentedAsync(continuationToken); + + continuationToken = response.ContinuationToken; + results.AddRange(response.Results); + } + while (continuationToken != null); + + var files = results.Where(t => t is CloudBlockBlob).Cast().ToList(); + foreach (var file in files) + await this.DeleteFileAsync(file.Name, cancellationToken); + } + + public Task DeleteFileAsync(string path, CancellationToken cancellationToken) + { + return GetContainer() + .GetBlobReference(path) + .DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, null, null, cancellationToken); + } + + public Task FileExistsAsync(string path, CancellationToken cancellationToken) => + GetContainer() + .GetBlobReference(path) + .ExistsAsync(null, null, cancellationToken); + + public async Task> GetDirectoriesAsync(string path, CancellationToken cancellationToken) => + (await GetListAsync(path, cancellationToken)) + .Where(t => t.IsDirectory) + .Cast() + .ToList(); + + public async Task> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default) + { + if (pattern != null) + throw new NotSupportedException("Blob Storage does not support glob searching only prefix."); + + var result = await GetListAsync(path, cancellationToken); + + return result + .Where(file => false == file.IsDirectory) + .Cast() + .ToList(); + } + + private static string? CleanDirectoryPath(string? path) + { + if (path == null) + return null; + + path = path.TrimEnd('/'); + + if (path != "") + path += "/"; + + return path; + } + + private CloudBlobContainer GetContainer() + { + var account = CloudStorageAccount.Parse(_connectionString); + var client = account.CreateCloudBlobClient(); + var container = client.GetContainerReference(_containerName); + return container; + } + + public async Task> GetListAsync(string path, CancellationToken cancellationToken) + { + var ret = new List(); + var container = GetContainer(); + var finalPath = CleanDirectoryPath(path); + + BlobContinuationToken? continuationToken = null; + var results = new List(); + + do + { + BlobResultSegment response; + if (continuationToken == null) + response = await container.ListBlobsSegmentedAsync(finalPath, false, BlobListingDetails.None, new int?(), + continuationToken, null, null, cancellationToken); + else + response = await container.ListBlobsSegmentedAsync(continuationToken); + + continuationToken = response.ContinuationToken; + results.AddRange(response.Results); + } + while (continuationToken != null); + + foreach (var result in results) + { + if (result is CloudBlobDirectory blobDirectory) + ret.Add(new AzureBlobDirectoryInfo(blobDirectory)); + else if (result is CloudBlockBlob blobBlock) + ret.Add(new AzureBlobFileInfo(blobBlock)); + } + + return ret; + } + + public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(sourcePath, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(bytes, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(stream, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public async Task GetFileStreamAsync(string path, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + var container = GetContainer(); + var blob = container.GetBlockBlobReference(path); + return await blob.OpenReadAsync(null, null, null, cancellationToken); + } + + private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken) + { + if (false == await FileExistsAsync(path, cancellationToken)) + throw new FileDoesNotExistException(path); + } + + public async Task GetFileBytesAsync(string path, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + var container = GetContainer(); + var blob = container.GetBlockBlobReference(path); + var bytes = new byte[blob.Properties.Length]; + await blob.DownloadToByteArrayAsync(bytes, 0, null, null, null, cancellationToken); + return bytes; + } + + public async Task GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + return encoding.GetString(await GetFileBytesAsync(path, cancellationToken)); + } + + public bool IsFileNameAllowed(string fileName) => true; + + public string SanitizeFileName(string key, string replacement) => key; + + public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + if (options is null) + throw new ArgumentNullException(nameof(options)); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + var container = GetContainer(); + var blob = container.GetBlockBlobReference(path); + await blob.UploadFromFileAsync(sourcePath, null, null, null, cancellationToken); + return new AzureBlobFileInfo(blob); + } + + public async Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + if (options is null) + throw new ArgumentNullException(nameof(options)); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + var container = GetContainer(); + var blob = container.GetBlockBlobReference(path); + + await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length, null, null, null, cancellationToken); + return new AzureBlobFileInfo(blob); + } + + public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + if (options is null) + throw new ArgumentNullException(nameof(options)); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + if (stream.CanSeek && stream.Position != 0) + stream.Seek(0, SeekOrigin.Begin); + + var container = GetContainer(); + var blob = container.GetBlockBlobReference(path); + + await blob.UploadFromStreamAsync(stream, null, null, null, cancellationToken); + return new AzureBlobFileInfo(blob); + } +} diff --git a/OpenHarbor.Storage.Azure/OpenHarbor.Storage.Azure.csproj b/OpenHarbor.Storage.Azure/OpenHarbor.Storage.Azure.csproj new file mode 100644 index 0000000..aa3bf59 --- /dev/null +++ b/OpenHarbor.Storage.Azure/OpenHarbor.Storage.Azure.csproj @@ -0,0 +1,17 @@ + + + net8.0 + enable + enable + Open Harbor + https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d + + + + + + + + + + diff --git a/OpenHarbor.Storage.Physical/OpenHarbor.Storage.Physical.csproj b/OpenHarbor.Storage.Physical/OpenHarbor.Storage.Physical.csproj new file mode 100644 index 0000000..11a70bc --- /dev/null +++ b/OpenHarbor.Storage.Physical/OpenHarbor.Storage.Physical.csproj @@ -0,0 +1,13 @@ + + + net8.0 + enable + enable + Open Harbor + https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d + + + + + + diff --git a/OpenHarbor.Storage.Physical/PhysicalDirectory.cs b/OpenHarbor.Storage.Physical/PhysicalDirectory.cs new file mode 100644 index 0000000..568d7c2 --- /dev/null +++ b/OpenHarbor.Storage.Physical/PhysicalDirectory.cs @@ -0,0 +1,9 @@ +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.Physical; + +public class PhysicalDirectoryInfo(string path) : IDirectoryInfo +{ + public string Path { get; } = path; + public bool IsDirectory => true; +} diff --git a/PoweredSoft.Storage.Physical/PhysicalFileInfo.cs b/OpenHarbor.Storage.Physical/PhysicalFileInfo.cs similarity index 92% rename from PoweredSoft.Storage.Physical/PhysicalFileInfo.cs rename to OpenHarbor.Storage.Physical/PhysicalFileInfo.cs index 809f539..5713c85 100644 --- a/PoweredSoft.Storage.Physical/PhysicalFileInfo.cs +++ b/OpenHarbor.Storage.Physical/PhysicalFileInfo.cs @@ -1,8 +1,8 @@ -using PoweredSoft.Storage.Core; +using OpenHarbor.Abstractions; using System; using System.IO; -namespace PoweredSoft.Storage.Physical +namespace OpenHarbor.Storage.Physical { public class PhysicalFileInfo : IFileInfo { diff --git a/OpenHarbor.Storage.Physical/PhysicalStorageProvider.cs b/OpenHarbor.Storage.Physical/PhysicalStorageProvider.cs new file mode 100644 index 0000000..27bdcb0 --- /dev/null +++ b/OpenHarbor.Storage.Physical/PhysicalStorageProvider.cs @@ -0,0 +1,189 @@ +using System.Text; +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.Physical; + +public class PhysicalStorageProvider : IStorageProvider +{ + public Task CreateDirectoryAsync(string path, CancellationToken cancellationToken) + { + Directory.CreateDirectory(path); + + var result = new PhysicalDirectoryInfo(path); + return Task.FromResult(result); + } + + public Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default) + { + if (force) + Directory.Delete(path, true); + else + Directory.Delete(path); + + return Task.CompletedTask; + } + + public Task DeleteFileAsync(string path, CancellationToken cancellationToken) + { + File.Delete(path); + return Task.CompletedTask; + } + + public Task FileExistsAsync(string path, CancellationToken cancellationToken) => + Task.FromResult(File.Exists(path)); + + public Task> GetDirectoriesAsync(string path, CancellationToken cancellationToken) + { + var directoryInfo = new DirectoryInfo(path); + var directories = directoryInfo.GetDirectories(); + + var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable().ToList(); + return Task.FromResult(directoriesConverted); + } + + public async Task GetFileBytesAsync(string path, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + return await File.ReadAllBytesAsync(path, cancellationToken); + } + + public async Task GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + return await File.ReadAllTextAsync(path, encoding, cancellationToken); + } + + public Task> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default) + { + var directoryInfo = new DirectoryInfo(path); + + FileInfo[] files; + if (string.IsNullOrWhiteSpace(pattern)) + files = directoryInfo.GetFiles(); + else + files = directoryInfo.GetFiles(pattern, searchOption); + + var result = files + .Select(fileInfo => new PhysicalFileInfo(fileInfo)) + .AsEnumerable() + .ToList(); + + return Task.FromResult(result); + } + + private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken) + { + if (false == await FileExistsAsync(path, cancellationToken)) + throw new FileDoesNotExistException(path); + } + + public async Task GetFileStreamAsync(string path, CancellationToken cancellationToken) + { + await ThrowNotExistingAsync(path, cancellationToken); + return new FileStream(path, FileMode.Open, FileAccess.Read); + } + + public async Task> GetListAsync(string path, CancellationToken cancellationToken) + { + var files = await GetFilesAsync(path, cancellationToken: cancellationToken); + var directories = await GetDirectoriesAsync(path, cancellationToken: cancellationToken); + var result = files.AsEnumerable() + .Concat(directories.AsEnumerable()) + .ToList(); + + return result; + } + + public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + CreateDirectoryIfNotExisting(path); + + File.Copy(sourcePath, path, options.OverrideIfExists); + var fileInfo = new FileInfo(path); + var ret = new PhysicalFileInfo(fileInfo); + return ret; + } + + public async Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + if (options is null) + throw new ArgumentNullException(nameof(options)); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + CreateDirectoryIfNotExisting(path); + + await File.WriteAllBytesAsync(path, bytes, cancellationToken); + var fileInfo = new FileInfo(path); + var physicalFileInfo = new PhysicalFileInfo(fileInfo); + return physicalFileInfo; + } + + public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + CreateDirectoryIfNotExisting(path); + + if (stream.CanSeek && stream.Position != 0) + stream.Seek(0, SeekOrigin.Begin); + + await using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write)) + { + await stream.CopyToAsync(fileStream, cancellationToken); + fileStream.Close(); + } + + var fileInfo = new FileInfo(path); + var physicalinfo = new PhysicalFileInfo(fileInfo); + return physicalinfo; + } + + private static void CreateDirectoryIfNotExisting(string path) + { + var directoryPath = Path.GetDirectoryName(path); + + if (directoryPath == null) + return; + + if (false == Directory.Exists(directoryPath)) + Directory.CreateDirectory(directoryPath); + } + + public bool IsFileNameAllowed(string fileName) => true; + + public string SanitizeFileName(string key, string replacement) => key; + + public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(sourcePath, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(bytes, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(stream, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } +} diff --git a/PoweredSoft.Storage.S3/IS3FileWriteOptions.cs b/OpenHarbor.Storage.S3/IS3FileWriteOptions.cs similarity index 81% rename from PoweredSoft.Storage.S3/IS3FileWriteOptions.cs rename to OpenHarbor.Storage.S3/IS3FileWriteOptions.cs index 7f436ea..5f4ba4f 100644 --- a/PoweredSoft.Storage.S3/IS3FileWriteOptions.cs +++ b/OpenHarbor.Storage.S3/IS3FileWriteOptions.cs @@ -1,9 +1,9 @@ -using PoweredSoft.Storage.Core; +using OpenHarbor.Abstractions; using System; using System.Collections.Generic; using System.Text; -namespace PoweredSoft.Storage.S3 +namespace OpenHarbor.Storage.S3 { public interface IS3FileWriteOptions { diff --git a/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj b/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj new file mode 100644 index 0000000..a421868 --- /dev/null +++ b/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj @@ -0,0 +1,17 @@ + + + net8.0 + enable + enable + Open Harbor + https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d + + + + + + + + + + diff --git a/PoweredSoft.Storage.S3/S3FileInfo.cs b/OpenHarbor.Storage.S3/S3FileInfo.cs similarity index 94% rename from PoweredSoft.Storage.S3/S3FileInfo.cs rename to OpenHarbor.Storage.S3/S3FileInfo.cs index b90456b..43c4ade 100644 --- a/PoweredSoft.Storage.S3/S3FileInfo.cs +++ b/OpenHarbor.Storage.S3/S3FileInfo.cs @@ -1,8 +1,8 @@ using System; using Amazon.S3.Model; -using PoweredSoft.Storage.Core; +using OpenHarbor.Abstractions; -namespace PoweredSoft.Storage.S3 +namespace OpenHarbor.Storage.S3 { public class S3FileInfo : IFileInfo { diff --git a/PoweredSoft.Storage.S3/S3NotExistingDirectoryInfo.cs b/OpenHarbor.Storage.S3/S3NotExistingDirectoryInfo.cs similarity index 78% rename from PoweredSoft.Storage.S3/S3NotExistingDirectoryInfo.cs rename to OpenHarbor.Storage.S3/S3NotExistingDirectoryInfo.cs index 3964fed..749d909 100644 --- a/PoweredSoft.Storage.S3/S3NotExistingDirectoryInfo.cs +++ b/OpenHarbor.Storage.S3/S3NotExistingDirectoryInfo.cs @@ -1,6 +1,6 @@ -using PoweredSoft.Storage.Core; +using OpenHarbor.Abstractions; -namespace PoweredSoft.Storage.S3 +namespace OpenHarbor.Storage.S3 { public class S3NotExistingDirectoryInfo : IDirectoryInfo { diff --git a/OpenHarbor.Storage.S3/S3StorageProvider.cs b/OpenHarbor.Storage.S3/S3StorageProvider.cs new file mode 100644 index 0000000..376fdb2 --- /dev/null +++ b/OpenHarbor.Storage.S3/S3StorageProvider.cs @@ -0,0 +1,260 @@ +using System.Text; +using System.Text.RegularExpressions; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using OpenHarbor.Abstractions; + +namespace OpenHarbor.Storage.S3; + +public partial class S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) : IStorageProvider +{ + private S3UsEast1RegionalEndpointValue? _s3UsEast1RegionalEndpointValue = null; + private bool _forcePathStyle = false; + + public void SetForcePathStyle(bool forcePathStyle) + { + _forcePathStyle = forcePathStyle; + } + + public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value) + { + _s3UsEast1RegionalEndpointValue = value; + } + + protected virtual IAmazonS3 GetClient() + { + var config = new AmazonS3Config + { + USEast1RegionalEndpointValue = _s3UsEast1RegionalEndpointValue, + ServiceURL = endpoint, + ForcePathStyle = _forcePathStyle + }; + + var client = new AmazonS3Client(accessKey, secret, config); + + return client; + } + + public Task CreateDirectoryAsync(string path, CancellationToken cancellationToken) + { + return Task.FromResult(new S3NotExistingDirectoryInfo(path)); + } + + /// + /// Can only delete 1000 at a time. + /// + /// + /// + /// + /// + public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default) + { + using var client = GetClient(); + var files = await this.GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken); + var next = files.AsQueryable(); + + while(next.Any()) + { + var next1000 = next.Take(1000); + var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList(); + await client.DeleteObjectsAsync(new DeleteObjectsRequest + { + BucketName = bucketName, + Objects = keys + }, cancellationToken); + + next = next.Skip(1000); + } + } + + public async Task DeleteFileAsync(string path, CancellationToken cancellationToken) + { + using var client = GetClient(); + var response = await client.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = bucketName, + Key = path + }, cancellationToken); + } + + public async Task FileExistsAsync(string path, CancellationToken cancellationToken = default) + { + var s3Object = await GetS3FileByPathAsync(path, cancellationToken); + return s3Object != null; + } + + public Task> GetDirectoriesAsync(string path, CancellationToken cancellationToken) + { + return Task.FromResult(new List()); + } + + public async Task GetFileBytesAsync(string path, CancellationToken cancellationToken = default) + { + await using var fileStream = await GetFileStreamAsync(path, cancellationToken); + using var memoryStream = new MemoryStream(); + await fileStream.CopyToAsync(memoryStream, cancellationToken); + return memoryStream.ToArray(); + } + + public async Task GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken = default) + { + await using var fileStream = await this.GetFileStreamAsync(path, cancellationToken); + using var streamReader = new StreamReader(fileStream, encoding); + return await streamReader.ReadToEndAsync(cancellationToken); + } + + public async Task> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default) + { + if (pattern != null) + throw new NotSupportedException(); + + var finalPath = SanitizeDirectoryRequest(path); + var s3Files = await GetS3FilesAsync(prefix: finalPath, delimiter: "/", cancellationToken: cancellationToken); + var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable().ToList(); + return ret; + } + + private static string SanitizeDirectoryRequest(string path) + { + string finalPath; + if (path == "/") + finalPath = ""; + else + finalPath = $"{path?.TrimEnd('/')}/"; + return finalPath; + } + + public async Task GetFileStreamAsync(string path, CancellationToken cancellationToken) + { + using var client = GetClient(); + return await client.GetObjectStreamAsync(bucketName, path, null, cancellationToken); + } + + protected virtual async Task> GetS3FilesAsync(string? prefix = null, string? delimiter = null, CancellationToken cancellationToken = default) + { + using var client = GetClient(); + + var items = new List(); + string? nextKey = null; + + do + { + var response = await client.ListObjectsV2Async(new ListObjectsV2Request + { + BucketName = bucketName, + Prefix = prefix, + Delimiter = delimiter, + MaxKeys = 1000, + ContinuationToken = nextKey + }, cancellationToken); + + items.AddRange(response.S3Objects); + nextKey = response.NextContinuationToken; + + } while (nextKey != null); + + return items; + } + + public async Task> GetListAsync(string path, CancellationToken cancellationToken) + { + var files = await GetFilesAsync(path, cancellationToken: cancellationToken); + return files.Cast().ToList(); + } + + public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + await using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read); + return await WriteFileAsync(fileStream, path, options, cancellationToken); + } + + public Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken) + { + return WriteFileAsync(new MemoryStream(bytes), path, options, cancellationToken); + } + + public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken)) + throw new FileAlreadyExistsException(path); + + using var client = GetClient(); + var request = new PutObjectRequest + { + BucketName = bucketName, + InputStream = stream, + Key = path + }; + + if (options is IS3FileWriteOptions s3FileWriteOptions) + request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl); + + // todo: unhandled response + var result = await client.PutObjectAsync(request, cancellationToken); + + var file = await GetFileInfoByPathAsync(path, cancellationToken); + return file; + } + + private async Task GetS3FileByPathAsync(string path, CancellationToken cancellationToken) + { + var files = await GetS3FilesAsync(path, cancellationToken: cancellationToken); + return files.FirstOrDefault(); + } + + private async Task GetFileInfoByPathAsync(string path, CancellationToken cancellationToken) + { + var s3Object = await GetS3FileByPathAsync(path, cancellationToken); + if (s3Object == null) + throw new FileDoesNotExistException(path); + + var ret = new S3FileInfo(s3Object); + return ret; + } + + public string SanitizeFileName(string key, string replacement) + { + var regex = S3FileNameRegex(); + + var result = regex.Replace(key, replacement); + return result; + } + + public bool IsFileNameAllowed(string fileName) + { + var regex = S3FileNameRegex(); + + var hasMatches = regex.IsMatch(fileName); + return false == hasMatches; + } + + public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(sourcePath, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(bytes, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default) + { + return WriteFileAsync(stream, path, new DefaultWriteOptions + { + OverrideIfExists = overrideIfExists + }, cancellationToken); + } + + [GeneratedRegex(@"[^a-zA-Z0-9.!\-_*'()]", RegexOptions.Multiline)] + private static partial Regex S3FileNameRegex(); +} \ No newline at end of file diff --git a/PoweredSoft.Storage.sln b/OpenHarbor.Storage.sln similarity index 61% rename from PoweredSoft.Storage.sln rename to OpenHarbor.Storage.sln index 346c592..07d7ed8 100644 --- a/PoweredSoft.Storage.sln +++ b/OpenHarbor.Storage.sln @@ -3,15 +3,13 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.30406.217 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Azure", "PoweredSoft.Storage.Azure\PoweredSoft.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Azure", "OpenHarbor.Storage.Azure\OpenHarbor.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Core", "PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Abstractions", "OpenHarbor.Abstractions\OpenHarbor.Abstractions.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Physical", "PoweredSoft.Storage.Physical\PoweredSoft.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Physical", "OpenHarbor.Storage.Physical\OpenHarbor.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.S3", "PoweredSoft.Storage.S3\PoweredSoft.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PoweredSoft.Storage.Test", "PoweredSoft.Storage.Test\PoweredSoft.Storage.Test.csproj", "{305416EE-51A4-4293-9262-87865D2784F4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.S3", "OpenHarbor.Storage.S3\OpenHarbor.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -35,10 +33,6 @@ Global {457912EA-48E3-4B2E-941F-2116D18C6D88}.Debug|Any CPU.Build.0 = Debug|Any CPU {457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.ActiveCfg = Release|Any CPU {457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.Build.0 = Release|Any CPU - {305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PoweredSoft.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs b/PoweredSoft.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs deleted file mode 100644 index ca9c3f8..0000000 --- a/PoweredSoft.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs +++ /dev/null @@ -1,29 +0,0 @@ -using Microsoft.WindowsAzure.Storage.Blob; -using PoweredSoft.Storage.Core; - -namespace PoweredSoft.Storage.Azure.Blob -{ - public class AzureBlobDirectoryInfo : IDirectoryInfo - { - private CloudBlobDirectory blobDirectory; - - public AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory) - { - this.blobDirectory = blobDirectory; - } - - public string Path => blobDirectory.Prefix.TrimEnd('/'); - public bool IsDirectory => true; - } - - public class AzureBlobNotExistingDirectoryInfo : IDirectoryInfo - { - public AzureBlobNotExistingDirectoryInfo(string path) - { - Path = path; - } - - public string Path { get; } - public bool IsDirectory => true; - } -} \ No newline at end of file diff --git a/PoweredSoft.Storage.Azure/Blob/AzureBlobFileInfo.cs b/PoweredSoft.Storage.Azure/Blob/AzureBlobFileInfo.cs deleted file mode 100644 index 47bbfe6..0000000 --- a/PoweredSoft.Storage.Azure/Blob/AzureBlobFileInfo.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Microsoft.WindowsAzure.Storage.Blob; -using PoweredSoft.Storage.Core; -using System; -using System.Collections.Generic; -using System.Text; - -namespace PoweredSoft.Storage.Azure.Blob -{ - public class AzureBlobFileInfo : IFileInfo - { - private readonly CloudBlockBlob fileBlock; - - public AzureBlobFileInfo(CloudBlockBlob fileBlock) - { - this.fileBlock = fileBlock; - } - - public string FileName => System.IO.Path.GetFileName(fileBlock.Name); - public string Extension => System.IO.Path.GetExtension(fileBlock.Name); - public long FileSize => fileBlock.Properties.Length; - public DateTimeOffset? CreatedTime => fileBlock.Properties.Created; - public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified; - public DateTimeOffset? LastAccessTime => null; - public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime; - public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime; - public DateTime? LastAccessTimeUtc => null; - public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", ""); - public bool IsDirectory => false; - } -} diff --git a/PoweredSoft.Storage.Azure/Blob/AzureBlobStorageProvider.cs b/PoweredSoft.Storage.Azure/Blob/AzureBlobStorageProvider.cs deleted file mode 100644 index 232d500..0000000 --- a/PoweredSoft.Storage.Azure/Blob/AzureBlobStorageProvider.cs +++ /dev/null @@ -1,265 +0,0 @@ -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; -using PoweredSoft.Storage.Core; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace PoweredSoft.Storage.Azure.Blob -{ - public class AzureBlobStorageProvider : IStorageProvider - { - private string connectionString = null; - private string containerName = null; - - public AzureBlobStorageProvider() - { - - } - - public AzureBlobStorageProvider(string connectionString, string containerName) - { - this.SetConnectionString(connectionString); - this.SetContainerName(containerName); - } - - public void SetContainerName(string name) - { - this.containerName = name; - } - - public void SetConnectionString(string connectionString) - { - this.connectionString = connectionString; - } - - public Task CreateDirectoryAsync(string path) - { - var ret = new AzureBlobNotExistingDirectoryInfo(path); - return Task.FromResult(ret); - } - - public async Task DeleteDirectoryAsync(string path, bool force = false) - { - var ret = new List(); - var container = GetContainer(); - var finalPath = CleanDirectoryPath(path); - - BlobContinuationToken continuationToken = null; - List results = new List(); - do - { - BlobResultSegment response; - if (continuationToken == null) - response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null); - else - response = await container.ListBlobsSegmentedAsync(continuationToken); - - continuationToken = response.ContinuationToken; - results.AddRange(response.Results); - } - while (continuationToken != null); - - var files = results.Where(t => t is CloudBlockBlob).Cast().ToList(); - foreach (var file in files) - await this.DeleteFileAsync(file.Name); - } - - public Task DeleteFileAsync(string path) - { - return GetContainer().GetBlobReference(path).DeleteIfExistsAsync(); - } - - public Task FileExistsAsync(string path) - { - return GetContainer().GetBlobReference(path).ExistsAsync(); - } - - public async Task> GetDirectories(string path) - { - return (await this.GetListAsync(path)).Where(t => t.IsDirectory).Cast().ToList(); - } - - public async Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly) - { - if (pattern != null) - throw new NotSupportedException("Blob Storage does not support glob searching only prefix."); - - var result = await GetListAsync(path); - var finalResult = result.Where(t => !t.IsDirectory).Cast().ToList(); - return finalResult; - } - - private string CleanDirectoryPath(string path) - { - if (path == null) - return path; - - path = path.TrimEnd('/'); - - if (path != "") - path += "/"; - - return path; - } - - private CloudBlobContainer GetContainer() - { - var account = CloudStorageAccount.Parse(connectionString); - var client = account.CreateCloudBlobClient(); - var container = client.GetContainerReference(containerName); - return container; - } - - public async Task> GetListAsync(string path) - { - var ret = new List(); - var container = GetContainer(); - var finalPath = CleanDirectoryPath(path); - - BlobContinuationToken continuationToken = null; - List results = new List(); - do - { - BlobResultSegment response; - if (continuationToken == null) - response = await container.ListBlobsSegmentedAsync(finalPath, continuationToken); - else - response = await container.ListBlobsSegmentedAsync(continuationToken); - - continuationToken = response.ContinuationToken; - results.AddRange(response.Results); - } - while (continuationToken != null); - - foreach (var result in results) - { - if (result is CloudBlobDirectory blobDirectory) - ret.Add(new AzureBlobDirectoryInfo(blobDirectory)); - else if (result is CloudBlockBlob blobBlock) - ret.Add(new AzureBlobFileInfo(blobBlock)); - } - - return ret; - } - - public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true) - { - return WriteFileAsync(sourcePath, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true) - { - return WriteFileAsync(bytes, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true) - { - return WriteFileAsync(stream, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public async Task GetFileStreamAsync(string path) - { - await ThrowNotExistingAsync(path); - var container = GetContainer(); - var blob = container.GetBlockBlobReference(path); - return await blob.OpenReadAsync(); - } - - private async Task ThrowNotExistingAsync(string path) - { - if (false == await this.FileExistsAsync(path)) - throw new FileDoesNotExistException(path); - } - - public async Task GetFileBytesAsync(string path) - { - await ThrowNotExistingAsync(path); - var container = GetContainer(); - var blob = container.GetBlockBlobReference(path); - var bytes = new byte[blob.Properties.Length]; - await blob.DownloadToByteArrayAsync(bytes, 0); - return bytes; - } - - public async Task GetFileContentAsync(string path, Encoding encoding) - { - await ThrowNotExistingAsync(path); - var container = GetContainer(); - return encoding.GetString(await this.GetFileBytesAsync(path)); - } - - public bool IsFileNameAllowed(string fileName) - { - return true; - } - - public string SanitizeFileName(string key, string replacement) - { - return key; - } - - public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - var container = GetContainer(); - var blob = container.GetBlockBlobReference(path); - await blob.UploadFromFileAsync(sourcePath); - return new AzureBlobFileInfo(blob); - } - - public async Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - var container = GetContainer(); - var blob = container.GetBlockBlobReference(path); - await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length); - return new AzureBlobFileInfo(blob); - } - - public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - if (stream.CanSeek && stream.Position != 0) - stream.Seek(0, SeekOrigin.Begin); - - var container = GetContainer(); - var blob = container.GetBlockBlobReference(path); - await blob.UploadFromStreamAsync(stream); - return new AzureBlobFileInfo(blob); - } - } -} diff --git a/PoweredSoft.Storage.Azure/PoweredSoft.Storage.Azure.csproj b/PoweredSoft.Storage.Azure/PoweredSoft.Storage.Azure.csproj deleted file mode 100644 index bcb3336..0000000 --- a/PoweredSoft.Storage.Azure/PoweredSoft.Storage.Azure.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - netstandard2.0 - Powered Softwares Inc. - PoweredSoft - https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&r=g&d=retro - - - - - - - - - - - diff --git a/PoweredSoft.Storage.Core/FileAlreadyExistsException.cs b/PoweredSoft.Storage.Core/FileAlreadyExistsException.cs deleted file mode 100644 index 6cc4057..0000000 --- a/PoweredSoft.Storage.Core/FileAlreadyExistsException.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Runtime.Serialization; - -namespace PoweredSoft.Storage.Core -{ - public class FileAlreadyExistsException : Exception - { - public FileAlreadyExistsException(string path) : base($"{path} already exists..") - { - } - } -} \ No newline at end of file diff --git a/PoweredSoft.Storage.Core/FileDoesNotExistException.cs b/PoweredSoft.Storage.Core/FileDoesNotExistException.cs deleted file mode 100644 index 5a97697..0000000 --- a/PoweredSoft.Storage.Core/FileDoesNotExistException.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; - -namespace PoweredSoft.Storage.Core -{ - public class FileDoesNotExistException : Exception - { - public FileDoesNotExistException(string path) : base($"{path} does not exist.") - { - - } - } -} \ No newline at end of file diff --git a/PoweredSoft.Storage.Core/IDirectory.cs b/PoweredSoft.Storage.Core/IDirectory.cs deleted file mode 100644 index c710864..0000000 --- a/PoweredSoft.Storage.Core/IDirectory.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace PoweredSoft.Storage.Core -{ - public interface IDirectoryInfo : IDirectoryOrFile - { - - } -} diff --git a/PoweredSoft.Storage.Core/IDirectoryOrFile.cs b/PoweredSoft.Storage.Core/IDirectoryOrFile.cs deleted file mode 100644 index 947b10c..0000000 --- a/PoweredSoft.Storage.Core/IDirectoryOrFile.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace PoweredSoft.Storage.Core -{ - public interface IDirectoryOrFile - { - string Path { get; } - bool IsDirectory { get; } - } -} diff --git a/PoweredSoft.Storage.Core/IFileInfo.cs b/PoweredSoft.Storage.Core/IFileInfo.cs deleted file mode 100644 index 27778d7..0000000 --- a/PoweredSoft.Storage.Core/IFileInfo.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; - -namespace PoweredSoft.Storage.Core -{ - public interface IFileInfo : IDirectoryOrFile - { - string FileName { get; } - string Extension { get; } - long FileSize { get; } - DateTimeOffset? CreatedTime { get; } - DateTimeOffset? LastModifiedTime { get; } - DateTimeOffset? LastAccessTime { get; } - DateTime? CreatedTimeUtc { get; } - DateTime? LastModifiedTimeUtc { get; } - DateTime? LastAccessTimeUtc { get; } - } -} diff --git a/PoweredSoft.Storage.Core/IStorageProvider.cs b/PoweredSoft.Storage.Core/IStorageProvider.cs deleted file mode 100644 index a8a445d..0000000 --- a/PoweredSoft.Storage.Core/IStorageProvider.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Text; -using System.Threading.Tasks; - -namespace PoweredSoft.Storage.Core -{ - - public interface IStorageProvider - { - Task> GetListAsync(string path); - Task> GetDirectories(string path); - Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly); - Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true); - Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true); - Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true); - Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options); - Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options); - Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options); - Task GetFileStreamAsync(string path); - Task GetFileBytesAsync(string path); - Task GetFileContentAsync(string path, Encoding encoding); - Task FileExistsAsync(string path); - Task DeleteFileAsync(string path); - Task DeleteDirectoryAsync(string path, bool force = false); - Task CreateDirectoryAsync(string path); - - bool IsFileNameAllowed(string fileName); - string SanitizeFileName(string key, string replacement); - } -} diff --git a/PoweredSoft.Storage.Core/IWriteFileOptions.cs b/PoweredSoft.Storage.Core/IWriteFileOptions.cs deleted file mode 100644 index feaca64..0000000 --- a/PoweredSoft.Storage.Core/IWriteFileOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace PoweredSoft.Storage.Core -{ - public interface IWriteFileOptions - { - bool OverrideIfExists { get; } - } - - public class DefaultWriteOptions : IWriteFileOptions - { - public bool OverrideIfExists { get; set; } - } -} diff --git a/PoweredSoft.Storage.Core/PoweredSoft.Storage.Core.csproj b/PoweredSoft.Storage.Core/PoweredSoft.Storage.Core.csproj deleted file mode 100644 index 94ef261..0000000 --- a/PoweredSoft.Storage.Core/PoweredSoft.Storage.Core.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - - netstandard2.0 - Powered Softwares Inc. - PoweredSoft - https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&r=g&d=retro - - - diff --git a/PoweredSoft.Storage.Physical/PhysicalDirectory.cs b/PoweredSoft.Storage.Physical/PhysicalDirectory.cs deleted file mode 100644 index e449747..0000000 --- a/PoweredSoft.Storage.Physical/PhysicalDirectory.cs +++ /dev/null @@ -1,15 +0,0 @@ -using PoweredSoft.Storage.Core; - -namespace PoweredSoft.Storage.Physical -{ - public class PhysicalDirectoryInfo : IDirectoryInfo - { - public PhysicalDirectoryInfo(string path) - { - Path = path; - } - - public string Path { get; } - public bool IsDirectory => true; - } -} diff --git a/PoweredSoft.Storage.Physical/PhysicalStorageProvider.cs b/PoweredSoft.Storage.Physical/PhysicalStorageProvider.cs deleted file mode 100644 index 97816e4..0000000 --- a/PoweredSoft.Storage.Physical/PhysicalStorageProvider.cs +++ /dev/null @@ -1,199 +0,0 @@ -using PoweredSoft.Storage.Core; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace PoweredSoft.Storage.Physical -{ - public class PhysicalStorageProvider : IStorageProvider - { - public Task CreateDirectoryAsync(string path) - { - var directoryInfo = System.IO.Directory.CreateDirectory(path); - var result = new PhysicalDirectoryInfo(path); - return Task.FromResult(result); - } - - public Task DeleteDirectoryAsync(string path, bool force = false) - { - if (force) - Directory.Delete(path, true); - else - Directory.Delete(path); - - return Task.CompletedTask; - } - - public Task DeleteFileAsync(string path) - { - System.IO.File.Delete(path); - return Task.CompletedTask; - } - - public Task FileExistsAsync(string path) - { - return Task.FromResult(File.Exists(path)); - } - - public Task> GetDirectories(string path) - { - var directoryInfo = new DirectoryInfo(path); - var directories = directoryInfo.GetDirectories(); - var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable().ToList(); - return Task.FromResult(directoriesConverted); - } - - public async Task GetFileBytesAsync(string path) - { - await ThrowNotExistingAsync(path); - return File.ReadAllBytes(path); - } - - public async Task GetFileContentAsync(string path, Encoding encoding) - { - await ThrowNotExistingAsync(path); - return File.ReadAllText(path, encoding); - } - - public Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly) - { - var directoryInfo = new DirectoryInfo(path); - - FileInfo[] files; - if (string.IsNullOrWhiteSpace(pattern)) - files = directoryInfo.GetFiles(); - else - files = directoryInfo.GetFiles(pattern, searchOption); - - var result = files.Select(fileInfo => new PhysicalFileInfo(fileInfo)).AsEnumerable().ToList(); - return Task.FromResult(result); - } - - private async Task ThrowNotExistingAsync(string path) - { - if (false == await this.FileExistsAsync(path)) - throw new FileDoesNotExistException(path); - } - - public async Task GetFileStreamAsync(string path) - { - await ThrowNotExistingAsync(path); - return new FileStream(path, FileMode.Open, FileAccess.Read); - } - - public async Task> GetListAsync(string path) - { - var files = await this.GetFilesAsync(path); - var directories = await this.GetDirectories(path); - var result = files.AsEnumerable().Concat(directories.AsEnumerable()).ToList(); - return result; - } - - public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - - CreateDirectoryIfNotExisting(path); - - System.IO.File.Copy(sourcePath, path, options.OverrideIfExists); - var fileInfo = new FileInfo(path); - var ret = new PhysicalFileInfo(fileInfo); - return ret; - } - - public async Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - CreateDirectoryIfNotExisting(path); - - File.WriteAllBytes(path, bytes); - var fileInfo = new FileInfo(path); - var physicalinfo = new PhysicalFileInfo(fileInfo); - return physicalinfo; - } - - public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - CreateDirectoryIfNotExisting(path); - - if (stream.CanSeek && stream.Position != 0) - stream.Seek(0, SeekOrigin.Begin); - - using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write)) - { - await stream.CopyToAsync(fileStream); - fileStream.Close(); - } - - var fileInfo = new FileInfo(path); - var physicalinfo = new PhysicalFileInfo(fileInfo); - return physicalinfo; - } - - private void CreateDirectoryIfNotExisting(string path) - { - var directoryPath = Path.GetDirectoryName(path); - if (!Directory.Exists(directoryPath)) - Directory.CreateDirectory(directoryPath); - } - - public bool IsFileNameAllowed(string fileName) - { - return true; - } - - public string SanitizeFileName(string key, string replacement) - { - return key; - } - - public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true) - { - return WriteFileAsync(sourcePath, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true) - { - return WriteFileAsync(bytes, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true) - { - return WriteFileAsync(stream, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - } -} diff --git a/PoweredSoft.Storage.Physical/PoweredSoft.Storage.Physical.csproj b/PoweredSoft.Storage.Physical/PoweredSoft.Storage.Physical.csproj deleted file mode 100644 index 6d6f911..0000000 --- a/PoweredSoft.Storage.Physical/PoweredSoft.Storage.Physical.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - - netstandard2.0 - Powered Softwares Inc. - PoweredSoft - https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&r=g&d=retro - - - - - - - diff --git a/PoweredSoft.Storage.S3/PoweredSoft.Storage.S3.csproj b/PoweredSoft.Storage.S3/PoweredSoft.Storage.S3.csproj deleted file mode 100644 index ba1e437..0000000 --- a/PoweredSoft.Storage.S3/PoweredSoft.Storage.S3.csproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - netstandard2.0 - 8.0 - Powered Softwares Inc. - PoweredSoft - https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&r=g&d=retro - - - - - - - - - - - - - diff --git a/PoweredSoft.Storage.S3/S3StorageProvider.cs b/PoweredSoft.Storage.S3/S3StorageProvider.cs deleted file mode 100644 index 11d99f1..0000000 --- a/PoweredSoft.Storage.S3/S3StorageProvider.cs +++ /dev/null @@ -1,283 +0,0 @@ -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using PoweredSoft.Storage.Core; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace PoweredSoft.Storage.S3 -{ - public class S3StorageProvider : IStorageProvider - { - protected readonly string endpoint; - protected readonly string bucketName; - protected readonly string accessKey; - protected readonly string secret; - - protected S3UsEast1RegionalEndpointValue? s3UsEast1RegionalEndpointValue = null; - protected bool forcePathStyle = false; - - public S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) - { - this.endpoint = endpoint; - this.bucketName = bucketName; - this.accessKey = accessKey; - this.secret = secret; - } - - public void SetForcePathStyle(bool forcePathStyle) - { - this.forcePathStyle = forcePathStyle; - } - - public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value) - { - this.s3UsEast1RegionalEndpointValue = value; - } - - protected virtual IAmazonS3 GetClient() - { - var config = new AmazonS3Config - { - USEast1RegionalEndpointValue = s3UsEast1RegionalEndpointValue, - ServiceURL = endpoint, - ForcePathStyle = forcePathStyle - }; - var client = new AmazonS3Client(this.accessKey, this.secret, config); - return client; - } - - public Task CreateDirectoryAsync(string path) - { - return Task.FromResult(new S3NotExistingDirectoryInfo(path)); - } - - /// - /// Can only delete 1000 at a time. - /// - /// - /// - /// - public async Task DeleteDirectoryAsync(string path, bool force = false) - { - using var client = GetClient(); - var files = await this.GetS3FilesAsync(prefix: path, delimiter: null); - var next = files.AsQueryable(); - while(next.Any()) - { - var next1000 = next.Take(1000); - var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList(); - await client.DeleteObjectsAsync(new DeleteObjectsRequest - { - BucketName = this.bucketName, - Objects = keys - }); - - next = next.Skip(1000); - } - } - - public async Task DeleteFileAsync(string path) - { - using var client = GetClient(); - var response = await client.DeleteObjectAsync(new DeleteObjectRequest - { - BucketName = this.bucketName, - Key = path - }); - } - - public async Task FileExistsAsync(string path) - { - var item = await GetS3FileByPath(path); - return item != null; - } - - public Task> GetDirectories(string path) - { - return Task.FromResult(new List()); - } - - public async Task GetFileBytesAsync(string path) - { - using var fileStream = await this.GetFileStreamAsync(path); - using var memoryStream = new MemoryStream(); - await fileStream.CopyToAsync(memoryStream); - return memoryStream.ToArray(); - } - - public async Task GetFileContentAsync(string path, Encoding encoding) - { - using var fileStream = await this.GetFileStreamAsync(path); - using var streamReader = new StreamReader(fileStream, encoding); - return await streamReader.ReadToEndAsync(); - } - - public async Task> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly) - { - if (pattern != null) - throw new NotSupportedException(); - - string finalPath = SantizeDirectoryRequest(path); - var s3Files = await this.GetS3FilesAsync(prefix: finalPath, delimiter: "/"); - var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable().ToList(); - return ret; - } - - private static string SantizeDirectoryRequest(string path) - { - string finalPath; - if (path == "/") - finalPath = ""; - else - finalPath = $"{path?.TrimEnd('/')}/"; - return finalPath; - } - - public Task GetFileStreamAsync(string path) - { - using var client = GetClient(); - return client.GetObjectStreamAsync(this.bucketName, path, null); - } - - protected virtual async Task> GetS3FilesAsync(string prefix = null, string delimiter = null) - { - using var client = GetClient(); - - var items = new List(); - string nextKey = null; - - do - { - var response = await client.ListObjectsV2Async(new ListObjectsV2Request - { - BucketName = this.bucketName, - Prefix = prefix, - Delimiter = delimiter, - MaxKeys = 1000, - ContinuationToken = nextKey - }); - - items.AddRange(response.S3Objects); - nextKey = response.NextContinuationToken; - - } while (nextKey != null); - - return items; - } - - public async Task> GetListAsync(string path) - { - var files = await this.GetFilesAsync(path); - return files.Cast().ToList(); - } - - public async Task WriteFileAsync(string sourcePath, string path, IWriteFileOptions options) - { - using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read); - return await WriteFileAsync(fileStream, path, options); - } - - public Task WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options) - { - return WriteFileAsync(new MemoryStream(bytes), path, options); - } - - public async Task WriteFileAsync(Stream stream, string path, IWriteFileOptions options) - { - if (options is null) - { - throw new ArgumentNullException(nameof(options)); - } - - if (!options.OverrideIfExists && await FileExistsAsync(path)) - throw new FileAlreadyExistsException(path); - - using var client = GetClient(); - var request = new PutObjectRequest - { - BucketName = this.bucketName, - InputStream = stream, - Key = path - }; - - if (options is IS3FileWriteOptions s3FileWriteOptions) - { - if (s3FileWriteOptions.Acl != null) - request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl); - } - - var result = await client.PutObjectAsync(request); - var file = await GetFileInfoByPath(path); - return file; - } - - private async Task GetS3FileByPath(string path) - { - var files = await this.GetS3FilesAsync(path); - var s3o = files.FirstOrDefault(); - return s3o; - } - - private async Task GetFileInfoByPath(string path) - { - var s3o = await GetS3FileByPath(path); - if (s3o == null) - throw new FileDoesNotExistException(path); - - var ret = new S3FileInfo(s3o); - return ret; - } - - public string SanitizeFileName(string key, string replacement) - { - string pattern = @"[^a-zA-Z0-9.!\-_*'()]"; - string substitution = replacement; - string input = key; - RegexOptions options = RegexOptions.Multiline; - - Regex regex = new Regex(pattern, options); - - string result = regex.Replace(input, substitution); - return result; - } - - public bool IsFileNameAllowed(string fileName) - { - string pattern = @"[^a-zA-Z0-9.!\-_*'()]"; - RegexOptions options = RegexOptions.Multiline; - Regex regex = new Regex(pattern, options); - var hasMatches = regex.IsMatch(fileName); - return false == hasMatches; - } - - public Task WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true) - { - return WriteFileAsync(sourcePath, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true) - { - return WriteFileAsync(bytes, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - - public Task WriteFileAsync(Stream stream, string path, bool overrideIfExists = true) - { - return WriteFileAsync(stream, path, new DefaultWriteOptions - { - OverrideIfExists = overrideIfExists - }); - } - } -} diff --git a/PoweredSoft.Storage.Test/PoweredSoft.Storage.Test.csproj b/PoweredSoft.Storage.Test/PoweredSoft.Storage.Test.csproj deleted file mode 100644 index 9cc1395..0000000 --- a/PoweredSoft.Storage.Test/PoweredSoft.Storage.Test.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - netcoreapp3.1 - - false - - - - - - - - - - - - - - diff --git a/PoweredSoft.Storage.Test/S3Tests.cs b/PoweredSoft.Storage.Test/S3Tests.cs deleted file mode 100644 index 2028fab..0000000 --- a/PoweredSoft.Storage.Test/S3Tests.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using PoweredSoft.Storage.S3; -using System.Text; - -namespace PoweredSoft.Storage.Test -{ - [TestClass] - public class S3Tests - { - [TestMethod] - public async System.Threading.Tasks.Task S3AclWriteAsync() - { - var space = GetMockS3Space(); - await space.WriteFileAsync(Encoding.UTF8.GetBytes("Hello World"), "hello-world.txt", new S3FileWriteOptions - { - Acl = "public-read", - OverrideIfExists = true - }); - } - - [TestMethod] - public void NameValidation() - { - var space = GetMockS3Space(); - - Assert.IsFalse(space.IsFileNameAllowed("Operations .pdf"), "Should not be valid"); - Assert.IsFalse(space.IsFileNameAllowed("Operations$$.pdf"), "Should not be valid"); - } - - [TestMethod] - public void CanContainDash() - { - var space = GetMockS3Space(); - Assert.IsTrue(space.IsFileNameAllowed("Operations-yay.pdf"), "Should be allowed"); - } - - [TestMethod] - public void NameSanitation() - { - var space = GetMockS3Space(); - - Assert.AreEqual("Operations_.pdf", space.SanitizeFileName("Operations .pdf", "_"), "does not match sanitation expectations"); - Assert.AreEqual("Operations__.pdf", space.SanitizeFileName("Operations$$.pdf", "_"), "does not match sanitation expectations"); - } - - private static S3StorageProvider GetMockS3Space() - { - var space = new S3StorageProvider("http://localhost:9000", "mybucket", "minioadmin", "minioadmin"); - space.SetForcePathStyle(true); - space.SetS3UsEast1RegionalEndpointValue(Amazon.Runtime.S3UsEast1RegionalEndpointValue.Legacy); - - return space; - } - } -} From 247b0d356f0130a6adc1bef41552faf8cb92ec71 Mon Sep 17 00:00:00 2001 From: Mathias Beaulieu-Duncan Date: Sun, 10 Nov 2024 23:48:12 -0500 Subject: [PATCH 2/2] update aws s3 dependency --- OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj | 2 +- OpenHarbor.Storage.S3/S3StorageProvider.cs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj b/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj index a421868..b876719 100644 --- a/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj +++ b/OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj @@ -12,6 +12,6 @@ - + diff --git a/OpenHarbor.Storage.S3/S3StorageProvider.cs b/OpenHarbor.Storage.S3/S3StorageProvider.cs index 376fdb2..5508309 100644 --- a/OpenHarbor.Storage.S3/S3StorageProvider.cs +++ b/OpenHarbor.Storage.S3/S3StorageProvider.cs @@ -51,13 +51,15 @@ public partial class S3StorageProvider(string endpoint, string bucketName, strin public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default) { using var client = GetClient(); - var files = await this.GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken); + var files = await GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken); var next = files.AsQueryable(); while(next.Any()) { var next1000 = next.Take(1000); - var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList(); + var keys = next1000.Select(s3Object => new KeyVersion { Key = s3Object.Key }) + .ToList(); + await client.DeleteObjectsAsync(new DeleteObjectsRequest { BucketName = bucketName,