Compare commits

..

7 Commits

43 changed files with 928 additions and 1080 deletions
@@ -0,0 +1,3 @@
namespace OpenHarbor.Storage.Abstractions;
public class FileAlreadyExistsException(string path) : Exception($"{path} already exists..");
@@ -0,0 +1,3 @@
namespace OpenHarbor.Storage.Abstractions;
public class FileDoesNotExistException(string path) : Exception($"{path} does not exist.");
@@ -0,0 +1,6 @@
namespace OpenHarbor.Storage.Abstractions;
public interface IDirectoryInfo: IDirectoryOrFile
{
}
@@ -0,0 +1,7 @@
namespace OpenHarbor.Storage.Abstractions;
public interface IDirectoryOrFile
{
string Path { get; }
bool IsDirectory { get; }
}
@@ -0,0 +1,14 @@
namespace OpenHarbor.Storage.Abstractions;
public interface IFileInfo : IDirectoryOrFile
{
string FileName { get; }
string Extension { get; }
long FileSize { get; }
DateTimeOffset? CreatedTime { get; }
DateTimeOffset? LastModifiedTime { get; }
DateTimeOffset? LastAccessTime { get; }
DateTime? CreatedTimeUtc { get; }
DateTime? LastModifiedTimeUtc { get; }
DateTime? LastAccessTimeUtc { get; }
}
@@ -0,0 +1,26 @@
using System.Text;
namespace OpenHarbor.Storage.Abstractions;
public interface IStorageProvider
{
Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken);
Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken);
Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken);
Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken);
Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken);
Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken);
Task DeleteFileAsync(string path, CancellationToken cancellationToken);
Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default);
Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken);
bool IsFileNameAllowed(string fileName);
string SanitizeFileName(string key, string replacement);
}
@@ -0,0 +1,11 @@
namespace OpenHarbor.Storage.Abstractions;
public interface IWriteFileOptions
{
bool OverrideIfExists { get; }
}
public class DefaultWriteOptions : IWriteFileOptions
{
public bool OverrideIfExists { get; set; }
}
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
</Project>
@@ -0,0 +1,16 @@
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory) : IDirectoryInfo
{
public string Path => blobDirectory.Prefix.TrimEnd('/');
public bool IsDirectory => true;
}
public class AzureBlobNotExistingDirectoryInfo(string path) : IDirectoryInfo
{
public string Path { get; } = path;
public bool IsDirectory => true;
}
@@ -0,0 +1,19 @@
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobFileInfo(CloudBlockBlob fileBlock) : IFileInfo
{
public string FileName => System.IO.Path.GetFileName(fileBlock.Name);
public string Extension => System.IO.Path.GetExtension(fileBlock.Name);
public long FileSize => fileBlock.Properties.Length;
public DateTimeOffset? CreatedTime => fileBlock.Properties.Created;
public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified;
public DateTimeOffset? LastAccessTime => null;
public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime;
public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime;
public DateTime? LastAccessTimeUtc => null;
public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", "");
public bool IsDirectory => false;
}
@@ -0,0 +1,251 @@
using System.Text;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobStorageProvider : IStorageProvider
{
private string? _connectionString = null;
private string? _containerName = null;
public AzureBlobStorageProvider(string connectionString, string containerName)
{
SetConnectionString(connectionString);
SetContainerName(containerName);
}
public void SetContainerName(string name)
{
_containerName = name;
}
public void SetConnectionString(string connectionString)
{
_connectionString = connectionString;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
var ret = new AzureBlobNotExistingDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(ret);
}
public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken? continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null, cancellationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
var files = results.Where(t => t is CloudBlockBlob).Cast<CloudBlockBlob>().ToList();
foreach (var file in files)
await this.DeleteFileAsync(file.Name, cancellationToken);
}
public Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
return GetContainer()
.GetBlobReference(path)
.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, null, null, cancellationToken);
}
public Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken) =>
GetContainer()
.GetBlobReference(path)
.ExistsAsync(null, null, cancellationToken);
public async Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken) =>
(await GetListAsync(path, cancellationToken))
.Where(t => t.IsDirectory)
.Cast<IDirectoryInfo>()
.ToList();
public async Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
if (pattern != null)
throw new NotSupportedException("Blob Storage does not support glob searching only prefix.");
var result = await GetListAsync(path, cancellationToken);
return result
.Where(file => false == file.IsDirectory)
.Cast<IFileInfo>()
.ToList();
}
private static string? CleanDirectoryPath(string? path)
{
if (path == null)
return null;
path = path.TrimEnd('/');
if (path != "")
path += "/";
return path;
}
private CloudBlobContainer GetContainer()
{
var account = CloudStorageAccount.Parse(_connectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(_containerName);
return container;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken? continuationToken = null;
var results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, false, BlobListingDetails.None, new int?(),
continuationToken, null, null, cancellationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
foreach (var result in results)
{
if (result is CloudBlobDirectory blobDirectory)
ret.Add(new AzureBlobDirectoryInfo(blobDirectory));
else if (result is CloudBlockBlob blobBlock)
ret.Add(new AzureBlobFileInfo(blobBlock));
}
return ret;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
return await blob.OpenReadAsync(null, null, null, cancellationToken);
}
private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken)
{
if (false == await FileExistsAsync(path, cancellationToken))
throw new FileDoesNotExistException(path);
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
var bytes = new byte[blob.Properties.Length];
await blob.DownloadToByteArrayAsync(bytes, 0, null, null, null, cancellationToken);
return bytes;
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return encoding.GetString(await GetFileBytesAsync(path, cancellationToken));
}
public bool IsFileNameAllowed(string fileName) => true;
public string SanitizeFileName(string key, string replacement) => key;
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromFileAsync(sourcePath, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromStreamAsync(stream, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
}
@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Storage.Abstractions\OpenHarbor.Storage.Abstractions.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Storage.Abstractions\OpenHarbor.Storage.Abstractions.csproj" />
</ItemGroup>
</Project>
@@ -0,0 +1,9 @@
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.Physical;
public class PhysicalDirectoryInfo(string path) : IDirectoryInfo
{
public string Path { get; } = path;
public bool IsDirectory => true;
}
@@ -1,8 +1,8 @@
using PoweredSoft.Storage.Core; using OpenHarbor.Storage.Abstractions;
using System; using System;
using System.IO; using System.IO;
namespace PoweredSoft.Storage.Physical namespace OpenHarbor.Storage.Physical
{ {
public class PhysicalFileInfo : IFileInfo public class PhysicalFileInfo : IFileInfo
{ {
@@ -0,0 +1,189 @@
using System.Text;
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.Physical;
public class PhysicalStorageProvider : IStorageProvider
{
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
Directory.CreateDirectory(path);
var result = new PhysicalDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(result);
}
public Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
if (force)
Directory.Delete(path, true);
else
Directory.Delete(path);
return Task.CompletedTask;
}
public Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
File.Delete(path);
return Task.CompletedTask;
}
public Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken) =>
Task.FromResult(File.Exists(path));
public Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken)
{
var directoryInfo = new DirectoryInfo(path);
var directories = directoryInfo.GetDirectories();
var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable<IDirectoryInfo>().ToList();
return Task.FromResult(directoriesConverted);
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return await File.ReadAllBytesAsync(path, cancellationToken);
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return await File.ReadAllTextAsync(path, encoding, cancellationToken);
}
public Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
var directoryInfo = new DirectoryInfo(path);
FileInfo[] files;
if (string.IsNullOrWhiteSpace(pattern))
files = directoryInfo.GetFiles();
else
files = directoryInfo.GetFiles(pattern, searchOption);
var result = files
.Select(fileInfo => new PhysicalFileInfo(fileInfo))
.AsEnumerable<IFileInfo>()
.ToList();
return Task.FromResult(result);
}
private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken)
{
if (false == await FileExistsAsync(path, cancellationToken))
throw new FileDoesNotExistException(path);
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return new FileStream(path, FileMode.Open, FileAccess.Read);
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var files = await GetFilesAsync(path, cancellationToken: cancellationToken);
var directories = await GetDirectoriesAsync(path, cancellationToken: cancellationToken);
var result = files.AsEnumerable<IDirectoryOrFile>()
.Concat(directories.AsEnumerable<IDirectoryOrFile>())
.ToList();
return result;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
File.Copy(sourcePath, path, options.OverrideIfExists);
var fileInfo = new FileInfo(path);
var ret = new PhysicalFileInfo(fileInfo);
return ret;
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
await File.WriteAllBytesAsync(path, bytes, cancellationToken);
var fileInfo = new FileInfo(path);
var physicalFileInfo = new PhysicalFileInfo(fileInfo);
return physicalFileInfo;
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
await using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write))
{
await stream.CopyToAsync(fileStream, cancellationToken);
fileStream.Close();
}
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
private static void CreateDirectoryIfNotExisting(string path)
{
var directoryPath = Path.GetDirectoryName(path);
if (directoryPath == null)
return;
if (false == Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
}
public bool IsFileNameAllowed(string fileName) => true;
public string SanitizeFileName(string key, string replacement) => key;
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
}
@@ -1,9 +1,9 @@
using PoweredSoft.Storage.Core; using OpenHarbor.Storage.Abstractions;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
namespace PoweredSoft.Storage.S3 namespace OpenHarbor.Storage.S3
{ {
public interface IS3FileWriteOptions public interface IS3FileWriteOptions
{ {
@@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Storage.Abstractions\OpenHarbor.Storage.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.7.405.11" />
</ItemGroup>
</Project>
@@ -1,8 +1,8 @@
using System; using System;
using Amazon.S3.Model; using Amazon.S3.Model;
using PoweredSoft.Storage.Core; using OpenHarbor.Storage.Abstractions;
namespace PoweredSoft.Storage.S3 namespace OpenHarbor.Storage.S3
{ {
public class S3FileInfo : IFileInfo public class S3FileInfo : IFileInfo
{ {
@@ -1,6 +1,6 @@
using PoweredSoft.Storage.Core; using OpenHarbor.Storage.Abstractions;
namespace PoweredSoft.Storage.S3 namespace OpenHarbor.Storage.S3
{ {
public class S3NotExistingDirectoryInfo : IDirectoryInfo public class S3NotExistingDirectoryInfo : IDirectoryInfo
{ {
+262
View File
@@ -0,0 +1,262 @@
using System.Text;
using System.Text.RegularExpressions;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using OpenHarbor.Storage.Abstractions;
namespace OpenHarbor.Storage.S3;
public partial class S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) : IStorageProvider
{
private S3UsEast1RegionalEndpointValue? _s3UsEast1RegionalEndpointValue = null;
private bool _forcePathStyle = false;
public void SetForcePathStyle(bool forcePathStyle)
{
_forcePathStyle = forcePathStyle;
}
public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value)
{
_s3UsEast1RegionalEndpointValue = value;
}
protected virtual IAmazonS3 GetClient()
{
var config = new AmazonS3Config
{
USEast1RegionalEndpointValue = _s3UsEast1RegionalEndpointValue,
ServiceURL = endpoint,
ForcePathStyle = _forcePathStyle
};
var client = new AmazonS3Client(accessKey, secret, config);
return client;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
return Task.FromResult<IDirectoryInfo>(new S3NotExistingDirectoryInfo(path));
}
/// <summary>
/// Can only delete 1000 at a time.
/// </summary>
/// <param name="path"></param>
/// <param name="force"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
using var client = GetClient();
var files = await GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken);
var next = files.AsQueryable();
while(next.Any())
{
var next1000 = next.Take(1000);
var keys = next1000.Select(s3Object => new KeyVersion { Key = s3Object.Key })
.ToList();
await client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keys
}, cancellationToken);
next = next.Skip(1000);
}
}
public async Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
using var client = GetClient();
var response = await client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = bucketName,
Key = path
}, cancellationToken);
}
public async Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken = default)
{
var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
return s3Object != null;
}
public Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken)
{
return Task.FromResult(new List<IDirectoryInfo>());
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken = default)
{
await using var fileStream = await GetFileStreamAsync(path, cancellationToken);
using var memoryStream = new MemoryStream();
await fileStream.CopyToAsync(memoryStream, cancellationToken);
return memoryStream.ToArray();
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken = default)
{
await using var fileStream = await this.GetFileStreamAsync(path, cancellationToken);
using var streamReader = new StreamReader(fileStream, encoding);
return await streamReader.ReadToEndAsync(cancellationToken);
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
if (pattern != null)
throw new NotSupportedException();
var finalPath = SanitizeDirectoryRequest(path);
var s3Files = await GetS3FilesAsync(prefix: finalPath, delimiter: "/", cancellationToken: cancellationToken);
var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable<IFileInfo>().ToList();
return ret;
}
private static string SanitizeDirectoryRequest(string path)
{
string finalPath;
if (path == "/")
finalPath = "";
else
finalPath = $"{path?.TrimEnd('/')}/";
return finalPath;
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
using var client = GetClient();
return await client.GetObjectStreamAsync(bucketName, path, null, cancellationToken);
}
protected virtual async Task<IEnumerable<S3Object>> GetS3FilesAsync(string? prefix = null, string? delimiter = null, CancellationToken cancellationToken = default)
{
using var client = GetClient();
var items = new List<S3Object>();
string? nextKey = null;
do
{
var response = await client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = bucketName,
Prefix = prefix,
Delimiter = delimiter,
MaxKeys = 1000,
ContinuationToken = nextKey
}, cancellationToken);
items.AddRange(response.S3Objects);
nextKey = response.NextContinuationToken;
} while (nextKey != null);
return items;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var files = await GetFilesAsync(path, cancellationToken: cancellationToken);
return files.Cast<IDirectoryOrFile>().ToList();
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
await using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read);
return await WriteFileAsync(fileStream, path, options, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
return WriteFileAsync(new MemoryStream(bytes), path, options, cancellationToken);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
using var client = GetClient();
var request = new PutObjectRequest
{
BucketName = bucketName,
InputStream = stream,
Key = path
};
if (options is IS3FileWriteOptions s3FileWriteOptions)
request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl);
// todo: unhandled response
var result = await client.PutObjectAsync(request, cancellationToken);
var file = await GetFileInfoByPathAsync(path, cancellationToken);
return file;
}
private async Task<S3Object?> GetS3FileByPathAsync(string path, CancellationToken cancellationToken)
{
var files = await GetS3FilesAsync(path, cancellationToken: cancellationToken);
return files.FirstOrDefault();
}
private async Task<IFileInfo> GetFileInfoByPathAsync(string path, CancellationToken cancellationToken)
{
var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
if (s3Object == null)
throw new FileDoesNotExistException(path);
var ret = new S3FileInfo(s3Object);
return ret;
}
public string SanitizeFileName(string key, string replacement)
{
var regex = S3FileNameRegex();
var result = regex.Replace(key, replacement);
return result;
}
public bool IsFileNameAllowed(string fileName)
{
var regex = S3FileNameRegex();
var hasMatches = regex.IsMatch(fileName);
return false == hasMatches;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
[GeneratedRegex(@"[^a-zA-Z0-9.!\-_*'()]", RegexOptions.Multiline)]
private static partial Regex S3FileNameRegex();
}
@@ -3,15 +3,13 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16 # Visual Studio Version 16
VisualStudioVersion = 16.0.30406.217 VisualStudioVersion = 16.0.30406.217
MinimumVisualStudioVersion = 10.0.40219.1 MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Azure", "PoweredSoft.Storage.Azure\PoweredSoft.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Azure", "OpenHarbor.Storage.Azure\OpenHarbor.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Core", "PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Abstractions", "OpenHarbor.Storage.Abstractions\OpenHarbor.Storage.Abstractions.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Physical", "PoweredSoft.Storage.Physical\PoweredSoft.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Physical", "OpenHarbor.Storage.Physical\OpenHarbor.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.S3", "PoweredSoft.Storage.S3\PoweredSoft.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.S3", "OpenHarbor.Storage.S3\OpenHarbor.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PoweredSoft.Storage.Test", "PoweredSoft.Storage.Test\PoweredSoft.Storage.Test.csproj", "{305416EE-51A4-4293-9262-87865D2784F4}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -35,10 +33,6 @@ Global
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Debug|Any CPU.Build.0 = Debug|Any CPU {457912EA-48E3-4B2E-941F-2116D18C6D88}.Debug|Any CPU.Build.0 = Debug|Any CPU
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.ActiveCfg = Release|Any CPU {457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.ActiveCfg = Release|Any CPU
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.Build.0 = Release|Any CPU {457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.Build.0 = Release|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@@ -1,29 +0,0 @@
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobDirectoryInfo : IDirectoryInfo
{
private CloudBlobDirectory blobDirectory;
public AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory)
{
this.blobDirectory = blobDirectory;
}
public string Path => blobDirectory.Prefix.TrimEnd('/');
public bool IsDirectory => true;
}
public class AzureBlobNotExistingDirectoryInfo : IDirectoryInfo
{
public AzureBlobNotExistingDirectoryInfo(string path)
{
Path = path;
}
public string Path { get; }
public bool IsDirectory => true;
}
}
@@ -1,30 +0,0 @@
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.Text;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobFileInfo : IFileInfo
{
private readonly CloudBlockBlob fileBlock;
public AzureBlobFileInfo(CloudBlockBlob fileBlock)
{
this.fileBlock = fileBlock;
}
public string FileName => System.IO.Path.GetFileName(fileBlock.Name);
public string Extension => System.IO.Path.GetExtension(fileBlock.Name);
public long FileSize => fileBlock.Properties.Length;
public DateTimeOffset? CreatedTime => fileBlock.Properties.Created;
public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified;
public DateTimeOffset? LastAccessTime => null;
public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime;
public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime;
public DateTime? LastAccessTimeUtc => null;
public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", "");
public bool IsDirectory => false;
}
}
@@ -1,265 +0,0 @@
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobStorageProvider : IStorageProvider
{
private string connectionString = null;
private string containerName = null;
public AzureBlobStorageProvider()
{
}
public AzureBlobStorageProvider(string connectionString, string containerName)
{
this.SetConnectionString(connectionString);
this.SetContainerName(containerName);
}
public void SetContainerName(string name)
{
this.containerName = name;
}
public void SetConnectionString(string connectionString)
{
this.connectionString = connectionString;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
var ret = new AzureBlobNotExistingDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(ret);
}
public async Task DeleteDirectoryAsync(string path, bool force = false)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
var files = results.Where(t => t is CloudBlockBlob).Cast<CloudBlockBlob>().ToList();
foreach (var file in files)
await this.DeleteFileAsync(file.Name);
}
public Task DeleteFileAsync(string path)
{
return GetContainer().GetBlobReference(path).DeleteIfExistsAsync();
}
public Task<bool> FileExistsAsync(string path)
{
return GetContainer().GetBlobReference(path).ExistsAsync();
}
public async Task<List<IDirectoryInfo>> GetDirectories(string path)
{
return (await this.GetListAsync(path)).Where(t => t.IsDirectory).Cast<IDirectoryInfo>().ToList();
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
if (pattern != null)
throw new NotSupportedException("Blob Storage does not support glob searching only prefix.");
var result = await GetListAsync(path);
var finalResult = result.Where(t => !t.IsDirectory).Cast<IFileInfo>().ToList();
return finalResult;
}
private string CleanDirectoryPath(string path)
{
if (path == null)
return path;
path = path.TrimEnd('/');
if (path != "")
path += "/";
return path;
}
private CloudBlobContainer GetContainer()
{
var account = CloudStorageAccount.Parse(connectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
return container;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, continuationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
foreach (var result in results)
{
if (result is CloudBlobDirectory blobDirectory)
ret.Add(new AzureBlobDirectoryInfo(blobDirectory));
else if (result is CloudBlockBlob blobBlock)
ret.Add(new AzureBlobFileInfo(blobBlock));
}
return ret;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public async Task<Stream> GetFileStreamAsync(string path)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
return await blob.OpenReadAsync();
}
private async Task ThrowNotExistingAsync(string path)
{
if (false == await this.FileExistsAsync(path))
throw new FileDoesNotExistException(path);
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
var bytes = new byte[blob.Properties.Length];
await blob.DownloadToByteArrayAsync(bytes, 0);
return bytes;
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
return encoding.GetString(await this.GetFileBytesAsync(path));
}
public bool IsFileNameAllowed(string fileName)
{
return true;
}
public string SanitizeFileName(string key, string replacement)
{
return key;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromFileAsync(sourcePath);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromStreamAsync(stream);
return new AzureBlobFileInfo(blob);
}
}
}
@@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
</Project>
@@ -1,12 +0,0 @@
using System;
using System.Runtime.Serialization;
namespace PoweredSoft.Storage.Core
{
public class FileAlreadyExistsException : Exception
{
public FileAlreadyExistsException(string path) : base($"{path} already exists..")
{
}
}
}
@@ -1,12 +0,0 @@
using System;
namespace PoweredSoft.Storage.Core
{
public class FileDoesNotExistException : Exception
{
public FileDoesNotExistException(string path) : base($"{path} does not exist.")
{
}
}
}
-7
View File
@@ -1,7 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IDirectoryInfo : IDirectoryOrFile
{
}
}
@@ -1,8 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IDirectoryOrFile
{
string Path { get; }
bool IsDirectory { get; }
}
}
-17
View File
@@ -1,17 +0,0 @@
using System;
namespace PoweredSoft.Storage.Core
{
public interface IFileInfo : IDirectoryOrFile
{
string FileName { get; }
string Extension { get; }
long FileSize { get; }
DateTimeOffset? CreatedTime { get; }
DateTimeOffset? LastModifiedTime { get; }
DateTimeOffset? LastAccessTime { get; }
DateTime? CreatedTimeUtc { get; }
DateTime? LastModifiedTimeUtc { get; }
DateTime? LastAccessTimeUtc { get; }
}
}
@@ -1,31 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Core
{
public interface IStorageProvider
{
Task<List<IDirectoryOrFile>> GetListAsync(string path);
Task<List<IDirectoryInfo>> GetDirectories(string path);
Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options);
Task<Stream> GetFileStreamAsync(string path);
Task<byte[]> GetFileBytesAsync(string path);
Task<string> GetFileContentAsync(string path, Encoding encoding);
Task<bool> FileExistsAsync(string path);
Task DeleteFileAsync(string path);
Task DeleteDirectoryAsync(string path, bool force = false);
Task<IDirectoryInfo> CreateDirectoryAsync(string path);
bool IsFileNameAllowed(string fileName);
string SanitizeFileName(string key, string replacement);
}
}
@@ -1,12 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IWriteFileOptions
{
bool OverrideIfExists { get; }
}
public class DefaultWriteOptions : IWriteFileOptions
{
public bool OverrideIfExists { get; set; }
}
}
@@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
</Project>
@@ -1,15 +0,0 @@
using PoweredSoft.Storage.Core;
namespace PoweredSoft.Storage.Physical
{
public class PhysicalDirectoryInfo : IDirectoryInfo
{
public PhysicalDirectoryInfo(string path)
{
Path = path;
}
public string Path { get; }
public bool IsDirectory => true;
}
}
@@ -1,199 +0,0 @@
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Physical
{
public class PhysicalStorageProvider : IStorageProvider
{
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
var directoryInfo = System.IO.Directory.CreateDirectory(path);
var result = new PhysicalDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(result);
}
public Task DeleteDirectoryAsync(string path, bool force = false)
{
if (force)
Directory.Delete(path, true);
else
Directory.Delete(path);
return Task.CompletedTask;
}
public Task DeleteFileAsync(string path)
{
System.IO.File.Delete(path);
return Task.CompletedTask;
}
public Task<bool> FileExistsAsync(string path)
{
return Task.FromResult(File.Exists(path));
}
public Task<List<IDirectoryInfo>> GetDirectories(string path)
{
var directoryInfo = new DirectoryInfo(path);
var directories = directoryInfo.GetDirectories();
var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable<IDirectoryInfo>().ToList();
return Task.FromResult(directoriesConverted);
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
await ThrowNotExistingAsync(path);
return File.ReadAllBytes(path);
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
await ThrowNotExistingAsync(path);
return File.ReadAllText(path, encoding);
}
public Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
var directoryInfo = new DirectoryInfo(path);
FileInfo[] files;
if (string.IsNullOrWhiteSpace(pattern))
files = directoryInfo.GetFiles();
else
files = directoryInfo.GetFiles(pattern, searchOption);
var result = files.Select(fileInfo => new PhysicalFileInfo(fileInfo)).AsEnumerable<IFileInfo>().ToList();
return Task.FromResult(result);
}
private async Task ThrowNotExistingAsync(string path)
{
if (false == await this.FileExistsAsync(path))
throw new FileDoesNotExistException(path);
}
public async Task<Stream> GetFileStreamAsync(string path)
{
await ThrowNotExistingAsync(path);
return new FileStream(path, FileMode.Open, FileAccess.Read);
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var files = await this.GetFilesAsync(path);
var directories = await this.GetDirectories(path);
var result = files.AsEnumerable<IDirectoryOrFile>().Concat(directories.AsEnumerable<IDirectoryOrFile>()).ToList();
return result;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
System.IO.File.Copy(sourcePath, path, options.OverrideIfExists);
var fileInfo = new FileInfo(path);
var ret = new PhysicalFileInfo(fileInfo);
return ret;
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
File.WriteAllBytes(path, bytes);
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write))
{
await stream.CopyToAsync(fileStream);
fileStream.Close();
}
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
private void CreateDirectoryIfNotExisting(string path)
{
var directoryPath = Path.GetDirectoryName(path);
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
}
public bool IsFileNameAllowed(string fileName)
{
return true;
}
public string SanitizeFileName(string key, string replacement)
{
return key;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
}
}
@@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
</Project>
@@ -1,21 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<LangVersion>8.0</LangVersion>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.3.110.10" />
</ItemGroup>
</Project>
-283
View File
@@ -1,283 +0,0 @@
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.S3
{
public class S3StorageProvider : IStorageProvider
{
protected readonly string endpoint;
protected readonly string bucketName;
protected readonly string accessKey;
protected readonly string secret;
protected S3UsEast1RegionalEndpointValue? s3UsEast1RegionalEndpointValue = null;
protected bool forcePathStyle = false;
public S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret)
{
this.endpoint = endpoint;
this.bucketName = bucketName;
this.accessKey = accessKey;
this.secret = secret;
}
public void SetForcePathStyle(bool forcePathStyle)
{
this.forcePathStyle = forcePathStyle;
}
public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value)
{
this.s3UsEast1RegionalEndpointValue = value;
}
protected virtual IAmazonS3 GetClient()
{
var config = new AmazonS3Config
{
USEast1RegionalEndpointValue = s3UsEast1RegionalEndpointValue,
ServiceURL = endpoint,
ForcePathStyle = forcePathStyle
};
var client = new AmazonS3Client(this.accessKey, this.secret, config);
return client;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
return Task.FromResult<IDirectoryInfo>(new S3NotExistingDirectoryInfo(path));
}
/// <summary>
/// Can only delete 1000 at a time.
/// </summary>
/// <param name="path"></param>
/// <param name="force"></param>
/// <returns></returns>
public async Task DeleteDirectoryAsync(string path, bool force = false)
{
using var client = GetClient();
var files = await this.GetS3FilesAsync(prefix: path, delimiter: null);
var next = files.AsQueryable();
while(next.Any())
{
var next1000 = next.Take(1000);
var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList();
await client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = this.bucketName,
Objects = keys
});
next = next.Skip(1000);
}
}
public async Task DeleteFileAsync(string path)
{
using var client = GetClient();
var response = await client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = this.bucketName,
Key = path
});
}
public async Task<bool> FileExistsAsync(string path)
{
var item = await GetS3FileByPath(path);
return item != null;
}
public Task<List<IDirectoryInfo>> GetDirectories(string path)
{
return Task.FromResult(new List<IDirectoryInfo>());
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
using var fileStream = await this.GetFileStreamAsync(path);
using var memoryStream = new MemoryStream();
await fileStream.CopyToAsync(memoryStream);
return memoryStream.ToArray();
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
using var fileStream = await this.GetFileStreamAsync(path);
using var streamReader = new StreamReader(fileStream, encoding);
return await streamReader.ReadToEndAsync();
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
if (pattern != null)
throw new NotSupportedException();
string finalPath = SantizeDirectoryRequest(path);
var s3Files = await this.GetS3FilesAsync(prefix: finalPath, delimiter: "/");
var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable<IFileInfo>().ToList();
return ret;
}
private static string SantizeDirectoryRequest(string path)
{
string finalPath;
if (path == "/")
finalPath = "";
else
finalPath = $"{path?.TrimEnd('/')}/";
return finalPath;
}
public Task<Stream> GetFileStreamAsync(string path)
{
using var client = GetClient();
return client.GetObjectStreamAsync(this.bucketName, path, null);
}
protected virtual async Task<IEnumerable<S3Object>> GetS3FilesAsync(string prefix = null, string delimiter = null)
{
using var client = GetClient();
var items = new List<S3Object>();
string nextKey = null;
do
{
var response = await client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = this.bucketName,
Prefix = prefix,
Delimiter = delimiter,
MaxKeys = 1000,
ContinuationToken = nextKey
});
items.AddRange(response.S3Objects);
nextKey = response.NextContinuationToken;
} while (nextKey != null);
return items;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var files = await this.GetFilesAsync(path);
return files.Cast<IDirectoryOrFile>().ToList();
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read);
return await WriteFileAsync(fileStream, path, options);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
return WriteFileAsync(new MemoryStream(bytes), path, options);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
using var client = GetClient();
var request = new PutObjectRequest
{
BucketName = this.bucketName,
InputStream = stream,
Key = path
};
if (options is IS3FileWriteOptions s3FileWriteOptions)
{
if (s3FileWriteOptions.Acl != null)
request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl);
}
var result = await client.PutObjectAsync(request);
var file = await GetFileInfoByPath(path);
return file;
}
private async Task<S3Object> GetS3FileByPath(string path)
{
var files = await this.GetS3FilesAsync(path);
var s3o = files.FirstOrDefault();
return s3o;
}
private async Task<IFileInfo> GetFileInfoByPath(string path)
{
var s3o = await GetS3FileByPath(path);
if (s3o == null)
throw new FileDoesNotExistException(path);
var ret = new S3FileInfo(s3o);
return ret;
}
public string SanitizeFileName(string key, string replacement)
{
string pattern = @"[^a-zA-Z0-9.!\-_*'()]";
string substitution = replacement;
string input = key;
RegexOptions options = RegexOptions.Multiline;
Regex regex = new Regex(pattern, options);
string result = regex.Replace(input, substitution);
return result;
}
public bool IsFileNameAllowed(string fileName)
{
string pattern = @"[^a-zA-Z0-9.!\-_*'()]";
RegexOptions options = RegexOptions.Multiline;
Regex regex = new Regex(pattern, options);
var hasMatches = regex.IsMatch(fileName);
return false == hasMatches;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
}
}
@@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.S3\PoweredSoft.Storage.S3.csproj" />
</ItemGroup>
</Project>
-55
View File
@@ -1,55 +0,0 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using PoweredSoft.Storage.S3;
using System.Text;
namespace PoweredSoft.Storage.Test
{
[TestClass]
public class S3Tests
{
[TestMethod]
public async System.Threading.Tasks.Task S3AclWriteAsync()
{
var space = GetMockS3Space();
await space.WriteFileAsync(Encoding.UTF8.GetBytes("Hello World"), "hello-world.txt", new S3FileWriteOptions
{
Acl = "public-read",
OverrideIfExists = true
});
}
[TestMethod]
public void NameValidation()
{
var space = GetMockS3Space();
Assert.IsFalse(space.IsFileNameAllowed("Operations .pdf"), "Should not be valid");
Assert.IsFalse(space.IsFileNameAllowed("Operations$$.pdf"), "Should not be valid");
}
[TestMethod]
public void CanContainDash()
{
var space = GetMockS3Space();
Assert.IsTrue(space.IsFileNameAllowed("Operations-yay.pdf"), "Should be allowed");
}
[TestMethod]
public void NameSanitation()
{
var space = GetMockS3Space();
Assert.AreEqual("Operations_.pdf", space.SanitizeFileName("Operations .pdf", "_"), "does not match sanitation expectations");
Assert.AreEqual("Operations__.pdf", space.SanitizeFileName("Operations$$.pdf", "_"), "does not match sanitation expectations");
}
private static S3StorageProvider GetMockS3Space()
{
var space = new S3StorageProvider("http://localhost:9000", "mybucket", "minioadmin", "minioadmin");
space.SetForcePathStyle(true);
space.SetS3UsEast1RegionalEndpointValue(Amazon.Runtime.S3UsEast1RegionalEndpointValue.Legacy);
return space;
}
}
}
+6 -4
View File
@@ -1,3 +1,5 @@
> This project was originally initiated by [Powered Software Inc.](https://poweredsoft.com/) and was forked from the [PoweredSoft/Storage](https://github.com/PoweredSoft/Storage) Repository
# Storage # Storage
Abstraction allowing you to use different storage options on the cloud or locally with the same interface. Abstraction allowing you to use different storage options on the cloud or locally with the same interface.
@@ -10,7 +12,7 @@ Abstraction allowing you to use different storage options on the cloud or locall
| Full Version | NuGet | NuGet Install | | Full Version | NuGet | NuGet Install |
| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -----------------------------------------------------: | | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -----------------------------------------------------: |
| PoweredSoft.Storage.Core | <a href="https://www.nuget.org/packages/PoweredSoft.Storage.Core/" target="_blank">[![NuGet](https://img.shields.io/nuget/v/PoweredSoft.Storage.Core.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/PoweredSoft.Storage.Core/)</a> | ```PM> Install-Package PoweredSoft.Storage.Core``` | | OpenHarbor.Storage.Abstractions | <a href="https://www.nuget.org/packages/OpenHarbor.Storage.Abstractions" target="_blank">[![NuGet](https://img.shields.io/nuget/v/OpenHarbor.Storage.Abstractions.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/OpenHarbor.Storage.Abstractions)</a> | ```PM> Install-Package OpenHarbor.Storage.Abstractions``` |
| PoweredSoft.Storage.Azure | <a href="https://www.nuget.org/packages/PoweredSoft.Storage.Azure/" target="_blank">[![NuGet](https://img.shields.io/nuget/v/PoweredSoft.Storage.Azure.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/PoweredSoft.Storage.Azure/)</a> | ```PM> Install-Package PoweredSoft.Storage.Azure``` | | OpenHarbor.Storage.Physical | <a href="https://www.nuget.org/packages/OpenHarbor.Storage.Physical" target="_blank">[![NuGet](https://img.shields.io/nuget/v/OpenHarbor.Storage.Physical.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/OpenHarbor.Storage.Physical)</a> | ```PM> Install-Package OpenHarbor.Storage.Physical``` |
| PoweredSoft.Storage.S3 | <a href="https://www.nuget.org/packages/PoweredSoft.Storage.S3/" target="_blank">[![NuGet](https://img.shields.io/nuget/v/PoweredSoft.Storage.S3.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/PoweredSoft.Storage.S3/)</a> | ```PM> Install-Package PoweredSoft.Storage.S3``` | | OpenHarbor.Storage.Azure | <a href="https://www.nuget.org/packages/OpenHarbor.Storage.Azure" target="_blank">[![NuGet](https://img.shields.io/nuget/v/OpenHarbor.Storage.Azure.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/OpenHarbor.Storage.Azure)</a> | ```PM> Install-Package OpenHarbor.Storage.S3``` |
| PoweredSoft.Storage.Physical | <a href="https://www.nuget.org/packages/PoweredSoft.Storage.Physical/" target="_blank">[![NuGet](https://img.shields.io/nuget/v/PoweredSoft.Storage.Physical.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/PoweredSoft.Storage.Physical/)</a> | ```PM> Install-Package PoweredSoft.Storage.Physical``` | | OpenHarbor.Storage.S3 | <a href="https://www.nuget.org/packages/OpenHarbor.Storage.S3" target="_blank">[![NuGet](https://img.shields.io/nuget/v/OpenHarbor.Storage.S3.svg?style=flat-square&label=nuget)](https://www.nuget.org/packages/OpenHarbor.Storage.S3)</a> | ```PM> Install-Package OpenHarbor.Storage.S3 ``` |
+38
View File
@@ -0,0 +1,38 @@
name: Publish NuGets
on:
release:
published:
- tag
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Extract Release Version
shell: bash
run: echo "RELEASE_VERSION=${GITEA_TAG_NAME}" >> $GITEA_ENV
- name: Debug Release Version
run: echo "RELEASE_VERSION=${RELEASE_VERSION}"
- name: Setup .NET
uses: actions/setup-dotnet@v3
with:
dotnet-version: 8.x
- name: Restore dependencies
run: dotnet restore
- name: Build and Pack NuGet Package
run: dotnet pack -c Release -o ./artifacts -p:Version=${RELEASE_VERSION}
- name: Publish to NuGet.org
env:
NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }}
run: |
dotnet nuget push ./artifacts/*.nupkg --source https://api.nuget.org/v3/index.json --api-key ${NUGET_API_KEY}