Open-Harbor/dotnet-storage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature/next #1

Merged
mathias merged 2 commits from feature/next into master 2024-11-11 11:26:31 -05:00
Conversation 0 Commits 2 Files Changed 41 +882 -1076
41 changed files with 882 additions and 1076 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c0b8b92c21 - Show all commits

3
OpenHarbor.Abstractions/FileAlreadyExistsException.cs Normal file
View File

@ -0,0 +1,3 @@
namespace OpenHarbor.Abstractions;
public class FileAlreadyExistsException(string path) : Exception($"{path} already exists..");

3
OpenHarbor.Abstractions/FileDoesNotExistException.cs Normal file
View File

@ -0,0 +1,3 @@
namespace OpenHarbor.Abstractions;
public class FileDoesNotExistException(string path) : Exception($"{path} does not exist.");

6
OpenHarbor.Abstractions/IDirectory.cs Normal file
View File

@ -0,0 +1,6 @@
namespace OpenHarbor.Abstractions;
public interface IDirectoryInfo: IDirectoryOrFile
{
}

7
OpenHarbor.Abstractions/IDirectoryOrFile.cs Normal file
View File

@ -0,0 +1,7 @@
namespace OpenHarbor.Abstractions;
public interface IDirectoryOrFile
{
string Path { get; }
bool IsDirectory { get; }
}

14
OpenHarbor.Abstractions/IFileInfo.cs Normal file
View File

@ -0,0 +1,14 @@
namespace OpenHarbor.Abstractions;
public interface IFileInfo : IDirectoryOrFile
{
string FileName { get; }
string Extension { get; }
long FileSize { get; }
DateTimeOffset? CreatedTime { get; }
DateTimeOffset? LastModifiedTime { get; }
DateTimeOffset? LastAccessTime { get; }
DateTime? CreatedTimeUtc { get; }
DateTime? LastModifiedTimeUtc { get; }
DateTime? LastAccessTimeUtc { get; }
}

26
OpenHarbor.Abstractions/IStorageProvider.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Text;
namespace OpenHarbor.Abstractions;
public interface IStorageProvider
{
Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken);
Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken);
Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken);
Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken);
Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken);
Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken);
Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken);
Task DeleteFileAsync(string path, CancellationToken cancellationToken);
Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default);
Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken);
bool IsFileNameAllowed(string fileName);
string SanitizeFileName(string key, string replacement);
}

11
OpenHarbor.Abstractions/IWriteFileOptions.cs Normal file
View File

@ -0,0 +1,11 @@
namespace OpenHarbor.Abstractions;
public interface IWriteFileOptions
{
bool OverrideIfExists { get; }
}
public class DefaultWriteOptions : IWriteFileOptions
{
public bool OverrideIfExists { get; set; }
}

9
OpenHarbor.Abstractions/OpenHarbor.Abstractions.csproj Normal file
View File

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
</Project>

16
OpenHarbor.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs Normal file
View File

@ -0,0 +1,16 @@
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory) : IDirectoryInfo
{
public string Path => blobDirectory.Prefix.TrimEnd('/');
public bool IsDirectory => true;
}
public class AzureBlobNotExistingDirectoryInfo(string path) : IDirectoryInfo
{
public string Path { get; } = path;
public bool IsDirectory => true;
}

19
OpenHarbor.Storage.Azure/Blob/AzureBlobFileInfo.cs Normal file
View File

@ -0,0 +1,19 @@
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobFileInfo(CloudBlockBlob fileBlock) : IFileInfo
{
public string FileName => System.IO.Path.GetFileName(fileBlock.Name);
public string Extension => System.IO.Path.GetExtension(fileBlock.Name);
public long FileSize => fileBlock.Properties.Length;
public DateTimeOffset? CreatedTime => fileBlock.Properties.Created;
public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified;
public DateTimeOffset? LastAccessTime => null;
public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime;
public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime;
public DateTime? LastAccessTimeUtc => null;
public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", "");
public bool IsDirectory => false;
}

251
OpenHarbor.Storage.Azure/Blob/AzureBlobStorageProvider.cs Normal file
View File

@ -0,0 +1,251 @@
using System.Text;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.Azure.Blob;
public class AzureBlobStorageProvider : IStorageProvider
{
private string? _connectionString = null;
private string? _containerName = null;
public AzureBlobStorageProvider(string connectionString, string containerName)
{
SetConnectionString(connectionString);
SetContainerName(containerName);
}
public void SetContainerName(string name)
{
_containerName = name;
}
public void SetConnectionString(string connectionString)
{
_connectionString = connectionString;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
var ret = new AzureBlobNotExistingDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(ret);
}
public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken? continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null, cancellationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
var files = results.Where(t => t is CloudBlockBlob).Cast<CloudBlockBlob>().ToList();
foreach (var file in files)
await this.DeleteFileAsync(file.Name, cancellationToken);
}
public Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
return GetContainer()
.GetBlobReference(path)
.DeleteIfExistsAsync(DeleteSnapshotsOption.None, null, null, null, cancellationToken);
}
public Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken) =>
GetContainer()
.GetBlobReference(path)
.ExistsAsync(null, null, cancellationToken);
public async Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken) =>
(await GetListAsync(path, cancellationToken))
.Where(t => t.IsDirectory)
.Cast<IDirectoryInfo>()
.ToList();
public async Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
if (pattern != null)
throw new NotSupportedException("Blob Storage does not support glob searching only prefix.");
var result = await GetListAsync(path, cancellationToken);
return result
.Where(file => false == file.IsDirectory)
.Cast<IFileInfo>()
.ToList();
}
private static string? CleanDirectoryPath(string? path)
{
if (path == null)
return null;
path = path.TrimEnd('/');
if (path != "")
path += "/";
return path;
}
private CloudBlobContainer GetContainer()
{
var account = CloudStorageAccount.Parse(_connectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(_containerName);
return container;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken? continuationToken = null;
var results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, false, BlobListingDetails.None, new int?(),
continuationToken, null, null, cancellationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
foreach (var result in results)
{
if (result is CloudBlobDirectory blobDirectory)
ret.Add(new AzureBlobDirectoryInfo(blobDirectory));
else if (result is CloudBlockBlob blobBlock)
ret.Add(new AzureBlobFileInfo(blobBlock));
}
return ret;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
return await blob.OpenReadAsync(null, null, null, cancellationToken);
}
private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken)
{
if (false == await FileExistsAsync(path, cancellationToken))
throw new FileDoesNotExistException(path);
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
var bytes = new byte[blob.Properties.Length];
await blob.DownloadToByteArrayAsync(bytes, 0, null, null, null, cancellationToken);
return bytes;
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return encoding.GetString(await GetFileBytesAsync(path, cancellationToken));
}
public bool IsFileNameAllowed(string fileName) => true;
public string SanitizeFileName(string key, string replacement) => key;
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromFileAsync(sourcePath, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromStreamAsync(stream, null, null, null, cancellationToken);
return new AzureBlobFileInfo(blob);
}
}

17
OpenHarbor.Storage.Azure/OpenHarbor.Storage.Azure.csproj Normal file
View File

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Abstractions\OpenHarbor.Abstractions.csproj" />
</ItemGroup>
</Project>

13
OpenHarbor.Storage.Physical/OpenHarbor.Storage.Physical.csproj Normal file
View File

@ -0,0 +1,13 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Abstractions\OpenHarbor.Abstractions.csproj" />
</ItemGroup>
</Project>

9
OpenHarbor.Storage.Physical/PhysicalDirectory.cs Normal file
View File

@ -0,0 +1,9 @@
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.Physical;
public class PhysicalDirectoryInfo(string path) : IDirectoryInfo
{
public string Path { get; } = path;
public bool IsDirectory => true;
}

4
PoweredSoft.Storage.Physical/PhysicalFileInfo.cs → OpenHarbor.Storage.Physical/PhysicalFileInfo.cs
View File

@ -1,8 +1,8 @@
using PoweredSoft.Storage.Core;
using OpenHarbor.Abstractions;
using System;
using System.IO;
namespace PoweredSoft.Storage.Physical
namespace OpenHarbor.Storage.Physical
{
public class PhysicalFileInfo : IFileInfo
{

189
OpenHarbor.Storage.Physical/PhysicalStorageProvider.cs Normal file
View File

@ -0,0 +1,189 @@
using System.Text;
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.Physical;
public class PhysicalStorageProvider : IStorageProvider
{
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
Directory.CreateDirectory(path);
var result = new PhysicalDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(result);
}
public Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
if (force)
Directory.Delete(path, true);
else
Directory.Delete(path);
return Task.CompletedTask;
}
public Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
File.Delete(path);
return Task.CompletedTask;
}
public Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken) =>
Task.FromResult(File.Exists(path));
public Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken)
{
var directoryInfo = new DirectoryInfo(path);
var directories = directoryInfo.GetDirectories();
var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable<IDirectoryInfo>().ToList();
return Task.FromResult(directoriesConverted);
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return await File.ReadAllBytesAsync(path, cancellationToken);
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return await File.ReadAllTextAsync(path, encoding, cancellationToken);
}
public Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
var directoryInfo = new DirectoryInfo(path);
FileInfo[] files;
if (string.IsNullOrWhiteSpace(pattern))
files = directoryInfo.GetFiles();
else
files = directoryInfo.GetFiles(pattern, searchOption);
var result = files
.Select(fileInfo => new PhysicalFileInfo(fileInfo))
.AsEnumerable<IFileInfo>()
.ToList();
return Task.FromResult(result);
}
private async Task ThrowNotExistingAsync(string path, CancellationToken cancellationToken)
{
if (false == await FileExistsAsync(path, cancellationToken))
throw new FileDoesNotExistException(path);
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
await ThrowNotExistingAsync(path, cancellationToken);
return new FileStream(path, FileMode.Open, FileAccess.Read);
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var files = await GetFilesAsync(path, cancellationToken: cancellationToken);
var directories = await GetDirectoriesAsync(path, cancellationToken: cancellationToken);
var result = files.AsEnumerable<IDirectoryOrFile>()
.Concat(directories.AsEnumerable<IDirectoryOrFile>())
.ToList();
return result;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
File.Copy(sourcePath, path, options.OverrideIfExists);
var fileInfo = new FileInfo(path);
var ret = new PhysicalFileInfo(fileInfo);
return ret;
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
if (options is null)
throw new ArgumentNullException(nameof(options));
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
await File.WriteAllBytesAsync(path, bytes, cancellationToken);
var fileInfo = new FileInfo(path);
var physicalFileInfo = new PhysicalFileInfo(fileInfo);
return physicalFileInfo;
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
await using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write))
{
await stream.CopyToAsync(fileStream, cancellationToken);
fileStream.Close();
}
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
private static void CreateDirectoryIfNotExisting(string path)
{
var directoryPath = Path.GetDirectoryName(path);
if (directoryPath == null)
return;
if (false == Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
}
public bool IsFileNameAllowed(string fileName) => true;
public string SanitizeFileName(string key, string replacement) => key;
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
}

4
PoweredSoft.Storage.S3/IS3FileWriteOptions.cs → OpenHarbor.Storage.S3/IS3FileWriteOptions.cs
View File

@ -1,9 +1,9 @@
using PoweredSoft.Storage.Core;
using OpenHarbor.Abstractions;
using System;
using System.Collections.Generic;
using System.Text;
namespace PoweredSoft.Storage.S3
namespace OpenHarbor.Storage.S3
{
public interface IS3FileWriteOptions
{

17
OpenHarbor.Storage.S3/OpenHarbor.Storage.S3.csproj Normal file
View File

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<Company>Open Harbor</Company>
<PackageIconUrl>https://www.gravatar.com/avatar/9cecda5822fc5d4d2e61ec03da571b3d</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OpenHarbor.Abstractions\OpenHarbor.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.3.110.10" />
</ItemGroup>
</Project>

4
PoweredSoft.Storage.S3/S3FileInfo.cs → OpenHarbor.Storage.S3/S3FileInfo.cs
View File

@ -1,8 +1,8 @@
using System;
using Amazon.S3.Model;
using PoweredSoft.Storage.Core;
using OpenHarbor.Abstractions;
namespace PoweredSoft.Storage.S3
namespace OpenHarbor.Storage.S3
{
public class S3FileInfo : IFileInfo
{

4
PoweredSoft.Storage.S3/S3NotExistingDirectoryInfo.cs → OpenHarbor.Storage.S3/S3NotExistingDirectoryInfo.cs
View File

@ -1,6 +1,6 @@
using PoweredSoft.Storage.Core;
using OpenHarbor.Abstractions;
namespace PoweredSoft.Storage.S3
namespace OpenHarbor.Storage.S3
{
public class S3NotExistingDirectoryInfo : IDirectoryInfo
{

260
OpenHarbor.Storage.S3/S3StorageProvider.cs Normal file
View File

@ -0,0 +1,260 @@
using System.Text;
using System.Text.RegularExpressions;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using OpenHarbor.Abstractions;
namespace OpenHarbor.Storage.S3;
public partial class S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) : IStorageProvider
{
private S3UsEast1RegionalEndpointValue? _s3UsEast1RegionalEndpointValue = null;
private bool _forcePathStyle = false;
public void SetForcePathStyle(bool forcePathStyle)
{
_forcePathStyle = forcePathStyle;
}
public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value)
{
_s3UsEast1RegionalEndpointValue = value;
}
protected virtual IAmazonS3 GetClient()
{
var config = new AmazonS3Config
{
USEast1RegionalEndpointValue = _s3UsEast1RegionalEndpointValue,
ServiceURL = endpoint,
ForcePathStyle = _forcePathStyle
};
var client = new AmazonS3Client(accessKey, secret, config);
return client;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
{
return Task.FromResult<IDirectoryInfo>(new S3NotExistingDirectoryInfo(path));
}
/// <summary>
/// Can only delete 1000 at a time.
/// </summary>
/// <param name="path"></param>
/// <param name="force"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
{
using var client = GetClient();
var files = await this.GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken);
var next = files.AsQueryable();
while(next.Any())
{
var next1000 = next.Take(1000);
var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList();
await client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = bucketName,
Objects = keys
}, cancellationToken);
next = next.Skip(1000);
}
}
public async Task DeleteFileAsync(string path, CancellationToken cancellationToken)
{
using var client = GetClient();
var response = await client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = bucketName,
Key = path
}, cancellationToken);
}
public async Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken = default)
{
var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
return s3Object != null;
}
public Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken)
{
return Task.FromResult(new List<IDirectoryInfo>());
}
public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken = default)
{
await using var fileStream = await GetFileStreamAsync(path, cancellationToken);
using var memoryStream = new MemoryStream();
await fileStream.CopyToAsync(memoryStream, cancellationToken);
return memoryStream.ToArray();
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken = default)
{
await using var fileStream = await this.GetFileStreamAsync(path, cancellationToken);
using var streamReader = new StreamReader(fileStream, encoding);
return await streamReader.ReadToEndAsync(cancellationToken);
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
{
if (pattern != null)
throw new NotSupportedException();
var finalPath = SanitizeDirectoryRequest(path);
var s3Files = await GetS3FilesAsync(prefix: finalPath, delimiter: "/", cancellationToken: cancellationToken);
var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable<IFileInfo>().ToList();
return ret;
}
private static string SanitizeDirectoryRequest(string path)
{
string finalPath;
if (path == "/")
finalPath = "";
else
finalPath = $"{path?.TrimEnd('/')}/";
return finalPath;
}
public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
{
using var client = GetClient();
return await client.GetObjectStreamAsync(bucketName, path, null, cancellationToken);
}
protected virtual async Task<IEnumerable<S3Object>> GetS3FilesAsync(string? prefix = null, string? delimiter = null, CancellationToken cancellationToken = default)
{
using var client = GetClient();
var items = new List<S3Object>();
string? nextKey = null;
do
{
var response = await client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = bucketName,
Prefix = prefix,
Delimiter = delimiter,
MaxKeys = 1000,
ContinuationToken = nextKey
}, cancellationToken);
items.AddRange(response.S3Objects);
nextKey = response.NextContinuationToken;
} while (nextKey != null);
return items;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
{
var files = await GetFilesAsync(path, cancellationToken: cancellationToken);
return files.Cast<IDirectoryOrFile>().ToList();
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
await using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read);
return await WriteFileAsync(fileStream, path, options, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
{
return WriteFileAsync(new MemoryStream(bytes), path, options, cancellationToken);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(options);
if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
throw new FileAlreadyExistsException(path);
using var client = GetClient();
var request = new PutObjectRequest
{
BucketName = bucketName,
InputStream = stream,
Key = path
};
if (options is IS3FileWriteOptions s3FileWriteOptions)
request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl);
// todo: unhandled response
var result = await client.PutObjectAsync(request, cancellationToken);
var file = await GetFileInfoByPathAsync(path, cancellationToken);
return file;
}
private async Task<S3Object?> GetS3FileByPathAsync(string path, CancellationToken cancellationToken)
{
var files = await GetS3FilesAsync(path, cancellationToken: cancellationToken);
return files.FirstOrDefault();
}
private async Task<IFileInfo> GetFileInfoByPathAsync(string path, CancellationToken cancellationToken)
{
var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
if (s3Object == null)
throw new FileDoesNotExistException(path);
var ret = new S3FileInfo(s3Object);
return ret;
}
public string SanitizeFileName(string key, string replacement)
{
var regex = S3FileNameRegex();
var result = regex.Replace(key, replacement);
return result;
}
public bool IsFileNameAllowed(string fileName)
{
var regex = S3FileNameRegex();
var hasMatches = regex.IsMatch(fileName);
return false == hasMatches;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
}, cancellationToken);
}
[GeneratedRegex(@"[^a-zA-Z0-9.!\-_*'()]", RegexOptions.Multiline)]
private static partial Regex S3FileNameRegex();
}

14
PoweredSoft.Storage.sln → OpenHarbor.Storage.sln
View File

@ -3,15 +3,13 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30406.217
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Azure", "PoweredSoft.Storage.Azure\PoweredSoft.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Azure", "OpenHarbor.Storage.Azure\OpenHarbor.Storage.Azure.csproj", "{B937F389-07BE-4235-B2A8-7D1229B3D0FC}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Core", "PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Abstractions", "OpenHarbor.Abstractions\OpenHarbor.Abstractions.csproj", "{C9CBCC98-B38E-4949-AF7C-BD291E09A1F4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.Physical", "PoweredSoft.Storage.Physical\PoweredSoft.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.Physical", "OpenHarbor.Storage.Physical\OpenHarbor.Storage.Physical.csproj", "{349E6B89-BEBB-4883-95C8-9E28F9FEF24C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PoweredSoft.Storage.S3", "PoweredSoft.Storage.S3\PoweredSoft.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PoweredSoft.Storage.Test", "PoweredSoft.Storage.Test\PoweredSoft.Storage.Test.csproj", "{305416EE-51A4-4293-9262-87865D2784F4}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenHarbor.Storage.S3", "OpenHarbor.Storage.S3\OpenHarbor.Storage.S3.csproj", "{457912EA-48E3-4B2E-941F-2116D18C6D88}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -35,10 +33,6 @@ Global
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Debug|Any CPU.Build.0 = Debug|Any CPU
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.ActiveCfg = Release|Any CPU
{457912EA-48E3-4B2E-941F-2116D18C6D88}.Release|Any CPU.Build.0 = Release|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{305416EE-51A4-4293-9262-87865D2784F4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

29
PoweredSoft.Storage.Azure/Blob/AzureBlobDirectoryInfo.cs
View File

@ -1,29 +0,0 @@
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobDirectoryInfo : IDirectoryInfo
{
private CloudBlobDirectory blobDirectory;
public AzureBlobDirectoryInfo(CloudBlobDirectory blobDirectory)
{
this.blobDirectory = blobDirectory;
}
public string Path => blobDirectory.Prefix.TrimEnd('/');
public bool IsDirectory => true;
}
public class AzureBlobNotExistingDirectoryInfo : IDirectoryInfo
{
public AzureBlobNotExistingDirectoryInfo(string path)
{
Path = path;
}
public string Path { get; }
public bool IsDirectory => true;
}
}

30
PoweredSoft.Storage.Azure/Blob/AzureBlobFileInfo.cs
View File

@ -1,30 +0,0 @@
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.Text;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobFileInfo : IFileInfo
{
private readonly CloudBlockBlob fileBlock;
public AzureBlobFileInfo(CloudBlockBlob fileBlock)
{
this.fileBlock = fileBlock;
}
public string FileName => System.IO.Path.GetFileName(fileBlock.Name);
public string Extension => System.IO.Path.GetExtension(fileBlock.Name);
public long FileSize => fileBlock.Properties.Length;
public DateTimeOffset? CreatedTime => fileBlock.Properties.Created;
public DateTimeOffset? LastModifiedTime => fileBlock.Properties.LastModified;
public DateTimeOffset? LastAccessTime => null;
public DateTime? CreatedTimeUtc => CreatedTime?.UtcDateTime;
public DateTime? LastModifiedTimeUtc => LastModifiedTime?.UtcDateTime;
public DateTime? LastAccessTimeUtc => null;
public string Path => fileBlock.Uri.LocalPath.Replace($"/{fileBlock.Container.Name}/", "");
public bool IsDirectory => false;
}
}

265
PoweredSoft.Storage.Azure/Blob/AzureBlobStorageProvider.cs
View File

@ -1,265 +0,0 @@
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Azure.Blob
{
public class AzureBlobStorageProvider : IStorageProvider
{
private string connectionString = null;
private string containerName = null;
public AzureBlobStorageProvider()
{
}
public AzureBlobStorageProvider(string connectionString, string containerName)
{
this.SetConnectionString(connectionString);
this.SetContainerName(containerName);
}
public void SetContainerName(string name)
{
this.containerName = name;
}
public void SetConnectionString(string connectionString)
{
this.connectionString = connectionString;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
var ret = new AzureBlobNotExistingDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(ret);
}
public async Task DeleteDirectoryAsync(string path, bool force = false)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, true, BlobListingDetails.All, null, continuationToken, null, null);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
var files = results.Where(t => t is CloudBlockBlob).Cast<CloudBlockBlob>().ToList();
foreach (var file in files)
await this.DeleteFileAsync(file.Name);
}
public Task DeleteFileAsync(string path)
{
return GetContainer().GetBlobReference(path).DeleteIfExistsAsync();
}
public Task<bool> FileExistsAsync(string path)
{
return GetContainer().GetBlobReference(path).ExistsAsync();
}
public async Task<List<IDirectoryInfo>> GetDirectories(string path)
{
return (await this.GetListAsync(path)).Where(t => t.IsDirectory).Cast<IDirectoryInfo>().ToList();
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
if (pattern != null)
throw new NotSupportedException("Blob Storage does not support glob searching only prefix.");
var result = await GetListAsync(path);
var finalResult = result.Where(t => !t.IsDirectory).Cast<IFileInfo>().ToList();
return finalResult;
}
private string CleanDirectoryPath(string path)
{
if (path == null)
return path;
path = path.TrimEnd('/');
if (path != "")
path += "/";
return path;
}
private CloudBlobContainer GetContainer()
{
var account = CloudStorageAccount.Parse(connectionString);
var client = account.CreateCloudBlobClient();
var container = client.GetContainerReference(containerName);
return container;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var ret = new List<IDirectoryOrFile>();
var container = GetContainer();
var finalPath = CleanDirectoryPath(path);
BlobContinuationToken continuationToken = null;
List<IListBlobItem> results = new List<IListBlobItem>();
do
{
BlobResultSegment response;
if (continuationToken == null)
response = await container.ListBlobsSegmentedAsync(finalPath, continuationToken);
else
response = await container.ListBlobsSegmentedAsync(continuationToken);
continuationToken = response.ContinuationToken;
results.AddRange(response.Results);
}
while (continuationToken != null);
foreach (var result in results)
{
if (result is CloudBlobDirectory blobDirectory)
ret.Add(new AzureBlobDirectoryInfo(blobDirectory));
else if (result is CloudBlockBlob blobBlock)
ret.Add(new AzureBlobFileInfo(blobBlock));
}
return ret;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public async Task<Stream> GetFileStreamAsync(string path)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
return await blob.OpenReadAsync();
}
private async Task ThrowNotExistingAsync(string path)
{
if (false == await this.FileExistsAsync(path))
throw new FileDoesNotExistException(path);
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
var bytes = new byte[blob.Properties.Length];
await blob.DownloadToByteArrayAsync(bytes, 0);
return bytes;
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
await ThrowNotExistingAsync(path);
var container = GetContainer();
return encoding.GetString(await this.GetFileBytesAsync(path));
}
public bool IsFileNameAllowed(string fileName)
{
return true;
}
public string SanitizeFileName(string key, string replacement)
{
return key;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromFileAsync(sourcePath);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromByteArrayAsync(bytes, 0, bytes.Length);
return new AzureBlobFileInfo(blob);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
var container = GetContainer();
var blob = container.GetBlockBlobReference(path);
await blob.UploadFromStreamAsync(stream);
return new AzureBlobFileInfo(blob);
}
}
}

18
PoweredSoft.Storage.Azure/PoweredSoft.Storage.Azure.csproj
View File

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
</Project>

12
PoweredSoft.Storage.Core/FileAlreadyExistsException.cs
View File

@ -1,12 +0,0 @@
using System;
using System.Runtime.Serialization;
namespace PoweredSoft.Storage.Core
{
public class FileAlreadyExistsException : Exception
{
public FileAlreadyExistsException(string path) : base($"{path} already exists..")
{
}
}
}

12
PoweredSoft.Storage.Core/FileDoesNotExistException.cs
View File

@ -1,12 +0,0 @@
using System;
namespace PoweredSoft.Storage.Core
{
public class FileDoesNotExistException : Exception
{
public FileDoesNotExistException(string path) : base($"{path} does not exist.")
{
}
}
}

7
PoweredSoft.Storage.Core/IDirectory.cs
View File

@ -1,7 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IDirectoryInfo : IDirectoryOrFile
{
}
}

8
PoweredSoft.Storage.Core/IDirectoryOrFile.cs
View File

@ -1,8 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IDirectoryOrFile
{
string Path { get; }
bool IsDirectory { get; }
}
}

17
PoweredSoft.Storage.Core/IFileInfo.cs
View File

@ -1,17 +0,0 @@
using System;
namespace PoweredSoft.Storage.Core
{
public interface IFileInfo : IDirectoryOrFile
{
string FileName { get; }
string Extension { get; }
long FileSize { get; }
DateTimeOffset? CreatedTime { get; }
DateTimeOffset? LastModifiedTime { get; }
DateTimeOffset? LastAccessTime { get; }
DateTime? CreatedTimeUtc { get; }
DateTime? LastModifiedTimeUtc { get; }
DateTime? LastAccessTimeUtc { get; }
}
}

31
PoweredSoft.Storage.Core/IStorageProvider.cs
View File

@ -1,31 +0,0 @@
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Core
{
public interface IStorageProvider
{
Task<List<IDirectoryOrFile>> GetListAsync(string path);
Task<List<IDirectoryInfo>> GetDirectories(string path);
Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true);
Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options);
Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options);
Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options);
Task<Stream> GetFileStreamAsync(string path);
Task<byte[]> GetFileBytesAsync(string path);
Task<string> GetFileContentAsync(string path, Encoding encoding);
Task<bool> FileExistsAsync(string path);
Task DeleteFileAsync(string path);
Task DeleteDirectoryAsync(string path, bool force = false);
Task<IDirectoryInfo> CreateDirectoryAsync(string path);
bool IsFileNameAllowed(string fileName);
string SanitizeFileName(string key, string replacement);
}
}

12
PoweredSoft.Storage.Core/IWriteFileOptions.cs
View File

@ -1,12 +0,0 @@
namespace PoweredSoft.Storage.Core
{
public interface IWriteFileOptions
{
bool OverrideIfExists { get; }
}
public class DefaultWriteOptions : IWriteFileOptions
{
public bool OverrideIfExists { get; set; }
}
}

10
PoweredSoft.Storage.Core/PoweredSoft.Storage.Core.csproj
View File

@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
</Project>

15
PoweredSoft.Storage.Physical/PhysicalDirectory.cs
View File

@ -1,15 +0,0 @@
using PoweredSoft.Storage.Core;
namespace PoweredSoft.Storage.Physical
{
public class PhysicalDirectoryInfo : IDirectoryInfo
{
public PhysicalDirectoryInfo(string path)
{
Path = path;
}
public string Path { get; }
public bool IsDirectory => true;
}
}

199
PoweredSoft.Storage.Physical/PhysicalStorageProvider.cs
View File

@ -1,199 +0,0 @@
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.Physical
{
public class PhysicalStorageProvider : IStorageProvider
{
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
var directoryInfo = System.IO.Directory.CreateDirectory(path);
var result = new PhysicalDirectoryInfo(path);
return Task.FromResult<IDirectoryInfo>(result);
}
public Task DeleteDirectoryAsync(string path, bool force = false)
{
if (force)
Directory.Delete(path, true);
else
Directory.Delete(path);
return Task.CompletedTask;
}
public Task DeleteFileAsync(string path)
{
System.IO.File.Delete(path);
return Task.CompletedTask;
}
public Task<bool> FileExistsAsync(string path)
{
return Task.FromResult(File.Exists(path));
}
public Task<List<IDirectoryInfo>> GetDirectories(string path)
{
var directoryInfo = new DirectoryInfo(path);
var directories = directoryInfo.GetDirectories();
var directoriesConverted = directories.Select(t => new PhysicalDirectoryInfo(t.FullName)).AsEnumerable<IDirectoryInfo>().ToList();
return Task.FromResult(directoriesConverted);
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
await ThrowNotExistingAsync(path);
return File.ReadAllBytes(path);
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
await ThrowNotExistingAsync(path);
return File.ReadAllText(path, encoding);
}
public Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
var directoryInfo = new DirectoryInfo(path);
FileInfo[] files;
if (string.IsNullOrWhiteSpace(pattern))
files = directoryInfo.GetFiles();
else
files = directoryInfo.GetFiles(pattern, searchOption);
var result = files.Select(fileInfo => new PhysicalFileInfo(fileInfo)).AsEnumerable<IFileInfo>().ToList();
return Task.FromResult(result);
}
private async Task ThrowNotExistingAsync(string path)
{
if (false == await this.FileExistsAsync(path))
throw new FileDoesNotExistException(path);
}
public async Task<Stream> GetFileStreamAsync(string path)
{
await ThrowNotExistingAsync(path);
return new FileStream(path, FileMode.Open, FileAccess.Read);
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var files = await this.GetFilesAsync(path);
var directories = await this.GetDirectories(path);
var result = files.AsEnumerable<IDirectoryOrFile>().Concat(directories.AsEnumerable<IDirectoryOrFile>()).ToList();
return result;
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
System.IO.File.Copy(sourcePath, path, options.OverrideIfExists);
var fileInfo = new FileInfo(path);
var ret = new PhysicalFileInfo(fileInfo);
return ret;
}
public async Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
File.WriteAllBytes(path, bytes);
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
CreateDirectoryIfNotExisting(path);
if (stream.CanSeek && stream.Position != 0)
stream.Seek(0, SeekOrigin.Begin);
using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.Write))
{
await stream.CopyToAsync(fileStream);
fileStream.Close();
}
var fileInfo = new FileInfo(path);
var physicalinfo = new PhysicalFileInfo(fileInfo);
return physicalinfo;
}
private void CreateDirectoryIfNotExisting(string path)
{
var directoryPath = Path.GetDirectoryName(path);
if (!Directory.Exists(directoryPath))
Directory.CreateDirectory(directoryPath);
}
public bool IsFileNameAllowed(string fileName)
{
return true;
}
public string SanitizeFileName(string key, string replacement)
{
return key;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
}
}

14
PoweredSoft.Storage.Physical/PoweredSoft.Storage.Physical.csproj
View File

@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
</Project>

21
PoweredSoft.Storage.S3/PoweredSoft.Storage.S3.csproj
View File

@ -1,21 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<LangVersion>8.0</LangVersion>
<Copyright>Powered Softwares Inc.</Copyright>
<Company>PoweredSoft</Company>
<PackageIconUrl>https://secure.gravatar.com/avatar/4e32f73820c16718909a06c2927f1f8b?s=512&amp;amp;r=g&amp;amp;d=retro</PackageIconUrl>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.Core\PoweredSoft.Storage.Core.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="3.3.110.10" />
</ItemGroup>
</Project>

283
PoweredSoft.Storage.S3/S3StorageProvider.cs
View File

@ -1,283 +0,0 @@
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using PoweredSoft.Storage.Core;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace PoweredSoft.Storage.S3
{
public class S3StorageProvider : IStorageProvider
{
protected readonly string endpoint;
protected readonly string bucketName;
protected readonly string accessKey;
protected readonly string secret;
protected S3UsEast1RegionalEndpointValue? s3UsEast1RegionalEndpointValue = null;
protected bool forcePathStyle = false;
public S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret)
{
this.endpoint = endpoint;
this.bucketName = bucketName;
this.accessKey = accessKey;
this.secret = secret;
}
public void SetForcePathStyle(bool forcePathStyle)
{
this.forcePathStyle = forcePathStyle;
}
public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value)
{
this.s3UsEast1RegionalEndpointValue = value;
}
protected virtual IAmazonS3 GetClient()
{
var config = new AmazonS3Config
{
USEast1RegionalEndpointValue = s3UsEast1RegionalEndpointValue,
ServiceURL = endpoint,
ForcePathStyle = forcePathStyle
};
var client = new AmazonS3Client(this.accessKey, this.secret, config);
return client;
}
public Task<IDirectoryInfo> CreateDirectoryAsync(string path)
{
return Task.FromResult<IDirectoryInfo>(new S3NotExistingDirectoryInfo(path));
}
/// <summary>
/// Can only delete 1000 at a time.
/// </summary>
/// <param name="path"></param>
/// <param name="force"></param>
/// <returns></returns>
public async Task DeleteDirectoryAsync(string path, bool force = false)
{
using var client = GetClient();
var files = await this.GetS3FilesAsync(prefix: path, delimiter: null);
var next = files.AsQueryable();
while(next.Any())
{
var next1000 = next.Take(1000);
var keys = next1000.Select(s3o => new KeyVersion { Key = s3o.Key }).ToList();
await client.DeleteObjectsAsync(new DeleteObjectsRequest
{
BucketName = this.bucketName,
Objects = keys
});
next = next.Skip(1000);
}
}
public async Task DeleteFileAsync(string path)
{
using var client = GetClient();
var response = await client.DeleteObjectAsync(new DeleteObjectRequest
{
BucketName = this.bucketName,
Key = path
});
}
public async Task<bool> FileExistsAsync(string path)
{
var item = await GetS3FileByPath(path);
return item != null;
}
public Task<List<IDirectoryInfo>> GetDirectories(string path)
{
return Task.FromResult(new List<IDirectoryInfo>());
}
public async Task<byte[]> GetFileBytesAsync(string path)
{
using var fileStream = await this.GetFileStreamAsync(path);
using var memoryStream = new MemoryStream();
await fileStream.CopyToAsync(memoryStream);
return memoryStream.ToArray();
}
public async Task<string> GetFileContentAsync(string path, Encoding encoding)
{
using var fileStream = await this.GetFileStreamAsync(path);
using var streamReader = new StreamReader(fileStream, encoding);
return await streamReader.ReadToEndAsync();
}
public async Task<List<IFileInfo>> GetFilesAsync(string path, string pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
if (pattern != null)
throw new NotSupportedException();
string finalPath = SantizeDirectoryRequest(path);
var s3Files = await this.GetS3FilesAsync(prefix: finalPath, delimiter: "/");
var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable<IFileInfo>().ToList();
return ret;
}
private static string SantizeDirectoryRequest(string path)
{
string finalPath;
if (path == "/")
finalPath = "";
else
finalPath = $"{path?.TrimEnd('/')}/";
return finalPath;
}
public Task<Stream> GetFileStreamAsync(string path)
{
using var client = GetClient();
return client.GetObjectStreamAsync(this.bucketName, path, null);
}
protected virtual async Task<IEnumerable<S3Object>> GetS3FilesAsync(string prefix = null, string delimiter = null)
{
using var client = GetClient();
var items = new List<S3Object>();
string nextKey = null;
do
{
var response = await client.ListObjectsV2Async(new ListObjectsV2Request
{
BucketName = this.bucketName,
Prefix = prefix,
Delimiter = delimiter,
MaxKeys = 1000,
ContinuationToken = nextKey
});
items.AddRange(response.S3Objects);
nextKey = response.NextContinuationToken;
} while (nextKey != null);
return items;
}
public async Task<List<IDirectoryOrFile>> GetListAsync(string path)
{
var files = await this.GetFilesAsync(path);
return files.Cast<IDirectoryOrFile>().ToList();
}
public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options)
{
using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read);
return await WriteFileAsync(fileStream, path, options);
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options)
{
return WriteFileAsync(new MemoryStream(bytes), path, options);
}
public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options)
{
if (options is null)
{
throw new ArgumentNullException(nameof(options));
}
if (!options.OverrideIfExists && await FileExistsAsync(path))
throw new FileAlreadyExistsException(path);
using var client = GetClient();
var request = new PutObjectRequest
{
BucketName = this.bucketName,
InputStream = stream,
Key = path
};
if (options is IS3FileWriteOptions s3FileWriteOptions)
{
if (s3FileWriteOptions.Acl != null)
request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl);
}
var result = await client.PutObjectAsync(request);
var file = await GetFileInfoByPath(path);
return file;
}
private async Task<S3Object> GetS3FileByPath(string path)
{
var files = await this.GetS3FilesAsync(path);
var s3o = files.FirstOrDefault();
return s3o;
}
private async Task<IFileInfo> GetFileInfoByPath(string path)
{
var s3o = await GetS3FileByPath(path);
if (s3o == null)
throw new FileDoesNotExistException(path);
var ret = new S3FileInfo(s3o);
return ret;
}
public string SanitizeFileName(string key, string replacement)
{
string pattern = @"[^a-zA-Z0-9.!\-_*'()]";
string substitution = replacement;
string input = key;
RegexOptions options = RegexOptions.Multiline;
Regex regex = new Regex(pattern, options);
string result = regex.Replace(input, substitution);
return result;
}
public bool IsFileNameAllowed(string fileName)
{
string pattern = @"[^a-zA-Z0-9.!\-_*'()]";
RegexOptions options = RegexOptions.Multiline;
Regex regex = new Regex(pattern, options);
var hasMatches = regex.IsMatch(fileName);
return false == hasMatches;
}
public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true)
{
return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true)
{
return WriteFileAsync(bytes, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true)
{
return WriteFileAsync(stream, path, new DefaultWriteOptions
{
OverrideIfExists = overrideIfExists
});
}
}
}

20
PoweredSoft.Storage.Test/PoweredSoft.Storage.Test.csproj
View File

@ -1,20 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
<PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
<PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
<PackageReference Include="coverlet.collector" Version="1.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\PoweredSoft.Storage.S3\PoweredSoft.Storage.S3.csproj" />
</ItemGroup>
</Project>

55
PoweredSoft.Storage.Test/S3Tests.cs
View File

@ -1,55 +0,0 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using PoweredSoft.Storage.S3;
using System.Text;
namespace PoweredSoft.Storage.Test
{
[TestClass]
public class S3Tests
{
[TestMethod]
public async System.Threading.Tasks.Task S3AclWriteAsync()
{
var space = GetMockS3Space();
await space.WriteFileAsync(Encoding.UTF8.GetBytes("Hello World"), "hello-world.txt", new S3FileWriteOptions
{
Acl = "public-read",
OverrideIfExists = true
});
}
[TestMethod]
public void NameValidation()
{
var space = GetMockS3Space();
Assert.IsFalse(space.IsFileNameAllowed("Operations .pdf"), "Should not be valid");
Assert.IsFalse(space.IsFileNameAllowed("Operations$$.pdf"), "Should not be valid");
}
[TestMethod]
public void CanContainDash()
{
var space = GetMockS3Space();
Assert.IsTrue(space.IsFileNameAllowed("Operations-yay.pdf"), "Should be allowed");
}
[TestMethod]
public void NameSanitation()
{
var space = GetMockS3Space();
Assert.AreEqual("Operations_.pdf", space.SanitizeFileName("Operations .pdf", "_"), "does not match sanitation expectations");
Assert.AreEqual("Operations__.pdf", space.SanitizeFileName("Operations$$.pdf", "_"), "does not match sanitation expectations");
}
private static S3StorageProvider GetMockS3Space()
{
var space = new S3StorageProvider("http://localhost:9000", "mybucket", "minioadmin", "minioadmin");
space.SetForcePathStyle(true);
space.SetS3UsEast1RegionalEndpointValue(Amazon.Runtime.S3UsEast1RegionalEndpointValue.Legacy);
return space;
}
}
}