dotnet-storage/OpenHarbor.Storage.S3/S3StorageProvider.cs

using System.Text;
using System.Text.RegularExpressions;
using Amazon.Runtime;
using Amazon.S3;
using Amazon.S3.Model;
using OpenHarbor.Abstractions;

namespace OpenHarbor.Storage.S3;
    
public partial class S3StorageProvider(string endpoint, string bucketName, string accessKey, string secret) : IStorageProvider
{
  private S3UsEast1RegionalEndpointValue? _s3UsEast1RegionalEndpointValue = null;
  private bool _forcePathStyle = false;

  public void SetForcePathStyle(bool forcePathStyle)
  {
    _forcePathStyle = forcePathStyle;
  }

  public void SetS3UsEast1RegionalEndpointValue(S3UsEast1RegionalEndpointValue value)
  {
    _s3UsEast1RegionalEndpointValue = value;
  }

  protected virtual IAmazonS3 GetClient()
  {
    var config = new AmazonS3Config
    {
      USEast1RegionalEndpointValue = _s3UsEast1RegionalEndpointValue,
      ServiceURL = endpoint,
      ForcePathStyle = _forcePathStyle
    };
    
    var client = new AmazonS3Client(accessKey, secret, config);
    
    return client;
  }

  public Task<IDirectoryInfo> CreateDirectoryAsync(string path, CancellationToken cancellationToken)
  {
    return Task.FromResult<IDirectoryInfo>(new S3NotExistingDirectoryInfo(path));
  }

  /// <summary>
  /// Can only delete 1000 at a time.
  /// </summary>
  /// <param name="path"></param>
  /// <param name="force"></param>
  /// <param name="cancellationToken"></param>
  /// <returns></returns>
  public async Task DeleteDirectoryAsync(string path, bool force = false, CancellationToken cancellationToken = default)
  {
    using var client = GetClient();
    var files = await GetS3FilesAsync(prefix: path, delimiter: null, cancellationToken: cancellationToken);
    var next = files.AsQueryable();
    
    while(next.Any())
    {
      var next1000 = next.Take(1000);
      var keys = next1000.Select(s3Object => new KeyVersion { Key = s3Object.Key })
        .ToList();
      
      await client.DeleteObjectsAsync(new DeleteObjectsRequest
      {
        BucketName = bucketName,
        Objects = keys
      }, cancellationToken);

      next = next.Skip(1000);
    }
  }

  public async Task DeleteFileAsync(string path, CancellationToken cancellationToken)
  {
    using var client = GetClient();
    var response = await client.DeleteObjectAsync(new DeleteObjectRequest
    {
      BucketName = bucketName,
      Key = path
    }, cancellationToken);
  }

  public async Task<bool> FileExistsAsync(string path, CancellationToken cancellationToken = default)
  {
    var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
    return s3Object != null;
  }

  public Task<List<IDirectoryInfo>> GetDirectoriesAsync(string path, CancellationToken cancellationToken)
  {
    return Task.FromResult(new List<IDirectoryInfo>());
  }

  public async Task<byte[]> GetFileBytesAsync(string path, CancellationToken cancellationToken = default)
  {
    await using var fileStream = await GetFileStreamAsync(path, cancellationToken);
    using var memoryStream = new MemoryStream();
    await fileStream.CopyToAsync(memoryStream, cancellationToken);
    return memoryStream.ToArray();
  }

  public async Task<string> GetFileContentAsync(string path, Encoding encoding, CancellationToken cancellationToken = default)
  {
    await using var fileStream = await this.GetFileStreamAsync(path, cancellationToken);
    using var streamReader = new StreamReader(fileStream, encoding);
    return await streamReader.ReadToEndAsync(cancellationToken);
  }

  public async Task<List<IFileInfo>> GetFilesAsync(string path, string? pattern = null, SearchOption searchOption = SearchOption.TopDirectoryOnly, CancellationToken cancellationToken = default)
  {
    if (pattern != null)
      throw new NotSupportedException();

    var finalPath = SanitizeDirectoryRequest(path);
    var s3Files = await GetS3FilesAsync(prefix: finalPath, delimiter: "/", cancellationToken: cancellationToken);
    var ret = s3Files.Select(s3 => new S3FileInfo(s3)).AsEnumerable<IFileInfo>().ToList();
    return ret;
  }

  private static string SanitizeDirectoryRequest(string path)
  {
    string finalPath;
    if (path == "/")
      finalPath = "";
    else
      finalPath = $"{path?.TrimEnd('/')}/";
    return finalPath;
  }

  public async Task<Stream> GetFileStreamAsync(string path, CancellationToken cancellationToken)
  {
    using var client = GetClient();
    return await client.GetObjectStreamAsync(bucketName, path, null, cancellationToken);
  }

  protected virtual async Task<IEnumerable<S3Object>> GetS3FilesAsync(string? prefix = null, string? delimiter = null, CancellationToken cancellationToken = default)
  {
    using var client = GetClient();

    var items = new List<S3Object>();
    string? nextKey = null;

    do
    {
      var response = await client.ListObjectsV2Async(new ListObjectsV2Request
      {
        BucketName = bucketName,
        Prefix = prefix,
        Delimiter = delimiter,
        MaxKeys = 1000,
        ContinuationToken = nextKey
      }, cancellationToken);

      items.AddRange(response.S3Objects);
      nextKey = response.NextContinuationToken;

    } while (nextKey != null);

    return items;
  }

  public async Task<List<IDirectoryOrFile>> GetListAsync(string path, CancellationToken cancellationToken)
  {
    var files = await GetFilesAsync(path, cancellationToken: cancellationToken);
    return files.Cast<IDirectoryOrFile>().ToList();
  }

  public async Task<IFileInfo> WriteFileAsync(string sourcePath, string path, IWriteFileOptions options, CancellationToken cancellationToken)
  {
    await using var fileStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read);
    return await WriteFileAsync(fileStream, path, options, cancellationToken);
  }

  public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, IWriteFileOptions options, CancellationToken cancellationToken)
  {
    return WriteFileAsync(new MemoryStream(bytes), path, options, cancellationToken);
  }

  public async Task<IFileInfo> WriteFileAsync(Stream stream, string path, IWriteFileOptions options, CancellationToken cancellationToken = default)
  {
    ArgumentNullException.ThrowIfNull(options);

    if (!options.OverrideIfExists && await FileExistsAsync(path, cancellationToken))
      throw new FileAlreadyExistsException(path);

    using var client = GetClient();
    var request = new PutObjectRequest
    {
      BucketName = bucketName,
      InputStream = stream,
      Key = path
    };

    if (options is IS3FileWriteOptions s3FileWriteOptions)
      request.CannedACL = new S3CannedACL(s3FileWriteOptions.Acl);

    // todo: unhandled response
    var result = await client.PutObjectAsync(request, cancellationToken);
    
    var file = await GetFileInfoByPathAsync(path, cancellationToken);
    return file;
  }

  private async Task<S3Object?> GetS3FileByPathAsync(string path, CancellationToken cancellationToken)
  {
    var files = await GetS3FilesAsync(path, cancellationToken: cancellationToken);
    return files.FirstOrDefault();
  }

  private async Task<IFileInfo> GetFileInfoByPathAsync(string path, CancellationToken cancellationToken)
  {
    var s3Object = await GetS3FileByPathAsync(path, cancellationToken);
    if (s3Object == null)
      throw new FileDoesNotExistException(path);

    var ret = new S3FileInfo(s3Object);
    return ret;
  }

  public string SanitizeFileName(string key, string replacement)
  {
    var regex = S3FileNameRegex();

    var result = regex.Replace(key, replacement);
    return result;
  }

  public bool IsFileNameAllowed(string fileName)
  {
    var regex = S3FileNameRegex();
    
    var hasMatches = regex.IsMatch(fileName);
    return false == hasMatches;
  }

  public Task<IFileInfo> WriteFileAsync(string sourcePath, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
  {
    return WriteFileAsync(sourcePath, path, new DefaultWriteOptions
    {
      OverrideIfExists = overrideIfExists
    }, cancellationToken);
  }

  public Task<IFileInfo> WriteFileAsync(byte[] bytes, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
  {
    return WriteFileAsync(bytes, path, new DefaultWriteOptions
    {
      OverrideIfExists = overrideIfExists
    }, cancellationToken);
  }

  public Task<IFileInfo> WriteFileAsync(Stream stream, string path, bool overrideIfExists = true, CancellationToken cancellationToken = default)
  {
    return WriteFileAsync(stream, path, new DefaultWriteOptions
    {
      OverrideIfExists = overrideIfExists
    }, cancellationToken);
  }

    [GeneratedRegex(@"[^a-zA-Z0-9.!\-_*'()]", RegexOptions.Multiline)]
    private static partial Regex S3FileNameRegex();
}