From f16bfe53497bd1dc0f1dd5e97c5789e3c5d31e12 Mon Sep 17 00:00:00 2001 From: Caelan Sayler Date: Thu, 28 Mar 2024 09:43:27 +0000 Subject: [PATCH] Remove ObjectRepository container concept --- src/Velopack.Deployment/AzureRepository.cs | 32 ++++----- src/Velopack.Deployment/S3Repository.cs | 70 +++++++++++++++---- src/Velopack.Deployment/_ObjectRepository.cs | 17 +++-- .../Commands/Deployment/AzureBaseCommand.cs | 4 +- .../Commands/Deployment/S3BaseCommand.cs | 4 +- .../Commands/AzureCommandTests.cs | 2 +- .../Commands/S3CommandTests.cs | 4 +- .../AzureDeploymentTests.cs | 2 +- .../S3DeploymentTests.cs | 2 +- 9 files changed, 87 insertions(+), 50 deletions(-) diff --git a/src/Velopack.Deployment/AzureRepository.cs b/src/Velopack.Deployment/AzureRepository.cs index e6163991..e4438ff7 100644 --- a/src/Velopack.Deployment/AzureRepository.cs +++ b/src/Velopack.Deployment/AzureRepository.cs @@ -12,7 +12,7 @@ public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions public string Endpoint { get; set; } - public string ContainerName { get; set; } + public string Container { get; set; } public string SasToken { get; set; } } @@ -22,40 +22,40 @@ public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions public int KeepMaxReleases { get; set; } } -public class AzureRepository : ObjectRepository +public class AzureRepository : ObjectRepository { public AzureRepository(ILogger logger) : base(logger) { } - protected override BlobServiceClient CreateClient(AzureDownloadOptions options) + protected override BlobContainerClient CreateClient(AzureDownloadOptions options) { var serviceUrl = options.Endpoint ?? "https://" + options.Account + ".blob.core.windows.net"; if (options.Endpoint == null) { Log.Info($"Endpoint not specified, default to: {serviceUrl}"); } + BlobServiceClient client; if (!String.IsNullOrEmpty(options.SasToken)) { - return new BlobServiceClient(new Uri(serviceUrl), new Azure.AzureSasCredential(options.SasToken)); + client = new BlobServiceClient(new Uri(serviceUrl), new Azure.AzureSasCredential(options.SasToken)); + } else { + client = new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key)); } - - return new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key)); + return client.GetBlobContainerClient(options.Container); } - protected override async Task DeleteObject(BlobServiceClient client, string container, string key) + protected override async Task DeleteObject(BlobContainerClient client, string key) { await RetryAsync(async () => { - var containerClient = client.GetBlobContainerClient(container); - await containerClient.DeleteBlobIfExistsAsync(key); + await client.DeleteBlobIfExistsAsync(key); }, "Deleting " + key); } - protected override async Task GetObjectBytes(BlobServiceClient client, string container, string key) + protected override async Task GetObjectBytes(BlobContainerClient client, string key) { return await RetryAsyncRet(async () => { try { - var containerClient = client.GetBlobContainerClient(container); - var obj = containerClient.GetBlobClient(key); + var obj = client.GetBlobClient(key); var ms = new MemoryStream(); using var response = await obj.DownloadToAsync(ms, CancellationToken.None); return ms.ToArray(); @@ -69,16 +69,14 @@ public class AzureRepository : ObjectRepository { var client = CreateClient(options); - var containerClient = client.GetBlobContainerClient(options.ContainerName); - var obj = containerClient.GetBlobClient(entry.FileName); + var obj = client.GetBlobClient(entry.FileName); using var response = await obj.DownloadToAsync(filePath, CancellationToken.None); }, $"Downloading {entry.FileName}..."); } - protected override async Task UploadObject(BlobServiceClient client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache) + protected override async Task UploadObject(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote, bool noCache) { - var containerClient = client.GetBlobContainerClient(container); - var blobClient = containerClient.GetBlobClient(key); + var blobClient = client.GetBlobClient(key); try { var properties = await blobClient.GetPropertiesAsync(); var md5 = GetFileMD5Checksum(f.FullName); diff --git a/src/Velopack.Deployment/S3Repository.cs b/src/Velopack.Deployment/S3Repository.cs index 0e1fee00..0415e7db 100644 --- a/src/Velopack.Deployment/S3Repository.cs +++ b/src/Velopack.Deployment/S3Repository.cs @@ -17,7 +17,7 @@ public class S3DownloadOptions : RepositoryOptions, IObjectDownloadOptions public string Endpoint { get; set; } - public string ContainerName { get; set; } + public string Bucket { get; set; } } public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions @@ -25,13 +25,50 @@ public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions public int KeepMaxReleases { get; set; } } -public class S3Repository : ObjectRepository +public class S3BucketClient +{ + public AmazonS3Client Amazon { get; } + + public string Bucket { get; } + + public S3BucketClient(AmazonS3Client client, string bucket) + { + Amazon = client; + Bucket = bucket; + } + + public virtual Task DeleteObjectAsync(string key, CancellationToken cancellationToken = default) + { + var request = new DeleteObjectRequest(); + request.BucketName = Bucket; + request.Key = key; + return Amazon.DeleteObjectAsync(request, cancellationToken); + } + + public virtual Task GetObjectAsync(string key, CancellationToken cancellationToken = default) + { + var request = new GetObjectRequest(); + request.BucketName = Bucket; + request.Key = key; + return Amazon.GetObjectAsync(request, cancellationToken); + } + + public virtual Task GetObjectMetadataAsync(string key, CancellationToken cancellationToken = default) + { + var request = new GetObjectMetadataRequest(); + request.BucketName = Bucket; + request.Key = key; + return Amazon.GetObjectMetadataAsync(request, cancellationToken); + } +} + +public class S3Repository : ObjectRepository { public S3Repository(ILogger logger) : base(logger) { } - protected override AmazonS3Client CreateClient(S3DownloadOptions options) + protected override S3BucketClient CreateClient(S3DownloadOptions options) { var config = new AmazonS3Config() { ServiceURL = options.Endpoint }; if (options.Endpoint != null) { @@ -42,24 +79,26 @@ public class S3Repository : ObjectRepository client.DeleteObjectAsync(container, key), "Deleting " + key); + await RetryAsync(() => client.DeleteObjectAsync(key), "Deleting " + key); } - protected override async Task GetObjectBytes(AmazonS3Client client, string container, string key) + protected override async Task GetObjectBytes(S3BucketClient client, string key) { return await RetryAsyncRet(async () => { try { var ms = new MemoryStream(); - using (var obj = await client.GetObjectAsync(container, key)) + using (var obj = await client.GetObjectAsync(key)) using (var stream = obj.ResponseStream) { await stream.CopyToAsync(ms); } @@ -74,19 +113,19 @@ public class S3Repository : ObjectRepository { - using (var obj = await client.GetObjectAsync(options.ContainerName, entry.FileName)) { + using (var obj = await client.GetObjectAsync(entry.FileName)) { await obj.WriteResponseStreamToFileAsync(filePath, false, CancellationToken.None); } }, $"Downloading {entry.FileName}..."); } - protected override async Task UploadObject(AmazonS3Client client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache) + protected override async Task UploadObject(S3BucketClient client, string key, FileInfo f, bool overwriteRemote, bool noCache) { string deleteOldVersionId = null; // try to detect an existing remote file of the same name try { - var metadata = await client.GetObjectMetadataAsync(container, key); + var metadata = await client.GetObjectMetadataAsync(key); var md5bytes = GetFileMD5Checksum(f.FullName); var md5 = BitConverter.ToString(md5bytes).Replace("-", String.Empty); var stored = metadata?.ETag?.Trim().Trim('"'); @@ -108,8 +147,9 @@ public class S3Repository : ObjectRepository client.PutObjectAsync(req), "Uploading " + key + (noCache ? " (no-cache)" : "")); + await RetryAsync(() => client.Amazon.PutObjectAsync(req), "Uploading " + key + (noCache ? " (no-cache)" : "")); if (deleteOldVersionId != null) { try { - await RetryAsync(() => client.DeleteObjectAsync(container, key, deleteOldVersionId), + await RetryAsync(() => client.Amazon.DeleteObjectAsync(bucket, key, deleteOldVersionId), "Removing old version of " + key); } catch { } } diff --git a/src/Velopack.Deployment/_ObjectRepository.cs b/src/Velopack.Deployment/_ObjectRepository.cs index 5b400e18..4be1aff2 100644 --- a/src/Velopack.Deployment/_ObjectRepository.cs +++ b/src/Velopack.Deployment/_ObjectRepository.cs @@ -11,7 +11,6 @@ public interface IObjectUploadOptions public interface IObjectDownloadOptions { - string ContainerName { get; set; } } public abstract class ObjectRepository : DownRepository, IRepositoryCanUpload @@ -22,9 +21,9 @@ public abstract class ObjectRepository : DownRepository GetObjectBytes(TClient client, string container, string key); + protected abstract Task UploadObject(TClient client, string key, FileInfo f, bool overwriteRemote, bool noCache); + protected abstract Task DeleteObject(TClient client, string key); + protected abstract Task GetObjectBytes(TClient client, string key); protected abstract TClient CreateClient(TDown options); protected byte[] GetFileMD5Checksum(string filePath) @@ -40,7 +39,7 @@ public abstract class ObjectRepository : DownRepository : DownRepository : DownRepository 0) { Log.Info($"Retention policy about to delete {toDelete.Length} releases..."); foreach (var del in toDelete) { - await DeleteObject(client, options.ContainerName, del.FileName); + await DeleteObject(client, del.FileName); } } diff --git a/src/Velopack.Vpk/Commands/Deployment/AzureBaseCommand.cs b/src/Velopack.Vpk/Commands/Deployment/AzureBaseCommand.cs index c3dc92c7..dae9016a 100644 --- a/src/Velopack.Vpk/Commands/Deployment/AzureBaseCommand.cs +++ b/src/Velopack.Vpk/Commands/Deployment/AzureBaseCommand.cs @@ -8,7 +8,7 @@ public class AzureBaseCommand : OutputCommand public string Endpoint { get; private set; } - public string ContainerName { get; private set; } + public string Container { get; private set; } public string SasToken { get; private set; } @@ -28,7 +28,7 @@ public class AzureBaseCommand : OutputCommand .SetDescription("Shared access signature token (not the url)") .SetArgumentHelpName("TOKEN"); - AddOption((v) => ContainerName = v, "--container") + AddOption((v) => Container = v, "--container") .SetDescription("Azure container name") .SetArgumentHelpName("NAME") .SetRequired(); diff --git a/src/Velopack.Vpk/Commands/Deployment/S3BaseCommand.cs b/src/Velopack.Vpk/Commands/Deployment/S3BaseCommand.cs index b6d08d2d..8c654d73 100644 --- a/src/Velopack.Vpk/Commands/Deployment/S3BaseCommand.cs +++ b/src/Velopack.Vpk/Commands/Deployment/S3BaseCommand.cs @@ -12,7 +12,7 @@ public class S3BaseCommand : OutputCommand public string Endpoint { get; private set; } - public string ContainerName { get; private set; } // Bucket + public string Bucket { get; private set; } protected S3BaseCommand(string name, string description) : base(name, description) @@ -43,7 +43,7 @@ public class S3BaseCommand : OutputCommand this.AreMutuallyExclusive(region, endpoint); this.AtLeastOneRequired(region, endpoint); - AddOption((v) => ContainerName = v, "--bucket") + AddOption((v) => Bucket = v, "--bucket") .SetDescription("Name of the S3 bucket.") .SetArgumentHelpName("NAME") .SetRequired(); diff --git a/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs b/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs index 692b989b..d8377f7d 100644 --- a/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs +++ b/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs @@ -18,7 +18,7 @@ public abstract class AzureCommandTests : BaseCommandTests Assert.Equal("account-name", command.Account); Assert.Equal("shhhh", command.Key); Assert.Equal("https://endpoint/", command.Endpoint); - Assert.Equal("mycontainer", command.ContainerName); + Assert.Equal("mycontainer", command.Container); } } diff --git a/test/Velopack.CommandLine.Tests/Commands/S3CommandTests.cs b/test/Velopack.CommandLine.Tests/Commands/S3CommandTests.cs index 92a05452..b0a9b53c 100644 --- a/test/Velopack.CommandLine.Tests/Commands/S3CommandTests.cs +++ b/test/Velopack.CommandLine.Tests/Commands/S3CommandTests.cs @@ -18,7 +18,7 @@ public abstract class S3CommandTests : BaseCommandTests Assert.Equal("some key", command.KeyId); Assert.Equal("shhhh", command.Secret); Assert.Equal("http://endpoint/", command.Endpoint); - Assert.Equal("a-bucket", command.ContainerName); + Assert.Equal("a-bucket", command.Bucket); } [Fact] @@ -33,7 +33,7 @@ public abstract class S3CommandTests : BaseCommandTests Assert.Equal("some key", command.KeyId); Assert.Equal("shhhh", command.Secret); Assert.Equal("us-west-1", command.Region); - Assert.Equal("a-bucket", command.ContainerName); + Assert.Equal("a-bucket", command.Bucket); } [Fact] diff --git a/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs b/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs index c2d9a71a..3b1d64cc 100644 --- a/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs +++ b/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs @@ -47,7 +47,7 @@ public class AzureDeploymentTests var repo = new AzureRepository(logger); var options = new AzureUploadOptions { ReleaseDir = new DirectoryInfo(releaseDir), - ContainerName = AZ_CONTAINER, + Container = AZ_CONTAINER, Channel = channel, Account = AZ_ACCOUNT, Key = AZ_KEY, diff --git a/test/Velopack.Packaging.Tests/S3DeploymentTests.cs b/test/Velopack.Packaging.Tests/S3DeploymentTests.cs index 8559911b..cadeae9f 100644 --- a/test/Velopack.Packaging.Tests/S3DeploymentTests.cs +++ b/test/Velopack.Packaging.Tests/S3DeploymentTests.cs @@ -47,7 +47,7 @@ public class S3DeploymentTests var repo = new S3Repository(logger); var options = new S3UploadOptions { ReleaseDir = new DirectoryInfo(releaseDir), - ContainerName = B2_BUCKET, + Bucket = B2_BUCKET, Channel = channel, Endpoint = "https://" + B2_ENDPOINT, KeyId = B2_KEYID,