Mirrors/velopack
1
0
Fork 0
mirror of https://github.com/velopack/velopack.git synced 2025-10-25 15:19:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove ObjectRepository container concept

Browse Source
This commit is contained in:
Caelan Sayler
2024-03-28 09:43:27 +00:00
parent 980caf0493
commit f16bfe5349
9 changed files with 87 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/Velopack.Deployment/AzureRepository.cs
View File

@@ -12,7 +12,7 @@ public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions
public string Endpoint { get; set; }
public string ContainerName { get; set; }
public string Container { get; set; }
public string SasToken { get; set; }
}
@@ -22,40 +22,40 @@ public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions
public int KeepMaxReleases { get; set; }
}
public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploadOptions, BlobServiceClient>
public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploadOptions, BlobContainerClient>
{
public AzureRepository(ILogger logger) : base(logger)
{
}
protected override BlobServiceClient CreateClient(AzureDownloadOptions options)
protected override BlobContainerClient CreateClient(AzureDownloadOptions options)
{
var serviceUrl = options.Endpoint ?? "https://" + options.Account + ".blob.core.windows.net";
if (options.Endpoint == null) {
Log.Info($"Endpoint not specified, default to: {serviceUrl}");
}
BlobServiceClient client;
if (!String.IsNullOrEmpty(options.SasToken)) {
return new BlobServiceClient(new Uri(serviceUrl), new Azure.AzureSasCredential(options.SasToken));
client = new BlobServiceClient(new Uri(serviceUrl), new Azure.AzureSasCredential(options.SasToken));
} else {
client = new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key));
}
return client.GetBlobContainerClient(options.Container);
}
return new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key));
}
protected override async Task DeleteObject(BlobServiceClient client, string container, string key)
protected override async Task DeleteObject(BlobContainerClient client, string key)
{
await RetryAsync(async () => {
var containerClient = client.GetBlobContainerClient(container);
await containerClient.DeleteBlobIfExistsAsync(key);
await client.DeleteBlobIfExistsAsync(key);
}, "Deleting " + key);
}
protected override async Task<byte[]> GetObjectBytes(BlobServiceClient client, string container, string key)
protected override async Task<byte[]> GetObjectBytes(BlobContainerClient client, string key)
{
return await RetryAsyncRet(async () => {
try {
var containerClient = client.GetBlobContainerClient(container);
var obj = containerClient.GetBlobClient(key);
var obj = client.GetBlobClient(key);
var ms = new MemoryStream();
using var response = await obj.DownloadToAsync(ms, CancellationToken.None);
return ms.ToArray();
@@ -69,16 +69,14 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
{
await RetryAsync(async () => {
var client = CreateClient(options);
var containerClient = client.GetBlobContainerClient(options.ContainerName);
var obj = containerClient.GetBlobClient(entry.FileName);
var obj = client.GetBlobClient(entry.FileName);
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
}, $"Downloading {entry.FileName}...");
}
protected override async Task UploadObject(BlobServiceClient client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache)
protected override async Task UploadObject(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote, bool noCache)
{
var containerClient = client.GetBlobContainerClient(container);
var blobClient = containerClient.GetBlobClient(key);
var blobClient = client.GetBlobClient(key);
try {
var properties = await blobClient.GetPropertiesAsync();
var md5 = GetFileMD5Checksum(f.FullName);

70
src/Velopack.Deployment/S3Repository.cs
View File

@@ -17,7 +17,7 @@ public class S3DownloadOptions : RepositoryOptions, IObjectDownloadOptions
public string Endpoint { get; set; }
public string ContainerName { get; set; }
public string Bucket { get; set; }
}
public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions
@@ -25,13 +25,50 @@ public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions
public int KeepMaxReleases { get; set; }
}
public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions, AmazonS3Client>
public class S3BucketClient
{
public AmazonS3Client Amazon { get; }
public string Bucket { get; }
public S3BucketClient(AmazonS3Client client, string bucket)
{
Amazon = client;
Bucket = bucket;
}
public virtual Task<DeleteObjectResponse> DeleteObjectAsync(string key, CancellationToken cancellationToken = default)
{
var request = new DeleteObjectRequest();
request.BucketName = Bucket;
request.Key = key;
return Amazon.DeleteObjectAsync(request, cancellationToken);
}
public virtual Task<GetObjectResponse> GetObjectAsync(string key, CancellationToken cancellationToken = default)
{
var request = new GetObjectRequest();
request.BucketName = Bucket;
request.Key = key;
return Amazon.GetObjectAsync(request, cancellationToken);
}
public virtual Task<GetObjectMetadataResponse> GetObjectMetadataAsync(string key, CancellationToken cancellationToken = default)
{
var request = new GetObjectMetadataRequest();
request.BucketName = Bucket;
request.Key = key;
return Amazon.GetObjectMetadataAsync(request, cancellationToken);
}
}
public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions, S3BucketClient>
{
public S3Repository(ILogger logger) : base(logger)
{
}
protected override AmazonS3Client CreateClient(S3DownloadOptions options)
protected override S3BucketClient CreateClient(S3DownloadOptions options)
{
var config = new AmazonS3Config() { ServiceURL = options.Endpoint };
if (options.Endpoint != null) {
@@ -42,24 +79,26 @@ public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions,
throw new InvalidOperationException("Missing endpoint");
}
AmazonS3Client client;
if (options.Session != null) {
return new AmazonS3Client(options.KeyId, options.Secret, options.Session, config);
client = new AmazonS3Client(options.KeyId, options.Secret, options.Session, config);
} else {
return new AmazonS3Client(options.KeyId, options.Secret, config);
client = new AmazonS3Client(options.KeyId, options.Secret, config);
}
return new S3BucketClient(client, options.Bucket);
}
protected override async Task DeleteObject(AmazonS3Client client, string container, string key)
protected override async Task DeleteObject(S3BucketClient client, string key)
{
await RetryAsync(() => client.DeleteObjectAsync(container, key), "Deleting " + key);
await RetryAsync(() => client.DeleteObjectAsync(key), "Deleting " + key);
}
protected override async Task<byte[]> GetObjectBytes(AmazonS3Client client, string container, string key)
protected override async Task<byte[]> GetObjectBytes(S3BucketClient client, string key)
{
return await RetryAsyncRet(async () => {
try {
var ms = new MemoryStream();
using (var obj = await client.GetObjectAsync(container, key))
using (var obj = await client.GetObjectAsync(key))
using (var stream = obj.ResponseStream) {
await stream.CopyToAsync(ms);
}
@@ -74,19 +113,19 @@ public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions,
{
var client = CreateClient(options);
await RetryAsync(async () => {
using (var obj = await client.GetObjectAsync(options.ContainerName, entry.FileName)) {
using (var obj = await client.GetObjectAsync(entry.FileName)) {
await obj.WriteResponseStreamToFileAsync(filePath, false, CancellationToken.None);
}
}, $"Downloading {entry.FileName}...");
}
protected override async Task UploadObject(AmazonS3Client client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache)
protected override async Task UploadObject(S3BucketClient client, string key, FileInfo f, bool overwriteRemote, bool noCache)
{
string deleteOldVersionId = null;
// try to detect an existing remote file of the same name
try {
var metadata = await client.GetObjectMetadataAsync(container, key);
var metadata = await client.GetObjectMetadataAsync(key);
var md5bytes = GetFileMD5Checksum(f.FullName);
var md5 = BitConverter.ToString(md5bytes).Replace("-", String.Empty);
var stored = metadata?.ETag?.Trim().Trim('"');
@@ -108,8 +147,9 @@ public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions,
// already exists. storage providers should prefer the newer file of the same name.
}
var bucket = client.Bucket;
var req = new PutObjectRequest {
BucketName = container,
BucketName = bucket,
FilePath = f.FullName,
Key = key,
};
@@ -118,11 +158,11 @@ public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions,
req.Headers.CacheControl = "no-cache";
}
await RetryAsync(() => client.PutObjectAsync(req), "Uploading " + key + (noCache ? " (no-cache)" : ""));
await RetryAsync(() => client.Amazon.PutObjectAsync(req), "Uploading " + key + (noCache ? " (no-cache)" : ""));
if (deleteOldVersionId != null) {
try {
await RetryAsync(() => client.DeleteObjectAsync(container, key, deleteOldVersionId),
await RetryAsync(() => client.Amazon.DeleteObjectAsync(bucket, key, deleteOldVersionId),
"Removing old version of " + key);
} catch { }
}

17
src/Velopack.Deployment/_ObjectRepository.cs
View File

@@ -11,7 +11,6 @@ public interface IObjectUploadOptions
public interface IObjectDownloadOptions
{
string ContainerName { get; set; }
}
public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDown>, IRepositoryCanUpload<TUp>
@@ -22,9 +21,9 @@ public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDo
{
}
protected abstract Task UploadObject(TClient client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache);
protected abstract Task DeleteObject(TClient client, string container, string key);
protected abstract Task<byte[]> GetObjectBytes(TClient client, string container, string key);
protected abstract Task UploadObject(TClient client, string key, FileInfo f, bool overwriteRemote, bool noCache);
protected abstract Task DeleteObject(TClient client, string key);
protected abstract Task<byte[]> GetObjectBytes(TClient client, string key);
protected abstract TClient CreateClient(TDown options);
protected byte[] GetFileMD5Checksum(string filePath)
@@ -40,7 +39,7 @@ public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDo
{
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
var client = CreateClient(options);
var bytes = await GetObjectBytes(client, options.ContainerName, releasesName);
var bytes = await GetObjectBytes(client, releasesName);
if (bytes == null || bytes.Length == 0) {
return new VelopackAssetFeed();
}
@@ -82,13 +81,13 @@ public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDo
}
foreach (var asset in build.Files) {
await UploadObject(client, options.ContainerName, Path.GetFileName(asset), new FileInfo(asset), true, noCache: false);
await UploadObject(client, Path.GetFileName(asset), new FileInfo(asset), true, noCache: false);
}
using var _1 = Utility.GetTempFileName(out var tmpReleases);
File.WriteAllText(tmpReleases, ReleaseEntryHelper.GetAssetFeedJson(new VelopackAssetFeed { Assets = releaseEntries }));
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
await UploadObject(client, options.ContainerName, releasesName, new FileInfo(tmpReleases), true, noCache: true);
await UploadObject(client, releasesName, new FileInfo(tmpReleases), true, noCache: true);
#pragma warning disable CS0612 // Type or member is obsolete
#pragma warning disable CS0618 // Type or member is obsolete
@@ -97,14 +96,14 @@ public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDo
using (var fs = File.Create(tmpReleases2)) {
ReleaseEntry.WriteReleaseFile(releaseEntries.Select(ReleaseEntry.FromVelopackAsset), fs);
}
await UploadObject(client, options.ContainerName, legacyKey, new FileInfo(tmpReleases2), true, noCache: true);
await UploadObject(client, legacyKey, new FileInfo(tmpReleases2), true, noCache: true);
#pragma warning restore CS0618 // Type or member is obsolete
#pragma warning restore CS0612 // Type or member is obsolete
if (toDelete.Length > 0) {
Log.Info($"Retention policy about to delete {toDelete.Length} releases...");
foreach (var del in toDelete) {
await DeleteObject(client, options.ContainerName, del.FileName);
await DeleteObject(client, del.FileName);
}
}

4
src/Velopack.Vpk/Commands/Deployment/AzureBaseCommand.cs
View File

@@ -8,7 +8,7 @@ public class AzureBaseCommand : OutputCommand
public string Endpoint { get; private set; }
public string ContainerName { get; private set; }
public string Container { get; private set; }
public string SasToken { get; private set; }
@@ -28,7 +28,7 @@ public class AzureBaseCommand : OutputCommand
.SetDescription("Shared access signature token (not the url)")
.SetArgumentHelpName("TOKEN");
AddOption<string>((v) => ContainerName = v, "--container")
AddOption<string>((v) => Container = v, "--container")
.SetDescription("Azure container name")
.SetArgumentHelpName("NAME")
.SetRequired();

4
src/Velopack.Vpk/Commands/Deployment/S3BaseCommand.cs
View File

@@ -12,7 +12,7 @@ public class S3BaseCommand : OutputCommand
public string Endpoint { get; private set; }
public string ContainerName { get; private set; } // Bucket
public string Bucket { get; private set; }
protected S3BaseCommand(string name, string description)
: base(name, description)
@@ -43,7 +43,7 @@ public class S3BaseCommand : OutputCommand
this.AreMutuallyExclusive(region, endpoint);
this.AtLeastOneRequired(region, endpoint);
AddOption<string>((v) => ContainerName = v, "--bucket")
AddOption<string>((v) => Bucket = v, "--bucket")
.SetDescription("Name of the S3 bucket.")
.SetArgumentHelpName("NAME")
.SetRequired();

2
test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs
View File

@@ -18,7 +18,7 @@ public abstract class AzureCommandTests<T> : BaseCommandTests<T>
Assert.Equal("account-name", command.Account);
Assert.Equal("shhhh", command.Key);
Assert.Equal("https://endpoint/", command.Endpoint);
Assert.Equal("mycontainer", command.ContainerName);
Assert.Equal("mycontainer", command.Container);
}
}

4
test/Velopack.CommandLine.Tests/Commands/S3CommandTests.cs
View File

@@ -18,7 +18,7 @@ public abstract class S3CommandTests<T> : BaseCommandTests<T>
Assert.Equal("some key", command.KeyId);
Assert.Equal("shhhh", command.Secret);
Assert.Equal("http://endpoint/", command.Endpoint);
Assert.Equal("a-bucket", command.ContainerName);
Assert.Equal("a-bucket", command.Bucket);
}
[Fact]
@@ -33,7 +33,7 @@ public abstract class S3CommandTests<T> : BaseCommandTests<T>
Assert.Equal("some key", command.KeyId);
Assert.Equal("shhhh", command.Secret);
Assert.Equal("us-west-1", command.Region);
Assert.Equal("a-bucket", command.ContainerName);
Assert.Equal("a-bucket", command.Bucket);
}
[Fact]

2
test/Velopack.Packaging.Tests/AzureDeploymentTests.cs
View File

@@ -47,7 +47,7 @@ public class AzureDeploymentTests
var repo = new AzureRepository(logger);
var options = new AzureUploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
ContainerName = AZ_CONTAINER,
Container = AZ_CONTAINER,
Channel = channel,
Account = AZ_ACCOUNT,
Key = AZ_KEY,

2
test/Velopack.Packaging.Tests/S3DeploymentTests.cs
View File

@@ -47,7 +47,7 @@ public class S3DeploymentTests
var repo = new S3Repository(logger);
var options = new S3UploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
ContainerName = B2_BUCKET,
Bucket = B2_BUCKET,
Channel = channel,
Endpoint = "https://" + B2_ENDPOINT,
KeyId = B2_KEYID,