mirror of
https://github.com/velopack/velopack.git
synced 2025-10-25 15:19:22 +00:00
Refactor S3 and Azure common code into ObjectRepository
This commit is contained in:
@@ -1,14 +1,10 @@
|
|||||||
using System.Net;
|
using Azure.Storage;
|
||||||
using System.Text;
|
|
||||||
using Azure;
|
|
||||||
using Azure.Storage;
|
|
||||||
using Azure.Storage.Blobs;
|
using Azure.Storage.Blobs;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Velopack.Packaging;
|
|
||||||
|
|
||||||
namespace Velopack.Deployment;
|
namespace Velopack.Deployment;
|
||||||
|
|
||||||
public class AzureDownloadOptions : RepositoryOptions
|
public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions
|
||||||
{
|
{
|
||||||
public string Account { get; set; }
|
public string Account { get; set; }
|
||||||
|
|
||||||
@@ -16,128 +12,58 @@ public class AzureDownloadOptions : RepositoryOptions
|
|||||||
|
|
||||||
public string Endpoint { get; set; }
|
public string Endpoint { get; set; }
|
||||||
|
|
||||||
public string Container { get; set; }
|
public string ContainerName { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class AzureUploadOptions : AzureDownloadOptions
|
public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions
|
||||||
{
|
{
|
||||||
public int KeepMaxReleases { get; set; }
|
public int KeepMaxReleases { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class AzureRepository : DownRepository<AzureDownloadOptions>, IRepositoryCanUpload<AzureUploadOptions>
|
public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploadOptions, BlobServiceClient>
|
||||||
{
|
{
|
||||||
public AzureRepository(ILogger logger) : base(logger)
|
public AzureRepository(ILogger logger) : base(logger)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task UploadMissingAssetsAsync(AzureUploadOptions options)
|
protected override BlobServiceClient CreateClient(AzureDownloadOptions options)
|
||||||
{
|
|
||||||
var build = BuildAssets.Read(options.ReleaseDir.FullName, options.Channel);
|
|
||||||
var client = GetBlobContainerClient(options);
|
|
||||||
|
|
||||||
Log.Info($"Preparing to upload {build.Files.Count} local assets to Azure endpoint {options.Endpoint ?? ""}");
|
|
||||||
|
|
||||||
var remoteReleases = await GetReleasesAsync(options);
|
|
||||||
Log.Info($"There are {remoteReleases.Assets.Length} assets in remote RELEASES file.");
|
|
||||||
|
|
||||||
var localEntries = build.GetReleaseEntries();
|
|
||||||
var releaseEntries = ReleaseEntryHelper.MergeAssets(localEntries, remoteReleases.Assets).ToArray();
|
|
||||||
|
|
||||||
Log.Info($"{releaseEntries.Length} merged local/remote releases.");
|
|
||||||
|
|
||||||
VelopackAsset[] toDelete = new VelopackAsset[0];
|
|
||||||
|
|
||||||
if (options.KeepMaxReleases > 0) {
|
|
||||||
var fullReleases = releaseEntries
|
|
||||||
.OrderByDescending(x => x.Version)
|
|
||||||
.Where(x => x.Type == VelopackAssetType.Full)
|
|
||||||
.ToArray();
|
|
||||||
if (fullReleases.Length > options.KeepMaxReleases) {
|
|
||||||
var minVersion = fullReleases[options.KeepMaxReleases - 1].Version;
|
|
||||||
toDelete = releaseEntries
|
|
||||||
.Where(x => x.Version < minVersion)
|
|
||||||
.ToArray();
|
|
||||||
releaseEntries = releaseEntries.Except(toDelete).ToArray();
|
|
||||||
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will delete {toDelete.Length} releases.");
|
|
||||||
} else {
|
|
||||||
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will not be applied, because there will only be {fullReleases.Length} full releases when this upload has completed.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var asset in build.Files) {
|
|
||||||
await UploadFile(client, Path.GetFileName(asset), new FileInfo(asset), true);
|
|
||||||
}
|
|
||||||
|
|
||||||
using var _1 = Utility.GetTempFileName(out var tmpReleases);
|
|
||||||
File.WriteAllText(tmpReleases, ReleaseEntryHelper.GetAssetFeedJson(new VelopackAssetFeed { Assets = releaseEntries }));
|
|
||||||
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
|
||||||
await UploadFile(client, releasesName, new FileInfo(tmpReleases), true);
|
|
||||||
|
|
||||||
#pragma warning disable CS0612 // Type or member is obsolete
|
|
||||||
#pragma warning disable CS0618 // Type or member is obsolete
|
|
||||||
var legacyKey = Utility.GetReleasesFileName(options.Channel);
|
|
||||||
using var _2 = Utility.GetTempFileName(out var tmpReleases2);
|
|
||||||
using (var fs = File.Create(tmpReleases2)) {
|
|
||||||
ReleaseEntry.WriteReleaseFile(releaseEntries.Select(ReleaseEntry.FromVelopackAsset), fs);
|
|
||||||
}
|
|
||||||
await UploadFile(client, legacyKey, new FileInfo(tmpReleases2), true);
|
|
||||||
#pragma warning restore CS0618 // Type or member is obsolete
|
|
||||||
#pragma warning restore CS0612 // Type or member is obsolete
|
|
||||||
|
|
||||||
if (toDelete.Length > 0) {
|
|
||||||
Log.Info($"Retention policy about to delete {toDelete.Length} releases...");
|
|
||||||
foreach (var del in toDelete) {
|
|
||||||
await RetryAsync(() => client.DeleteBlobIfExistsAsync(del.FileName), "Deleting " + del.FileName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Log.Info("Done.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override async Task<VelopackAssetFeed> GetReleasesAsync(AzureDownloadOptions options)
|
|
||||||
{
|
|
||||||
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
|
||||||
var client = GetBlobContainerClient(options);
|
|
||||||
|
|
||||||
var ms = new MemoryStream();
|
|
||||||
|
|
||||||
try {
|
|
||||||
await RetryAsync(async () => {
|
|
||||||
var obj = client.GetBlobClient(releasesName);
|
|
||||||
using var response = await obj.DownloadToAsync(ms);
|
|
||||||
}, $"Fetching {releasesName}...");
|
|
||||||
} catch (RequestFailedException ex) when (ex.Status == 404) {
|
|
||||||
return new VelopackAssetFeed();
|
|
||||||
}
|
|
||||||
|
|
||||||
return VelopackAssetFeed.FromJson(Encoding.UTF8.GetString(ms.ToArray()));
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override async Task SaveEntryToFileAsync(AzureDownloadOptions options, VelopackAsset entry, string filePath)
|
|
||||||
{
|
|
||||||
var client = GetBlobContainerClient(options);
|
|
||||||
await RetryAsync(async () => {
|
|
||||||
var obj = client.GetBlobClient(entry.FileName);
|
|
||||||
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
|
|
||||||
}, $"Downloading {entry.FileName}...");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static BlobServiceClient GetBlobServiceClient(AzureDownloadOptions options)
|
|
||||||
{
|
{
|
||||||
return new BlobServiceClient(new Uri(options.Endpoint), new StorageSharedKeyCredential(options.Account, options.Key));
|
return new BlobServiceClient(new Uri(options.Endpoint), new StorageSharedKeyCredential(options.Account, options.Key));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static BlobContainerClient GetBlobContainerClient(AzureDownloadOptions options)
|
protected override async Task DeleteObject(BlobServiceClient client, string container, string key)
|
||||||
{
|
{
|
||||||
var client = GetBlobServiceClient(options);
|
await RetryAsync(async () => {
|
||||||
var containerClient = client.GetBlobContainerClient(options.Container);
|
var containerClient = client.GetBlobContainerClient(container);
|
||||||
return containerClient;
|
await containerClient.DeleteBlobIfExistsAsync(key);
|
||||||
|
}, "Deleting " + key);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task UploadFile(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote)
|
protected override async Task<byte[]> GetObjectBytes(BlobServiceClient client, string container, string key)
|
||||||
{
|
{
|
||||||
// try to detect an existing remote file of the same name
|
return await RetryAsyncRet(async () => {
|
||||||
var blobClient = client.GetBlobClient(key);
|
var containerClient = client.GetBlobContainerClient(container);
|
||||||
|
var obj = containerClient.GetBlobClient(key);
|
||||||
|
var ms = new MemoryStream();
|
||||||
|
using var response = await obj.DownloadToAsync(ms, CancellationToken.None);
|
||||||
|
return ms.ToArray();
|
||||||
|
}, $"Downloading {key}...");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task SaveEntryToFileAsync(AzureDownloadOptions options, VelopackAsset entry, string filePath)
|
||||||
|
{
|
||||||
|
await RetryAsync(async () => {
|
||||||
|
var client = CreateClient(options);
|
||||||
|
var containerClient = client.GetBlobContainerClient(options.ContainerName);
|
||||||
|
var obj = containerClient.GetBlobClient(entry.FileName);
|
||||||
|
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
|
||||||
|
}, $"Downloading {entry.FileName}...");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task UploadObject(BlobServiceClient client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache)
|
||||||
|
{
|
||||||
|
var containerClient = client.GetBlobContainerClient(container);
|
||||||
|
var blobClient = containerClient.GetBlobClient(key);
|
||||||
try {
|
try {
|
||||||
var properties = await blobClient.GetPropertiesAsync();
|
var properties = await blobClient.GetPropertiesAsync();
|
||||||
var md5 = GetFileMD5Checksum(f.FullName);
|
var md5 = GetFileMD5Checksum(f.FullName);
|
||||||
@@ -161,13 +87,4 @@ public class AzureRepository : DownRepository<AzureDownloadOptions>, IRepository
|
|||||||
|
|
||||||
await RetryAsync(() => blobClient.UploadAsync(f.FullName, overwriteRemote), "Uploading " + key);
|
await RetryAsync(() => blobClient.UploadAsync(f.FullName, overwriteRemote), "Uploading " + key);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static byte[] GetFileMD5Checksum(string filePath)
|
|
||||||
{
|
|
||||||
var sha = System.Security.Cryptography.MD5.Create();
|
|
||||||
byte[] checksum;
|
|
||||||
using (var fs = File.OpenRead(filePath))
|
|
||||||
checksum = sha.ComputeHash(fs);
|
|
||||||
return checksum;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -1,14 +1,11 @@
|
|||||||
using System.Net;
|
using Amazon;
|
||||||
using System.Text;
|
|
||||||
using Amazon;
|
|
||||||
using Amazon.S3;
|
using Amazon.S3;
|
||||||
using Amazon.S3.Model;
|
using Amazon.S3.Model;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Velopack.Packaging;
|
|
||||||
|
|
||||||
namespace Velopack.Deployment;
|
namespace Velopack.Deployment;
|
||||||
|
|
||||||
public class S3DownloadOptions : RepositoryOptions
|
public class S3DownloadOptions : RepositoryOptions, IObjectDownloadOptions
|
||||||
{
|
{
|
||||||
public string KeyId { get; set; }
|
public string KeyId { get; set; }
|
||||||
|
|
||||||
@@ -20,117 +17,21 @@ public class S3DownloadOptions : RepositoryOptions
|
|||||||
|
|
||||||
public string Endpoint { get; set; }
|
public string Endpoint { get; set; }
|
||||||
|
|
||||||
public string Bucket { get; set; }
|
public string ContainerName { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class S3UploadOptions : S3DownloadOptions
|
public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions
|
||||||
{
|
{
|
||||||
public int KeepMaxReleases { get; set; }
|
public int KeepMaxReleases { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class S3Repository : DownRepository<S3DownloadOptions>, IRepositoryCanUpload<S3UploadOptions>
|
public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions, AmazonS3Client>
|
||||||
{
|
{
|
||||||
public S3Repository(ILogger logger) : base(logger)
|
public S3Repository(ILogger logger) : base(logger)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task UploadMissingAssetsAsync(S3UploadOptions options)
|
protected override AmazonS3Client CreateClient(S3DownloadOptions options)
|
||||||
{
|
|
||||||
var build = BuildAssets.Read(options.ReleaseDir.FullName, options.Channel);
|
|
||||||
var client = GetS3Client(options);
|
|
||||||
|
|
||||||
Log.Info($"Preparing to upload {build.Files.Count} local assets to S3 endpoint {options.Endpoint ?? ""}");
|
|
||||||
|
|
||||||
var remoteReleases = await GetReleasesAsync(options);
|
|
||||||
Log.Info($"There are {remoteReleases.Assets.Length} assets in remote RELEASES file.");
|
|
||||||
|
|
||||||
var localEntries = build.GetReleaseEntries();
|
|
||||||
var releaseEntries = ReleaseEntryHelper.MergeAssets(localEntries, remoteReleases.Assets).ToArray();
|
|
||||||
|
|
||||||
Log.Info($"{releaseEntries.Length} merged local/remote releases.");
|
|
||||||
|
|
||||||
VelopackAsset[] toDelete = new VelopackAsset[0];
|
|
||||||
|
|
||||||
if (options.KeepMaxReleases > 0) {
|
|
||||||
var fullReleases = releaseEntries
|
|
||||||
.OrderByDescending(x => x.Version)
|
|
||||||
.Where(x => x.Type == VelopackAssetType.Full)
|
|
||||||
.ToArray();
|
|
||||||
if (fullReleases.Length > options.KeepMaxReleases) {
|
|
||||||
var minVersion = fullReleases[options.KeepMaxReleases - 1].Version;
|
|
||||||
toDelete = releaseEntries
|
|
||||||
.Where(x => x.Version < minVersion)
|
|
||||||
.ToArray();
|
|
||||||
releaseEntries = releaseEntries.Except(toDelete).ToArray();
|
|
||||||
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will delete {toDelete.Length} releases.");
|
|
||||||
} else {
|
|
||||||
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will not be applied, because there will only be {fullReleases.Length} full releases when this upload has completed.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var asset in build.Files) {
|
|
||||||
await UploadFile(client, options.Bucket, Path.GetFileName(asset), new FileInfo(asset), true);
|
|
||||||
}
|
|
||||||
|
|
||||||
using var _1 = Utility.GetTempFileName(out var tmpReleases);
|
|
||||||
File.WriteAllText(tmpReleases, ReleaseEntryHelper.GetAssetFeedJson(new VelopackAssetFeed { Assets = releaseEntries }));
|
|
||||||
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
|
||||||
await UploadFile(client, options.Bucket, releasesName, new FileInfo(tmpReleases), true, noCache: true);
|
|
||||||
|
|
||||||
#pragma warning disable CS0612 // Type or member is obsolete
|
|
||||||
#pragma warning disable CS0618 // Type or member is obsolete
|
|
||||||
var legacyKey = Utility.GetReleasesFileName(options.Channel);
|
|
||||||
using var _2 = Utility.GetTempFileName(out var tmpReleases2);
|
|
||||||
using (var fs = File.Create(tmpReleases2)) {
|
|
||||||
ReleaseEntry.WriteReleaseFile(releaseEntries.Select(ReleaseEntry.FromVelopackAsset), fs);
|
|
||||||
}
|
|
||||||
await UploadFile(client, options.Bucket, legacyKey, new FileInfo(tmpReleases2), true, noCache: true);
|
|
||||||
#pragma warning restore CS0618 // Type or member is obsolete
|
|
||||||
#pragma warning restore CS0612 // Type or member is obsolete
|
|
||||||
|
|
||||||
if (toDelete.Length > 0) {
|
|
||||||
Log.Info($"Retention policy about to delete {toDelete.Length} releases...");
|
|
||||||
foreach (var del in toDelete) {
|
|
||||||
//var metadata = await client.GetObjectMetadataAsync(options.Bucket, del.FileName);
|
|
||||||
await RetryAsync(() => client.DeleteObjectAsync(options.Bucket, del.FileName), "Deleting " + del.FileName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Log.Info("Done.");
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override async Task<VelopackAssetFeed> GetReleasesAsync(S3DownloadOptions options)
|
|
||||||
{
|
|
||||||
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
|
||||||
var client = GetS3Client(options);
|
|
||||||
|
|
||||||
var ms = new MemoryStream();
|
|
||||||
|
|
||||||
try {
|
|
||||||
await RetryAsync(async () => {
|
|
||||||
using (var obj = await client.GetObjectAsync(options.Bucket, releasesName))
|
|
||||||
using (var stream = obj.ResponseStream) {
|
|
||||||
await stream.CopyToAsync(ms);
|
|
||||||
}
|
|
||||||
}, $"Fetching {releasesName}...");
|
|
||||||
} catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) {
|
|
||||||
return new VelopackAssetFeed();
|
|
||||||
}
|
|
||||||
|
|
||||||
return VelopackAssetFeed.FromJson(Encoding.UTF8.GetString(ms.ToArray()));
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override async Task SaveEntryToFileAsync(S3DownloadOptions options, VelopackAsset entry, string filePath)
|
|
||||||
{
|
|
||||||
var client = GetS3Client(options);
|
|
||||||
await RetryAsync(async () => {
|
|
||||||
using (var obj = await client.GetObjectAsync(options.Bucket, entry.FileName)) {
|
|
||||||
await obj.WriteResponseStreamToFileAsync(filePath, false, CancellationToken.None);
|
|
||||||
}
|
|
||||||
}, $"Downloading {entry.FileName}...");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static AmazonS3Client GetS3Client(S3DownloadOptions options)
|
|
||||||
{
|
{
|
||||||
var config = new AmazonS3Config() { ServiceURL = options.Endpoint };
|
var config = new AmazonS3Config() { ServiceURL = options.Endpoint };
|
||||||
if (options.Endpoint != null) {
|
if (options.Endpoint != null) {
|
||||||
@@ -148,14 +49,42 @@ public class S3Repository : DownRepository<S3DownloadOptions>, IRepositoryCanUpl
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task UploadFile(AmazonS3Client client, string bucket, string key, FileInfo f, bool overwriteRemote, bool noCache = false)
|
protected override async Task DeleteObject(AmazonS3Client client, string container, string key)
|
||||||
|
{
|
||||||
|
await RetryAsync(() => client.DeleteObjectAsync(container, key), "Deleting " + key);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task<byte[]> GetObjectBytes(AmazonS3Client client, string container, string key)
|
||||||
|
{
|
||||||
|
return await RetryAsyncRet(async () => {
|
||||||
|
var ms = new MemoryStream();
|
||||||
|
using (var obj = await client.GetObjectAsync(container, key))
|
||||||
|
using (var stream = obj.ResponseStream) {
|
||||||
|
await stream.CopyToAsync(ms);
|
||||||
|
}
|
||||||
|
return ms.ToArray();
|
||||||
|
}, $"Downloading {key}...");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task SaveEntryToFileAsync(S3DownloadOptions options, VelopackAsset entry, string filePath)
|
||||||
|
{
|
||||||
|
var client = CreateClient(options);
|
||||||
|
await RetryAsync(async () => {
|
||||||
|
using (var obj = await client.GetObjectAsync(options.ContainerName, entry.FileName)) {
|
||||||
|
await obj.WriteResponseStreamToFileAsync(filePath, false, CancellationToken.None);
|
||||||
|
}
|
||||||
|
}, $"Downloading {entry.FileName}...");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task UploadObject(AmazonS3Client client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache)
|
||||||
{
|
{
|
||||||
string deleteOldVersionId = null;
|
string deleteOldVersionId = null;
|
||||||
|
|
||||||
// try to detect an existing remote file of the same name
|
// try to detect an existing remote file of the same name
|
||||||
try {
|
try {
|
||||||
var metadata = await client.GetObjectMetadataAsync(bucket, key);
|
var metadata = await client.GetObjectMetadataAsync(container, key);
|
||||||
var md5 = GetFileMD5Checksum(f.FullName);
|
var md5bytes = GetFileMD5Checksum(f.FullName);
|
||||||
|
var md5 = BitConverter.ToString(md5bytes).Replace("-", String.Empty);
|
||||||
var stored = metadata?.ETag?.Trim().Trim('"');
|
var stored = metadata?.ETag?.Trim().Trim('"');
|
||||||
|
|
||||||
if (stored != null) {
|
if (stored != null) {
|
||||||
@@ -176,7 +105,7 @@ public class S3Repository : DownRepository<S3DownloadOptions>, IRepositoryCanUpl
|
|||||||
}
|
}
|
||||||
|
|
||||||
var req = new PutObjectRequest {
|
var req = new PutObjectRequest {
|
||||||
BucketName = bucket,
|
BucketName = container,
|
||||||
FilePath = f.FullName,
|
FilePath = f.FullName,
|
||||||
Key = key,
|
Key = key,
|
||||||
};
|
};
|
||||||
@@ -189,18 +118,9 @@ public class S3Repository : DownRepository<S3DownloadOptions>, IRepositoryCanUpl
|
|||||||
|
|
||||||
if (deleteOldVersionId != null) {
|
if (deleteOldVersionId != null) {
|
||||||
try {
|
try {
|
||||||
await RetryAsync(() => client.DeleteObjectAsync(bucket, key, deleteOldVersionId),
|
await RetryAsync(() => client.DeleteObjectAsync(container, key, deleteOldVersionId),
|
||||||
"Removing old version of " + key);
|
"Removing old version of " + key);
|
||||||
} catch { }
|
} catch { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string GetFileMD5Checksum(string filePath)
|
|
||||||
{
|
|
||||||
var sha = System.Security.Cryptography.MD5.Create();
|
|
||||||
byte[] checksum;
|
|
||||||
using (var fs = File.OpenRead(filePath))
|
|
||||||
checksum = sha.ComputeHash(fs);
|
|
||||||
return BitConverter.ToString(checksum).Replace("-", String.Empty);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
122
src/Velopack.Deployment/_ObjectRepository.cs
Normal file
122
src/Velopack.Deployment/_ObjectRepository.cs
Normal file
@@ -0,0 +1,122 @@
|
|||||||
|
using System.Text;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Velopack.Packaging;
|
||||||
|
|
||||||
|
namespace Velopack.Deployment;
|
||||||
|
|
||||||
|
public interface IObjectUploadOptions
|
||||||
|
{
|
||||||
|
public int KeepMaxReleases { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public interface IObjectDownloadOptions
|
||||||
|
{
|
||||||
|
string ContainerName { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ObjectNotFoundException : Exception
|
||||||
|
{
|
||||||
|
public ObjectNotFoundException(string message) : base(message) { }
|
||||||
|
}
|
||||||
|
|
||||||
|
public abstract class ObjectRepository<TDown, TUp, TClient> : DownRepository<TDown>, IRepositoryCanUpload<TUp>
|
||||||
|
where TDown : RepositoryOptions, IObjectDownloadOptions
|
||||||
|
where TUp : IObjectUploadOptions, TDown
|
||||||
|
{
|
||||||
|
protected ObjectRepository(ILogger logger) : base(logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
protected abstract Task UploadObject(TClient client, string container, string key, FileInfo f, bool overwriteRemote, bool noCache);
|
||||||
|
protected abstract Task DeleteObject(TClient client, string container, string key);
|
||||||
|
protected abstract Task<byte[]> GetObjectBytes(TClient client, string container, string key);
|
||||||
|
protected abstract TClient CreateClient(TDown options);
|
||||||
|
|
||||||
|
protected byte[] GetFileMD5Checksum(string filePath)
|
||||||
|
{
|
||||||
|
var sha = System.Security.Cryptography.MD5.Create();
|
||||||
|
byte[] checksum;
|
||||||
|
using (var fs = File.OpenRead(filePath))
|
||||||
|
checksum = sha.ComputeHash(fs);
|
||||||
|
return checksum;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override async Task<VelopackAssetFeed> GetReleasesAsync(TDown options)
|
||||||
|
{
|
||||||
|
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
||||||
|
var client = CreateClient(options);
|
||||||
|
return await RetryAsyncRet(async () => {
|
||||||
|
try {
|
||||||
|
var bytes = await GetObjectBytes(client, options.ContainerName, releasesName);
|
||||||
|
return VelopackAssetFeed.FromJson(Encoding.UTF8.GetString(bytes));
|
||||||
|
} catch (ObjectNotFoundException) {
|
||||||
|
return new VelopackAssetFeed();
|
||||||
|
}
|
||||||
|
}, $"Fetching {releasesName}...");
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UploadMissingAssetsAsync(TUp options)
|
||||||
|
{
|
||||||
|
var build = BuildAssets.Read(options.ReleaseDir.FullName, options.Channel);
|
||||||
|
var client = CreateClient(options);
|
||||||
|
|
||||||
|
Log.Info($"Preparing to upload {build.Files.Count} local assets.");
|
||||||
|
|
||||||
|
var remoteReleases = await GetReleasesAsync(options);
|
||||||
|
Log.Info($"There are {remoteReleases.Assets.Length} assets in remote RELEASES file.");
|
||||||
|
|
||||||
|
var localEntries = build.GetReleaseEntries();
|
||||||
|
var releaseEntries = ReleaseEntryHelper.MergeAssets(localEntries, remoteReleases.Assets).ToArray();
|
||||||
|
|
||||||
|
Log.Info($"{releaseEntries.Length} merged local/remote releases.");
|
||||||
|
|
||||||
|
var toDelete = new VelopackAsset[0];
|
||||||
|
|
||||||
|
if (options.KeepMaxReleases > 0) {
|
||||||
|
var fullReleases = releaseEntries
|
||||||
|
.OrderByDescending(x => x.Version)
|
||||||
|
.Where(x => x.Type == VelopackAssetType.Full)
|
||||||
|
.ToArray();
|
||||||
|
if (fullReleases.Length > options.KeepMaxReleases) {
|
||||||
|
var minVersion = fullReleases[options.KeepMaxReleases - 1].Version;
|
||||||
|
toDelete = releaseEntries
|
||||||
|
.Where(x => x.Version < minVersion)
|
||||||
|
.ToArray();
|
||||||
|
releaseEntries = releaseEntries.Except(toDelete).ToArray();
|
||||||
|
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will delete {toDelete.Length} releases.");
|
||||||
|
} else {
|
||||||
|
Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will not be applied, because there will only be {fullReleases.Length} full releases when this upload has completed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var asset in build.Files) {
|
||||||
|
await UploadObject(client, options.ContainerName, Path.GetFileName(asset), new FileInfo(asset), true, noCache: false);
|
||||||
|
}
|
||||||
|
|
||||||
|
using var _1 = Utility.GetTempFileName(out var tmpReleases);
|
||||||
|
File.WriteAllText(tmpReleases, ReleaseEntryHelper.GetAssetFeedJson(new VelopackAssetFeed { Assets = releaseEntries }));
|
||||||
|
var releasesName = Utility.GetVeloReleaseIndexName(options.Channel);
|
||||||
|
await UploadObject(client, options.ContainerName, releasesName, new FileInfo(tmpReleases), true, noCache: true);
|
||||||
|
|
||||||
|
#pragma warning disable CS0612 // Type or member is obsolete
|
||||||
|
#pragma warning disable CS0618 // Type or member is obsolete
|
||||||
|
var legacyKey = Utility.GetReleasesFileName(options.Channel);
|
||||||
|
using var _2 = Utility.GetTempFileName(out var tmpReleases2);
|
||||||
|
using (var fs = File.Create(tmpReleases2)) {
|
||||||
|
ReleaseEntry.WriteReleaseFile(releaseEntries.Select(ReleaseEntry.FromVelopackAsset), fs);
|
||||||
|
}
|
||||||
|
await UploadObject(client, options.ContainerName, legacyKey, new FileInfo(tmpReleases2), true, noCache: true);
|
||||||
|
#pragma warning restore CS0618 // Type or member is obsolete
|
||||||
|
#pragma warning restore CS0612 // Type or member is obsolete
|
||||||
|
|
||||||
|
if (toDelete.Length > 0) {
|
||||||
|
Log.Info($"Retention policy about to delete {toDelete.Length} releases...");
|
||||||
|
foreach (var del in toDelete) {
|
||||||
|
//var metadata = await client.GetObjectMetadataAsync(options.Bucket, del.FileName);
|
||||||
|
await RetryAsync(() => DeleteObject(client, options.ContainerName, del.FileName), "Deleting " + del.FileName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.Info("Done.");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,7 +9,7 @@ public class AzureBaseCommand : OutputCommand
|
|||||||
|
|
||||||
public string Endpoint { get; private set; }
|
public string Endpoint { get; private set; }
|
||||||
|
|
||||||
public string Container { get; private set; }
|
public string ContainerName { get; private set; }
|
||||||
|
|
||||||
protected AzureBaseCommand(string name, string description)
|
protected AzureBaseCommand(string name, string description)
|
||||||
: base(name, description)
|
: base(name, description)
|
||||||
@@ -24,7 +24,7 @@ public class AzureBaseCommand : OutputCommand
|
|||||||
.SetArgumentHelpName("KEY")
|
.SetArgumentHelpName("KEY")
|
||||||
.SetRequired();
|
.SetRequired();
|
||||||
|
|
||||||
AddOption<string>((v) => Container = v, "--container")
|
AddOption<string>((v) => ContainerName = v, "--container")
|
||||||
.SetDescription("Azure container name")
|
.SetDescription("Azure container name")
|
||||||
.SetArgumentHelpName("CONTAINER")
|
.SetArgumentHelpName("CONTAINER")
|
||||||
.SetRequired();
|
.SetRequired();
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ public class S3BaseCommand : OutputCommand
|
|||||||
|
|
||||||
public string Endpoint { get; private set; }
|
public string Endpoint { get; private set; }
|
||||||
|
|
||||||
public string Bucket { get; private set; }
|
public string ContainerName { get; private set; } // Bucket
|
||||||
|
|
||||||
protected S3BaseCommand(string name, string description)
|
protected S3BaseCommand(string name, string description)
|
||||||
: base(name, description)
|
: base(name, description)
|
||||||
@@ -44,7 +44,7 @@ public class S3BaseCommand : OutputCommand
|
|||||||
this.AreMutuallyExclusive(region, endpoint);
|
this.AreMutuallyExclusive(region, endpoint);
|
||||||
this.AtLeastOneRequired(region, endpoint);
|
this.AtLeastOneRequired(region, endpoint);
|
||||||
|
|
||||||
AddOption<string>((v) => Bucket = v, "--bucket")
|
AddOption<string>((v) => ContainerName = v, "--bucket")
|
||||||
.SetDescription("Name of the S3 bucket.")
|
.SetDescription("Name of the S3 bucket.")
|
||||||
.SetArgumentHelpName("NAME")
|
.SetArgumentHelpName("NAME")
|
||||||
.SetRequired();
|
.SetRequired();
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
using Riok.Mapperly.Abstractions;
|
using Riok.Mapperly.Abstractions;
|
||||||
using Velopack.Deployment;
|
using Velopack.Deployment;
|
||||||
using Velopack.Packaging.Commands;
|
using Velopack.Packaging.Commands;
|
||||||
using Velopack.Packaging.Unix.Commands;
|
using Velopack.Packaging.Unix.Commands;
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ public abstract class AzureCommandTests<T> : BaseCommandTests<T>
|
|||||||
Assert.Equal("account-name", command.Account);
|
Assert.Equal("account-name", command.Account);
|
||||||
Assert.Equal("shhhh", command.Key);
|
Assert.Equal("shhhh", command.Key);
|
||||||
Assert.Equal("https://endpoint/", command.Endpoint);
|
Assert.Equal("https://endpoint/", command.Endpoint);
|
||||||
Assert.Equal("mycontainer", command.Container);
|
Assert.Equal("mycontainer", command.ContainerName);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ public abstract class S3CommandTests<T> : BaseCommandTests<T>
|
|||||||
Assert.Equal("some key", command.KeyId);
|
Assert.Equal("some key", command.KeyId);
|
||||||
Assert.Equal("shhhh", command.Secret);
|
Assert.Equal("shhhh", command.Secret);
|
||||||
Assert.Equal("http://endpoint/", command.Endpoint);
|
Assert.Equal("http://endpoint/", command.Endpoint);
|
||||||
Assert.Equal("a-bucket", command.Bucket);
|
Assert.Equal("a-bucket", command.ContainerName);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -33,7 +33,7 @@ public abstract class S3CommandTests<T> : BaseCommandTests<T>
|
|||||||
Assert.Equal("some key", command.KeyId);
|
Assert.Equal("some key", command.KeyId);
|
||||||
Assert.Equal("shhhh", command.Secret);
|
Assert.Equal("shhhh", command.Secret);
|
||||||
Assert.Equal("us-west-1", command.Region);
|
Assert.Equal("us-west-1", command.Region);
|
||||||
Assert.Equal("a-bucket", command.Bucket);
|
Assert.Equal("a-bucket", command.ContainerName);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ public class AzureDeploymentTests
|
|||||||
var repo = new AzureRepository(logger);
|
var repo = new AzureRepository(logger);
|
||||||
var options = new AzureUploadOptions {
|
var options = new AzureUploadOptions {
|
||||||
ReleaseDir = new DirectoryInfo(releaseDir),
|
ReleaseDir = new DirectoryInfo(releaseDir),
|
||||||
Container = B2_BUCKET,
|
ContainerName = B2_BUCKET,
|
||||||
Channel = channel,
|
Channel = channel,
|
||||||
Endpoint = "https://" + B2_ENDPOINT,
|
Endpoint = "https://" + B2_ENDPOINT,
|
||||||
Account = B2_KEYID,
|
Account = B2_KEYID,
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ public class S3DeploymentTests
|
|||||||
var repo = new S3Repository(logger);
|
var repo = new S3Repository(logger);
|
||||||
var options = new S3UploadOptions {
|
var options = new S3UploadOptions {
|
||||||
ReleaseDir = new DirectoryInfo(releaseDir),
|
ReleaseDir = new DirectoryInfo(releaseDir),
|
||||||
Bucket = B2_BUCKET,
|
ContainerName = B2_BUCKET,
|
||||||
Channel = channel,
|
Channel = channel,
|
||||||
Endpoint = "https://" + B2_ENDPOINT,
|
Endpoint = "https://" + B2_ENDPOINT,
|
||||||
KeyId = B2_KEYID,
|
KeyId = B2_KEYID,
|
||||||
|
|||||||
Reference in New Issue
Block a user