Mirrors/velopack
1
0
Fork 0
mirror of https://github.com/velopack/velopack.git synced 2025-10-25 15:19:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Combine AZ and B2 test together

Browse Source
This commit is contained in:
Caelan Sayler
2024-12-01 21:10:41 +00:00
committed by Caelan
parent ea0a6ce8b7
commit ec7ea0196d
11 changed files with 183 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/vpk/Velopack.Deployment/AzureRepository.cs
View File

@@ -18,8 +18,6 @@ public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions
public string Container { get; set; }
public string SasToken { get; set; }
public double Timeout { get; set; }
}
public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions
@@ -51,37 +49,44 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
} else {
client = new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key), clientOptions);
}
return client.GetBlobContainerClient(options.Container);
}
protected override async Task DeleteObject(BlobContainerClient client, string key)
{
await RetryAsync(async () => {
await client.DeleteBlobIfExistsAsync(key);
}, "Deleting " + key);
await RetryAsync(
async () => {
await client.DeleteBlobIfExistsAsync(key);
},
"Deleting " + key);
}
protected override async Task<byte[]> GetObjectBytes(BlobContainerClient client, string key)
{
return await RetryAsyncRet(async () => {
try {
var obj = client.GetBlobClient(key);
var ms = new MemoryStream();
using var response = await obj.DownloadToAsync(ms, CancellationToken.None);
return ms.ToArray();
} catch (Azure.RequestFailedException ex) when (ex.Status == 404) {
return null;
}
}, $"Downloading {key}...");
return await RetryAsyncRet(
async () => {
try {
var obj = client.GetBlobClient(key);
var ms = new MemoryStream();
using var response = await obj.DownloadToAsync(ms, CancellationToken.None);
return ms.ToArray();
} catch (Azure.RequestFailedException ex) when (ex.Status == 404) {
return null;
}
},
$"Downloading {key}...");
}
protected override async Task SaveEntryToFileAsync(AzureDownloadOptions options, VelopackAsset entry, string filePath)
{
await RetryAsync(async () => {
var client = CreateClient(options);
var obj = client.GetBlobClient(entry.FileName);
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
}, $"Downloading {entry.FileName}...");
await RetryAsync(
async () => {
var client = CreateClient(options);
var obj = client.GetBlobClient(entry.FileName);
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
},
$"Downloading {entry.FileName}...");
}
protected override async Task UploadObject(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote, bool noCache)

2
src/vpk/Velopack.Deployment/GitHubRepository.cs
View File

@@ -16,8 +16,6 @@ public class GitHubDownloadOptions : RepositoryOptions
public string RepoUrl { get; set; }
public string Token { get; set; }
public double Timeout { get; set; }
}
public class GitHubUploadOptions : GitHubDownloadOptions

2
src/vpk/Velopack.Deployment/GiteaRepository.cs
View File

@@ -20,8 +20,6 @@ public class GiteaDownloadOptions : RepositoryOptions
public string Token { get; set; }
public double Timeout { get; set; }
///// <summary>
///// Example https://gitea.com
///// </summary>

2
src/vpk/Velopack.Deployment/HttpRepository.cs
View File

@@ -6,8 +6,6 @@ namespace Velopack.Deployment;
public class HttpDownloadOptions : RepositoryOptions
{
public string Url { get; set; }
public double Timeout { get; set; }
}
public class HttpRepository : SourceRepository<HttpDownloadOptions, SimpleWebSource>

3
src/vpk/Velopack.Deployment/LocalRepository.cs
View File

@@ -13,6 +13,7 @@ public class LocalDownloadOptions : RepositoryOptions, IObjectDownloadOptions
public class LocalUploadOptions : LocalDownloadOptions, IObjectUploadOptions
{
public bool ForceRegenerate { get; set; }
public int KeepMaxReleases { get; set; }
}
@@ -75,4 +76,4 @@ public class LocalRepository(ILogger logger) : ObjectRepository<LocalDownloadOpt
var source = new SimpleFileSource(options.TargetPath);
return source.GetReleaseFeed(channel: options.Channel, logger: Log);
}
}
}

4
src/vpk/Velopack.Deployment/S3Repository.cs
View File

@@ -21,8 +21,6 @@ public class S3DownloadOptions : RepositoryOptions, IObjectDownloadOptions
public string Bucket { get; set; }
public string Prefix { get; set; }
public double Timeout { get; set; }
}
public class S3UploadOptions : S3DownloadOptions, IObjectUploadOptions
@@ -101,7 +99,7 @@ public class S3Repository : ObjectRepository<S3DownloadOptions, S3UploadOptions,
ForcePathStyle = true, // support for MINIO
Timeout = TimeSpan.FromMinutes(options.Timeout)
};
if (options.Endpoint != null) {
config.ServiceURL = options.Endpoint;
// if the endpoint is using https, and is _not_ an AWS endpoint, we can disable signing

10
src/vpk/Velopack.Deployment/_Repository.cs
View File

@@ -19,6 +19,8 @@ public class RepositoryOptions : IOutputOptions
}
public DirectoryInfo ReleaseDir { get; set; }
public double Timeout { get; set; } = 30d;
}
public interface IRepositoryCanUpload<TUp> where TUp : RepositoryOptions
@@ -37,7 +39,8 @@ public abstract class SourceRepository<TDown, TSource> : DownRepository<TDown>
{
public SourceRepository(ILogger logger)
: base(logger)
{ }
{
}
protected override Task<VelopackAssetFeed> GetReleasesAsync(TDown options)
{
@@ -104,8 +107,7 @@ public abstract class DownRepository<TDown> : IRepositoryCanDownload<TDown>
Log.Error($"Checksum mismatch, expected {latest.SHA256}, got {newHash}");
return;
}
}
else if (latest.SHA1 != (newHash = IoUtil.CalculateFileSHA1(incomplete))) {
} else if (latest.SHA1 != (newHash = IoUtil.CalculateFileSHA1(incomplete))) {
Log.Error($"Checksum mismatch, expected {latest.SHA1}, got {newHash}");
return;
}
@@ -156,4 +158,4 @@ public abstract class DownRepository<TDown> : IRepositoryCanDownload<TDown>
}
}
}
}
}

24
src/vpk/Velopack.Vpk/OptionMapper.cs
View File

@@ -17,25 +17,47 @@ public static partial class OptionMapper
public static partial TDest Map<TDest>(object source);
public static partial OsxPackOptions ToOptions(this OsxPackCommand cmd);
public static partial WindowsPackOptions ToOptions(this WindowsPackCommand cmd);
public static partial LinuxPackOptions ToOptions(this LinuxPackCommand cmd);
public static partial OsxBundleOptions ToOptions(this OsxBundleCommand cmd);
public static partial GitHubDownloadOptions ToOptions(this GitHubDownloadCommand cmd);
public static partial GitHubUploadOptions ToOptions(this GitHubUploadCommand cmd);
public static partial GiteaDownloadOptions ToOptions(this GiteaDownloadCommand cmd);
public static partial GiteaUploadOptions ToOptions(this GiteaUploadCommand cmd);
public static partial HttpDownloadOptions ToOptions(this HttpDownloadCommand cmd);
[MapperIgnoreTarget(nameof(LocalDownloadOptions.Timeout))]
public static partial LocalDownloadOptions ToOptions(this LocalDownloadCommand cmd);
[MapperIgnoreTarget(nameof(LocalDownloadOptions.Timeout))]
public static partial LocalUploadOptions ToOptions(this LocalUploadCommand cmd);
public static partial S3DownloadOptions ToOptions(this S3DownloadCommand cmd);
public static partial S3UploadOptions ToOptions(this S3UploadCommand cmd);
public static partial AzureDownloadOptions ToOptions(this AzureDownloadCommand cmd);
public static partial AzureUploadOptions ToOptions(this AzureUploadCommand cmd);
public static partial DeltaGenOptions ToOptions(this DeltaGenCommand cmd);
public static partial DeltaPatchOptions ToOptions(this DeltaPatchCommand cmd);
public static partial LoginOptions ToOptions(this LoginCommand cmd);
public static partial LogoutOptions ToOptions(this LogoutCommand cmd);
public static partial PublishOptions ToOptions(this PublishCommand cmd);
public static partial ApiOptions ToOptions(this ApiCommand cmd);
private static DirectoryInfo StringToDirectoryInfo(string t)
@@ -51,4 +73,4 @@ public static partial class OptionMapper
if (t == null) return null;
return RID.Parse(t);
}
}
}

86
test/Velopack.Packaging.Tests/AzureDeploymentTests.cs
View File

@@ -1,86 +0,0 @@
using NuGet.Versioning;
using Velopack.Deployment;
using Velopack.Sources;
using Velopack.Util;
namespace Velopack.Packaging.Tests;
public class AzureDeploymentTests
{
public readonly static string AZ_ACCOUNT = "velopacktesting";
public readonly static string AZ_KEY = Environment.GetEnvironmentVariable("VELOPACK_AZ_TEST_TOKEN");
public readonly static string AZ_CONTAINER = "ci-deployment";
public readonly static string AZ_ENDPOINT = "velopacktesting.blob.core.windows.net";
private readonly ITestOutputHelper _output;
public AzureDeploymentTests(ITestOutputHelper output)
{
_output = output;
}
[SkippableFact]
public void CanDeployToAzure()
{
Skip.If(String.IsNullOrWhiteSpace(AZ_KEY), "VELOPACK_AZ_TEST_TOKEN is not set.");
using var logger = _output.BuildLoggerFor<S3DeploymentTests>();
using var _1 = TempUtil.GetTempDirectory(out var releaseDir);
string channel = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI"))
? VelopackRuntimeInfo.SystemOs.GetOsShortName()
: "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName();
// get latest version, and increment patch by one
var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}";
var source = new SimpleWebSource(updateUrl);
VelopackAssetFeed feed = new VelopackAssetFeed();
try {
feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult();
} catch (Exception ex) {
logger.Warn(ex, "Failed to fetch release feed.");
}
var latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full)
.OrderByDescending(a => a.Version)
.FirstOrDefault();
var newVer = latest != null ? new SemanticVersion(1, 0, latest.Version.Patch + 1) : new SemanticVersion(1, 0, 0);
// create repo
var repo = new AzureRepository(logger);
var options = new AzureUploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
Container = AZ_CONTAINER,
Channel = channel,
Account = AZ_ACCOUNT,
Key = AZ_KEY,
KeepMaxReleases = 4,
};
// download latest version and create delta
repo.DownloadLatestFullPackageAsync(options).GetAwaiterResult();
if (options.ReleaseDir.EnumerateFiles("*.incomplete").Any()) {
logger.Warn("A previous package was not downloaded, probably due to invalid checksum. This is a race condition in this test.");
latest = null;
}
var id = "AZTestApp";
TestApp.PackTestApp(id, newVer.ToFullString(), $"az-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: channel);
if (latest != null) {
// check delta was created
Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created.");
}
// upload new files
repo.UploadMissingAssetsAsync(options).GetAwaiterResult();
// verify that new version has been uploaded
feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult();
latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full)
.OrderByDescending(a => a.Version)
.FirstOrDefault();
Assert.True(latest != null, "No latest version found.");
Assert.Equal(newVer, latest.Version);
Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept.");
}
}

126
test/Velopack.Packaging.Tests/DeploymentTests.cs Normal file
View File

@@ -0,0 +1,126 @@
using Azure.Storage.Blobs;
using NuGet.Versioning;
using Velopack.Deployment;
using Velopack.Sources;
using Velopack.Util;
namespace Velopack.Packaging.Tests;
public class DeploymentTests
{
private static string CHANNEL = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI"))
? VelopackRuntimeInfo.SystemOs.GetOsShortName()
: "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName();
private static readonly string B2_KEYID = "0035016844a4188000000000b";
private static readonly string B2_SECRET = Environment.GetEnvironmentVariable("VELOPACK_B2_TEST_TOKEN");
private static readonly string B2_BUCKET = "velopack-testing";
private static readonly string B2_ENDPOINT = "s3.eu-central-003.backblazeb2.com";
private static readonly string AZ_ACCOUNT = "velopacktesting";
private static readonly string AZ_KEY = Environment.GetEnvironmentVariable("VELOPACK_AZ_TEST_TOKEN");
private static readonly string AZ_CONTAINER = "ci-deployment";
private static readonly string AZ_ENDPOINT = "velopacktesting.blob.core.windows.net";
private readonly ITestOutputHelper _output;
public DeploymentTests(ITestOutputHelper output)
{
_output = output;
}
[SkippableFact]
public async Task CanDeployToBackBlazeB2()
{
Skip.If(String.IsNullOrWhiteSpace(B2_SECRET), "VELOPACK_B2_TEST_TOKEN is not set.");
using var logger = _output.BuildLoggerFor<DeploymentTests>();
using var _1 = TempUtil.GetTempDirectory(out var releaseDir);
var repo = new S3Repository(logger);
var options = new S3UploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
Bucket = B2_BUCKET,
Channel = CHANNEL,
Endpoint = "https://" + B2_ENDPOINT,
KeyId = B2_KEYID,
Secret = B2_SECRET,
KeepMaxReleases = 4,
};
var updateUrl = $"https://{B2_BUCKET}.{B2_ENDPOINT}/";
await Deploy<S3Repository, S3DownloadOptions, S3UploadOptions, S3BucketClient>("B2TestApp", repo, options, releaseDir, updateUrl, logger);
}
[SkippableFact]
public async Task CanDeployToAzure()
{
Skip.If(String.IsNullOrWhiteSpace(AZ_KEY), "VELOPACK_AZ_TEST_TOKEN is not set.");
using var logger = _output.BuildLoggerFor<DeploymentTests>();
using var _1 = TempUtil.GetTempDirectory(out var releaseDir);
var repo = new AzureRepository(logger);
var options = new AzureUploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
Container = AZ_CONTAINER,
Channel = CHANNEL,
Account = AZ_ACCOUNT,
Key = AZ_KEY,
KeepMaxReleases = 4,
};
var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}";
await Deploy<AzureRepository, AzureDownloadOptions, AzureUploadOptions, BlobContainerClient>("AZTestApp", repo, options, releaseDir, updateUrl, logger);
}
static SemanticVersion GenerateSemverFromDateTime()
{
DateTime now = DateTime.Now;
int major = now.Year; // YYYY
int minor = now.Month * 100 + now.Day; // MMDD
int patch = now.Hour * 3600 + now.Minute * 60 + now.Second; // Seconds of the day
return new SemanticVersion(major, minor, patch);
}
private async Task Deploy<TRepo, TDown, TUp, TClient>(string id, TRepo repo, TUp options, string releaseDir, string updateUrl, ILogger logger)
where TDown : RepositoryOptions, IObjectDownloadOptions
where TUp : IObjectUploadOptions, TDown
where TRepo : ObjectRepository<TDown, TUp, TClient>
{
var targetVer = GenerateSemverFromDateTime();
logger.Info($"Target version: {targetVer}");
// get the latest
var source = new SimpleWebSource(updateUrl);
VelopackAssetFeed feed = new VelopackAssetFeed();
try {
feed = await source.GetReleaseFeed(logger, CHANNEL);
} catch (Exception ex) {
logger.Warn(ex, "Failed to fetch release feed.");
}
var latestOnline = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full).MaxBy(a => a.Version);
if (latestOnline != null) {
logger.Info($"Latest online version: {latestOnline.Version}");
Assert.True(targetVer > latestOnline.Version, "New version is not greater than the latest online version.");
}
// download latest version and create delta
await repo.DownloadLatestFullPackageAsync(options);
TestApp.PackTestApp(id, targetVer.ToFullString(), $"b2-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: CHANNEL);
if (latestOnline != null) {
// check delta was created
Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created.");
}
// upload new files
await repo.UploadMissingAssetsAsync(options);
// verify that new version has been uploaded
feed = await source.GetReleaseFeed(logger, CHANNEL);
latestOnline = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full).MaxBy(a => a.Version);
Assert.True(latestOnline != null, "No latest version found.");
Assert.Equal(targetVer, latestOnline.Version);
Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept.");
}
}

81
test/Velopack.Packaging.Tests/S3DeploymentTests.cs
View File

@@ -1,81 +0,0 @@
using NuGet.Versioning;
using Velopack.Deployment;
using Velopack.Sources;
using Velopack.Util;
namespace Velopack.Packaging.Tests;
public class S3DeploymentTests
{
public readonly static string B2_KEYID = "0035016844a4188000000000a";
public readonly static string B2_SECRET = Environment.GetEnvironmentVariable("VELOPACK_B2_TEST_TOKEN");
public readonly static string B2_BUCKET = "velopack-testing";
public readonly static string B2_ENDPOINT = "s3.eu-central-003.backblazeb2.com";
private readonly ITestOutputHelper _output;
public S3DeploymentTests(ITestOutputHelper output)
{
_output = output;
}
[SkippableFact]
public void CanDeployToBackBlazeB2()
{
Skip.If(String.IsNullOrWhiteSpace(B2_SECRET), "VELOPACK_B2_TEST_TOKEN is not set.");
using var logger = _output.BuildLoggerFor<S3DeploymentTests>();
using var _1 = TempUtil.GetTempDirectory(out var releaseDir);
string channel = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI"))
? VelopackRuntimeInfo.SystemOs.GetOsShortName()
: "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName();
// get latest version, and increment patch by one
var updateUrl = $"https://{B2_BUCKET}.{B2_ENDPOINT}/";
var source = new SimpleWebSource(updateUrl);
VelopackAssetFeed feed = new VelopackAssetFeed();
try {
feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult();
} catch (Exception ex) {
logger.Warn(ex, "Failed to fetch release feed.");
}
var latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full)
.OrderByDescending(a => a.Version)
.FirstOrDefault();
var newVer = latest != null ? new SemanticVersion(1, 0, latest.Version.Patch + 1) : new SemanticVersion(1, 0, 0);
// create repo
var repo = new S3Repository(logger);
var options = new S3UploadOptions {
ReleaseDir = new DirectoryInfo(releaseDir),
Bucket = B2_BUCKET,
Channel = channel,
Endpoint = "https://" + B2_ENDPOINT,
KeyId = B2_KEYID,
Secret = B2_SECRET,
KeepMaxReleases = 4,
};
// download latest version and create delta
repo.DownloadLatestFullPackageAsync(options).GetAwaiterResult();
var id = "B2TestApp";
TestApp.PackTestApp(id, newVer.ToFullString(), $"b2-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: channel);
if (latest != null) {
// check delta was created
Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created.");
}
// upload new files
repo.UploadMissingAssetsAsync(options).GetAwaiterResult();
// verify that new version has been uploaded
feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult();
latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full)
.OrderByDescending(a => a.Version)
.FirstOrDefault();
Assert.True(latest != null, "No latest version found.");
Assert.Equal(newVer, latest.Version);
Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept.");
}
}