From ec7ea0196dc4c663db7a47708778841d9c9fb179 Mon Sep 17 00:00:00 2001 From: Caelan Sayler Date: Sun, 1 Dec 2024 21:10:41 +0000 Subject: [PATCH] Combine AZ and B2 test together --- .../Velopack.Deployment/AzureRepository.cs | 45 ++++--- .../Velopack.Deployment/GitHubRepository.cs | 2 - .../Velopack.Deployment/GiteaRepository.cs | 2 - src/vpk/Velopack.Deployment/HttpRepository.cs | 2 - .../Velopack.Deployment/LocalRepository.cs | 3 +- src/vpk/Velopack.Deployment/S3Repository.cs | 4 +- src/vpk/Velopack.Deployment/_Repository.cs | 10 +- src/vpk/Velopack.Vpk/OptionMapper.cs | 24 +++- .../AzureDeploymentTests.cs | 86 ------------ .../DeploymentTests.cs | 126 ++++++++++++++++++ .../S3DeploymentTests.cs | 81 ----------- 11 files changed, 183 insertions(+), 202 deletions(-) delete mode 100644 test/Velopack.Packaging.Tests/AzureDeploymentTests.cs create mode 100644 test/Velopack.Packaging.Tests/DeploymentTests.cs delete mode 100644 test/Velopack.Packaging.Tests/S3DeploymentTests.cs diff --git a/src/vpk/Velopack.Deployment/AzureRepository.cs b/src/vpk/Velopack.Deployment/AzureRepository.cs index 2930646b..ff0a3aa0 100644 --- a/src/vpk/Velopack.Deployment/AzureRepository.cs +++ b/src/vpk/Velopack.Deployment/AzureRepository.cs @@ -18,8 +18,6 @@ public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions public string Container { get; set; } public string SasToken { get; set; } - - public double Timeout { get; set; } } public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions @@ -51,37 +49,44 @@ public class AzureRepository : ObjectRepository { - await client.DeleteBlobIfExistsAsync(key); - }, "Deleting " + key); + await RetryAsync( + async () => { + await client.DeleteBlobIfExistsAsync(key); + }, + "Deleting " + key); } protected override async Task GetObjectBytes(BlobContainerClient client, string key) { - return await RetryAsyncRet(async () => { - try { - var obj = client.GetBlobClient(key); - var ms = new MemoryStream(); - using var response = await obj.DownloadToAsync(ms, CancellationToken.None); - return ms.ToArray(); - } catch (Azure.RequestFailedException ex) when (ex.Status == 404) { - return null; - } - }, $"Downloading {key}..."); + return await RetryAsyncRet( + async () => { + try { + var obj = client.GetBlobClient(key); + var ms = new MemoryStream(); + using var response = await obj.DownloadToAsync(ms, CancellationToken.None); + return ms.ToArray(); + } catch (Azure.RequestFailedException ex) when (ex.Status == 404) { + return null; + } + }, + $"Downloading {key}..."); } protected override async Task SaveEntryToFileAsync(AzureDownloadOptions options, VelopackAsset entry, string filePath) { - await RetryAsync(async () => { - var client = CreateClient(options); - var obj = client.GetBlobClient(entry.FileName); - using var response = await obj.DownloadToAsync(filePath, CancellationToken.None); - }, $"Downloading {entry.FileName}..."); + await RetryAsync( + async () => { + var client = CreateClient(options); + var obj = client.GetBlobClient(entry.FileName); + using var response = await obj.DownloadToAsync(filePath, CancellationToken.None); + }, + $"Downloading {entry.FileName}..."); } protected override async Task UploadObject(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote, bool noCache) diff --git a/src/vpk/Velopack.Deployment/GitHubRepository.cs b/src/vpk/Velopack.Deployment/GitHubRepository.cs index 488e0d0c..2f6d85fa 100644 --- a/src/vpk/Velopack.Deployment/GitHubRepository.cs +++ b/src/vpk/Velopack.Deployment/GitHubRepository.cs @@ -16,8 +16,6 @@ public class GitHubDownloadOptions : RepositoryOptions public string RepoUrl { get; set; } public string Token { get; set; } - - public double Timeout { get; set; } } public class GitHubUploadOptions : GitHubDownloadOptions diff --git a/src/vpk/Velopack.Deployment/GiteaRepository.cs b/src/vpk/Velopack.Deployment/GiteaRepository.cs index aa5fc7a9..ad25b27a 100644 --- a/src/vpk/Velopack.Deployment/GiteaRepository.cs +++ b/src/vpk/Velopack.Deployment/GiteaRepository.cs @@ -20,8 +20,6 @@ public class GiteaDownloadOptions : RepositoryOptions public string Token { get; set; } - public double Timeout { get; set; } - ///// ///// Example https://gitea.com ///// diff --git a/src/vpk/Velopack.Deployment/HttpRepository.cs b/src/vpk/Velopack.Deployment/HttpRepository.cs index 15001111..bdbf0700 100644 --- a/src/vpk/Velopack.Deployment/HttpRepository.cs +++ b/src/vpk/Velopack.Deployment/HttpRepository.cs @@ -6,8 +6,6 @@ namespace Velopack.Deployment; public class HttpDownloadOptions : RepositoryOptions { public string Url { get; set; } - - public double Timeout { get; set; } } public class HttpRepository : SourceRepository diff --git a/src/vpk/Velopack.Deployment/LocalRepository.cs b/src/vpk/Velopack.Deployment/LocalRepository.cs index 8289ced3..7a58fc7e 100644 --- a/src/vpk/Velopack.Deployment/LocalRepository.cs +++ b/src/vpk/Velopack.Deployment/LocalRepository.cs @@ -13,6 +13,7 @@ public class LocalDownloadOptions : RepositoryOptions, IObjectDownloadOptions public class LocalUploadOptions : LocalDownloadOptions, IObjectUploadOptions { public bool ForceRegenerate { get; set; } + public int KeepMaxReleases { get; set; } } @@ -75,4 +76,4 @@ public class LocalRepository(ILogger logger) : ObjectRepository where TUp : RepositoryOptions @@ -37,7 +39,8 @@ public abstract class SourceRepository : DownRepository { public SourceRepository(ILogger logger) : base(logger) - { } + { + } protected override Task GetReleasesAsync(TDown options) { @@ -104,8 +107,7 @@ public abstract class DownRepository : IRepositoryCanDownload Log.Error($"Checksum mismatch, expected {latest.SHA256}, got {newHash}"); return; } - } - else if (latest.SHA1 != (newHash = IoUtil.CalculateFileSHA1(incomplete))) { + } else if (latest.SHA1 != (newHash = IoUtil.CalculateFileSHA1(incomplete))) { Log.Error($"Checksum mismatch, expected {latest.SHA1}, got {newHash}"); return; } @@ -156,4 +158,4 @@ public abstract class DownRepository : IRepositoryCanDownload } } } -} +} \ No newline at end of file diff --git a/src/vpk/Velopack.Vpk/OptionMapper.cs b/src/vpk/Velopack.Vpk/OptionMapper.cs index 253c0d59..91b99e15 100644 --- a/src/vpk/Velopack.Vpk/OptionMapper.cs +++ b/src/vpk/Velopack.Vpk/OptionMapper.cs @@ -17,25 +17,47 @@ public static partial class OptionMapper public static partial TDest Map(object source); public static partial OsxPackOptions ToOptions(this OsxPackCommand cmd); + public static partial WindowsPackOptions ToOptions(this WindowsPackCommand cmd); + public static partial LinuxPackOptions ToOptions(this LinuxPackCommand cmd); + public static partial OsxBundleOptions ToOptions(this OsxBundleCommand cmd); + public static partial GitHubDownloadOptions ToOptions(this GitHubDownloadCommand cmd); + public static partial GitHubUploadOptions ToOptions(this GitHubUploadCommand cmd); + public static partial GiteaDownloadOptions ToOptions(this GiteaDownloadCommand cmd); + public static partial GiteaUploadOptions ToOptions(this GiteaUploadCommand cmd); + public static partial HttpDownloadOptions ToOptions(this HttpDownloadCommand cmd); + + [MapperIgnoreTarget(nameof(LocalDownloadOptions.Timeout))] public static partial LocalDownloadOptions ToOptions(this LocalDownloadCommand cmd); + + [MapperIgnoreTarget(nameof(LocalDownloadOptions.Timeout))] public static partial LocalUploadOptions ToOptions(this LocalUploadCommand cmd); + public static partial S3DownloadOptions ToOptions(this S3DownloadCommand cmd); + public static partial S3UploadOptions ToOptions(this S3UploadCommand cmd); + public static partial AzureDownloadOptions ToOptions(this AzureDownloadCommand cmd); + public static partial AzureUploadOptions ToOptions(this AzureUploadCommand cmd); + public static partial DeltaGenOptions ToOptions(this DeltaGenCommand cmd); + public static partial DeltaPatchOptions ToOptions(this DeltaPatchCommand cmd); + public static partial LoginOptions ToOptions(this LoginCommand cmd); + public static partial LogoutOptions ToOptions(this LogoutCommand cmd); + public static partial PublishOptions ToOptions(this PublishCommand cmd); + public static partial ApiOptions ToOptions(this ApiCommand cmd); private static DirectoryInfo StringToDirectoryInfo(string t) @@ -51,4 +73,4 @@ public static partial class OptionMapper if (t == null) return null; return RID.Parse(t); } -} +} \ No newline at end of file diff --git a/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs b/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs deleted file mode 100644 index 4d36b54c..00000000 --- a/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs +++ /dev/null @@ -1,86 +0,0 @@ -using NuGet.Versioning; -using Velopack.Deployment; -using Velopack.Sources; -using Velopack.Util; - -namespace Velopack.Packaging.Tests; - -public class AzureDeploymentTests -{ - public readonly static string AZ_ACCOUNT = "velopacktesting"; - public readonly static string AZ_KEY = Environment.GetEnvironmentVariable("VELOPACK_AZ_TEST_TOKEN"); - public readonly static string AZ_CONTAINER = "ci-deployment"; - public readonly static string AZ_ENDPOINT = "velopacktesting.blob.core.windows.net"; - - private readonly ITestOutputHelper _output; - - public AzureDeploymentTests(ITestOutputHelper output) - { - _output = output; - } - - [SkippableFact] - public void CanDeployToAzure() - { - Skip.If(String.IsNullOrWhiteSpace(AZ_KEY), "VELOPACK_AZ_TEST_TOKEN is not set."); - using var logger = _output.BuildLoggerFor(); - using var _1 = TempUtil.GetTempDirectory(out var releaseDir); - - string channel = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) - ? VelopackRuntimeInfo.SystemOs.GetOsShortName() - : "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName(); - - // get latest version, and increment patch by one - var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}"; - var source = new SimpleWebSource(updateUrl); - VelopackAssetFeed feed = new VelopackAssetFeed(); - try { - feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); - } catch (Exception ex) { - logger.Warn(ex, "Failed to fetch release feed."); - } - var latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) - .OrderByDescending(a => a.Version) - .FirstOrDefault(); - var newVer = latest != null ? new SemanticVersion(1, 0, latest.Version.Patch + 1) : new SemanticVersion(1, 0, 0); - - // create repo - var repo = new AzureRepository(logger); - var options = new AzureUploadOptions { - ReleaseDir = new DirectoryInfo(releaseDir), - Container = AZ_CONTAINER, - Channel = channel, - Account = AZ_ACCOUNT, - Key = AZ_KEY, - KeepMaxReleases = 4, - }; - - // download latest version and create delta - repo.DownloadLatestFullPackageAsync(options).GetAwaiterResult(); - - if (options.ReleaseDir.EnumerateFiles("*.incomplete").Any()) { - logger.Warn("A previous package was not downloaded, probably due to invalid checksum. This is a race condition in this test."); - latest = null; - } - - var id = "AZTestApp"; - TestApp.PackTestApp(id, newVer.ToFullString(), $"az-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: channel); - if (latest != null) { - // check delta was created - Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created."); - } - - // upload new files - repo.UploadMissingAssetsAsync(options).GetAwaiterResult(); - - // verify that new version has been uploaded - feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); - latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) - .OrderByDescending(a => a.Version) - .FirstOrDefault(); - - Assert.True(latest != null, "No latest version found."); - Assert.Equal(newVer, latest.Version); - Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept."); - } -} diff --git a/test/Velopack.Packaging.Tests/DeploymentTests.cs b/test/Velopack.Packaging.Tests/DeploymentTests.cs new file mode 100644 index 00000000..d128b7f5 --- /dev/null +++ b/test/Velopack.Packaging.Tests/DeploymentTests.cs @@ -0,0 +1,126 @@ +using Azure.Storage.Blobs; +using NuGet.Versioning; +using Velopack.Deployment; +using Velopack.Sources; +using Velopack.Util; + +namespace Velopack.Packaging.Tests; + +public class DeploymentTests +{ + private static string CHANNEL = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) + ? VelopackRuntimeInfo.SystemOs.GetOsShortName() + : "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName(); + + private static readonly string B2_KEYID = "0035016844a4188000000000b"; + private static readonly string B2_SECRET = Environment.GetEnvironmentVariable("VELOPACK_B2_TEST_TOKEN"); + private static readonly string B2_BUCKET = "velopack-testing"; + private static readonly string B2_ENDPOINT = "s3.eu-central-003.backblazeb2.com"; + + private static readonly string AZ_ACCOUNT = "velopacktesting"; + private static readonly string AZ_KEY = Environment.GetEnvironmentVariable("VELOPACK_AZ_TEST_TOKEN"); + private static readonly string AZ_CONTAINER = "ci-deployment"; + private static readonly string AZ_ENDPOINT = "velopacktesting.blob.core.windows.net"; + + private readonly ITestOutputHelper _output; + + public DeploymentTests(ITestOutputHelper output) + { + _output = output; + } + + [SkippableFact] + public async Task CanDeployToBackBlazeB2() + { + Skip.If(String.IsNullOrWhiteSpace(B2_SECRET), "VELOPACK_B2_TEST_TOKEN is not set."); + using var logger = _output.BuildLoggerFor(); + using var _1 = TempUtil.GetTempDirectory(out var releaseDir); + + var repo = new S3Repository(logger); + var options = new S3UploadOptions { + ReleaseDir = new DirectoryInfo(releaseDir), + Bucket = B2_BUCKET, + Channel = CHANNEL, + Endpoint = "https://" + B2_ENDPOINT, + KeyId = B2_KEYID, + Secret = B2_SECRET, + KeepMaxReleases = 4, + }; + + var updateUrl = $"https://{B2_BUCKET}.{B2_ENDPOINT}/"; + await Deploy("B2TestApp", repo, options, releaseDir, updateUrl, logger); + } + + [SkippableFact] + public async Task CanDeployToAzure() + { + Skip.If(String.IsNullOrWhiteSpace(AZ_KEY), "VELOPACK_AZ_TEST_TOKEN is not set."); + using var logger = _output.BuildLoggerFor(); + using var _1 = TempUtil.GetTempDirectory(out var releaseDir); + + var repo = new AzureRepository(logger); + var options = new AzureUploadOptions { + ReleaseDir = new DirectoryInfo(releaseDir), + Container = AZ_CONTAINER, + Channel = CHANNEL, + Account = AZ_ACCOUNT, + Key = AZ_KEY, + KeepMaxReleases = 4, + }; + + var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}"; + await Deploy("AZTestApp", repo, options, releaseDir, updateUrl, logger); + } + + static SemanticVersion GenerateSemverFromDateTime() + { + DateTime now = DateTime.Now; + int major = now.Year; // YYYY + int minor = now.Month * 100 + now.Day; // MMDD + int patch = now.Hour * 3600 + now.Minute * 60 + now.Second; // Seconds of the day + return new SemanticVersion(major, minor, patch); + } + + private async Task Deploy(string id, TRepo repo, TUp options, string releaseDir, string updateUrl, ILogger logger) + where TDown : RepositoryOptions, IObjectDownloadOptions + where TUp : IObjectUploadOptions, TDown + where TRepo : ObjectRepository + { + var targetVer = GenerateSemverFromDateTime(); + logger.Info($"Target version: {targetVer}"); + + // get the latest + var source = new SimpleWebSource(updateUrl); + VelopackAssetFeed feed = new VelopackAssetFeed(); + try { + feed = await source.GetReleaseFeed(logger, CHANNEL); + } catch (Exception ex) { + logger.Warn(ex, "Failed to fetch release feed."); + } + + var latestOnline = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full).MaxBy(a => a.Version); + if (latestOnline != null) { + logger.Info($"Latest online version: {latestOnline.Version}"); + Assert.True(targetVer > latestOnline.Version, "New version is not greater than the latest online version."); + } + + // download latest version and create delta + await repo.DownloadLatestFullPackageAsync(options); + TestApp.PackTestApp(id, targetVer.ToFullString(), $"b2-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: CHANNEL); + if (latestOnline != null) { + // check delta was created + Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created."); + } + + // upload new files + await repo.UploadMissingAssetsAsync(options); + + // verify that new version has been uploaded + feed = await source.GetReleaseFeed(logger, CHANNEL); + latestOnline = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full).MaxBy(a => a.Version); + + Assert.True(latestOnline != null, "No latest version found."); + Assert.Equal(targetVer, latestOnline.Version); + Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept."); + } +} \ No newline at end of file diff --git a/test/Velopack.Packaging.Tests/S3DeploymentTests.cs b/test/Velopack.Packaging.Tests/S3DeploymentTests.cs deleted file mode 100644 index ab2fb873..00000000 --- a/test/Velopack.Packaging.Tests/S3DeploymentTests.cs +++ /dev/null @@ -1,81 +0,0 @@ -using NuGet.Versioning; -using Velopack.Deployment; -using Velopack.Sources; -using Velopack.Util; - -namespace Velopack.Packaging.Tests; - -public class S3DeploymentTests -{ - public readonly static string B2_KEYID = "0035016844a4188000000000a"; - public readonly static string B2_SECRET = Environment.GetEnvironmentVariable("VELOPACK_B2_TEST_TOKEN"); - public readonly static string B2_BUCKET = "velopack-testing"; - public readonly static string B2_ENDPOINT = "s3.eu-central-003.backblazeb2.com"; - - private readonly ITestOutputHelper _output; - - public S3DeploymentTests(ITestOutputHelper output) - { - _output = output; - } - - [SkippableFact] - public void CanDeployToBackBlazeB2() - { - Skip.If(String.IsNullOrWhiteSpace(B2_SECRET), "VELOPACK_B2_TEST_TOKEN is not set."); - using var logger = _output.BuildLoggerFor(); - using var _1 = TempUtil.GetTempDirectory(out var releaseDir); - - string channel = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) - ? VelopackRuntimeInfo.SystemOs.GetOsShortName() - : "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName(); - - // get latest version, and increment patch by one - var updateUrl = $"https://{B2_BUCKET}.{B2_ENDPOINT}/"; - var source = new SimpleWebSource(updateUrl); - VelopackAssetFeed feed = new VelopackAssetFeed(); - try { - feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); - } catch (Exception ex) { - logger.Warn(ex, "Failed to fetch release feed."); - } - var latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) - .OrderByDescending(a => a.Version) - .FirstOrDefault(); - var newVer = latest != null ? new SemanticVersion(1, 0, latest.Version.Patch + 1) : new SemanticVersion(1, 0, 0); - - // create repo - var repo = new S3Repository(logger); - var options = new S3UploadOptions { - ReleaseDir = new DirectoryInfo(releaseDir), - Bucket = B2_BUCKET, - Channel = channel, - Endpoint = "https://" + B2_ENDPOINT, - KeyId = B2_KEYID, - Secret = B2_SECRET, - KeepMaxReleases = 4, - }; - - // download latest version and create delta - repo.DownloadLatestFullPackageAsync(options).GetAwaiterResult(); - var id = "B2TestApp"; - TestApp.PackTestApp(id, newVer.ToFullString(), $"b2-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: channel); - if (latest != null) { - // check delta was created - Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created."); - } - - // upload new files - repo.UploadMissingAssetsAsync(options).GetAwaiterResult(); - - // verify that new version has been uploaded - feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); - latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) - .OrderByDescending(a => a.Version) - .FirstOrDefault(); - - Assert.True(latest != null, "No latest version found."); - Assert.Equal(newVer, latest.Version); - Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept."); - } -}