diff --git a/src/Velopack.Deployment/AzureRepository.cs b/src/Velopack.Deployment/AzureRepository.cs new file mode 100644 index 00000000..0f3b5a3b --- /dev/null +++ b/src/Velopack.Deployment/AzureRepository.cs @@ -0,0 +1,173 @@ +using System.Net; +using System.Text; +using Azure; +using Azure.Storage; +using Azure.Storage.Blobs; +using Microsoft.Extensions.Logging; +using Velopack.Packaging; + +namespace Velopack.Deployment; + +public class AzureDownloadOptions : RepositoryOptions +{ + public string Account { get; set; } + + public string Key { get; set; } + + public string Endpoint { get; set; } + + public string Container { get; set; } +} + +public class AzureUploadOptions : AzureDownloadOptions +{ + public int KeepMaxReleases { get; set; } +} + +public class AzureRepository : DownRepository, IRepositoryCanUpload +{ + public AzureRepository(ILogger logger) : base(logger) + { + } + + public async Task UploadMissingAssetsAsync(AzureUploadOptions options) + { + var build = BuildAssets.Read(options.ReleaseDir.FullName, options.Channel); + var client = GetBlobContainerClient(options); + + Log.Info($"Preparing to upload {build.Files.Count} local assets to Azure endpoint {options.Endpoint ?? ""}"); + + var remoteReleases = await GetReleasesAsync(options); + Log.Info($"There are {remoteReleases.Assets.Length} assets in remote RELEASES file."); + + var localEntries = build.GetReleaseEntries(); + var releaseEntries = ReleaseEntryHelper.MergeAssets(localEntries, remoteReleases.Assets).ToArray(); + + Log.Info($"{releaseEntries.Length} merged local/remote releases."); + + VelopackAsset[] toDelete = new VelopackAsset[0]; + + if (options.KeepMaxReleases > 0) { + var fullReleases = releaseEntries + .OrderByDescending(x => x.Version) + .Where(x => x.Type == VelopackAssetType.Full) + .ToArray(); + if (fullReleases.Length > options.KeepMaxReleases) { + var minVersion = fullReleases[options.KeepMaxReleases - 1].Version; + toDelete = releaseEntries + .Where(x => x.Version < minVersion) + .ToArray(); + releaseEntries = releaseEntries.Except(toDelete).ToArray(); + Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will delete {toDelete.Length} releases."); + } else { + Log.Info($"Retention policy (keepMaxReleases={options.KeepMaxReleases}) will not be applied, because there will only be {fullReleases.Length} full releases when this upload has completed."); + } + } + + foreach (var asset in build.Files) { + await UploadFile(client, Path.GetFileName(asset), new FileInfo(asset), true); + } + + using var _1 = Utility.GetTempFileName(out var tmpReleases); + File.WriteAllText(tmpReleases, ReleaseEntryHelper.GetAssetFeedJson(new VelopackAssetFeed { Assets = releaseEntries })); + var releasesName = Utility.GetVeloReleaseIndexName(options.Channel); + await UploadFile(client, releasesName, new FileInfo(tmpReleases), true); + +#pragma warning disable CS0612 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete + var legacyKey = Utility.GetReleasesFileName(options.Channel); + using var _2 = Utility.GetTempFileName(out var tmpReleases2); + using (var fs = File.Create(tmpReleases2)) { + ReleaseEntry.WriteReleaseFile(releaseEntries.Select(ReleaseEntry.FromVelopackAsset), fs); + } + await UploadFile(client, legacyKey, new FileInfo(tmpReleases2), true); +#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning restore CS0612 // Type or member is obsolete + + if (toDelete.Length > 0) { + Log.Info($"Retention policy about to delete {toDelete.Length} releases..."); + foreach (var del in toDelete) { + await RetryAsync(() => client.DeleteBlobIfExistsAsync(del.FileName), "Deleting " + del.FileName); + } + } + + Log.Info("Done."); + } + + protected override async Task GetReleasesAsync(AzureDownloadOptions options) + { + var releasesName = Utility.GetVeloReleaseIndexName(options.Channel); + var client = GetBlobContainerClient(options); + + var ms = new MemoryStream(); + + try { + await RetryAsync(async () => { + var obj = client.GetBlobClient(releasesName); + using var response = await obj.DownloadToAsync(ms); + }, $"Fetching {releasesName}..."); + } catch (RequestFailedException ex) when (ex.Status == 404) { + return new VelopackAssetFeed(); + } + + return VelopackAssetFeed.FromJson(Encoding.UTF8.GetString(ms.ToArray())); + } + + protected override async Task SaveEntryToFileAsync(AzureDownloadOptions options, VelopackAsset entry, string filePath) + { + var client = GetBlobContainerClient(options); + await RetryAsync(async () => { + var obj = client.GetBlobClient(entry.FileName); + using var response = await obj.DownloadToAsync(filePath, CancellationToken.None); + }, $"Downloading {entry.FileName}..."); + } + + private static BlobServiceClient GetBlobServiceClient(AzureDownloadOptions options) + { + return new BlobServiceClient(new Uri(options.Endpoint), new StorageSharedKeyCredential(options.Account, options.Key)); + } + + private static BlobContainerClient GetBlobContainerClient(AzureDownloadOptions options) + { + var client = GetBlobServiceClient(options); + var containerClient = client.GetBlobContainerClient(options.Container); + return containerClient; + } + + private async Task UploadFile(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote) + { + // try to detect an existing remote file of the same name + var blobClient = client.GetBlobClient(key); + try { + var properties = await blobClient.GetPropertiesAsync(); + var md5 = GetFileMD5Checksum(f.FullName); + var stored = properties.Value.ContentHash; + + if (stored != null) { + if (Enumerable.SequenceEqual(md5, stored)) { + Log.Info($"Upload file '{key}' skipped (already exists in remote)"); + return; + } else if (overwriteRemote) { + Log.Info($"File '{key}' exists in remote, replacing..."); + } else { + Log.Warn($"File '{key}' exists in remote and checksum does not match local file. Use 'overwrite' argument to replace remote file."); + return; + } + } + } catch { + // don't care if this check fails. worst case, we end up re-uploading a file that + // already exists. storage providers should prefer the newer file of the same name. + } + + await RetryAsync(() => blobClient.UploadAsync(f.FullName, overwriteRemote), "Uploading " + key); + } + + private static byte[] GetFileMD5Checksum(string filePath) + { + var sha = System.Security.Cryptography.MD5.Create(); + byte[] checksum; + using (var fs = File.OpenRead(filePath)) + checksum = sha.ComputeHash(fs); + return checksum; + } +} \ No newline at end of file diff --git a/src/Velopack.Deployment/Velopack.Deployment.csproj b/src/Velopack.Deployment/Velopack.Deployment.csproj index 2295154d..dc7256f9 100644 --- a/src/Velopack.Deployment/Velopack.Deployment.csproj +++ b/src/Velopack.Deployment/Velopack.Deployment.csproj @@ -8,6 +8,7 @@ + diff --git a/src/Velopack.Vpk/Commands/AzureBaseCommand.cs b/src/Velopack.Vpk/Commands/AzureBaseCommand.cs new file mode 100644 index 00000000..41bf1146 --- /dev/null +++ b/src/Velopack.Vpk/Commands/AzureBaseCommand.cs @@ -0,0 +1,63 @@ + +namespace Velopack.Vpk.Commands; + +public class AzureBaseCommand : OutputCommand +{ + public string Account { get; private set; } + + public string Key { get; private set; } + + public string Endpoint { get; private set; } + + public string Container { get; private set; } + + protected AzureBaseCommand(string name, string description) + : base(name, description) + { + AddOption((v) => Account = v, "--account") + .SetDescription("Account name") + .SetArgumentHelpName("ACCOUNT") + .SetRequired(); + + AddOption((v) => Key = v, "--key") + .SetDescription("Account secret key") + .SetArgumentHelpName("KEY") + .SetRequired(); + + AddOption((v) => Container = v, "--container") + .SetDescription("Azure container name") + .SetArgumentHelpName("CONTAINER") + .SetRequired(); + + AddOption((v) => Endpoint = v.ToAbsoluteOrNull(), "--endpoint") + .SetDescription("Service url (eg. https://.blob.core.windows.net)") + .SetArgumentHelpName("URL") + .MustBeValidHttpUri() + .SetRequired(); + + } +} + +public class AzureDownloadCommand : AzureBaseCommand +{ + public AzureDownloadCommand() + : base("az", "Download latest release from an AZ container.") + { + } +} + +public class AzureUploadCommand : AzureBaseCommand +{ + public int KeepMaxReleases { get; private set; } + + public AzureUploadCommand() + : base("az", "Upload releases to an Azure container.") + { + AddOption((x) => KeepMaxReleases = x, "--keepMaxReleases") + .SetDescription("The maximum number of releases to keep in the bucket, anything older will be deleted.") + .SetArgumentHelpName("COUNT"); + + ReleaseDirectoryOption.SetRequired(); + ReleaseDirectoryOption.MustNotBeEmpty(); + } +} diff --git a/src/Velopack.Vpk/OptionMapper.cs b/src/Velopack.Vpk/OptionMapper.cs index 49530662..56116143 100644 --- a/src/Velopack.Vpk/OptionMapper.cs +++ b/src/Velopack.Vpk/OptionMapper.cs @@ -24,6 +24,8 @@ public static partial class OptionMapper public static partial LocalDownloadOptions ToOptions(this LocalDownloadCommand cmd); public static partial S3DownloadOptions ToOptions(this S3DownloadCommand cmd); public static partial S3UploadOptions ToOptions(this S3UploadCommand cmd); + public static partial AzureDownloadOptions ToOptions(this AzureDownloadCommand cmd); + public static partial AzureUploadOptions ToOptions(this AzureUploadCommand cmd); public static partial DeltaGenOptions ToOptions(this DeltaGenCommand cmd); public static partial DeltaPatchOptions ToOptions(this DeltaPatchCommand cmd); public static partial LoginOptions ToOptions(this LoginCommand cmd); diff --git a/src/Velopack.Vpk/Program.cs b/src/Velopack.Vpk/Program.cs index 1583842c..ffb94e7c 100644 --- a/src/Velopack.Vpk/Program.cs +++ b/src/Velopack.Vpk/Program.cs @@ -84,6 +84,7 @@ public class Program var downloadCommand = new CliCommand("download", "Download's the latest release from a remote update source."); downloadCommand.AddRepositoryDownload(provider); downloadCommand.AddRepositoryDownload(provider); + downloadCommand.AddRepositoryDownload(provider); downloadCommand.AddRepositoryDownload(provider); downloadCommand.AddRepositoryDownload(provider); rootCommand.Add(downloadCommand); @@ -91,6 +92,7 @@ public class Program var uploadCommand = new CliCommand("upload", "Upload local package(s) to a remote update source."); uploadCommand.AddRepositoryUpload(provider); uploadCommand.AddRepositoryUpload(provider); + uploadCommand.AddRepositoryUpload(provider); rootCommand.Add(uploadCommand); var deltaCommand = new CliCommand("delta", "Utilities for creating or applying delta packages."); diff --git a/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs b/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs new file mode 100644 index 00000000..f721e7ef --- /dev/null +++ b/test/Velopack.CommandLine.Tests/Commands/AzureCommandTests.cs @@ -0,0 +1,42 @@ +using System.CommandLine; +using Velopack.Vpk.Commands; + +namespace Velopack.CommandLine.Tests.Commands; + +public abstract class AzureCommandTests : BaseCommandTests + where T : AzureBaseCommand, new() +{ + [Fact] + public void Command_WithRequiredEndpointOptions_ParsesValue() + { + AzureBaseCommand command = new T(); + + string cli = $"--account \"account-name\" --key \"shhhh\" --endpoint \"https://endpoint\" --container \"mycontainer\""; + ParseResult parseResult = command.ParseAndApply(cli); + + Assert.Empty(parseResult.Errors); + Assert.Equal("account-name", command.Account); + Assert.Equal("shhhh", command.Key); + Assert.Equal("https://endpoint/", command.Endpoint); + Assert.Equal("mycontainer", command.Container); + } +} + +public class AzureDownloadCommandTests : AzureCommandTests +{ } + +public class AzureUploadCommandTests : AzureCommandTests +{ + public override bool ShouldBeNonEmptyReleaseDir => true; + + //[Fact] + //public void KeepMaxReleases_WithNumber_ParsesValue() + //{ + // var command = new S3UploadCommand(); + + // string cli = GetRequiredDefaultOptions() + "--keepMaxReleases 42"; + // ParseResult parseResult = command.ParseAndApply(cli); + + // Assert.Equal(42, command.KeepMaxReleases); + //} +} diff --git a/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs b/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs new file mode 100644 index 00000000..4d568ac7 --- /dev/null +++ b/test/Velopack.Packaging.Tests/AzureDeploymentTests.cs @@ -0,0 +1,80 @@ +using NuGet.Versioning; +using Velopack.Deployment; +using Velopack.Sources; + +namespace Velopack.Packaging.Tests; + +public class AzureDeploymentTests +{ + public readonly static string B2_KEYID = "xstg"; + public readonly static string B2_SECRET = Environment.GetEnvironmentVariable("VELOPACK_AZ_TEST_TOKEN"); + public readonly static string B2_BUCKET = "test-releases"; + public readonly static string B2_ENDPOINT = "xstg.blob.core.windows.net"; + + private readonly ITestOutputHelper _output; + + public AzureDeploymentTests(ITestOutputHelper output) + { + _output = output; + } + + [SkippableFact] + public void CanDeployToAzure() + { + Skip.If(String.IsNullOrWhiteSpace(B2_SECRET), "VELOPACK_AZ_TEST_TOKEN is not set."); + using var logger = _output.BuildLoggerFor(); + using var _1 = Utility.GetTempDirectory(out var releaseDir); + + string channel = String.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("CI")) + ? VelopackRuntimeInfo.SystemOs.GetOsShortName() + : "ci-" + VelopackRuntimeInfo.SystemOs.GetOsShortName(); + + // get latest version, and increment patch by one + var updateUrl = $"https://{B2_ENDPOINT}/{B2_BUCKET}"; + var source = new SimpleWebSource(updateUrl); + VelopackAssetFeed feed = new VelopackAssetFeed(); + try { + feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); + } catch (Exception ex) { + logger.Warn(ex, "Failed to fetch release feed."); + } + var latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) + .OrderByDescending(a => a.Version) + .FirstOrDefault(); + var newVer = latest != null ? new SemanticVersion(1, 0, latest.Version.Patch + 1) : new SemanticVersion(1, 0, 0); + + // create repo + var repo = new AzureRepository(logger); + var options = new AzureUploadOptions { + ReleaseDir = new DirectoryInfo(releaseDir), + Container = B2_BUCKET, + Channel = channel, + Endpoint = "https://" + B2_ENDPOINT, + Account = B2_KEYID, + Key = B2_SECRET, + KeepMaxReleases = 4, + }; + + // download latest version and create delta + repo.DownloadLatestFullPackageAsync(options).GetAwaiterResult(); + var id = "B2TestApp"; + TestApp.PackTestApp(id, newVer.ToFullString(), $"b2-{DateTime.UtcNow.ToLongDateString()}", releaseDir, logger, channel: channel); + if (latest != null) { + // check delta was created + Assert.True(Directory.EnumerateFiles(releaseDir, "*-delta.nupkg").Any(), "No delta package was created."); + } + + // upload new files + repo.UploadMissingAssetsAsync(options).GetAwaiterResult(); + + // verify that new version has been uploaded + feed = source.GetReleaseFeed(logger, channel).GetAwaiterResult(); + latest = feed.Assets.Where(a => a.Version != null && a.Type == VelopackAssetType.Full) + .OrderByDescending(a => a.Version) + .FirstOrDefault(); + + Assert.True(latest != null, "No latest version found."); + Assert.Equal(newVer, latest.Version); + Assert.True(feed.Assets.Count(x => x.Type == VelopackAssetType.Full) <= options.KeepMaxReleases, "Too many releases were kept."); + } +}