Fix azure folder uploading and align with s3 implementation

This commit is contained in:
Caelan Sayler
2025-09-24 23:56:33 +01:00
committed by Caelan
parent 095567789a
commit 1d5c984c14
8 changed files with 51 additions and 150 deletions

View File

@@ -21,7 +21,7 @@ public class AzureDownloadOptions : RepositoryOptions, IObjectDownloadOptions
public string SasToken { get; set; } public string SasToken { get; set; }
public string Folder { get; set; } public string Prefix { get; set; }
} }
public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions
@@ -29,13 +29,26 @@ public class AzureUploadOptions : AzureDownloadOptions, IObjectUploadOptions
public int KeepMaxReleases { get; set; } public int KeepMaxReleases { get; set; }
} }
public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploadOptions, BlobContainerClient> public class AzureBlobClient(BlobContainerClient client, string prefix)
{
public virtual Task DeleteBlobIfExistsAsync(string key, CancellationToken cancellationToken = default)
{
return client.DeleteBlobIfExistsAsync(prefix + key, cancellationToken: cancellationToken);
}
public virtual BlobClient GetBlobClient(string key)
{
return client.GetBlobClient(prefix + key);
}
}
public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploadOptions, AzureBlobClient>
{ {
public AzureRepository(ILogger logger) : base(logger) public AzureRepository(ILogger logger) : base(logger)
{ {
} }
protected override BlobContainerClient CreateClient(AzureDownloadOptions options) protected override AzureBlobClient CreateClient(AzureDownloadOptions options)
{ {
var serviceUrl = options.Endpoint ?? "https://" + options.Account + ".blob.core.windows.net"; var serviceUrl = options.Endpoint ?? "https://" + options.Account + ".blob.core.windows.net";
if (options.Endpoint == null) { if (options.Endpoint == null) {
@@ -54,16 +67,22 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
client = new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key), clientOptions); client = new BlobServiceClient(new Uri(serviceUrl), new StorageSharedKeyCredential(options.Account, options.Key), clientOptions);
} }
return client.GetBlobContainerClient(options.Container); var containerClient = client.GetBlobContainerClient(options.Container);
}
protected override async Task DeleteObject(BlobContainerClient client, string key) var prefix = options.Prefix?.Trim();
{ if (prefix == null) {
// Prepend folder path if specified (using _uploadFolder since this is called during upload) prefix = "";
if (!string.IsNullOrEmpty(_uploadFolder)) {
key = _uploadFolder.TrimEnd('/') + "/" + key;
} }
if (!string.IsNullOrEmpty(prefix) && !prefix.EndsWith("/")) {
prefix += "/";
}
return new AzureBlobClient(containerClient, prefix);
}
protected override async Task DeleteObject(AzureBlobClient client, string key)
{
await RetryAsync( await RetryAsync(
async () => { async () => {
await client.DeleteBlobIfExistsAsync(key); await client.DeleteBlobIfExistsAsync(key);
@@ -71,7 +90,7 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
"Deleting " + key); "Deleting " + key);
} }
protected override async Task<byte[]> GetObjectBytes(BlobContainerClient client, string key) protected override async Task<byte[]> GetObjectBytes(AzureBlobClient client, string key)
{ {
return await RetryAsyncRet( return await RetryAsyncRet(
async () => { async () => {
@@ -90,12 +109,6 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
protected override async Task<VelopackAssetFeed> GetReleasesAsync(AzureDownloadOptions options) protected override async Task<VelopackAssetFeed> GetReleasesAsync(AzureDownloadOptions options)
{ {
var releasesName = CoreUtil.GetVeloReleaseIndexName(options.Channel); var releasesName = CoreUtil.GetVeloReleaseIndexName(options.Channel);
// Prepend folder path if specified
if (!string.IsNullOrEmpty(options.Folder)) {
releasesName = options.Folder.TrimEnd('/') + "/" + releasesName;
}
var client = CreateClient(options); var client = CreateClient(options);
var bytes = await GetObjectBytes(client, releasesName); var bytes = await GetObjectBytes(client, releasesName);
if (bytes == null || bytes.Length == 0) { if (bytes == null || bytes.Length == 0) {
@@ -109,40 +122,15 @@ public class AzureRepository : ObjectRepository<AzureDownloadOptions, AzureUploa
await RetryAsync( await RetryAsync(
async () => { async () => {
var client = CreateClient(options); var client = CreateClient(options);
var key = entry.FileName; var obj = client.GetBlobClient(entry.FileName);
// Prepend folder path if specified
if (!string.IsNullOrEmpty(options.Folder)) {
key = options.Folder.TrimEnd('/') + "/" + key;
}
var obj = client.GetBlobClient(key);
using var response = await obj.DownloadToAsync(filePath, CancellationToken.None); using var response = await obj.DownloadToAsync(filePath, CancellationToken.None);
}, },
$"Downloading {entry.FileName}..."); $"Downloading {entry.FileName}...");
} }
public override async Task UploadMissingAssetsAsync(AzureUploadOptions options)
protected override async Task UploadObject(AzureBlobClient client, string key, FileInfo f, bool overwriteRemote, bool noCache)
{ {
// Store the folder in a private field for use in UploadObject
// Note: Azure Blob Storage will handle path validation and reject invalid paths
_uploadFolder = options.Folder;
try {
await base.UploadMissingAssetsAsync(options);
} finally {
_uploadFolder = null;
}
}
private string _uploadFolder;
protected override async Task UploadObject(BlobContainerClient client, string key, FileInfo f, bool overwriteRemote, bool noCache)
{
// Prepend folder path if specified
if (!string.IsNullOrEmpty(_uploadFolder)) {
key = _uploadFolder.TrimEnd('/') + "/" + key;
}
var blobClient = client.GetBlobClient(key); var blobClient = client.GetBlobClient(key);
try { try {
var properties = await blobClient.GetPropertiesAsync(); var properties = await blobClient.GetPropertiesAsync();

View File

@@ -12,31 +12,37 @@ public class AzureBaseCommand : OutputCommand
public string SasToken { get; private set; } public string SasToken { get; private set; }
public string Prefix { get; private set; }
public double Timeout { get; private set; } public double Timeout { get; private set; }
protected AzureBaseCommand(string name, string description) protected AzureBaseCommand(string name, string description)
: base(name, description) : base(name, description)
{ {
AddOption<string>((v) => Account = v, "--account") AddOption<string>((v) => Account = v, "--account")
.SetDescription("Account name") .SetDescription("Account name.")
.SetArgumentHelpName("ACCOUNT") .SetArgumentHelpName("ACCOUNT")
.SetRequired(); .SetRequired();
var key = AddOption<string>((v) => Key = v, "--key") var key = AddOption<string>((v) => Key = v, "--key")
.SetDescription("Account secret key") .SetDescription("Account secret key.")
.SetArgumentHelpName("KEY"); .SetArgumentHelpName("KEY");
var sas = AddOption<string>((v) => SasToken = v, "--sas") var sas = AddOption<string>((v) => SasToken = v, "--sas")
.SetDescription("Shared access signature token (not the url)") .SetDescription("Shared access signature token (not the url).")
.SetArgumentHelpName("TOKEN"); .SetArgumentHelpName("TOKEN");
AddOption<string>((v) => Container = v, "--container") AddOption<string>((v) => Container = v, "--container")
.SetDescription("Azure container name") .SetDescription("Azure storage container name.")
.SetArgumentHelpName("NAME") .SetArgumentHelpName("NAME")
.SetRequired(); .SetRequired();
AddOption<string>((v) => Prefix = v, "--prefix")
.SetDescription("Optional blob filename path prefix.")
.SetArgumentHelpName("PREFIX");
AddOption<Uri>((v) => Endpoint = v.ToAbsoluteOrNull(), "--endpoint") AddOption<Uri>((v) => Endpoint = v.ToAbsoluteOrNull(), "--endpoint")
.SetDescription("Service url (eg. https://<account-name>.blob.core.windows.net)") .SetDescription("Service url (eg. https://<account-name>.blob.core.windows.net).")
.SetArgumentHelpName("URL") .SetArgumentHelpName("URL")
.MustBeValidHttpUri(); .MustBeValidHttpUri();

View File

@@ -2,13 +2,8 @@
public class AzureDownloadCommand : AzureBaseCommand public class AzureDownloadCommand : AzureBaseCommand
{ {
public string Folder { get; private set; }
public AzureDownloadCommand() public AzureDownloadCommand()
: base("az", "Download latest release from an Azure Blob Storage container.") : base("az", "Download latest release from an Azure Blob Storage container.")
{ {
AddOption<string>((x) => Folder = x, "--folder")
.SetDescription("The folder path within the container where files are stored.")
.SetArgumentHelpName("PATH");
} }
} }

View File

@@ -4,8 +4,6 @@ public class AzureUploadCommand : AzureBaseCommand
{ {
public int KeepMaxReleases { get; private set; } public int KeepMaxReleases { get; private set; }
public string Folder { get; private set; }
public AzureUploadCommand() public AzureUploadCommand()
: base("az", "Upload releases to an Azure Blob Storage container.") : base("az", "Upload releases to an Azure Blob Storage container.")
{ {
@@ -13,10 +11,6 @@ public class AzureUploadCommand : AzureBaseCommand
.SetDescription("The maximum number of releases to keep in the container, anything older will be deleted.") .SetDescription("The maximum number of releases to keep in the container, anything older will be deleted.")
.SetArgumentHelpName("COUNT"); .SetArgumentHelpName("COUNT");
AddOption<string>((x) => Folder = x, "--folder")
.SetDescription("The folder path within the container where files should be uploaded.")
.SetArgumentHelpName("PATH");
ReleaseDirectoryOption.SetRequired(); ReleaseDirectoryOption.SetRequired();
ReleaseDirectoryOption.MustNotBeEmpty(); ReleaseDirectoryOption.MustNotBeEmpty();
} }

View File

@@ -55,7 +55,7 @@ public class S3BaseCommand : OutputCommand
.SetRequired(); .SetRequired();
AddOption<string>((v) => Prefix = v, "--prefix") AddOption<string>((v) => Prefix = v, "--prefix")
.SetDescription("Prefix to the S3 url.") .SetDescription("Optional filename path prefix.")
.SetArgumentHelpName("PREFIX"); .SetArgumentHelpName("PREFIX");
AddOption<bool>((v) => DisablePathStyle = v, "--disablePathStyle") AddOption<bool>((v) => DisablePathStyle = v, "--disablePathStyle")

View File

@@ -34,10 +34,10 @@ public class AzureDownloadCommandTests : AzureCommandTests<AzureDownloadCommand>
{ {
var command = new AzureDownloadCommand(); var command = new AzureDownloadCommand();
string cli = GetRequiredDefaultOptions() + " --folder \"releases/v1\""; string cli = GetRequiredDefaultOptions() + " --prefix \"releases/v1\"";
ParseResult parseResult = command.ParseAndApply(cli); ParseResult parseResult = command.ParseAndApply(cli);
Assert.Equal("releases/v1", command.Folder); Assert.Equal("releases/v1", command.Prefix);
} }
} }
@@ -55,10 +55,10 @@ public class AzureUploadCommandTests : AzureCommandTests<AzureUploadCommand>
{ {
var command = new AzureUploadCommand(); var command = new AzureUploadCommand();
string cli = GetRequiredDefaultOptions() + " --folder \"releases/v1\""; string cli = GetRequiredDefaultOptions() + " --prefix \"releases/v1\"";
ParseResult parseResult = command.ParseAndApply(cli); ParseResult parseResult = command.ParseAndApply(cli);
Assert.Equal("releases/v1", command.Folder); Assert.Equal("releases/v1", command.Prefix);
} }
} }

View File

@@ -1,82 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using Velopack.Deployment;
using Velopack.Core;
using Xunit;
using Microsoft.Extensions.Logging.Abstractions;
namespace Velopack.Packaging.Tests;
public class AzureFolderTests
{
[Fact]
public void AzureRepository_UploadWithFolder_PrependsPathToKeys()
{
// This test verifies that when a folder is specified,
// all blob keys (filenames) are prepended with the folder path
var options = new AzureUploadOptions
{
Folder = "releases/v1",
Account = "testaccount",
Key = "testkey",
Container = "testcontainer"
};
Assert.Equal("releases/v1", options.Folder);
}
[Fact]
public void AzureRepository_DownloadWithFolder_PrependsPathToKeys()
{
// This test verifies that download options can have a folder
var options = new AzureDownloadOptions
{
Folder = "releases/v1",
Account = "testaccount",
Key = "testkey",
Container = "testcontainer"
};
Assert.Equal("releases/v1", options.Folder);
}
[Fact]
public void AzureRepository_FolderPath_NormalizesSlashes()
{
// Test that trailing slashes are handled correctly
var options1 = new AzureUploadOptions { Folder = "releases/v1/" };
var options2 = new AzureUploadOptions { Folder = "releases/v1" };
// Both should work correctly when used
Assert.Equal("releases/v1/", options1.Folder);
Assert.Equal("releases/v1", options2.Folder);
}
[Theory]
[InlineData("releases/v1", "releases/v1/file.nupkg")]
[InlineData("releases/v1/", "releases/v1/file.nupkg")]
[InlineData("releases/v1//", "releases/v1/file.nupkg")]
[InlineData("/releases/v1", "/releases/v1/file.nupkg")]
[InlineData("", "file.nupkg")]
[InlineData(null, "file.nupkg")]
public void AzureRepository_FolderPath_ProducesCorrectBlobKey(string folder, string expectedKey)
{
// Test the logic we use to combine folder and filename
string filename = "file.nupkg";
string result;
if (!string.IsNullOrEmpty(folder))
{
result = folder.TrimEnd('/') + "/" + filename;
}
else
{
result = filename;
}
Assert.Equal(expectedKey, result);
}
}

View File

@@ -71,7 +71,7 @@ public class DeploymentTests
}; };
var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}"; var updateUrl = $"https://{AZ_ENDPOINT}/{AZ_CONTAINER}";
await Deploy<AzureRepository, AzureDownloadOptions, AzureUploadOptions, BlobContainerClient>("AZTestApp", repo, options, releaseDir, updateUrl, logger); await Deploy<AzureRepository, AzureDownloadOptions, AzureUploadOptions, AzureBlobClient>("AZTestApp", repo, options, releaseDir, updateUrl, logger);
} }
static SemanticVersion GenerateSemverFromDateTime() static SemanticVersion GenerateSemverFromDateTime()