Mirrors/velopack
1
0
Fork 0
mirror of https://github.com/velopack/velopack.git synced 2025-10-25 15:19:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch to using Zstd for delta patch/apply.

Browse Source
This commit is contained in:
Caelan Sayler
2023-12-30 13:28:20 +00:00
parent 1c7eb5e1f4
commit 1e08addb52
26 changed files with 343 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
src/Rust/Cargo.lock generated
View File

@@ -173,6 +173,7 @@ version = "1.0.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
dependencies = [
"jobserver",
"libc",
]
@@ -263,6 +264,7 @@ dependencies = [
"regex",
"remove_dir_all",
"semver",
"sha1_smol",
"simple-stopwatch",
"simplelog",
"strum",
@@ -276,6 +278,7 @@ dependencies = [
"winsafe",
"xml",
"zip",
"zstd",
]
[[package]]
@@ -728,6 +731,15 @@ version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]]
name = "jobserver"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
dependencies = [
"libc",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.0"
@@ -1328,6 +1340,12 @@ dependencies = [
"opaque-debug",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sha2"
version = "0.9.9"
@@ -2053,3 +2071,31 @@ dependencies = [
"crossbeam-utils",
"flate2",
]
[[package]]
name = "zstd"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.9+zstd.1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
dependencies = [
"cc",
"pkg-config",
]

2
src/Rust/Cargo.toml
View File

@@ -66,6 +66,8 @@ remove_dir_all = { git = "https://github.com/caesay/remove_dir_all.git", feature
"log",
] }
ntest = "0.9.0"
zstd = "0.13"
sha1_smol = "1.0.0"
[target.'cfg(target_os = "macos")'.dependencies]
native-dialog = "0.7"

3
src/Rust/src/commands/mod.rs
View File

@@ -1,6 +1,9 @@
mod apply;
pub use apply::*;
mod patch;
pub use patch::*;
#[cfg(target_os = "windows")]
mod start;
#[cfg(target_os = "windows")]

66
src/Rust/src/commands/patch.rs Normal file
View File

@@ -0,0 +1,66 @@
use anyhow::{bail, Result};
use std::{fs, io, path::PathBuf};
pub fn patch(old_file: &PathBuf, patch_file: &PathBuf, output_file: &PathBuf) -> Result<()> {
if !old_file.exists() {
bail!("Old file does not exist: {}", old_file.to_string_lossy());
}
if !patch_file.exists() {
bail!("Patch file does not exist: {}", patch_file.to_string_lossy());
}
let dict = fs::read(old_file)?;
let patch = fs::OpenOptions::new().read(true).open(patch_file)?;
let patch_reader = io::BufReader::new(patch);
let mut output = fs::OpenOptions::new().write(true).create(true).open(output_file)?;
let mut decoder = zstd::Decoder::with_dictionary(patch_reader, &dict)?;
info!("Dictionary Size: {}", dict.len());
info!("Decoder loaded. Beginning patch...");
io::copy(&mut decoder, &mut output)?;
Ok(())
}
#[test]
fn test_patch_apply() {
crate::logging::trace_logger();
let mut path = std::env::current_exe().unwrap();
path.pop();
path.pop();
path.pop();
path.pop();
path.pop();
path.pop();
path.push("test");
path.push("Squirrel.Tests");
path.push("fixtures");
info!("Path: {}", path.to_string_lossy());
let old_file = path.join("obs29.1.2.dll");
let new_file = path.join("obs30.0.2.dll");
let p1 = path.join("obs-size.patch");
let p2 = path.join("obs-speed.patch");
fn get_sha1(file: &PathBuf) -> String {
let file_bytes = fs::read(file).unwrap();
let mut sha1 = sha1_smol::Sha1::new();
sha1.update(&file_bytes);
sha1.digest().to_string()
}
let expected_sha1 = get_sha1(&new_file);
let tmp_file = std::path::Path::new("temp.patch").to_path_buf();
patch(&old_file, &p1, &tmp_file).unwrap();
let tmp_sha1 = get_sha1(&tmp_file);
fs::remove_file(&tmp_file).unwrap();
assert_eq!(expected_sha1, tmp_sha1);
patch(&old_file, &p2, &tmp_file).unwrap();
let tmp_sha1 = get_sha1(&tmp_file);
fs::remove_file(&tmp_file).unwrap();
assert_eq!(expected_sha1, tmp_sha1);
}

20
src/Rust/src/update.rs
View File

@@ -30,6 +30,12 @@ fn root_command() -> Command {
.arg(arg!(-p --package <FILE> "Update package to apply").value_parser(value_parser!(PathBuf)))
.arg(arg!([EXE_ARGS] "Arguments to pass to the started executable. Must be preceeded by '--'.").required(false).last(true).num_args(0..))
)
.subcommand(Command::new("patch")
.about("Applies a Zstd patch file")
.arg(arg!(--old <FILE> "Base / old file to apply the patch to").required(true).value_parser(value_parser!(PathBuf)))
.arg(arg!(--patch <FILE> "The Zstd patch to apply to the old file").required(true).value_parser(value_parser!(PathBuf)))
.arg(arg!(--output <FILE> "The file to create with the patch applied").required(true).value_parser(value_parser!(PathBuf)))
)
.arg(arg!(--verbose "Print debug messages to console / log").global(true))
.arg(arg!(--nocolor "Disable colored output").hide(true).global(true))
.arg(arg!(-s --silent "Don't show any prompts / dialogs").global(true))
@@ -103,6 +109,7 @@ fn main() -> Result<()> {
#[cfg(target_os = "windows")]
"start" => start(&subcommand_matches).map_err(|e| anyhow!("Start error: {}", e)),
"apply" => apply(subcommand_matches).map_err(|e| anyhow!("Apply error: {}", e)),
"patch" => patch(subcommand_matches).map_err(|e| anyhow!("Patch error: {}", e)),
_ => bail!("Unknown subcommand. Try `--help` for more information."),
};
@@ -114,6 +121,19 @@ fn main() -> Result<()> {
Ok(())
}
fn patch(matches: &ArgMatches) -> Result<()> {
let old_file = matches.get_one::<PathBuf>("old").unwrap();
let patch_file = matches.get_one::<PathBuf>("patch").unwrap();
let output_file = matches.get_one::<PathBuf>("output").unwrap();
info!("Command: Patch");
info!(" Old File: {:?}", old_file);
info!(" Patch File: {:?}", patch_file);
info!(" Output File: {:?}", output_file);
commands::patch(old_file, patch_file, output_file)
}
fn apply(matches: &ArgMatches) -> Result<()> {
let restart = matches.get_flag("restart");
let wait_for_parent = matches.get_flag("wait");

11
src/Squirrel.Csq/Commands/OsxReleasifyCommand.cs
View File

@@ -1,4 +1,6 @@
namespace Squirrel.Csq.Commands;
using Squirrel.Packaging;
namespace Squirrel.Csq.Commands;
public class OsxReleasifyCommand : BaseCommand
{
@@ -8,7 +10,7 @@ public class OsxReleasifyCommand : BaseCommand
public string ReleaseNotes { get; private set; }
public bool NoDelta { get; private set; }
public DeltaMode Delta { get; private set; }
public bool NoPackage { get; private set; }
@@ -48,8 +50,9 @@ public class OsxReleasifyCommand : BaseCommand
.SetArgumentHelpName("PATH")
.MustExist();
AddOption<bool>((v) => NoDelta = v, "--noDelta")
.SetDescription("Skip the generation of delta packages.");
AddOption<DeltaMode>((v) => Delta = v, "--delta")
.SetDefault(DeltaMode.BestSpeed)
.SetDescription("Set the delta generation mode.");
AddOption<string>((v) => Channel = v, "-c", "--channel")
.SetDescription("Release channel to use when creating the package.")

11
src/Squirrel.Csq/Commands/WindowsReleasifyCommand.cs
View File

@@ -1,10 +1,12 @@
namespace Squirrel.Csq.Commands;
using Squirrel.Packaging;
namespace Squirrel.Csq.Commands;
public class WindowsReleasifyCommand : WindowsSigningCommand
{
public string Package { get; set; }
public bool NoDelta { get; private set; }
public DeltaMode Delta { get; private set; }
public string Runtimes { get; private set; }
@@ -34,8 +36,9 @@ public class WindowsReleasifyCommand : WindowsSigningCommand
protected WindowsReleasifyCommand(string name, string description)
: base(name, description)
{
AddOption<bool>((v) => NoDelta = v, "--noDelta")
.SetDescription("Skip the generation of delta packages.");
AddOption<DeltaMode>((v) => Delta = v, "--delta")
.SetDefault(DeltaMode.BestSpeed)
.SetDescription("Set the delta generation mode.");
AddOption<string>((v) => Runtimes = v, "-f", "--framework")
.SetDescription("List of required runtimes to install during setup. example: 'net6,vcredist143'.")

6
src/Squirrel.Csq/Compat/EmbeddedRunner.cs
View File

@@ -40,7 +40,7 @@ public class EmbeddedRunner : ICommandRunner
ReleaseDir = command.GetReleaseDirectory(),
BundleDirectory = command.BundleDirectory,
IncludePdb = command.IncludePdb,
NoDelta = command.NoDelta,
DeltaMode = command.Delta,
NoPackage = command.NoPackage,
NotaryProfile = command.NotaryProfile,
PackageConclusion = command.PackageConclusion,
@@ -64,7 +64,7 @@ public class EmbeddedRunner : ICommandRunner
ReleaseDir = command.GetReleaseDirectory(),
Package = command.Package,
Icon = command.Icon,
NoDelta = command.NoDelta,
DeltaMode = command.Delta,
IncludePdb = command.IncludePdb,
SignParameters = command.SignParameters,
EntryExecutableName = command.EntryExecutableName,
@@ -91,7 +91,7 @@ public class EmbeddedRunner : ICommandRunner
ReleaseDir = command.GetReleaseDirectory(),
Package = command.Package,
Icon = command.Icon,
NoDelta = command.NoDelta,
DeltaMode = command.Delta,
SignParameters = command.SignParameters,
EntryExecutableName = command.EntryExecutableName,
Runtimes = command.Runtimes,

4
src/Squirrel.Csq/Compat/V2CompatRunner.cs
View File

@@ -28,7 +28,7 @@ public class V2CompatRunner : ICommandRunner
framework = command.Runtimes,
splashImage = command.SplashImage,
icon = command.Icon,
noDelta = command.NoDelta,
noDelta = command.Delta == Packaging.DeltaMode.None,
allowUnaware = false,
signParams = command.SignParameters,
signTemplate = command.SignTemplate,
@@ -59,7 +59,7 @@ public class V2CompatRunner : ICommandRunner
framework = command.Runtimes,
splashImage = command.SplashImage,
icon = command.Icon,
noDelta = command.NoDelta,
noDelta = command.Delta == Packaging.DeltaMode.None,
allowUnaware = false,
signParams = command.SignParameters,
signTemplate = command.SignTemplate,

4
src/Squirrel.Packaging.OSX/Commands/OsxReleasifyCommandRunner.cs
View File

@@ -126,10 +126,10 @@ public class OsxReleasifyCommandRunner
_logger.Info("Creating Delta Packages");
var prev = ReleasePackageBuilder.GetPreviousRelease(_logger, releases.Values, rp, releaseDir.FullName);
if (prev != null && !options.NoDelta) {
if (prev != null && options.DeltaMode != DeltaMode.None) {
var deltaBuilder = new DeltaPackageBuilder(_logger);
var deltaFile = rp.ReleasePackageFile.Replace("-full", "-delta");
var dp = deltaBuilder.CreateDeltaPackage(prev, rp, deltaFile);
var dp = deltaBuilder.CreateDeltaPackage(prev, rp, deltaFile, options.DeltaMode);
var deltaEntry = ReleaseEntry.GenerateFromFile(deltaFile);
releases[deltaEntry.OriginalFilename] = deltaEntry;
}

2
src/Squirrel.Packaging.OSX/Commands/OsxReleasifyOptions.cs
View File

@@ -12,7 +12,7 @@ public class OsxReleasifyOptions
public string ReleaseNotes { get; set; }
public bool NoDelta { get; set; }
public DeltaMode DeltaMode { get; set; }
public bool NoPackage { get; set; }

5
src/Squirrel.Packaging.Windows/Commands/WindowsReleasifyCommandRunner.cs
View File

@@ -18,7 +18,6 @@ public class WindowsReleasifyCommandRunner
{
var targetDir = options.ReleaseDir.FullName;
var package = options.Package;
var generateDeltas = !options.NoDelta;
var backgroundGif = options.SplashImage;
var setupIcon = options.Icon;
@@ -129,10 +128,10 @@ public class WindowsReleasifyCommandRunner
processed.Add(rp.ReleasePackageFile);
var prev = ReleasePackageBuilder.GetPreviousRelease(_logger, previousReleases, rp, targetDir);
if (prev != null && generateDeltas) {
if (prev != null && options.DeltaMode != DeltaMode.None) {
var deltaBuilder = new DeltaPackageBuilder(_logger);
var deltaOutputPath = rp.ReleasePackageFile.Replace("-full", "-delta");
var dp = deltaBuilder.CreateDeltaPackage(prev, rp, deltaOutputPath);
var dp = deltaBuilder.CreateDeltaPackage(prev, rp, deltaOutputPath, options.DeltaMode);
processed.Insert(0, dp.InputPackageFile);
}

2
src/Squirrel.Packaging.Windows/Commands/WindowsReleasifyOptions.cs
View File

@@ -8,7 +8,7 @@ public class WindowsReleasifyOptions : WindowsSigningOptions
public string Package { get; set; }
public bool NoDelta { get; set; }
public DeltaMode DeltaMode { get; set; }
public string Runtimes { get; set; }

63
src/Squirrel.Packaging/DeltaPackageBuilder.cs
View File

@@ -1,6 +1,7 @@
using System.Text;
using Squirrel.Compression;
using Microsoft.Extensions.Logging;
using System.IO.MemoryMappedFiles;
namespace Squirrel.Packaging;
@@ -13,7 +14,7 @@ public class DeltaPackageBuilder
_logger = logger;
}
public ReleasePackageBuilder CreateDeltaPackage(ReleasePackageBuilder basePackage, ReleasePackageBuilder newPackage, string outputFile)
public ReleasePackageBuilder CreateDeltaPackage(ReleasePackageBuilder basePackage, ReleasePackageBuilder newPackage, string outputFile, DeltaMode mode)
{
if (basePackage == null) throw new ArgumentNullException(nameof(basePackage));
if (newPackage == null) throw new ArgumentNullException(nameof(newPackage));
@@ -65,11 +66,7 @@ public class DeltaPackageBuilder
var newLibFiles = newLibDir.GetAllFilesRecursively().ToArray();
int fNew = 0, fSame = 0, fChanged = 0, fWarnings = 0;
bool bytesAreIdentical(ReadOnlySpan<byte> a1, ReadOnlySpan<byte> a2)
{
return a1.SequenceEqual(a2);
}
var helper = new HelperFile(_logger);
void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory)
{
@@ -95,10 +92,7 @@ public class DeltaPackageBuilder
var oldFilePath = baseLibFiles[relativePath];
_logger.Debug($"Delta patching {oldFilePath} => {targetFile.FullName}");
var oldData = File.ReadAllBytes(oldFilePath);
var newData = File.ReadAllBytes(targetFile.FullName);
if (bytesAreIdentical(oldData, newData)) {
if (AreFilesEqualFast(oldFilePath, targetFile.FullName)) {
// 2. exists in both, keep it the same
_logger.Debug($"{relativePath} hasn't changed, writing dummy file");
File.Create(targetFile.FullName + ".bsdiff").Dispose();
@@ -106,10 +100,10 @@ public class DeltaPackageBuilder
fSame++;
} else {
// 3. changed, write a delta in new
using (FileStream of = File.Create(targetFile.FullName + ".bsdiff")) {
BinaryPatchUtility.Create(oldData, newData, of);
}
var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum");
var outputFile = targetFile.FullName + ".zsdiff";
helper.CreateZstdPatch(oldFilePath, targetFile.FullName, outputFile, mode);
using var newfs = File.OpenRead(targetFile.FullName);
var rl = ReleaseEntry.GenerateFromFile(newfs, targetFile.Name + ".shasum");
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
fChanged++;
}
@@ -173,4 +167,45 @@ public class DeltaPackageBuilder
return new ReleasePackageBuilder(_logger, outputFile);
}
public unsafe static bool AreFilesEqualFast(string filePath1, string filePath2)
{
var fileInfo1 = new FileInfo(filePath1);
var fileInfo2 = new FileInfo(filePath2);
if (fileInfo1.Length != fileInfo2.Length) {
return false;
}
long length = fileInfo1.Length;
using var mmf1 = MemoryMappedFile.CreateFromFile(filePath1, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
using var mmf2 = MemoryMappedFile.CreateFromFile(filePath2, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
const long chunkSize = 10 * 1024 * 1024; // 10 MB
for (long offset = 0; offset < length; offset += chunkSize) {
long size = Math.Min(chunkSize, length - offset);
using var accessor1 = mmf1.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);
using var accessor2 = mmf2.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);
byte* ptr1 = null;
byte* ptr2 = null;
accessor1.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr1);
accessor2.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr2);
try {
var span1 = new ReadOnlySpan<byte>(ptr1, (int) accessor1.SafeMemoryMappedViewHandle.ByteLength);
var span2 = new ReadOnlySpan<byte>(ptr2, (int) accessor2.SafeMemoryMappedViewHandle.ByteLength);
if (!span1.SequenceEqual(span2)) {
return false;
}
} finally {
if (ptr1 != null) accessor1.SafeMemoryMappedViewHandle.ReleasePointer();
if (ptr2 != null) accessor2.SafeMemoryMappedViewHandle.ReleasePointer();
}
}
return true;
}
}

63
src/Squirrel.Packaging/HelperFile.cs
View File

@@ -3,10 +3,19 @@ using System.Diagnostics;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
namespace Squirrel.Packaging;
public enum DeltaMode
{
None,
BestSpeed,
BestSize,
}
public class HelperFile
{
private static List<string> _searchPaths = new List<string>();
@@ -42,6 +51,60 @@ public class HelperFile
_searchPaths.Insert(0, path);
}
public void CreateZstdPatch(string oldFile, string newFile, string outputFile, DeltaMode mode)
{
if (mode == DeltaMode.None)
throw new ArgumentException("DeltaMode.None is not supported.", nameof(mode));
List<string> args = new() {
"--patch-from", oldFile,
newFile,
"-o", outputFile,
"--force",
};
if (mode == DeltaMode.BestSize) {
args.Add("-19");
args.Add("--single-thread");
args.Add("--zstd");
args.Add("targetLength=4096");
args.Add("--zstd");
args.Add("chainLog=30");
}
var deltaMode = mode switch {
DeltaMode.None => "none",
DeltaMode.BestSpeed => "bsdiff",
DeltaMode.BestSize => "xdelta",
_ => throw new InvalidEnumArgumentException(nameof(mode), (int) mode, typeof(DeltaMode)),
};
string zstdPath;
if (SquirrelRuntimeInfo.IsWindows) {
zstdPath = FindHelperFile("zstd.exe");
} else {
zstdPath = "zstd";
AssertSystemBinaryExists(zstdPath);
}
InvokeAndThrowIfNonZero(zstdPath, args, null);
}
public void AssertSystemBinaryExists(string binaryName)
{
try {
if (SquirrelRuntimeInfo.IsWindows) {
var output = InvokeAndThrowIfNonZero("where", new[] { binaryName }, null);
if (String.IsNullOrWhiteSpace(output) || !File.Exists(output))
throw new ProcessFailedException("", "");
} else {
InvokeAndThrowIfNonZero("command", new[] { "-v", binaryName }, null);
}
} catch (ProcessFailedException) {
throw new Exception($"Could not find '{binaryName}' on the system, ensure it is installed and on the PATH.");
}
}
// protected static string FindAny(params string[] names)
// {
// var findCommand = SquirrelRuntimeInfo.IsWindows ? "where" : "which";

1
src/Squirrel.Packaging/Squirrel.Packaging.csproj
View File

@@ -4,6 +4,7 @@
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<NoWarn>$(NoWarn);CA2007;CS8002</NoWarn>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>

10
src/Squirrel/Compression/BZip2Stream.cs
View File

@@ -1,9 +1,11 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
namespace Squirrel.Compression
{
[ExcludeFromCodeCoverage]
internal sealed class BZip2Stream : Stream
{
private readonly Stream stream;
@@ -66,9 +68,9 @@ namespace Squirrel.Compression
#if !NETFRAMEWORK && !NETSTANDARD2_0
public override int Read(Span<byte> buffer) => stream.Read(buffer);
public override int Read(Span<byte> buffer) => stream.Read(buffer);
public override void Write(ReadOnlySpan<byte> buffer) => stream.Write(buffer);
public override void Write(ReadOnlySpan<byte> buffer) => stream.Write(buffer);
#endif
public override void Write(byte[] buffer, int offset, int count) =>
@@ -125,6 +127,7 @@ namespace Squirrel.Compression
* start of the BZIP2 stream to make it compatible with other PGP programs.
*/
[ExcludeFromCodeCoverage]
internal class CBZip2InputStream : Stream
{
private static void Cadvise()
@@ -1085,6 +1088,7 @@ namespace Squirrel.Compression
* start of the BZIP2 stream to make it compatible with other PGP programs.
*/
[ExcludeFromCodeCoverage]
internal sealed class CBZip2OutputStream : Stream
{
private const int SETMASK = (1 << 21);
@@ -2845,6 +2849,7 @@ namespace Squirrel.Compression
* @author <a href="mailto:keiron@aftexsw.com">Keiron Liddle</a>
*/
[ExcludeFromCodeCoverage]
internal class BZip2Constants
{
public const int baseBlockSize = 100000;
@@ -3382,6 +3387,7 @@ namespace Squirrel.Compression
* @author <a href="mailto:keiron@aftexsw.com">Keiron Liddle</a>
*/
[ExcludeFromCodeCoverage]
internal class CRC
{
public static int[] crc32Table =

6
src/Squirrel/Compression/BinaryPatchUtility.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.IO.Compression;
using System.Threading;
@@ -35,6 +36,7 @@ namespace Squirrel.Compression
IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
*/
[ExcludeFromCodeCoverage]
internal class BinaryPatchUtility
{
/// <summary>
@@ -44,7 +46,7 @@ namespace Squirrel.Compression
/// <param name="oldData">The original binary data.</param>
/// <param name="newData">The new binary data.</param>
/// <param name="output">A <see cref="Stream"/> to which the patch will be written.</param>
public static void Create(byte[] oldData, byte[] newData, Stream output)
private static void Create(byte[] oldData, byte[] newData, Stream output)
{
// NB: If you diff a file big enough, we blow the stack. This doesn't
// solve it, just buys us more space. The solution is to rewrite Split
@@ -648,6 +650,7 @@ namespace Squirrel.Compression
/// <see cref="System.Security.Cryptography.CryptoStream"/> that take ownership of the stream passed to their constructors.
/// </summary>
/// <remarks>See <a href="http://code.logos.com/blog/2009/05/wrappingstream_implementation.html">WrappingStream Implementation</a>.</remarks>
[ExcludeFromCodeCoverage]
class WrappingStream : Stream
{
/// <summary>
@@ -875,6 +878,7 @@ namespace Squirrel.Compression
/// <summary>
/// Provides helper methods for working with <see cref="Stream"/>.
/// </summary>
[ExcludeFromCodeCoverage]
static class StreamUtility
{
/// <summary>

32
src/Squirrel/Compression/DeltaPackage.cs
View File

@@ -1,11 +1,13 @@
#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using Squirrel.Locators;
// https://dev.to/emrahsungu/how-to-compare-two-files-using-net-really-really-fast-2pd9
// https://github.com/SnowflakePowered/vcdiff
@@ -15,12 +17,15 @@ namespace Squirrel.Compression
internal class DeltaPackage
{
private readonly ILogger _log;
private readonly string _updatePath;
private readonly string _baseTempDir;
private static Regex DIFF_SUFFIX = new Regex(@"\.(bs|zs)?diff$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public DeltaPackage(ILogger logger, string baseTempDir = null)
public DeltaPackage(ILogger logger, ISquirrelLocator locator)
{
_log = logger;
_baseTempDir = baseTempDir ?? Utility.GetDefaultTempBaseDirectory();
_baseTempDir = locator.AppTempDir;
_updatePath = locator.UpdateExePath;
}
public void ApplyDeltaPackageFast(string workingPath, string deltaPackageZip, Action<int> progress = null)
@@ -45,16 +50,15 @@ namespace Squirrel.Compression
var files = deltaPathRelativePaths
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".shasum", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase) ||
!deltaPathRelativePaths.Contains(x.Replace(".diff", ".bsdiff")))
.Where(x => !DIFF_SUFFIX.IsMatch(x))
.ToArray();
for (var index = 0; index < files.Length; index++) {
var file = files[index];
pathsVisited.Add(Regex.Replace(file, @"\.(bs)?diff$", "").ToLowerInvariant());
pathsVisited.Add(DIFF_SUFFIX.Replace(file, "").ToLowerInvariant());
applyDiffToFile(deltaPath, file, workingPath);
var perc = (index + 1) / (double) files.Length * 100;
Utility.CalculateProgress((int) perc, 10, 90);
progress(Utility.CalculateProgress((int) perc, 10, 90));
}
progress(90);
@@ -86,7 +90,7 @@ namespace Squirrel.Compression
void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDirectory)
{
var inputFile = Path.Combine(deltaPath, relativeFilePath);
var finalTarget = Path.Combine(workingDirectory, Regex.Replace(relativeFilePath, @"\.(bs)?diff$", ""));
var finalTarget = Path.Combine(workingDirectory, DIFF_SUFFIX.Replace(relativeFilePath, ""));
using var _d = Utility.GetTempFileName(out var tempTargetFile, _baseTempDir);
@@ -96,7 +100,17 @@ namespace Squirrel.Compression
return;
}
if (relativeFilePath.EndsWith(".bsdiff", StringComparison.InvariantCultureIgnoreCase)) {
if (relativeFilePath.EndsWith(".zsdiff", StringComparison.InvariantCultureIgnoreCase)) {
var psi = new ProcessStartInfo(_updatePath);
psi.AppendArgumentListSafe(new string[] { "--old", finalTarget, "--patch", inputFile, "--output", tempTargetFile }, out var _);
_log.Trace($"Applying zstd diff to {relativeFilePath}");
var p = psi.StartRedirectOutputToILogger(_log);
if (!p.WaitForExit(60_000)) {
p.Kill();
throw new TimeoutException("zstd patch process timed out (60s).");
}
verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile);
} else if (relativeFilePath.EndsWith(".bsdiff", StringComparison.InvariantCultureIgnoreCase)) {
using (var of = File.OpenWrite(tempTargetFile))
using (var inf = File.OpenRead(finalTarget)) {
_log.Trace($"Applying bsdiff to {relativeFilePath}");
@@ -132,7 +146,7 @@ namespace Squirrel.Compression
void verifyPatchedFile(string relativeFilePath, string inputFile, string tempTargetFile)
{
var shaFile = Regex.Replace(inputFile, @"\.(bs)?diff$", ".shasum");
var shaFile = DIFF_SUFFIX.Replace(inputFile, ".shasum");
var expectedReleaseEntry = ReleaseEntry.ParseReleaseEntry(File.ReadAllText(shaFile, Encoding.UTF8));
var actualReleaseEntry = ReleaseEntry.GenerateFromFile(tempTargetFile);

30
src/Squirrel/Internal/ProcessArgumentListPolyfill.cs → src/Squirrel/Internal/ProcessExtensions.cs
View File

@@ -3,11 +3,12 @@ using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using Microsoft.Extensions.Logging;
namespace Squirrel
{
[ExcludeFromCodeCoverage]
internal static class ProcessArgumentListPolyfill
internal static class ProcessStartExtensions
{
#if NET5_0_OR_GREATER
@@ -30,6 +31,33 @@ namespace Squirrel
debug = psi.Arguments;
}
#endif
public static Process StartRedirectOutputToILogger(this ProcessStartInfo psi, ILogger log)
{
psi.RedirectStandardOutput = true;
psi.RedirectStandardError = true;
psi.UseShellExecute = false;
var p = Process.Start(psi);
p.BeginErrorReadLine();
p.BeginOutputReadLine();
p.ErrorDataReceived += (o, e) => {
if (e.Data != null) {
log.LogError(e.Data);
}
};
p.OutputDataReceived += (o, e) => {
if (e.Data != null) {
log.LogInformation(e.Data);
}
};
return p;
}
// https://source.dot.net/#System.Diagnostics.Process/System/Diagnostics/ProcessStartInfo.cs,204
private static void AppendArgumentsTo(StringBuilder stringBuilder, IEnumerable<string> args)
{

2
src/Squirrel/UpdateManager.cs
View File

@@ -342,7 +342,7 @@ namespace Squirrel
// applying deltas accounts for 50%-100% of progress
double progressStepSize = 100d / releasesToDownload.Length;
var builder = new DeltaPackage(Log, Locator.AppTempDir);
var builder = new DeltaPackage(Log, Locator);
for (var i = 0; i < releasesToDownload.Length; i++) {
var rel = releasesToDownload[i];
double baseProgress = i * progressStepSize;

4
test/Squirrel.CommandLine.Tests/Commands/WindowsCommandTests.cs
View File

@@ -37,10 +37,10 @@ public abstract class ReleaseCommandTests<T> : BaseCommandTests<T>
{
var command = new T();
string cli = GetRequiredDefaultOptions() + "--noDelta";
string cli = GetRequiredDefaultOptions() + "--delta none";
ParseResult parseResult = command.ParseAndApply(cli);
Assert.True(command.NoDelta);
Assert.True(command.Delta == Packaging.DeltaMode.None);
}
[Fact]

BIN
test/Squirrel.Tests/fixtures/obs-size.patch Normal file
View File

Binary file not shown.

BIN
test/Squirrel.Tests/fixtures/obs-speed.patch Normal file
View File

Binary file not shown.

BIN
test/Squirrel.Tests/fixtures/obs29.1.2.dll Normal file
View File

Binary file not shown.

BIN
test/Squirrel.Tests/fixtures/obs30.0.2.dll Normal file
View File

Binary file not shown.