mirror of
https://github.com/velopack/velopack.git
synced 2025-10-25 15:19:22 +00:00
256 lines
12 KiB
C#
256 lines
12 KiB
C#
using System.IO.MemoryMappedFiles;
|
|
using System.Text;
|
|
using Microsoft.Extensions.Logging;
|
|
using Velopack.Core;
|
|
using Velopack.Packaging.Exceptions;
|
|
using Velopack.Util;
|
|
|
|
namespace Velopack.Packaging.Compression;
|
|
|
|
public class DeltaPackageBuilder
|
|
{
|
|
private readonly ILogger _logger;
|
|
|
|
public DeltaPackageBuilder(ILogger logger)
|
|
{
|
|
_logger = logger;
|
|
}
|
|
|
|
public class DeltaStats
|
|
{
|
|
public int New { get; set; }
|
|
public int Same { get; set; }
|
|
public int Changed { get; set; }
|
|
public int Warnings { get; set; }
|
|
public int Processed { get; set; }
|
|
public int Removed { get; set; }
|
|
}
|
|
|
|
public (ReleasePackage package, DeltaStats stats) CreateDeltaPackage(ReleasePackage basePackage, ReleasePackage newPackage, string outputFile,
|
|
DeltaMode mode, Action<int> progress)
|
|
{
|
|
if (basePackage == null) throw new ArgumentNullException(nameof(basePackage));
|
|
if (newPackage == null) throw new ArgumentNullException(nameof(newPackage));
|
|
if (String.IsNullOrEmpty(outputFile) || File.Exists(outputFile))
|
|
throw new ArgumentException("The output file is null or already exists", nameof(outputFile));
|
|
|
|
Zstd zstd = null;
|
|
try {
|
|
zstd = new Zstd(HelperFile.GetZstdPath());
|
|
} catch (Exception ex) {
|
|
_logger.Error(ex.Message);
|
|
_logger.Warn("Zstd not available. Falling back to legacy bsdiff delta format. This will be a lot slower and more prone to breaking.");
|
|
}
|
|
|
|
if (basePackage.Version >= newPackage.Version) {
|
|
var message = String.Format(
|
|
"Cannot create a delta package based on version {0} as it is a later or equal to the base version {1}",
|
|
basePackage.Version,
|
|
newPackage.Version);
|
|
throw new InvalidOperationException(message);
|
|
}
|
|
|
|
if (basePackage.PackageFile == null) {
|
|
throw new ArgumentException("The base package's release file is null", "basePackage");
|
|
}
|
|
|
|
if (!File.Exists(basePackage.PackageFile)) {
|
|
throw new FileNotFoundException("The base package release does not exist", basePackage.PackageFile);
|
|
}
|
|
|
|
if (!File.Exists(newPackage.PackageFile)) {
|
|
throw new FileNotFoundException("The new package release does not exist", newPackage.PackageFile);
|
|
}
|
|
|
|
int fNew = 0, fSame = 0, fChanged = 0, fWarnings = 0, fProcessed = 0, fRemoved = 0;
|
|
|
|
using (TempUtil.GetTempDirectory(out var baseTempPath))
|
|
using (TempUtil.GetTempDirectory(out var tempPath)) {
|
|
var baseTempInfo = new DirectoryInfo(baseTempPath);
|
|
var tempInfo = new DirectoryInfo(tempPath);
|
|
|
|
// minThreads = 1, maxThreads = 8
|
|
int numParallel = Math.Min(Math.Max(Environment.ProcessorCount - 1, 1), 8);
|
|
|
|
_logger.Info($"Creating delta for {basePackage.Version} -> {newPackage.Version} with {numParallel} parallel threads.");
|
|
_logger.Debug($"Extracting {Path.GetFileName(basePackage.PackageFile)} and {Path.GetFileName(newPackage.PackageFile)} into {tempPath}");
|
|
|
|
var veloLogger = _logger.ToVelopackLogger();
|
|
EasyZip.ExtractZipToDirectory(veloLogger, basePackage.PackageFile, baseTempInfo.FullName);
|
|
EasyZip.ExtractZipToDirectory(veloLogger, newPackage.PackageFile, tempInfo.FullName);
|
|
|
|
// Collect a list of relative paths under 'lib' and map them
|
|
// to their full name. We'll use this later to determine in
|
|
// the new version of the package whether the file exists or
|
|
// not.
|
|
var baseLibFiles = baseTempInfo.GetAllFilesRecursively()
|
|
.Where(x => x.FullName.ToLowerInvariant().Contains("lib" + Path.DirectorySeparatorChar))
|
|
.ToDictionary(k => k.FullName.Replace(baseTempInfo.FullName, ""), v => v.FullName);
|
|
var newLibDir = tempInfo.GetDirectories().First(x => x.Name.ToLowerInvariant() == "lib");
|
|
var newLibFiles = newLibDir.GetAllFilesRecursively().ToArray();
|
|
var numNewFiles = newLibFiles.Length;
|
|
|
|
void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, bool useZstd)
|
|
{
|
|
// NB: There are three cases here that we'll handle:
|
|
//
|
|
// 1. Exists only in new => leave it alone, we'll use it directly.
|
|
// 2. Exists in both old and new => write a dummy file so we know
|
|
// to keep it.
|
|
// 3. Exists in old but changed in new => create a delta file
|
|
//
|
|
// The fourth case of "Exists only in old => delete it in new"
|
|
// is handled when we apply the delta package
|
|
try {
|
|
var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, "");
|
|
|
|
// 1. new file, leave it alone
|
|
if (!baseLibFiles.ContainsKey(relativePath)) {
|
|
_logger.Debug($"{relativePath} not found in base package, marking as new");
|
|
Interlocked.Increment(ref fNew);
|
|
return;
|
|
}
|
|
|
|
var oldFilePath = baseLibFiles[relativePath];
|
|
_logger.Debug($"Delta patching {oldFilePath} => {targetFile.FullName}");
|
|
|
|
if (AreFilesEqualFast(oldFilePath, targetFile.FullName)) {
|
|
// 2. exists in both, keep it the same
|
|
_logger.Debug($"{relativePath} hasn't changed, writing dummy file");
|
|
File.Create(targetFile.FullName + ".diff").Dispose();
|
|
File.Create(targetFile.FullName + ".shasum").Dispose();
|
|
Interlocked.Increment(ref fSame);
|
|
} else {
|
|
// 3. changed, write a delta in new
|
|
if (useZstd) {
|
|
var diffOut = targetFile.FullName + ".zsdiff";
|
|
zstd.CreatePatch(oldFilePath, targetFile.FullName, diffOut, mode);
|
|
} else {
|
|
var oldData = File.ReadAllBytes(oldFilePath);
|
|
var newData = File.ReadAllBytes(targetFile.FullName);
|
|
using (FileStream of = File.Create(targetFile.FullName + ".bsdiff")) {
|
|
BinaryPatchUtility.Create(oldData, newData, of);
|
|
}
|
|
}
|
|
|
|
using var newfs = File.OpenRead(targetFile.FullName);
|
|
#pragma warning disable CS0618 // Type or member is obsolete
|
|
var rl = ReleaseEntry.GenerateFromFile(newfs, targetFile.Name + ".shasum");
|
|
#pragma warning restore CS0618 // Type or member is obsolete
|
|
File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8);
|
|
Interlocked.Increment(ref fChanged);
|
|
}
|
|
|
|
targetFile.Delete();
|
|
baseLibFiles.Remove(relativePath);
|
|
var p = Interlocked.Increment(ref fProcessed);
|
|
progress(CoreUtil.CalculateProgress((int) ((double) p / numNewFiles * 100), 0, 70));
|
|
} catch (Exception ex) {
|
|
_logger.Debug(ex, String.Format("Failed to create a delta for {0}", targetFile.Name));
|
|
IoUtil.DeleteFileOrDirectoryHard(targetFile.FullName + ".bsdiff", throwOnFailure: false);
|
|
IoUtil.DeleteFileOrDirectoryHard(targetFile.FullName + ".diff", throwOnFailure: false);
|
|
IoUtil.DeleteFileOrDirectoryHard(targetFile.FullName + ".shasum", throwOnFailure: false);
|
|
Interlocked.Increment(ref fWarnings);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
try {
|
|
Parallel.ForEach(
|
|
newLibFiles,
|
|
new ParallelOptions() { MaxDegreeOfParallelism = numParallel },
|
|
(f) => {
|
|
// we try to use zstd first, if it fails we'll try bsdiff
|
|
if (zstd != null) {
|
|
try {
|
|
createDeltaForSingleFile(f, tempInfo, true);
|
|
return; // success, so return from this function
|
|
} catch (ProcessFailedException ex) {
|
|
_logger.Error(
|
|
$"Failed to create zstd diff for file '{f.FullName}' (will try to fallback to legacy bsdiff format - this will be much slower). " +
|
|
Environment.NewLine + ex.Message);
|
|
} catch (Exception ex) {
|
|
_logger.Error($"Failed to create zstd diff for file '{f.FullName}'. " + Environment.NewLine + ex.Message);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
// if we're here, either zstd is not available or it failed
|
|
try {
|
|
createDeltaForSingleFile(f, tempInfo, false);
|
|
if (zstd != null) {
|
|
_logger.Info($"Successfully created fallback bsdiff for file '{f.FullName}'.");
|
|
}
|
|
} catch (Exception ex) {
|
|
_logger.Error($"Failed to create bsdiff for file '{f.FullName}'. " + Environment.NewLine + ex.Message);
|
|
throw;
|
|
}
|
|
});
|
|
} catch {
|
|
throw new UserInfoException(
|
|
"Delta creation failed for one or more files. See log for details. To skip delta generation, use the '--delta none' argument.");
|
|
}
|
|
|
|
EasyZip.CreateZipFromDirectoryAsync(_logger.ToVelopackLogger(), outputFile, tempInfo.FullName, CoreUtil.CreateProgressDelegate(progress, 70, 100)).GetAwaiterResult();
|
|
progress(100);
|
|
fRemoved = baseLibFiles.Count;
|
|
|
|
_logger.Info(
|
|
$"Delta processed {fProcessed:D4} files. "
|
|
+ $"{fChanged:D4} patched, {fSame:D4} unchanged, {fNew:D4} new, {fRemoved:D4} removed");
|
|
|
|
_logger.Debug(
|
|
$"Successfully created delta package for {basePackage.Version} -> {newPackage.Version}" +
|
|
(fWarnings > 0 ? $" (with {fWarnings} retries)" : "") +
|
|
".");
|
|
}
|
|
|
|
return (new ReleasePackage(outputFile), new DeltaStats {
|
|
New = fNew, Same = fSame, Changed = fChanged, Warnings = fWarnings, Processed = fProcessed, Removed = fRemoved,
|
|
});
|
|
}
|
|
|
|
public unsafe static bool AreFilesEqualFast(string filePath1, string filePath2)
|
|
{
|
|
var fileInfo1 = new FileInfo(filePath1);
|
|
var fileInfo2 = new FileInfo(filePath2);
|
|
if (fileInfo1.Length != fileInfo2.Length) {
|
|
return false;
|
|
}
|
|
|
|
long length = fileInfo1.Length;
|
|
if (length == 0) {
|
|
return true;
|
|
}
|
|
|
|
using var mmf1 = MemoryMappedFile.CreateFromFile(filePath1, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
|
|
using var mmf2 = MemoryMappedFile.CreateFromFile(filePath2, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
|
|
|
|
const long chunkSize = 10 * 1024 * 1024; // 10 MB
|
|
|
|
for (long offset = 0; offset < length; offset += chunkSize) {
|
|
long size = Math.Min(chunkSize, length - offset);
|
|
|
|
using var accessor1 = mmf1.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);
|
|
using var accessor2 = mmf2.CreateViewAccessor(offset, size, MemoryMappedFileAccess.Read);
|
|
|
|
byte* ptr1 = null;
|
|
byte* ptr2 = null;
|
|
accessor1.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr1);
|
|
accessor2.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr2);
|
|
|
|
try {
|
|
var span1 = new ReadOnlySpan<byte>(ptr1, (int) accessor1.SafeMemoryMappedViewHandle.ByteLength);
|
|
var span2 = new ReadOnlySpan<byte>(ptr2, (int) accessor2.SafeMemoryMappedViewHandle.ByteLength);
|
|
if (!span1.SequenceEqual(span2)) {
|
|
return false;
|
|
}
|
|
} finally {
|
|
if (ptr1 != null) accessor1.SafeMemoryMappedViewHandle.ReleasePointer();
|
|
if (ptr2 != null) accessor2.SafeMemoryMappedViewHandle.ReleasePointer();
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
} |