From fedd102aa4db12215b237d7637bd0089613693ab Mon Sep 17 00:00:00 2001 From: Caelan Sayler Date: Sat, 19 Mar 2022 23:35:53 +0000 Subject: [PATCH] Add some parallelism and remove msdelta in delta package generator --- src/Squirrel/Internal/DeltaPackage.cs | 210 +++++++++++++------------ src/Squirrel/Lib/MsDeltaCompression.cs | 28 ++-- 2 files changed, 126 insertions(+), 112 deletions(-) diff --git a/src/Squirrel/Internal/DeltaPackage.cs b/src/Squirrel/Internal/DeltaPackage.cs index 3861ff2e..b12d12bb 100644 --- a/src/Squirrel/Internal/DeltaPackage.cs +++ b/src/Squirrel/Internal/DeltaPackage.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.IO; @@ -12,6 +12,9 @@ using SharpCompress.Archives.Zip; using SharpCompress.Common; using SharpCompress.Readers; using SharpCompress.Compressors.Deflate; +using System.Threading.Tasks; +using System.Threading; +using System.Runtime.InteropServices; namespace Squirrel { @@ -21,9 +24,6 @@ namespace Squirrel ReleasePackage ApplyDeltaPackage(ReleasePackage basePackage, ReleasePackage deltaPackage, string outputFile); } -#if NET5_0_OR_GREATER - [System.Runtime.Versioning.SupportedOSPlatform("windows")] -#endif internal class DeltaPackageBuilder : IEnableLogger, IDeltaPackageBuilder { readonly string localAppDirectory; @@ -65,7 +65,11 @@ namespace Squirrel var baseTempInfo = new DirectoryInfo(baseTempPath); var tempInfo = new DirectoryInfo(tempPath); - this.Log().Info("Extracting {0} and {1} into {2}", + const int numParallel = 4; + + this.Log().Info($"Creating delta for {basePackage.Version} -> {newPackage.Version} with {numParallel} parallel threads."); + + this.Log().Debug("Extracting {0} and {1} into {2}", basePackage.ReleasePackageFile, newPackage.ReleasePackageFile, tempPath); EasyZip.ExtractZipToDirectory(basePackage.ReleasePackageFile, baseTempInfo.FullName); @@ -80,13 +84,103 @@ namespace Squirrel .ToDictionary(k => k.FullName.Replace(baseTempInfo.FullName, ""), v => v.FullName); var newLibDir = tempInfo.GetDirectories().First(x => x.Name.ToLowerInvariant() == "lib"); + var newLibFiles = newLibDir.GetAllFilesRecursively().ToArray(); - foreach (var libFile in newLibDir.GetAllFilesRecursively()) { - createDeltaForSingleFile(libFile, tempInfo, baseLibFiles); + int fNew = 0, fSame = 0, fChanged = 0; + + bool bytesAreIdentical(ReadOnlySpan a1, ReadOnlySpan a2) + { + return a1.SequenceEqual(a2); } + void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary baseFileListing) + { + // NB: There are three cases here that we'll handle: + // + // 1. Exists only in new => leave it alone, we'll use it directly. + // 2. Exists in both old and new => write a dummy file so we know + // to keep it. + // 3. Exists in old but changed in new => create a delta file + // + // The fourth case of "Exists only in old => delete it in new" + // is handled when we apply the delta package + var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, ""); + + if (!baseFileListing.ContainsKey(relativePath)) { + this.Log().Debug("{0} not found in base package, marking as new", relativePath); + fNew++; + return; + } + + var oldFilePath = baseFileListing[relativePath]; + baseFileListing.Remove(relativePath); + + var oldData = File.ReadAllBytes(oldFilePath); + var newData = File.ReadAllBytes(targetFile.FullName); + + if (bytesAreIdentical(oldData, newData)) { + this.Log().Debug("{0} hasn't changed, writing dummy file", relativePath); + File.Create(targetFile.FullName + ".bsdiff").Dispose(); + File.Create(targetFile.FullName + ".shasum").Dispose(); + targetFile.Delete(); + fSame++; + return; + } + + this.Log().Debug("Delta patching {0} => {1}", oldFilePath, targetFile.FullName); + + try { + using (FileStream of = File.Create(targetFile.FullName + ".bsdiff")) { + BinaryPatchUtility.Create(oldData, newData, of); + } + var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum"); + File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8); + targetFile.Delete(); + fChanged++; + } catch (Exception ex) { + this.Log().DebugException(String.Format("We couldn't create a delta for {0}", targetFile.Name), ex); + Utility.DeleteFileOrDirectoryHardOrGiveUp(targetFile.FullName + ".bsdiff"); + Utility.DeleteFileOrDirectoryHardOrGiveUp(targetFile.FullName + ".diff"); + Utility.DeleteFileOrDirectoryHardOrGiveUp(targetFile.FullName + ".shasum"); + throw; + } + } + + var tResult = Task.Run(() => { + Parallel.ForEach(newLibFiles, new ParallelOptions() { MaxDegreeOfParallelism = numParallel }, (f) => { + try { + createDeltaForSingleFile(f, tempInfo, baseLibFiles); + } catch (Exception ex) { + this.Log().WarnException($"Unable to create delta for '{f}', retrying...", ex); + createDeltaForSingleFile(f, tempInfo, baseLibFiles); + } + }); + }); + + int prevCount = 0; + while (!tResult.IsCompleted) { + // sleep for 3 seconds (in 100ms intervals) + for (int i = 0; i < 30 && !tResult.IsCompleted; i++) + Thread.Sleep(100); + + int processed = fNew + fChanged + fSame; + if (prevCount == processed) { + // if there has been no progress, do not print another message + continue; + } + + this.Log().Info($"Processed {processed}/{newLibFiles.Length} files. {fChanged} patched, {fNew} new, {fSame} unchanged."); + prevCount = processed; + } + + if (tResult.Exception != null) + throw new Exception("Unable to create delta package.", tResult.Exception); + ReleasePackage.addDeltaFilesToContentTypes(tempInfo.FullName); EasyZip.CreateZipFromDirectory(outputFile, tempInfo.FullName); + + this.Log().Info($"Successfully created delta package for {basePackage.Version} -> {newPackage.Version}."); + this.Log().Info($"{newLibFiles.Length} files processed. {fChanged} patched, {fNew} new, {fSame} unchanged, {baseLibFiles.Count} removed."); } return new ReleasePackage(outputFile); @@ -177,75 +271,6 @@ namespace Squirrel return new ReleasePackage(outputFile); } - void createDeltaForSingleFile(FileInfo targetFile, DirectoryInfo workingDirectory, Dictionary baseFileListing) - { - // NB: There are three cases here that we'll handle: - // - // 1. Exists only in new => leave it alone, we'll use it directly. - // 2. Exists in both old and new => write a dummy file so we know - // to keep it. - // 3. Exists in old but changed in new => create a delta file - // - // The fourth case of "Exists only in old => delete it in new" - // is handled when we apply the delta package - var relativePath = targetFile.FullName.Replace(workingDirectory.FullName, ""); - - if (!baseFileListing.ContainsKey(relativePath)) { - this.Log().Info("{0} not found in base package, marking as new", relativePath); - return; - } - - var oldData = File.ReadAllBytes(baseFileListing[relativePath]); - var newData = File.ReadAllBytes(targetFile.FullName); - - if (bytesAreIdentical(oldData, newData)) { - this.Log().Debug("{0} hasn't changed, writing dummy file", relativePath); - - File.Create(targetFile.FullName + ".diff").Dispose(); - File.Create(targetFile.FullName + ".shasum").Dispose(); - targetFile.Delete(); - return; - } - - this.Log().Info("Delta patching {0} => {1}", baseFileListing[relativePath], targetFile.FullName); - var msDelta = new MsDeltaCompression(); - - if (targetFile.Extension.Equals(".exe", StringComparison.OrdinalIgnoreCase) || - targetFile.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase) || - targetFile.Extension.Equals(".node", StringComparison.OrdinalIgnoreCase)) { - try { - msDelta.CreateDelta(baseFileListing[relativePath], targetFile.FullName, targetFile.FullName + ".diff"); - goto exit; - } catch (Exception) { - this.Log().Warn("We couldn't create a delta for {0}, attempting to create bsdiff", targetFile.Name); - } - } - - try { - using (FileStream of = File.Create(targetFile.FullName + ".bsdiff")) { - BinaryPatchUtility.Create(oldData, newData, of); - - // NB: Create a dummy corrupt .diff file so that older - // versions which don't understand bsdiff will fail out - // until they get upgraded, instead of seeing the missing - // file and just removing it. - File.WriteAllText(targetFile.FullName + ".diff", "1"); - } - } catch (Exception ex) { - this.Log().WarnException(String.Format("We really couldn't create a delta for {0}", targetFile.Name), ex); - Utility.DeleteFileOrDirectoryHardOrGiveUp(targetFile.FullName + ".bsdiff"); - Utility.DeleteFileOrDirectoryHardOrGiveUp(targetFile.FullName + ".diff"); - return; - } - - exit: - - var rl = ReleaseEntry.GenerateFromFile(new MemoryStream(newData), targetFile.Name + ".shasum"); - File.WriteAllText(targetFile.FullName + ".shasum", rl.EntryAsString, Encoding.UTF8); - targetFile.Delete(); - } - - void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDirectory) { var inputFile = Path.Combine(deltaPath, relativeFilePath); @@ -264,16 +289,23 @@ namespace Squirrel if (relativeFilePath.EndsWith(".bsdiff", StringComparison.InvariantCultureIgnoreCase)) { using (var of = File.OpenWrite(tempTargetFile)) using (var inf = File.OpenRead(finalTarget)) { - this.Log().Info("Applying BSDiff to {0}", relativeFilePath); + this.Log().Info("Applying bsdiff to {0}", relativeFilePath); BinaryPatchUtility.Apply(inf, () => File.OpenRead(inputFile), of); } verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile); } else if (relativeFilePath.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase)) { - this.Log().Info("Applying MSDiff to {0}", relativeFilePath); - var msDelta = new MsDeltaCompression(); - msDelta.ApplyDelta(inputFile, finalTarget, tempTargetFile); + this.Log().Info("Applying msdiff to {0}", relativeFilePath); +#if NETFRAMEWORK + if (true) { +#else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { +#endif + MsDeltaCompression.ApplyDelta(inputFile, finalTarget, tempTargetFile); + } else { + throw new InvalidOperationException("msdiff is not supported on non-windows platforms."); + } verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile); } else { using (var of = File.OpenWrite(tempTargetFile)) @@ -312,23 +344,5 @@ namespace Squirrel throw new ChecksumFailedException() { Filename = relativeFilePath }; } } - - bool bytesAreIdentical(byte[] oldData, byte[] newData) - { - if (oldData == null || newData == null) { - return oldData == newData; - } - if (oldData.LongLength != newData.LongLength) { - return false; - } - - for (long i = 0; i < newData.LongLength; i++) { - if (oldData[i] != newData[i]) { - return false; - } - } - - return true; - } } } diff --git a/src/Squirrel/Lib/MsDeltaCompression.cs b/src/Squirrel/Lib/MsDeltaCompression.cs index 0d4a03f6..e138e489 100644 --- a/src/Squirrel/Lib/MsDeltaCompression.cs +++ b/src/Squirrel/Lib/MsDeltaCompression.cs @@ -10,22 +10,22 @@ namespace Squirrel #endif internal class MsDeltaCompression { - public void CreateDelta(string oldFilePath, string newFilePath, string deltaFilePath) - { - const string? sourceOptionsName = null; - const string? targetOptionsName = null; - var globalOptions = new DeltaInput(); - var targetFileTime = IntPtr.Zero; + //public void CreateDelta(string oldFilePath, string newFilePath, string deltaFilePath) + //{ + // const string? sourceOptionsName = null; + // const string? targetOptionsName = null; + // var globalOptions = new DeltaInput(); + // var targetFileTime = IntPtr.Zero; - if (!NativeMethods.CreateDelta( - FileTypeSet.Executables, CreateFlags.IgnoreFileSizeLimit, CreateFlags.None, oldFilePath, newFilePath, - sourceOptionsName, targetOptionsName, globalOptions, targetFileTime, HashAlgId.Crc32, deltaFilePath)) - { - throw new Win32Exception(); - } - } + // if (!NativeMethods.CreateDelta( + // FileTypeSet.Executables, CreateFlags.IgnoreFileSizeLimit, CreateFlags.None, oldFilePath, newFilePath, + // sourceOptionsName, targetOptionsName, globalOptions, targetFileTime, HashAlgId.Crc32, deltaFilePath)) + // { + // throw new Win32Exception(); + // } + //} - public void ApplyDelta(string deltaFilePath, string oldFilePath, string newFilePath) + public static void ApplyDelta(string deltaFilePath, string oldFilePath, string newFilePath) { if (!NativeMethods.ApplyDelta(ApplyFlags.AllowLegacy, oldFilePath, deltaFilePath, newFilePath)) throw new Win32Exception();