From e894991b5fcf0f58cbaf30d0f35245741654e1e0 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:24:46 -0800 Subject: [PATCH 01/37] Add SdkArchiveDiff task to verify the sdk archive has all the expected outputs --- src/SourceBuild/content/Directory.Build.props | 1 + src/SourceBuild/content/build.proj | 1 + src/SourceBuild/content/eng/build.targets | 27 ++++ .../content/eng/tools/init-build.proj | 9 ++ .../Archive.cs | 137 ++++++++++++++++++ .../FindArchiveDiffs.cs | 134 +++++++++++++++++ .../GetClosestOfficialSdk.cs | 82 +++++++++++ ...et.SourceBuild.Tasks.SdkArchiveDiff.csproj | 13 ++ .../PathWithVersions.cs | 113 +++++++++++++++ .../ZipArchiveExtensions.cs | 79 ++++++++++ 10 files changed, 596 insertions(+) create mode 100644 src/SourceBuild/content/eng/build.targets create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index bbd94253d7d2..c9a997e7a63a 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -199,6 +199,7 @@ $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll')) diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 04f6a25b6f6f..adc0a15bb530 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -20,5 +20,6 @@ + diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets new file mode 100644 index 000000000000..5f7b15caa398 --- /dev/null +++ b/src/SourceBuild/content/eng/build.targets @@ -0,0 +1,27 @@ + + + + + + + + $(OutDir) + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index 4d6b4d316651..b5201514e5ea 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -15,6 +15,7 @@ UnpackTarballs; BuildXPlatTasks; BuildMSBuildSdkResolver; + BuildTarballDiff; BuildLeakDetection; ExtractToolPackage; GenerateRootFs; @@ -116,6 +117,14 @@ + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs new file mode 100644 index 000000000000..e7943c0a30cd --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -0,0 +1,137 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Threading; +using System.Threading.Tasks; +using static ArchiveExtensions; + +public abstract class Archive : IDisposable +{ + public static async Task Create(string path) + { + if (path.EndsWith(".tar.gz")) + return await TarArchive.Create(path); + else if (path.EndsWith(".zip")) + return ZipFileArchive.Create(path); + else + throw new NotSupportedException("Unsupported archive type"); + } + + public abstract bool Contains(string relativePath); + + public abstract string[] GetFileNames(); + + public abstract string[] GetFileLines(string relativePath); + + public abstract Task GetFileBytesAsync(string relativePath); + + public abstract void Dispose(); + + public class TarArchive : Archive + { + private string _extractedFolder; + + private TarArchive(string extractedFolder) + { + _extractedFolder = extractedFolder; + } + + public static async Task Create(string path, CancellationToken cancellationToken = default) + { + var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs)); + using (var gzStream = File.OpenRead (path)) + using (var gzipStream = new GZipStream (gzStream, CompressionMode.Decompress)) + { + await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken); + } + return new TarArchive(tmpFolder.FullName); + } + + public override bool Contains(string relativePath) + { + return File.Exists(Path.Combine(_extractedFolder, relativePath)); + } + + public override string[] GetFileNames() + { + return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray(); + } + + public override string[] GetFileLines(string relativePath) + { + return File.ReadAllLines(Path.Combine(_extractedFolder, relativePath)); + } + + public override Task GetFileBytesAsync(string relativePath) + { + var filePath = Path.Combine(_extractedFolder, relativePath); + if (!File.Exists(filePath)) + return Task.FromResult([]); + return File.ReadAllBytesAsync(Path.Combine(_extractedFolder, relativePath)); + } + + public override void Dispose() + { + if (Directory.Exists(_extractedFolder)) + Directory.Delete(_extractedFolder, true); + } + } + + public class ZipFileArchive : Archive + { + private ZipArchive _archive; + + private ZipFileArchive(ZipArchive archive) + { + _archive = archive; + } + + public static new ZipFileArchive Create(string path) + { + return new ZipFileArchive(new ZipArchive(File.OpenRead(path))); + } + + public override bool Contains(string relativePath) + { + return _archive.GetEntry(relativePath) != null; + } + + public override string[] GetFileNames() + { + return _archive.Entries.Select(e => e.FullName).ToArray(); + } + + public override string[] GetFileLines(string relativePath) + { + var entry = _archive.GetEntry(relativePath); + if (entry == null) + throw new ArgumentException("File not found"); + return entry.Lines(); + } + public override Task GetFileBytesAsync(string relativePath) + { + using (var entry = _archive.GetEntry(relativePath)?.Open()) + { + if (entry == null) + { + return Task.FromResult([]); + } + return entry.ReadToEndAsync(); + } + } + + public override void Dispose() + { + _archive.Dispose(); + } + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs new file mode 100644 index 000000000000..8225c59f3acf --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Task = System.Threading.Tasks.Task; + +public class FindArchiveDiffs : Microsoft.Build.Utilities.Task +{ + public class ArchiveItem + { + public required string Path { get; init; } + } + + [Required] + public required ITaskItem BaselineArchive { get; init; } + + [Required] + public required ITaskItem TestArchive { get; init; } + + [Output] + public ITaskItem[] ContentDifferences { get; set; } = []; + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + var baselineTask = Archive.Create(BaselineArchive.ItemSpec); + var testTask = Archive.Create(TestArchive.ItemSpec); + Task.WaitAll(baselineTask, testTask); + using var baseline = await baselineTask; + using var test = await testTask; + var baselineFiles = baseline.GetFileNames(); + var testFiles = test.GetFileNames(); + ContentDifferences = + GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionAnonymousPath) + .Select(FromDiff) + .ToArray(); + return true; + } + + static ITaskItem FromDiff((string, DifferenceKind) diff) + { + var item = new TaskItem(diff.Item1); + item.SetMetadata("Kind", Enum.GetName(diff.Item2)); + return item; + } + + public enum DifferenceKind + { + Added, + Removed, + Unchanged + } + + public static List<(string, DifferenceKind DifferenceKind)> GetDiffs( + string[] originalPathsWithVersions, + string[] modifiedPathsWithVersions, + Func equalityComparer, + Func? formatter = null) + { + formatter ??= static s => s; + // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence + + int[,] dp = new int[originalPathsWithVersions.Length + 1, modifiedPathsWithVersions.Length + 1]; + + // Initialize first row and column + for (int i = 0; i <= originalPathsWithVersions.Length; i++) + { + dp[i, 0] = i; + } + for (int j = 0; j <= modifiedPathsWithVersions.Length; j++) + { + dp[0, j] = j; + } + + // Compute edit distance + for (int i = 1; i <= originalPathsWithVersions.Length; i++) + { + for (int j = 1; j <= modifiedPathsWithVersions.Length; j++) + { + if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1])) + { + dp[i, j] = dp[i - 1, j - 1]; + } + else + { + dp[i, j] = 1 + Math.Min(dp[i - 1, j], dp[i, j - 1]); + } + } + } + + // Trace back the edits + int row = originalPathsWithVersions.Length; + int col = modifiedPathsWithVersions.Length; + + List<(string, DifferenceKind)> formattedDiff = []; + while (row > 0 || col > 0) + { + var baselineItem = originalPathsWithVersions[row - 1]; + var testItem = modifiedPathsWithVersions[col - 1]; + if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem)) + { + formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Unchanged)); + row--; + col--; + } + else if (col > 0 && (row == 0 || dp[row, col - 1] <= dp[row - 1, col])) + { + formattedDiff.Add((formatter(modifiedPathsWithVersions[col - 1]), DifferenceKind.Added)); + col--; + } + else if (row > 0 && (col == 0 || dp[row, col - 1] > dp[row - 1, col])) + { + formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Removed)); + row--; + } + else + { + throw new UnreachableException(); + } + } + formattedDiff.Reverse(); + return formattedDiff; + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs new file mode 100644 index 000000000000..51b171da8136 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task +{ + [Required] + public required string BuiltSdkPath { get; init; } + + [Output] + public string ClosestOfficialSdkPath { get; set; } = ""; + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + var (versionString, rid, extension) = ExtractFromFilePath(BuiltSdkPath); + + string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); + + Log.LogMessage($"Downloading {downloadUrl}"); + var packageResponse = await new HttpClient().GetAsync(downloadUrl); + packageResponse.EnsureSuccessStatusCode(); + + var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; + string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString(); + + ClosestOfficialSdkPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".dotnet-sdk-{downloadedVersion}-{rid}{extension}"); + Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialSdkPath}"); + using (var file = File.Create(ClosestOfficialSdkPath)) + { + await packageResponse.Content.CopyToAsync(file); + } + + return true; + } + + string GetLatestOfficialSdkUrl(string versionString, string rid, string extension) + { + // Channel in the form of 9.0.1xx + var channel = versionString[..5] + "xx"; + return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}"; + } + + static (string Version, string Rid, string extension) ExtractFromFilePath(string path) + { + string extension; + if (path.EndsWith(".tar.gz")) + { + extension = ".tar.gz"; + } + else if (path.EndsWith(".zip")) + { + extension = ".zip"; + } + else + { + throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip"); + } + + string filename = Path.GetFileName(path)[..^extension.Length]; + var dashDelimitedParts = filename.Split('-'); + var (rid, versionString) = dashDelimitedParts switch + { + ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second), + ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth), + _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build full build full build full build full build full build full build full build full build version and rid") + }; + + return (versionString, rid, extension); + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj new file mode 100644 index 000000000000..a12ea4d87aa8 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -0,0 +1,13 @@ + + + + Exe + $(NetCurrent) + enable + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs new file mode 100644 index 000000000000..2d3eb29c3b93 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs @@ -0,0 +1,113 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +public static class PathWithVersions +{ + public const string VersionPlaceholder = "{VERSION}"; + + public static bool Equal(string path1, string path2) + { + if (path1 == path2) + { + return true; + } + + ReadOnlySpan directory = path1; + ReadOnlySpan directory2 = path2; + while (TryGetPathLeaf(directory, out var root, out var directoryPart) && TryGetPathLeaf(directory2, out var root2, out var directoryPart2)) + { + if (!ReplaceVersionString(directoryPart).SequenceEqual(ReplaceVersionString(directoryPart2))) + { + return false; + } + directory= Path.GetDirectoryName(directory); + directory2= Path.GetDirectoryName(directory2); + } + if (!directory.IsEmpty || !directory2.IsEmpty) + { + return false; + } + return true; + } + + public static bool IsVersionString(ReadOnlySpan directoryPart) + { + return directoryPart.Length >= 6 + && char.IsDigit(directoryPart[0]) + && directoryPart[1] == '.' + && char.IsDigit(directoryPart[2]) + && directoryPart[3] == '.' + && char.IsDigit(directoryPart[4]) + && ((char.IsDigit(directoryPart[5]) && char.IsDigit(directoryPart[6])) || directoryPart[5] == '-'); + } + + static ReadOnlySpan ReplaceVersionString(ReadOnlySpan directoryPart) + { + if (IsVersionString(directoryPart)) + { + return VersionPlaceholder; + } + else + { + return directoryPart; + } + } + + static bool TryGetPathLeaf(ReadOnlySpan path, out ReadOnlySpan root, out ReadOnlySpan leaf) + { + if (path.IsEmpty) + { + root = default; + leaf = default; + return false; + } + leaf = Path.GetFileName(path); + root = Path.GetDirectoryName(path); + return true; + } + + public static string GetVersionAnonymousPath(string path) + { + return GetVersionAnonymousPath(path).ToString(); + } + + public static ReadOnlySpan GetVersionAnonymousPath(ReadOnlySpan path) + { + StringBuilder sb = new StringBuilder(); + bool altered = false; + ReadOnlySpan myPath = path; + while (TryGetPathLeaf(myPath, out var directory, out var directoryPart)) + { + sb = sb.Insert(0, Path.DirectorySeparatorChar); + var versionOrDirectory = ReplaceVersionString(directoryPart); + if (versionOrDirectory == VersionPlaceholder) + { + altered = true; + } + sb = sb.Insert(0, versionOrDirectory); + myPath = directory; + } + if (!altered) + return path; + return sb.ToString(); + } + + public static ReadOnlySpan GetVersionInPath(ReadOnlySpan path) + { + ReadOnlySpan myPath = path; + while (TryGetPathLeaf(myPath, out var directory, out var directoryPart)) + { + if (IsVersionString(directoryPart)) + { + return directoryPart; + } + myPath = directory; + } + throw new ArgumentException("Path does not contain a version"); + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs new file mode 100644 index 000000000000..08c3c6cef578 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs @@ -0,0 +1,79 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +static class ArchiveExtensions +{ + public static string[] Lines(this ZipArchiveEntry entry, Encoding? encoding = null) + { + return entry.ReadToString(encoding).Replace("\r\n", "\n").Split('\n').ToArray(); + } + + public static string ReadToString(this ZipArchiveEntry entry, Encoding? encoding = null) + { + Stream stream = entry.Open(); + byte[] buffer = stream.ReadToEnd(); + // Remove UTF-8 BOM if present + int index = 0; + if (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) + { + index = 3; + } + encoding ??= Encoding.UTF8; + string fileText = encoding.GetString(buffer, index, buffer.Length - index); + return fileText; + } + + public static byte[] ReadToEnd(this Stream stream) + { + int bufferSize = 2048; + byte[] buffer = new byte[bufferSize]; + int offset = 0; + while (true) + { + int bytesRead = stream.Read(buffer, offset, bufferSize - offset); + offset += bytesRead; + if (bytesRead == 0) + { + break; + } + if (offset == bufferSize) + { + Array.Resize(ref buffer, bufferSize * 2); + bufferSize *= 2; + } + } + Array.Resize(ref buffer, offset); + return buffer; + } + + public static async Task ReadToEndAsync(this Stream stream) + { + int bufferSize = 2048; + byte[] buffer = new byte[bufferSize]; + int offset = 0; + while (true) + { + int bytesRead = await stream.ReadAsync(buffer, offset, bufferSize - offset); + offset += bytesRead; + if (bytesRead == 0) + { + break; + } + if (offset == bufferSize) + { + Array.Resize(ref buffer, bufferSize * 2); + bufferSize *= 2; + } + } + Array.Resize(ref buffer, offset); + return buffer; + } +} From cd1b907f804323cff92d8e7d93d6a049edb683c5 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:38:05 -0800 Subject: [PATCH 02/37] fix name --- src/SourceBuild/content/eng/tools/init-build.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index b5201514e5ea..95b45311372f 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -15,7 +15,7 @@ UnpackTarballs; BuildXPlatTasks; BuildMSBuildSdkResolver; - BuildTarballDiff; + BuildSdkArchiveDiff; BuildLeakDetection; ExtractToolPackage; GenerateRootFs; From a4bee9104a7c3e63a9e529e8df4dbdfbcb406ddb Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:47:34 -0800 Subject: [PATCH 03/37] Forgot to make the project a library after testing --- .../Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index a12ea4d87aa8..99de429359bc 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -1,7 +1,7 @@  - Exe + Library $(NetCurrent) enable From 5cc5ee053262af866b6c6f4673cc8bd74e996a20 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:52:31 -0800 Subject: [PATCH 04/37] Run the archive diff by default --- src/SourceBuild/content/eng/build.targets | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 5f7b15caa398..409f28339223 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -4,8 +4,7 @@ + DependsOnTargets="DetermineSourceBuiltSdkVersion"> $(OutDir) From 2862e1f75c24a066578c4c82641938624b81a5d5 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:14:53 -0800 Subject: [PATCH 05/37] Report if there is no sdk produced in the build --- src/SourceBuild/content/eng/build.targets | 14 ++++++-------- ....DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 3 +-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 409f28339223..c10d8c12efd1 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -4,22 +4,20 @@ + DependsOnTargets="DetermineSourceBuiltSdkVersion" > - - $(OutDir) - - - + + - + + - + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index 99de429359bc..a8a8731f5ef0 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -7,7 +7,6 @@ - - + From a336dd0f65fe7950066451ccd477a3ee78af7e8e Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:44:46 -0800 Subject: [PATCH 06/37] Update src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj Co-authored-by: Viktor Hofer --- .../Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index a8a8731f5ef0..2d59b695180b 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -1,7 +1,6 @@  - Library $(NetCurrent) enable From a3256e2d70944d9ef598f6f6386ed05f40ad1ba9 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:44:51 -0800 Subject: [PATCH 07/37] Update src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj Co-authored-by: Viktor Hofer --- .../Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index 2d59b695180b..ff3bf2402a01 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -8,4 +8,5 @@ + From 8c32f67bad8832a209abd3f09f8e865bf0bf09e8 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:32:02 -0800 Subject: [PATCH 08/37] Don't autoredirect and check for correct aka.ms redirect code --- .../GetClosestOfficialSdk.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 51b171da8136..6f130c100e47 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -29,8 +29,19 @@ public async Task ExecuteAsync() string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); Log.LogMessage($"Downloading {downloadUrl}"); - var packageResponse = await new HttpClient().GetAsync(downloadUrl); - packageResponse.EnsureSuccessStatusCode(); + var handler = new HttpClientHandler() + { + AllowAutoRedirect = false + }; + var client = new HttpClient(handler); + var redirectResponse = await client.GetAsync(downloadUrl); + // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs + if (redirectResponse.StatusCode != HttpStatusCode.Moved) + { + Log.LogMessage(MessageImportance.High, $"Failed to download '{downloadUrl}': invalid aka.ms URL"); + return true; + } + var packageResponse = await client.GetAsync(redirectResponse.Headers.Location!); var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString(); From 0eea7e057935d2bc99be27e824138d65f511b2ed Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:29:22 -0800 Subject: [PATCH 09/37] Fix stack overflow and rename method --- src/SourceBuild/content/eng/build.targets | 2 +- .../GetClosestOfficialSdk.cs | 4 ++-- .../PathWithVersions.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index c10d8c12efd1..63030702808b 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -16,7 +16,7 @@ - + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 6f130c100e47..708bea6d65d5 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -24,7 +24,7 @@ public override bool Execute() public async Task ExecuteAsync() { - var (versionString, rid, extension) = ExtractFromFilePath(BuiltSdkPath); + var (versionString, rid, extension) = GetInfoFromArchivePath(BuiltSdkPath); string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); @@ -63,7 +63,7 @@ string GetLatestOfficialSdkUrl(string versionString, string rid, string extensio return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}"; } - static (string Version, string Rid, string extension) ExtractFromFilePath(string path) + static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path) { string extension; if (path.EndsWith(".tar.gz")) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs index 2d3eb29c3b93..6baf40e86a67 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs @@ -73,7 +73,7 @@ static bool TryGetPathLeaf(ReadOnlySpan path, out ReadOnlySpan root, public static string GetVersionAnonymousPath(string path) { - return GetVersionAnonymousPath(path).ToString(); + return GetVersionAnonymousPath(path.AsSpan()).ToString(); } public static ReadOnlySpan GetVersionAnonymousPath(ReadOnlySpan path) From d30c7408d76042e2f3b73bd0a9d9c5fe7e99af6f Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:31:36 -0800 Subject: [PATCH 10/37] Rename GetVersionAnonymousPath --- .../FindArchiveDiffs.cs | 2 +- .../PathWithVersions.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs index 8225c59f3acf..f7bf07292cc5 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -41,7 +41,7 @@ public async Task ExecuteAsync() var baselineFiles = baseline.GetFileNames(); var testFiles = test.GetFileNames(); ContentDifferences = - GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionAnonymousPath) + GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath) .Select(FromDiff) .ToArray(); return true; diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs index 6baf40e86a67..e8e3d7c9da57 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs @@ -71,12 +71,12 @@ static bool TryGetPathLeaf(ReadOnlySpan path, out ReadOnlySpan root, return true; } - public static string GetVersionAnonymousPath(string path) + public static string GetVersionlessPath(string path) { - return GetVersionAnonymousPath(path.AsSpan()).ToString(); + return GetVersionlessPath(path.AsSpan()).ToString(); } - public static ReadOnlySpan GetVersionAnonymousPath(ReadOnlySpan path) + public static ReadOnlySpan GetVersionlessPath(ReadOnlySpan path) { StringBuilder sb = new StringBuilder(); bool altered = false; From f704a53482cd4ffab1962e11faa4ceb00c977d87 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 28 Feb 2024 07:02:52 -0800 Subject: [PATCH 11/37] Fix error message --- .../GetClosestOfficialSdk.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 708bea6d65d5..1eca113e16fa 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -85,7 +85,7 @@ string GetLatestOfficialSdkUrl(string versionString, string rid, string extensio { ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second), ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth), - _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build full build full build full build full build full build full build full build full build version and rid") + _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk--{extension} or dotnet-sdk--{extension}") }; return (versionString, rid, extension); From 45728f7ba535af71ab5ac3585c62ac6f38428caf Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 28 Feb 2024 07:36:59 -0800 Subject: [PATCH 12/37] Add task to find the single best tarball, and remove unused usings --- src/SourceBuild/content/eng/build.targets | 13 +++-- .../Archive.cs | 33 ++++++++++-- .../GetClosestOfficialSdk.cs | 33 +----------- .../GetSingleTarballItem.cs | 51 +++++++++++++++++++ .../PathWithVersions.cs | 1 - .../ZipArchiveExtensions.cs | 1 - 6 files changed, 90 insertions(+), 42 deletions(-) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 63030702808b..048d2b89b3ce 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -1,18 +1,23 @@ + - - + + + + + - - + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs index e7943c0a30cd..7a572b356261 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -2,17 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Immutable; using System.Formats.Tar; using System.IO; using System.IO.Compression; using System.Linq; -using System.Reflection; -using System.Reflection.Metadata; -using System.Reflection.PortableExecutable; using System.Threading; using System.Threading.Tasks; -using static ArchiveExtensions; public abstract class Archive : IDisposable { @@ -134,4 +129,32 @@ public override void Dispose() _archive.Dispose(); } } + + public static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path) + { + string extension; + if (path.EndsWith(".tar.gz")) + { + extension = ".tar.gz"; + } + else if (path.EndsWith(".zip")) + { + extension = ".zip"; + } + else + { + throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip"); + } + + string filename = Path.GetFileName(path)[..^extension.Length]; + var dashDelimitedParts = filename.Split('-'); + var (rid, versionString) = dashDelimitedParts switch + { + ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second), + ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth), + _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk--{extension} or dotnet-sdk--{extension}") + }; + + return (versionString, rid, extension); + } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 1eca113e16fa..8ea6b4f83465 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -1,14 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Diagnostics; using System.IO; using System.Net; using System.Net.Http; -using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Build.Framework; + public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task { [Required] @@ -24,7 +22,7 @@ public override bool Execute() public async Task ExecuteAsync() { - var (versionString, rid, extension) = GetInfoFromArchivePath(BuiltSdkPath); + var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(BuiltSdkPath); string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); @@ -63,31 +61,4 @@ string GetLatestOfficialSdkUrl(string versionString, string rid, string extensio return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}"; } - static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path) - { - string extension; - if (path.EndsWith(".tar.gz")) - { - extension = ".tar.gz"; - } - else if (path.EndsWith(".zip")) - { - extension = ".zip"; - } - else - { - throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip"); - } - - string filename = Path.GetFileName(path)[..^extension.Length]; - var dashDelimitedParts = filename.Split('-'); - var (rid, versionString) = dashDelimitedParts switch - { - ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second), - ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth), - _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk--{extension} or dotnet-sdk--{extension}") - }; - - return (versionString, rid, extension); - } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs new file mode 100644 index 000000000000..9641ad505c0d --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Build.Framework; + +public class GetSingleTarballItem : Microsoft.Build.Utilities.Task +{ + [Required] + public required ITaskItem[] SdkTarballItems { get; init; } + + [Output] + public string BestSdkTarballItem { get; set; } = ""; + + public override bool Execute() + { + List tarballItems = new (); + foreach(var item in SdkTarballItems) + { + try + { + var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(item.ItemSpec); + tarballItems.Add(item.ItemSpec); + } + catch (ArgumentException e) + { + Log.LogMessage(MessageImportance.High, e.Message); + continue; + } + } + switch (tarballItems.Count){ + case 0: + Log.LogMessage(MessageImportance.High, "No valid tarball items found"); + BestSdkTarballItem = ""; + break; + case 1: + Log.LogMessage(MessageImportance.High, $"{tarballItems[0]} is the only valid tarball item found"); + BestSdkTarballItem = tarballItems[0]; + break; + default: + tarballItems.Sort((a,b) => a.Length - b.Length); + Log.LogMessage(MessageImportance.High, $"Multiple valid tarball items found: '{string.Join("', '", tarballItems)}'"); + BestSdkTarballItem = tarballItems[0]; + Log.LogMessage(MessageImportance.High, $"Choosing '{BestSdkTarballItem}"); + break; + } + return true; + } + +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs index e8e3d7c9da57..bc2af0c6b705 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.IO; using System.Text; diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs index 08c3c6cef578..4b67efea0bc1 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; From b6fba7eb7cf93c3319174f2b99fa4ccf662b15cd Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 28 Feb 2024 09:56:49 -0800 Subject: [PATCH 13/37] Use property notation instead of item --- src/SourceBuild/content/eng/build.targets | 2 +- .../GetClosestOfficialSdk.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 048d2b89b3ce..bc77c38f94eb 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -11,7 +11,7 @@ - + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 8ea6b4f83465..8283cc09fc44 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -26,7 +26,7 @@ public async Task ExecuteAsync() string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); - Log.LogMessage($"Downloading {downloadUrl}"); + Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); var handler = new HttpClientHandler() { AllowAutoRedirect = false From 9ffe5c200da2bd0cd778fbcdbd4f1178acc07450 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 28 Feb 2024 10:06:38 -0800 Subject: [PATCH 14/37] Make long running tasks cancellable --- .../Archive.cs | 8 ++++---- .../FindArchiveDiffs.cs | 10 +++++++++- .../GetClosestOfficialSdk.cs | 17 +++++++++++++---- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs index 7a572b356261..2e89173b58ec 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -11,10 +11,10 @@ public abstract class Archive : IDisposable { - public static async Task Create(string path) + public static async Task Create(string path, CancellationToken cancellationToken = default) { if (path.EndsWith(".tar.gz")) - return await TarArchive.Create(path); + return await TarArchive.Create(path, cancellationToken); else if (path.EndsWith(".zip")) return ZipFileArchive.Create(path); else @@ -40,7 +40,7 @@ private TarArchive(string extractedFolder) _extractedFolder = extractedFolder; } - public static async Task Create(string path, CancellationToken cancellationToken = default) + public static new async Task Create(string path, CancellationToken cancellationToken = default) { var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs)); using (var gzStream = File.OpenRead (path)) @@ -90,7 +90,7 @@ private ZipFileArchive(ZipArchive archive) _archive = archive; } - public static new ZipFileArchive Create(string path) + public static ZipFileArchive Create(string path) { return new ZipFileArchive(new ZipArchive(File.OpenRead(path))); } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs index f7bf07292cc5..1d1c7108b254 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -5,12 +5,13 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Task = System.Threading.Tasks.Task; -public class FindArchiveDiffs : Microsoft.Build.Utilities.Task +public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask { public class ArchiveItem { @@ -26,6 +27,13 @@ public class ArchiveItem [Output] public ITaskItem[] ContentDifferences { get; set; } = []; + private CancellationTokenSource _cancellationTokenSource = new(); + private CancellationToken cancellationToken => _cancellationTokenSource.Token; + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + public override bool Execute() { return Task.Run(ExecuteAsync).Result; diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 8283cc09fc44..e5cd717a8fa5 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -4,10 +4,11 @@ using System.IO; using System.Net; using System.Net.Http; +using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; -public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task +public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task, ICancelableTask { [Required] public required string BuiltSdkPath { get; init; } @@ -20,8 +21,16 @@ public override bool Execute() return Task.Run(ExecuteAsync).Result; } + private CancellationTokenSource _cancellationTokenSource = new(); + private CancellationToken cancellationToken => _cancellationTokenSource.Token; + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + public async Task ExecuteAsync() { + cancellationToken.ThrowIfCancellationRequested(); var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(BuiltSdkPath); string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); @@ -32,14 +41,14 @@ public async Task ExecuteAsync() AllowAutoRedirect = false }; var client = new HttpClient(handler); - var redirectResponse = await client.GetAsync(downloadUrl); + var redirectResponse = await client.GetAsync(downloadUrl, cancellationToken); // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs if (redirectResponse.StatusCode != HttpStatusCode.Moved) { Log.LogMessage(MessageImportance.High, $"Failed to download '{downloadUrl}': invalid aka.ms URL"); return true; } - var packageResponse = await client.GetAsync(redirectResponse.Headers.Location!); + var packageResponse = await client.GetAsync(redirectResponse.Headers.Location!, cancellationToken); var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString(); @@ -48,7 +57,7 @@ public async Task ExecuteAsync() Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialSdkPath}"); using (var file = File.Create(ClosestOfficialSdkPath)) { - await packageResponse.Content.CopyToAsync(file); + await packageResponse.Content.CopyToAsync(file, cancellationToken); } return true; From 7632ab4d74cf3ba0cbd525c1a81d429a587a6e5f Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:48:31 -0800 Subject: [PATCH 15/37] PR Feedback: - Use 'archive' rather than 'tarball' - Remove dead code - Discard unused return values --- src/SourceBuild/content/eng/build.targets | 14 ++--- .../Archive.cs | 38 +------------- .../GetSingleArchiveItem.cs | 52 +++++++++++++++++++ .../GetSingleTarballItem.cs | 51 ------------------ 4 files changed, 60 insertions(+), 95 deletions(-) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs delete mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index bc77c38f94eb..8ea3c1a3c0a5 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -1,23 +1,23 @@ - + - - - + + + - + - + - + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs index 2e89173b58ec..32e1951aa86b 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -21,13 +21,9 @@ public static async Task Create(string path, CancellationToken cancella throw new NotSupportedException("Unsupported archive type"); } - public abstract bool Contains(string relativePath); - public abstract string[] GetFileNames(); - public abstract string[] GetFileLines(string relativePath); - - public abstract Task GetFileBytesAsync(string relativePath); + public abstract bool Contains(string relativePath); public abstract void Dispose(); @@ -61,19 +57,6 @@ public override string[] GetFileNames() return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray(); } - public override string[] GetFileLines(string relativePath) - { - return File.ReadAllLines(Path.Combine(_extractedFolder, relativePath)); - } - - public override Task GetFileBytesAsync(string relativePath) - { - var filePath = Path.Combine(_extractedFolder, relativePath); - if (!File.Exists(filePath)) - return Task.FromResult([]); - return File.ReadAllBytesAsync(Path.Combine(_extractedFolder, relativePath)); - } - public override void Dispose() { if (Directory.Exists(_extractedFolder)) @@ -105,25 +88,6 @@ public override string[] GetFileNames() return _archive.Entries.Select(e => e.FullName).ToArray(); } - public override string[] GetFileLines(string relativePath) - { - var entry = _archive.GetEntry(relativePath); - if (entry == null) - throw new ArgumentException("File not found"); - return entry.Lines(); - } - public override Task GetFileBytesAsync(string relativePath) - { - using (var entry = _archive.GetEntry(relativePath)?.Open()) - { - if (entry == null) - { - return Task.FromResult([]); - } - return entry.ReadToEndAsync(); - } - } - public override void Dispose() { _archive.Dispose(); diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs new file mode 100644 index 000000000000..a902a7ea3307 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Build.Framework; + +public class GetSingleArchiveItem : Microsoft.Build.Utilities.Task +{ + [Required] + public required ITaskItem[] SdkArchiveItems { get; init; } + + [Output] + public string BestSdkArchiveItem { get; set; } = ""; + + public override bool Execute() + { + List archiveItems = new (); + foreach(var item in SdkArchiveItems) + { + try + { + // Ensure the version and RID info can be parsed from the item + _ = Archive.GetInfoFromArchivePath(item.ItemSpec); + archiveItems.Add(item.ItemSpec); + } + catch (ArgumentException e) + { + Log.LogMessage(MessageImportance.High, e.Message); + continue; + } + } + switch (archiveItems.Count){ + case 0: + Log.LogMessage(MessageImportance.High, "No valid archive items found"); + BestSdkArchiveItem = ""; + break; + case 1: + Log.LogMessage(MessageImportance.High, $"{archiveItems[0]} is the only valid archive item found"); + BestSdkArchiveItem = archiveItems[0]; + break; + default: + archiveItems.Sort((a,b) => a.Length - b.Length); + Log.LogMessage(MessageImportance.High, $"Multiple valid archive items found: '{string.Join("', '", archiveItems)}'"); + BestSdkArchiveItem = archiveItems[0]; + Log.LogMessage(MessageImportance.High, $"Choosing '{BestSdkArchiveItem}"); + break; + } + return true; + } + +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs deleted file mode 100644 index 9641ad505c0d..000000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleTarballItem.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using Microsoft.Build.Framework; - -public class GetSingleTarballItem : Microsoft.Build.Utilities.Task -{ - [Required] - public required ITaskItem[] SdkTarballItems { get; init; } - - [Output] - public string BestSdkTarballItem { get; set; } = ""; - - public override bool Execute() - { - List tarballItems = new (); - foreach(var item in SdkTarballItems) - { - try - { - var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(item.ItemSpec); - tarballItems.Add(item.ItemSpec); - } - catch (ArgumentException e) - { - Log.LogMessage(MessageImportance.High, e.Message); - continue; - } - } - switch (tarballItems.Count){ - case 0: - Log.LogMessage(MessageImportance.High, "No valid tarball items found"); - BestSdkTarballItem = ""; - break; - case 1: - Log.LogMessage(MessageImportance.High, $"{tarballItems[0]} is the only valid tarball item found"); - BestSdkTarballItem = tarballItems[0]; - break; - default: - tarballItems.Sort((a,b) => a.Length - b.Length); - Log.LogMessage(MessageImportance.High, $"Multiple valid tarball items found: '{string.Join("', '", tarballItems)}'"); - BestSdkTarballItem = tarballItems[0]; - Log.LogMessage(MessageImportance.High, $"Choosing '{BestSdkTarballItem}"); - break; - } - return true; - } - -} From 3cf9a383be01e8423ca73b3ee40e4d332b969407 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:00:40 -0800 Subject: [PATCH 16/37] Use cancellation token in FindArchiveDiff and move directory separator in path builder --- .../FindArchiveDiffs.cs | 10 +++++++--- .../PathWithVersions.cs | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs index 1d1c7108b254..ac6a444627d6 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -43,13 +43,13 @@ public async Task ExecuteAsync() { var baselineTask = Archive.Create(BaselineArchive.ItemSpec); var testTask = Archive.Create(TestArchive.ItemSpec); - Task.WaitAll(baselineTask, testTask); + Task.WaitAll([baselineTask, testTask], cancellationToken); using var baseline = await baselineTask; using var test = await testTask; var baselineFiles = baseline.GetFileNames(); var testFiles = test.GetFileNames(); ContentDifferences = - GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath) + GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath, cancellationToken) .Select(FromDiff) .ToArray(); return true; @@ -73,8 +73,10 @@ public enum DifferenceKind string[] originalPathsWithVersions, string[] modifiedPathsWithVersions, Func equalityComparer, - Func? formatter = null) + Func? formatter = null, + CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); formatter ??= static s => s; // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence @@ -93,6 +95,7 @@ public enum DifferenceKind // Compute edit distance for (int i = 1; i <= originalPathsWithVersions.Length; i++) { + cancellationToken.ThrowIfCancellationRequested(); for (int j = 1; j <= modifiedPathsWithVersions.Length; j++) { if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1])) @@ -113,6 +116,7 @@ public enum DifferenceKind List<(string, DifferenceKind)> formattedDiff = []; while (row > 0 || col > 0) { + cancellationToken.ThrowIfCancellationRequested(); var baselineItem = originalPathsWithVersions[row - 1]; var testItem = modifiedPathsWithVersions[col - 1]; if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem)) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs index bc2af0c6b705..28a1f854ee59 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs @@ -80,16 +80,16 @@ public static ReadOnlySpan GetVersionlessPath(ReadOnlySpan path) StringBuilder sb = new StringBuilder(); bool altered = false; ReadOnlySpan myPath = path; - while (TryGetPathLeaf(myPath, out var directory, out var directoryPart)) + while (TryGetPathLeaf(myPath, out var root, out var leaf)) { - sb = sb.Insert(0, Path.DirectorySeparatorChar); - var versionOrDirectory = ReplaceVersionString(directoryPart); + var versionOrDirectory = ReplaceVersionString(leaf); if (versionOrDirectory == VersionPlaceholder) { altered = true; } sb = sb.Insert(0, versionOrDirectory); - myPath = directory; + sb = sb.Insert(0, Path.DirectorySeparatorChar); + myPath = root; } if (!altered) return path; From 478bb285c11482064a81b3029c330667103a9635 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:32:22 -0800 Subject: [PATCH 17/37] PR Feedback: use underscore prefix for 'local' properties --- src/SourceBuild/content/eng/build.targets | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 8ea3c1a3c0a5..7ddc69e09e05 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -12,17 +12,17 @@ - + - + - - + + - - + + From f84323905dfccd0f3d316e7735ca715674c3dde6 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 10:42:14 -0800 Subject: [PATCH 18/37] PR Feedback: - Extract common archive diffing into a base class - Move Diff utils to a separate class - Return all valid archives and assert only one valid SDK archive - Use Arcade VersionIdentifier util class --- src/SourceBuild/content/eng/build.targets | 39 ++- .../Archive.cs | 35 +-- .../Diff.cs | 127 +++++++++ .../FindArchiveDiffs.cs | 105 +------ .../GetClosestArchiveVersion.cs | 94 +++++++ .../GetClosestOfficialSdk.cs | 73 ++--- ...ArchiveItem.cs => GetValidArchiveItems.cs} | 31 ++- .../PathWithVersions.cs | 112 -------- .../VersionIdentifier.cs | 257 ++++++++++++++++++ .../ZipArchiveExtensions.cs | 78 ------ 10 files changed, 566 insertions(+), 385 deletions(-) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs rename src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/{GetSingleArchiveItem.cs => GetValidArchiveItems.cs} (58%) delete mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs delete mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 7ddc69e09e05..9a6eabd2be79 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -1,28 +1,41 @@ - + - + - - - + + + - - - + - + + + - - + + - - + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs index 32e1951aa86b..0b862af52aa9 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -39,8 +39,8 @@ private TarArchive(string extractedFolder) public static new async Task Create(string path, CancellationToken cancellationToken = default) { var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs)); - using (var gzStream = File.OpenRead (path)) - using (var gzipStream = new GZipStream (gzStream, CompressionMode.Decompress)) + using (var gzStream = File.OpenRead(path)) + using (var gzipStream = new GZipStream(gzStream, CompressionMode.Decompress)) { await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken); } @@ -94,31 +94,34 @@ public override void Dispose() } } - public static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path) + private static string GetArchiveExtension(string path) { - string extension; if (path.EndsWith(".tar.gz")) { - extension = ".tar.gz"; + return ".tar.gz"; } else if (path.EndsWith(".zip")) { - extension = ".zip"; + return ".zip"; } else { throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip"); } + } - string filename = Path.GetFileName(path)[..^extension.Length]; - var dashDelimitedParts = filename.Split('-'); - var (rid, versionString) = dashDelimitedParts switch - { - ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second), - ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth), - _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk--{extension} or dotnet-sdk--{extension}") - }; - - return (versionString, rid, extension); + public static (string Version, string Rid, string extension) GetInfoFromFileName(string filename, string packageName) + { + var extension = GetArchiveExtension(filename); + var Version = VersionIdentifier.GetVersion(filename); + if (Version is null) + throw new ArgumentException("Invalid archive file name '{filename}': No valid version found in file name."); + // Once we've removed the version, package name, and extension, we should be left with the RID + var Rid = filename + .Replace(extension, "") + .Replace(Version, "") + .Replace(packageName, "") + .Trim('-', '.', '_'); + return (Version, Rid, extension); } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs new file mode 100644 index 000000000000..ab1676f97fed --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs @@ -0,0 +1,127 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +static class Diff +{ + public static ITaskItem TaskItemFromDiff((string, DifferenceKind) diff) + { + var item = new TaskItem(diff.Item1); + item.SetMetadata("Kind", Enum.GetName(diff.Item2)); + return item; + } + + public enum DifferenceKind + { + /// + /// Present in the test but not in the baseline + /// + Added, + + /// + /// Present in the baseline but not in the test + /// + Removed, + + /// + /// Present in both the baseline and test + /// + Unchanged + } + + /// + /// Uses the Longest Common Subsequence algorithm (as used in 'git diff') to find the differences between two lists of strings. + /// Returns a list of the joined lists with the differences marked as either added or removed. + /// + public static List<(string, DifferenceKind DifferenceKind)> GetDiffs( + Span baselineSequence, + Span testSequence, + Func equalityComparer, + Func? formatter = null, + CancellationToken cancellationToken = default) + { + // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence + // cancellationToken.ThrowIfCancellationRequested(); + formatter ??= static s => s; + List<(string, DifferenceKind)> diff = []; + + // Optimization: remove common prefix + int i = 0; + while (i < baselineSequence.Length && i < testSequence.Length && equalityComparer(baselineSequence[i], testSequence[i])) + { + diff.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged)); + i++; + } + + baselineSequence = baselineSequence[i..]; + testSequence = testSequence[i..]; + + // Initialize first row and column + int[,] m = new int[baselineSequence.Length + 1, testSequence.Length + 1]; + for (i = 0; i <= baselineSequence.Length; i++) + { + m[i, 0] = i; + } + for (i = 0; i <= testSequence.Length; i++) + { + m[0, i] = i; + } + + // Compute edit distance + for (i = 1; i <= baselineSequence.Length; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + for (int j = 1; j <= testSequence.Length; j++) + { + if (equalityComparer(baselineSequence[i - 1], testSequence[j - 1])) + { + m[i, j] = m[i - 1, j - 1]; + } + else + { + m[i, j] = 1 + Math.Min(m[i - 1, j], m[i, j - 1]); + } + } + } + + // Trace back the edits + int row = baselineSequence.Length; + int col = testSequence.Length; + + while (row > 0 || col > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var baselineItem = baselineSequence[row - 1]; + var testItem = testSequence[col - 1]; + if (row > 0 && col > 0 && equalityComparer(baselineItem, testItem)) + { + diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Unchanged)); + row--; + col--; + } + else if (col > 0 && (row == 0 || m[row, col - 1] <= m[row - 1, col])) + { + diff.Add((formatter(testSequence[col - 1]), DifferenceKind.Added)); + col--; + } + else if (row > 0 && (col == 0 || m[row, col - 1] > m[row - 1, col])) + { + diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Removed)); + row--; + } + else + { + throw new UnreachableException(); + } + } + diff.Reverse(); + return diff; + } + +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs index ac6a444627d6..9fc19a7385c0 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -1,23 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Task = System.Threading.Tasks.Task; public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask { - public class ArchiveItem - { - public required string Path { get; init; } - } - [Required] public required ITaskItem BaselineArchive { get; init; } @@ -29,10 +20,6 @@ public class ArchiveItem private CancellationTokenSource _cancellationTokenSource = new(); private CancellationToken cancellationToken => _cancellationTokenSource.Token; - public void Cancel() - { - _cancellationTokenSource.Cancel(); - } public override bool Execute() { @@ -49,98 +36,14 @@ public async Task ExecuteAsync() var baselineFiles = baseline.GetFileNames(); var testFiles = test.GetFileNames(); ContentDifferences = - GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath, cancellationToken) - .Select(FromDiff) + Diff.GetDiffs(baselineFiles, testFiles, VersionIdentifier.AreVersionlessEqual, static p => VersionIdentifier.RemoveVersions(p, "{VERSION}"), cancellationToken) + .Select(Diff.TaskItemFromDiff) .ToArray(); return true; } - static ITaskItem FromDiff((string, DifferenceKind) diff) - { - var item = new TaskItem(diff.Item1); - item.SetMetadata("Kind", Enum.GetName(diff.Item2)); - return item; - } - - public enum DifferenceKind - { - Added, - Removed, - Unchanged - } - - public static List<(string, DifferenceKind DifferenceKind)> GetDiffs( - string[] originalPathsWithVersions, - string[] modifiedPathsWithVersions, - Func equalityComparer, - Func? formatter = null, - CancellationToken cancellationToken = default) + public void Cancel() { - cancellationToken.ThrowIfCancellationRequested(); - formatter ??= static s => s; - // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence - - int[,] dp = new int[originalPathsWithVersions.Length + 1, modifiedPathsWithVersions.Length + 1]; - - // Initialize first row and column - for (int i = 0; i <= originalPathsWithVersions.Length; i++) - { - dp[i, 0] = i; - } - for (int j = 0; j <= modifiedPathsWithVersions.Length; j++) - { - dp[0, j] = j; - } - - // Compute edit distance - for (int i = 1; i <= originalPathsWithVersions.Length; i++) - { - cancellationToken.ThrowIfCancellationRequested(); - for (int j = 1; j <= modifiedPathsWithVersions.Length; j++) - { - if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1])) - { - dp[i, j] = dp[i - 1, j - 1]; - } - else - { - dp[i, j] = 1 + Math.Min(dp[i - 1, j], dp[i, j - 1]); - } - } - } - - // Trace back the edits - int row = originalPathsWithVersions.Length; - int col = modifiedPathsWithVersions.Length; - - List<(string, DifferenceKind)> formattedDiff = []; - while (row > 0 || col > 0) - { - cancellationToken.ThrowIfCancellationRequested(); - var baselineItem = originalPathsWithVersions[row - 1]; - var testItem = modifiedPathsWithVersions[col - 1]; - if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem)) - { - formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Unchanged)); - row--; - col--; - } - else if (col > 0 && (row == 0 || dp[row, col - 1] <= dp[row - 1, col])) - { - formattedDiff.Add((formatter(modifiedPathsWithVersions[col - 1]), DifferenceKind.Added)); - col--; - } - else if (row > 0 && (col == 0 || dp[row, col - 1] > dp[row - 1, col])) - { - formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Removed)); - row--; - } - else - { - throw new UnreachableException(); - } - } - formattedDiff.Reverse(); - return formattedDiff; + _cancellationTokenSource.Cancel(); } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs new file mode 100644 index 000000000000..fc69a385a9ac --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask +{ + [Required] + public required string BuiltArchivePath { get; init; } + + [Output] + public string ClosestOfficialArchivePath { get; set; } = ""; + + private string? _builtVersion; + protected string BuiltVersion + { + get => _builtVersion ?? throw new InvalidOperationException(); + private set => _builtVersion = value; + } + + private string? _builtRid; + protected string BuiltRid + { + get => _builtRid ?? throw new InvalidOperationException(); + private set => _builtRid = value; + } + + private string? _archiveExtension; + protected string ArchiveExtension + { + get => _archiveExtension ?? throw new InvalidOperationException(); + private set => _archiveExtension = value; + } + + /// + /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime". + /// + protected abstract string ArchiveName { get; } + + private CancellationTokenSource _cancellationTokenSource = new(); + protected CancellationToken CancellationToken => _cancellationTokenSource.Token; + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + + /// + /// Get the URL of the latest official archive for the given version string and RID. + /// + public abstract Task GetLatestOfficialArchiveUrl(); + + public abstract Task GetClosestOfficialArchiveVersion(); + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + CancellationToken.ThrowIfCancellationRequested(); + var filename = Path.GetFileName(BuiltArchivePath); + (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName); + Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + + string? downloadUrl = await GetLatestOfficialArchiveUrl(); + if (downloadUrl == null) + { + Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + return false; + } + + HttpClient client = new HttpClient(); + + Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); + HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken); + + var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; + + ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".{ArchiveName}-{BuiltVersion}-{BuiltRid}.closest.{ArchiveExtension}"); + Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}"); + using (var file = File.Create(ClosestOfficialArchivePath)) + { + await packageResponse.Content.CopyToAsync(file, CancellationToken); + } + + return true; + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index e5cd717a8fa5..95819f7be314 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -1,73 +1,44 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.IO; using System.Net; using System.Net.Http; -using System.Threading; using System.Threading.Tasks; using Microsoft.Build.Framework; -public class GetClosestOfficialSdk : Microsoft.Build.Utilities.Task, ICancelableTask +public class GetClosestOfficialSdk : GetClosestArchive { - [Required] - public required string BuiltSdkPath { get; init; } + protected override string ArchiveName => "dotnet-sdk"; - [Output] - public string ClosestOfficialSdkPath { get; set; } = ""; + HttpClient client = new HttpClient(new HttpClientHandler() { AllowAutoRedirect = false }); - public override bool Execute() - { - return Task.Run(ExecuteAsync).Result; - } - - private CancellationTokenSource _cancellationTokenSource = new(); - private CancellationToken cancellationToken => _cancellationTokenSource.Token; - public void Cancel() - { - _cancellationTokenSource.Cancel(); - } + private string? closestVersion; + private string? closestUrl; - public async Task ExecuteAsync() + public override async Task GetLatestOfficialArchiveUrl() { - cancellationToken.ThrowIfCancellationRequested(); - var (versionString, rid, extension) = Archive.GetInfoFromArchivePath(BuiltSdkPath); - - string downloadUrl = GetLatestOfficialSdkUrl(versionString, rid, extension); - - Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); - var handler = new HttpClientHandler() - { - AllowAutoRedirect = false - }; - var client = new HttpClient(handler); - var redirectResponse = await client.GetAsync(downloadUrl, cancellationToken); + // Channel in the form of 9.0.1xx + var channel = BuiltVersion[..5] + "xx"; + var akaMsUrl = $"https://aka.ms/dotnet/{channel}/daily/{ArchiveName}-{BuiltRid}{ArchiveExtension}"; + var redirectResponse = await client.GetAsync(akaMsUrl, CancellationToken); // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs if (redirectResponse.StatusCode != HttpStatusCode.Moved) { - Log.LogMessage(MessageImportance.High, $"Failed to download '{downloadUrl}': invalid aka.ms URL"); - return true; + Log.LogMessage(MessageImportance.High, $"Failed to find package at '{akaMsUrl}': invalid aka.ms URL"); + return null; } - var packageResponse = await client.GetAsync(redirectResponse.Headers.Location!, cancellationToken); - - var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; - string downloadedVersion = PathWithVersions.GetVersionInPath(packageUriPath).ToString(); - - ClosestOfficialSdkPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".dotnet-sdk-{downloadedVersion}-{rid}{extension}"); - Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialSdkPath}"); - using (var file = File.Create(ClosestOfficialSdkPath)) - { - await packageResponse.Content.CopyToAsync(file, cancellationToken); - } - - return true; + closestUrl = redirectResponse.Headers.Location!.ToString(); + closestVersion = VersionIdentifier.GetVersion(closestUrl); + return closestUrl; } - string GetLatestOfficialSdkUrl(string versionString, string rid, string extension) + public override async Task GetClosestOfficialArchiveVersion() { - // Channel in the form of 9.0.1xx - var channel = versionString[..5] + "xx"; - return $"https://aka.ms/dotnet/{channel}/daily/dotnet-sdk-{rid}{extension}"; + if (closestUrl is not null) + { + return closestVersion; + } + _ = await GetLatestOfficialArchiveUrl(); + return closestVersion; } - } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs similarity index 58% rename from src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs rename to src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs index a902a7ea3307..3ebcc571a0f4 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetSingleArchiveItem.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs @@ -5,24 +5,27 @@ using System.Collections.Generic; using Microsoft.Build.Framework; -public class GetSingleArchiveItem : Microsoft.Build.Utilities.Task +public class GetValidArchiveItems : Microsoft.Build.Utilities.Task { [Required] - public required ITaskItem[] SdkArchiveItems { get; init; } + public required ITaskItem[] ArchiveItems { get; init; } + + [Required] + public required string ArchiveName { get; init; } [Output] - public string BestSdkArchiveItem { get; set; } = ""; + public ITaskItem[] ValidArchiveItems { get; set; } = []; public override bool Execute() { - List archiveItems = new (); - foreach(var item in SdkArchiveItems) + List archiveItems = new(); + foreach (var item in ArchiveItems) { try { // Ensure the version and RID info can be parsed from the item - _ = Archive.GetInfoFromArchivePath(item.ItemSpec); - archiveItems.Add(item.ItemSpec); + _ = Archive.GetInfoFromFileName(item.ItemSpec, ArchiveName); + archiveItems.Add(item); } catch (ArgumentException e) { @@ -30,20 +33,20 @@ public override bool Execute() continue; } } - switch (archiveItems.Count){ + switch (archiveItems.Count) + { case 0: Log.LogMessage(MessageImportance.High, "No valid archive items found"); - BestSdkArchiveItem = ""; - break; + ValidArchiveItems = []; + return false; case 1: Log.LogMessage(MessageImportance.High, $"{archiveItems[0]} is the only valid archive item found"); - BestSdkArchiveItem = archiveItems[0]; + ValidArchiveItems = archiveItems.ToArray(); break; default: - archiveItems.Sort((a,b) => a.Length - b.Length); + archiveItems.Sort((a, b) => a.ItemSpec.Length - b.ItemSpec.Length); Log.LogMessage(MessageImportance.High, $"Multiple valid archive items found: '{string.Join("', '", archiveItems)}'"); - BestSdkArchiveItem = archiveItems[0]; - Log.LogMessage(MessageImportance.High, $"Choosing '{BestSdkArchiveItem}"); + ValidArchiveItems = archiveItems.ToArray(); break; } return true; diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs deleted file mode 100644 index 28a1f854ee59..000000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/PathWithVersions.cs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Text; - -public static class PathWithVersions -{ - public const string VersionPlaceholder = "{VERSION}"; - - public static bool Equal(string path1, string path2) - { - if (path1 == path2) - { - return true; - } - - ReadOnlySpan directory = path1; - ReadOnlySpan directory2 = path2; - while (TryGetPathLeaf(directory, out var root, out var directoryPart) && TryGetPathLeaf(directory2, out var root2, out var directoryPart2)) - { - if (!ReplaceVersionString(directoryPart).SequenceEqual(ReplaceVersionString(directoryPart2))) - { - return false; - } - directory= Path.GetDirectoryName(directory); - directory2= Path.GetDirectoryName(directory2); - } - if (!directory.IsEmpty || !directory2.IsEmpty) - { - return false; - } - return true; - } - - public static bool IsVersionString(ReadOnlySpan directoryPart) - { - return directoryPart.Length >= 6 - && char.IsDigit(directoryPart[0]) - && directoryPart[1] == '.' - && char.IsDigit(directoryPart[2]) - && directoryPart[3] == '.' - && char.IsDigit(directoryPart[4]) - && ((char.IsDigit(directoryPart[5]) && char.IsDigit(directoryPart[6])) || directoryPart[5] == '-'); - } - - static ReadOnlySpan ReplaceVersionString(ReadOnlySpan directoryPart) - { - if (IsVersionString(directoryPart)) - { - return VersionPlaceholder; - } - else - { - return directoryPart; - } - } - - static bool TryGetPathLeaf(ReadOnlySpan path, out ReadOnlySpan root, out ReadOnlySpan leaf) - { - if (path.IsEmpty) - { - root = default; - leaf = default; - return false; - } - leaf = Path.GetFileName(path); - root = Path.GetDirectoryName(path); - return true; - } - - public static string GetVersionlessPath(string path) - { - return GetVersionlessPath(path.AsSpan()).ToString(); - } - - public static ReadOnlySpan GetVersionlessPath(ReadOnlySpan path) - { - StringBuilder sb = new StringBuilder(); - bool altered = false; - ReadOnlySpan myPath = path; - while (TryGetPathLeaf(myPath, out var root, out var leaf)) - { - var versionOrDirectory = ReplaceVersionString(leaf); - if (versionOrDirectory == VersionPlaceholder) - { - altered = true; - } - sb = sb.Insert(0, versionOrDirectory); - sb = sb.Insert(0, Path.DirectorySeparatorChar); - myPath = root; - } - if (!altered) - return path; - return sb.ToString(); - } - - public static ReadOnlySpan GetVersionInPath(ReadOnlySpan path) - { - ReadOnlySpan myPath = path; - while (TryGetPathLeaf(myPath, out var directory, out var directoryPart)) - { - if (IsVersionString(directoryPart)) - { - return directoryPart; - } - myPath = directory; - } - throw new ArgumentException("Path does not contain a version"); - } -} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs new file mode 100644 index 000000000000..82c58c8d449e --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs @@ -0,0 +1,257 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +// Copied from https://github.com/dotnet/arcade/blob/903d35c426248ade56719db5bc0947cd7634b297/eng/common/native/init-distro-rid.sh +// Conflicting MSBuild versions and some customizations make it difficult to use the Arcade assembly. +public static class VersionIdentifier +{ + private static readonly HashSet _knownTags = new HashSet + { + "alpha", + "beta", + "preview", + "prerelease", + "servicing", + "rtm", + "rc" + }; + + private static readonly SortedDictionary _sequencesToReplace = + new SortedDictionary + { + { "-.", "." }, + { "..", "." }, + { "--", "-" }, + { "//", "/" }, + { "_.", "." } + }; + + private const string _finalSuffix = "final"; + + private static readonly char[] _delimiters = new char[] { '.', '-', '_' }; + + /// + /// Identify the version of an asset. + /// + /// Asset names can come in two forms: + /// - Blobs that include the full path + /// - Packages that do not include any path elements. + /// + /// There may be multiple different version numbers in a blob path. + /// This method starts at the last segment of the path and works backward to find a version number. + /// + /// Asset name + /// Version, or null if none is found. + public static string? GetVersion(string assetName) + { + string[] pathSegments = assetName.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + string? potentialVersion = null; + for (int i = pathSegments.Length - 1; i >= 0; i--) + { + potentialVersion = GetVersionForSingleSegment(pathSegments[i]); + if (potentialVersion != null) + { + return potentialVersion; + } + } + + return potentialVersion; + } + + /// + /// Identify the version number of an asset segment. + /// + /// Asset segment + /// Version number, or null if none was found + /// + /// Identifying versions is not particularly easy. To constrain the problem, + /// we apply the following assumptions which are generally valid for .NET Core. + /// - We always have major.minor.patch, and it always begins the version string. + /// - The only pre-release or build metadata labels we use begin with the _knownTags shown above. + /// - We use additional numbers in our version numbers after the initial + /// major.minor.patch-prereleaselabel.prereleaseiteration segment, + /// but any non-numeric element will end the version string. + /// - The we use in versions and file names are ., -, and _. + /// + private static string? GetVersionForSingleSegment(string assetPathSegment) + { + + // Find the start of the version number by finding the major.minor.patch. + // Scan the string forward looking for a digit preceded by one of the delimiters, + // then look for a minor.patch, completing the major.minor.patch. Continue to do so until we get + // to something that is NOT major.minor.patch (this is necessary because we sometimes see things like: + // VS.Redist.Common.NetCore.Templates.x86.2.2.3.0.101-servicing-014320.nupkg + // Continue iterating until we find ALL potential versions. Return the one that is the latest in the segment + // This is to deal with files with multiple major.minor.patchs in the file name, for example: + // Microsoft.NET.Workload.Mono.ToolChain.Manifest-6.0.100.Msi.x64.6.0.0-rc.1.21380.2.symbols.nupkg + + int currentIndex = 0; + // Stack of major.minor.patch. + Stack<(int versionNumber, int index)> majorMinorPatchStack = new Stack<(int, int)>(3); + string? majorMinorPatch = null; + int majorMinorPatchIndex = 0; + StringBuilder versionSuffix = new StringBuilder(); + char prevDelimiterCharacter = char.MinValue; + char nextDelimiterCharacter = char.MinValue; + Dictionary majorMinorPatchDictionary = new Dictionary(); + while (true) + { + string nextSegment; + prevDelimiterCharacter = nextDelimiterCharacter; + int nextDelimiterIndex = assetPathSegment.IndexOfAny(_delimiters, currentIndex); + if (nextDelimiterIndex != -1) + { + nextDelimiterCharacter = assetPathSegment[nextDelimiterIndex]; + nextSegment = assetPathSegment.Substring(currentIndex, nextDelimiterIndex - currentIndex); + } + else + { + nextSegment = assetPathSegment.Substring(currentIndex); + } + + // If we have not yet found the major/minor/patch, then there are four cases: + // - There have been no potential major/minor/patch numbers found and the current segment is a number. Push onto the majorMinorPatch stack + // and continue. + // - There has been at least one number found, but less than 3, and the current segment not a number or not preceded by '.'. In this case, + // we should clear out the stack and continue the search. + // - There have been at least 2 numbers found and the current segment is a number and preceded by '.'. Push onto the majorMinorPatch stack and continue + // - There have been at least 3 numbers found and the current segment is not a number or not preceded by '-'. In this case, we can call this the major minor + // patch number and no longer need to continue searching + if (majorMinorPatch == null) + { + bool isNumber = int.TryParse(nextSegment, out int potentialVersionSegment); + if ((majorMinorPatchStack.Count == 0 && isNumber) || + (majorMinorPatchStack.Count > 0 && prevDelimiterCharacter == '.' && isNumber)) + { + majorMinorPatchStack.Push((potentialVersionSegment, currentIndex)); + } + // Check for partial major.minor.patch cases, like: 2.2.bar or 2.2-100.bleh + else if (majorMinorPatchStack.Count > 0 && majorMinorPatchStack.Count < 3 && + (prevDelimiterCharacter != '.' || !isNumber)) + { + majorMinorPatchStack.Clear(); + } + + // Determine whether we are done with major.minor.patch after this update. + if (majorMinorPatchStack.Count >= 3 && (prevDelimiterCharacter != '.' || !isNumber || nextDelimiterIndex == -1)) + { + // Done with major.minor.patch, found. Pop the top 3 elements off the stack. + (int patch, int patchIndex) = majorMinorPatchStack.Pop(); + (int minor, int minorIndex) = majorMinorPatchStack.Pop(); + (int major, int majorIndex) = majorMinorPatchStack.Pop(); + majorMinorPatch = $"{major}.{minor}.{patch}"; + majorMinorPatchIndex = majorIndex; + } + } + + // Don't use else, so that we don't miss segments + // in case we are just deciding that we've finished major minor patch. + if (majorMinorPatch != null) + { + // Now look at the next segment. If it looks like it could be part of a version, append to what we have + // and continue. If it can't, then we're done. + // + // Cases where we should break out and be done: + // - We have an empty pre-release label and the delimiter is not '-'. + // - We have an empty pre-release label and the next segment does not start with a known tag. + // - We have a non-empty pre-release label and the current segment is not a number and also not 'final' + // A corner case of versioning uses .final to represent a non-date suffixed final pre-release version: + // 3.1.0-preview.10.final + if (versionSuffix.Length == 0 && + (prevDelimiterCharacter != '-' || !_knownTags.Any(tag => nextSegment.StartsWith(tag, StringComparison.OrdinalIgnoreCase)))) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, majorMinorPatch); + majorMinorPatch = null; + versionSuffix = new StringBuilder(); + } + else if (versionSuffix.Length != 0 && !int.TryParse(nextSegment, out int potentialVersionSegment) && nextSegment != _finalSuffix) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}"); + majorMinorPatch = null; + versionSuffix = new StringBuilder(); + } + else + { + // Append the delimiter character and then the current segment + versionSuffix.Append(prevDelimiterCharacter); + versionSuffix.Append(nextSegment); + } + } + + if (nextDelimiterIndex != -1) + { + currentIndex = nextDelimiterIndex + 1; + } + else + { + break; + } + } + + if (majorMinorPatch != null) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}"); + } + + if (!majorMinorPatchDictionary.Any()) + { + return null; + } + + int maxKey = majorMinorPatchDictionary.Keys.Max(); + return majorMinorPatchDictionary[maxKey]; + } + + /// + /// Given an asset name, remove all .NET Core version numbers (as defined by GetVersionForSingleSegment) + /// from the string + /// + /// Asset + /// Asset name without versions + public static string RemoveVersions(string assetName, string replacement = "") + { + string[] pathSegments = assetName.Split('/'); + + // Remove the version number from each segment, then join back together and + // remove any useless character sequences. + + for (int i = 0; i < pathSegments.Length; i++) + { + if (!string.IsNullOrEmpty(pathSegments[i])) + { + string? versionForSegment = GetVersionForSingleSegment(pathSegments[i]); + if (versionForSegment != null) + { + pathSegments[i] = pathSegments[i].Replace(versionForSegment, replacement); + } + } + } + + // Continue replacing things until there is nothing left to replace. + string assetWithoutVersions = string.Join("/", pathSegments); + bool anyReplacements = true; + while (anyReplacements) + { + string replacementIterationResult = assetWithoutVersions; + foreach (var sequence in _sequencesToReplace) + { + replacementIterationResult = replacementIterationResult.Replace(sequence.Key, sequence.Value); + } + anyReplacements = replacementIterationResult != assetWithoutVersions; + assetWithoutVersions = replacementIterationResult; + } + + return assetWithoutVersions; + } + + + public static bool AreVersionlessEqual(string assetName1, string assetName2) + { + return RemoveVersions(assetName1) == RemoveVersions(assetName2); + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs deleted file mode 100644 index 4b67efea0bc1..000000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/ZipArchiveExtensions.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -static class ArchiveExtensions -{ - public static string[] Lines(this ZipArchiveEntry entry, Encoding? encoding = null) - { - return entry.ReadToString(encoding).Replace("\r\n", "\n").Split('\n').ToArray(); - } - - public static string ReadToString(this ZipArchiveEntry entry, Encoding? encoding = null) - { - Stream stream = entry.Open(); - byte[] buffer = stream.ReadToEnd(); - // Remove UTF-8 BOM if present - int index = 0; - if (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF) - { - index = 3; - } - encoding ??= Encoding.UTF8; - string fileText = encoding.GetString(buffer, index, buffer.Length - index); - return fileText; - } - - public static byte[] ReadToEnd(this Stream stream) - { - int bufferSize = 2048; - byte[] buffer = new byte[bufferSize]; - int offset = 0; - while (true) - { - int bytesRead = stream.Read(buffer, offset, bufferSize - offset); - offset += bytesRead; - if (bytesRead == 0) - { - break; - } - if (offset == bufferSize) - { - Array.Resize(ref buffer, bufferSize * 2); - bufferSize *= 2; - } - } - Array.Resize(ref buffer, offset); - return buffer; - } - - public static async Task ReadToEndAsync(this Stream stream) - { - int bufferSize = 2048; - byte[] buffer = new byte[bufferSize]; - int offset = 0; - while (true) - { - int bytesRead = await stream.ReadAsync(buffer, offset, bufferSize - offset); - offset += bytesRead; - if (bytesRead == 0) - { - break; - } - if (offset == bufferSize) - { - Array.Resize(ref buffer, bufferSize * 2); - bufferSize *= 2; - } - } - Array.Resize(ref buffer, offset); - return buffer; - } -} From 3ed90968ee5cf8ec20ea6859ffa781b3580cf964 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:21:40 -0800 Subject: [PATCH 19/37] Don't validate Sdk archive diffs on shortstack --- src/SourceBuild/content/eng/build.targets | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 9a6eabd2be79..fdb8c629aac4 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -5,16 +5,17 @@ + DependsOnTargets="DetermineSourceBuiltSdkVersion" + Condition="'$(ShortStack)' != 'true'" > + ArchiveName="dotnet-sdk"> - + Date: Fri, 1 Mar 2024 11:31:35 -0800 Subject: [PATCH 20/37] Rename file to match type --- .../GetClosestArchive.cs | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs new file mode 100644 index 000000000000..fc69a385a9ac --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask +{ + [Required] + public required string BuiltArchivePath { get; init; } + + [Output] + public string ClosestOfficialArchivePath { get; set; } = ""; + + private string? _builtVersion; + protected string BuiltVersion + { + get => _builtVersion ?? throw new InvalidOperationException(); + private set => _builtVersion = value; + } + + private string? _builtRid; + protected string BuiltRid + { + get => _builtRid ?? throw new InvalidOperationException(); + private set => _builtRid = value; + } + + private string? _archiveExtension; + protected string ArchiveExtension + { + get => _archiveExtension ?? throw new InvalidOperationException(); + private set => _archiveExtension = value; + } + + /// + /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime". + /// + protected abstract string ArchiveName { get; } + + private CancellationTokenSource _cancellationTokenSource = new(); + protected CancellationToken CancellationToken => _cancellationTokenSource.Token; + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + + /// + /// Get the URL of the latest official archive for the given version string and RID. + /// + public abstract Task GetLatestOfficialArchiveUrl(); + + public abstract Task GetClosestOfficialArchiveVersion(); + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + CancellationToken.ThrowIfCancellationRequested(); + var filename = Path.GetFileName(BuiltArchivePath); + (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName); + Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + + string? downloadUrl = await GetLatestOfficialArchiveUrl(); + if (downloadUrl == null) + { + Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + return false; + } + + HttpClient client = new HttpClient(); + + Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); + HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken); + + var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; + + ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".{ArchiveName}-{BuiltVersion}-{BuiltRid}.closest.{ArchiveExtension}"); + Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}"); + using (var file = File.Create(ClosestOfficialArchivePath)) + { + await packageResponse.Content.CopyToAsync(file, CancellationToken); + } + + return true; + } +} From 2b3389d8428552cf220e16bebd718c7433884d1c Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 11:43:57 -0800 Subject: [PATCH 21/37] Forgot to add the deletion of the old file --- .../GetClosestArchiveVersion.cs | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs deleted file mode 100644 index fc69a385a9ac..000000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchiveVersion.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Build.Framework; - -public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask -{ - [Required] - public required string BuiltArchivePath { get; init; } - - [Output] - public string ClosestOfficialArchivePath { get; set; } = ""; - - private string? _builtVersion; - protected string BuiltVersion - { - get => _builtVersion ?? throw new InvalidOperationException(); - private set => _builtVersion = value; - } - - private string? _builtRid; - protected string BuiltRid - { - get => _builtRid ?? throw new InvalidOperationException(); - private set => _builtRid = value; - } - - private string? _archiveExtension; - protected string ArchiveExtension - { - get => _archiveExtension ?? throw new InvalidOperationException(); - private set => _archiveExtension = value; - } - - /// - /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime". - /// - protected abstract string ArchiveName { get; } - - private CancellationTokenSource _cancellationTokenSource = new(); - protected CancellationToken CancellationToken => _cancellationTokenSource.Token; - public void Cancel() - { - _cancellationTokenSource.Cancel(); - } - - /// - /// Get the URL of the latest official archive for the given version string and RID. - /// - public abstract Task GetLatestOfficialArchiveUrl(); - - public abstract Task GetClosestOfficialArchiveVersion(); - - public override bool Execute() - { - return Task.Run(ExecuteAsync).Result; - } - - public async Task ExecuteAsync() - { - CancellationToken.ThrowIfCancellationRequested(); - var filename = Path.GetFileName(BuiltArchivePath); - (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName); - Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); - - string? downloadUrl = await GetLatestOfficialArchiveUrl(); - if (downloadUrl == null) - { - Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); - return false; - } - - HttpClient client = new HttpClient(); - - Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); - HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken); - - var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; - - ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".{ArchiveName}-{BuiltVersion}-{BuiltRid}.closest.{ArchiveExtension}"); - Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}"); - using (var file = File.Create(ClosestOfficialArchivePath)) - { - await packageResponse.Content.CopyToAsync(file, CancellationToken); - } - - return true; - } -} From eafa1ba7695cf06304c8c06d316f069b328054e6 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 12:42:27 -0800 Subject: [PATCH 22/37] Check index before indexing array, and add common prefix to start of diff --- src/SourceBuild/content/Directory.Build.props | 3 +-- .../Diff.cs | 13 ++++++------- ...t.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 1 + 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 216830039f4e..30a914855a7d 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -34,7 +34,7 @@ runtime - @@ -212,6 +212,5 @@ $(PackageReportDir)poisoned.txt - diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs index ab1676f97fed..008fe67f0e16 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs @@ -49,13 +49,13 @@ public enum DifferenceKind // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence // cancellationToken.ThrowIfCancellationRequested(); formatter ??= static s => s; - List<(string, DifferenceKind)> diff = []; // Optimization: remove common prefix int i = 0; + List<(string, DifferenceKind)> prefix = []; while (i < baselineSequence.Length && i < testSequence.Length && equalityComparer(baselineSequence[i], testSequence[i])) { - diff.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged)); + prefix.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged)); i++; } @@ -93,13 +93,11 @@ public enum DifferenceKind // Trace back the edits int row = baselineSequence.Length; int col = testSequence.Length; - + List<(string, DifferenceKind)> diff = []; while (row > 0 || col > 0) { cancellationToken.ThrowIfCancellationRequested(); - var baselineItem = baselineSequence[row - 1]; - var testItem = testSequence[col - 1]; - if (row > 0 && col > 0 && equalityComparer(baselineItem, testItem)) + if (row > 0 && col > 0 && equalityComparer(baselineSequence[row - 1], testSequence[col - 1])) { diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Unchanged)); row--; @@ -121,7 +119,8 @@ public enum DifferenceKind } } diff.Reverse(); - return diff; + prefix.AddRange(diff); + return prefix; } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index ff3bf2402a01..4098031757f7 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -1,6 +1,7 @@  + exe $(NetCurrent) enable From 5a2d878d2541e00ec16457f334afed2956d7f7fe Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 12:44:27 -0800 Subject: [PATCH 23/37] Accidentally added test changes --- src/SourceBuild/content/Directory.Build.props | 3 ++- .../Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 30a914855a7d..216830039f4e 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -34,7 +34,7 @@ runtime - @@ -212,5 +212,6 @@ $(PackageReportDir)poisoned.txt + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj index 4098031757f7..ff3bf2402a01 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -1,7 +1,6 @@  - exe $(NetCurrent) enable From 8eb0cd2a5c0f6360d694f7ac3e59c9bcc5464589 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Fri, 1 Mar 2024 15:19:42 -0800 Subject: [PATCH 24/37] Update logs, add check for valid RID --- src/SourceBuild/content/eng/build.targets | 3 +++ .../Archive.cs | 9 +++++++++ .../GetValidArchiveItems.cs | 6 ++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index fdb8c629aac4..406c4778e1ed 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -8,6 +8,9 @@ DependsOnTargets="DetermineSourceBuiltSdkVersion" Condition="'$(ShortStack)' != 'true'" > + + p.Split('.').Skip(1))) + { + if (!int.TryParse(item, out _)) + throw new ArgumentException($"Invalid Rid '{Rid}' in archive file name '{filename}'. Expected RID with '.' to be part of a version. This likely means the file is an archive of a different file type."); + } return (Version, Rid, extension); } } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs index 3ebcc571a0f4..bd27213b80b3 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.IO; using Microsoft.Build.Framework; public class GetValidArchiveItems : Microsoft.Build.Utilities.Task @@ -21,15 +22,16 @@ public override bool Execute() List archiveItems = new(); foreach (var item in ArchiveItems) { + var filename = Path.GetFileName(item.ItemSpec); try { // Ensure the version and RID info can be parsed from the item - _ = Archive.GetInfoFromFileName(item.ItemSpec, ArchiveName); + _ = Archive.GetInfoFromFileName(filename, ArchiveName); archiveItems.Add(item); } catch (ArgumentException e) { - Log.LogMessage(MessageImportance.High, e.Message); + Log.LogMessage($"'{item.ItemSpec}' is not a valid archive name: '{e.Message}'"); continue; } } From 3adb0cc25cb3fcd7c9c6ebcbd00c65a9284c5324 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Sat, 2 Mar 2024 09:28:13 -0800 Subject: [PATCH 25/37] Don't add another item in SdkTarballItem --- src/SourceBuild/content/eng/build.sourcebuild.targets | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets index 00b1439ef7d6..a05905856cf0 100644 --- a/src/SourceBuild/content/eng/build.sourcebuild.targets +++ b/src/SourceBuild/content/eng/build.sourcebuild.targets @@ -35,10 +35,6 @@ - - - - $(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) From 97d8e4d2b6e16292ff2990ae51ed2f7dc2200179 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Mon, 4 Mar 2024 11:51:45 -0800 Subject: [PATCH 26/37] Add message when there are no differences, don't validate non-portable builds --- src/SourceBuild/content/eng/build.targets | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 406c4778e1ed..2f2ea8e8c522 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -6,7 +6,8 @@ + Condition="'$(ShortStack)' != 'true' AND '$(PortableBuild)' == 'true'" > + @@ -34,9 +35,16 @@ ItemName="_ContentDifferences" /> - + <_changedFiles Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' != 'Unchanged'" /> + + + + + Condition="'@(_changedFiles->Count())' == '0'" /> From d49fa678427e95924a8a7922cf74908a3b74d801 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:06:35 -0800 Subject: [PATCH 27/37] Update link to copied source --- .../VersionIdentifier.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs index 82c58c8d449e..2150a721197f 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Text; -// Copied from https://github.com/dotnet/arcade/blob/903d35c426248ade56719db5bc0947cd7634b297/eng/common/native/init-distro-rid.sh +// Copied from https://github.com/dotnet/arcade/blob/db45698020f58f88eef75b23b2598a59872918f6/src/Microsoft.DotNet.VersionTools/lib/src/BuildManifest/VersionIdentifier.cs // Conflicting MSBuild versions and some customizations make it difficult to use the Arcade assembly. public static class VersionIdentifier { From 4f6bac312de56da573c8dc25229f9ed732ead707 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:14:53 -0800 Subject: [PATCH 28/37] Remove conditions on tasks --- src/SourceBuild/content/eng/build.targets | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/build.targets index 2f2ea8e8c522..008c7c2ad929 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/build.targets @@ -22,15 +22,13 @@ - + + TestArchive="$(_ClosestOfficialSdkPath)"> From 2075e8765b38206a5f336da2956c81eae3235e04 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:34:00 -0800 Subject: [PATCH 29/37] Use "Closest" sdk instead of "Latest" --- .../GetClosestArchive.cs | 2 +- .../GetClosestOfficialSdk.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs index fc69a385a9ac..8039301cc815 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs @@ -52,7 +52,7 @@ public void Cancel() /// /// Get the URL of the latest official archive for the given version string and RID. /// - public abstract Task GetLatestOfficialArchiveUrl(); + public abstract Task GetClosestOfficialArchiveUrl(); public abstract Task GetClosestOfficialArchiveVersion(); diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs index 95819f7be314..bb038516c434 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -15,7 +15,7 @@ public class GetClosestOfficialSdk : GetClosestArchive private string? closestVersion; private string? closestUrl; - public override async Task GetLatestOfficialArchiveUrl() + public override async Task GetClosestOfficialArchiveUrl() { // Channel in the form of 9.0.1xx var channel = BuiltVersion[..5] + "xx"; @@ -38,7 +38,7 @@ public class GetClosestOfficialSdk : GetClosestArchive { return closestVersion; } - _ = await GetLatestOfficialArchiveUrl(); + _ = await GetClosestOfficialArchiveUrl(); return closestVersion; } } From 3c848d51a36f157bd8ca1cccfdf2d39ccd4e3bd3 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Tue, 5 Mar 2024 16:00:00 -0800 Subject: [PATCH 30/37] Update missed reference --- .../GetClosestArchive.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs index 8039301cc815..b78f4c91b8f1 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs @@ -68,7 +68,7 @@ public async Task ExecuteAsync() (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName); Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); - string? downloadUrl = await GetLatestOfficialArchiveUrl(); + string? downloadUrl = await GetClosestOfficialArchiveUrl(); if (downloadUrl == null) { Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); From 97fc3b83423bf37256f02763bbd1f846f9ff383a Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:41:32 -0800 Subject: [PATCH 31/37] PR Feedback: Formatting, move/add conditions, output diffs to csv in logs --- src/SourceBuild/content/build.proj | 3 ++- .../{build.targets => sdkArchiveDiffs.targets} | 15 +++++++++------ src/SourceBuild/content/eng/tools/init-build.proj | 3 ++- 3 files changed, 13 insertions(+), 8 deletions(-) rename src/SourceBuild/content/eng/{build.targets => sdkArchiveDiffs.targets} (82%) diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index adc0a15bb530..d18fb9a508a7 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -20,6 +20,7 @@ - + + diff --git a/src/SourceBuild/content/eng/build.targets b/src/SourceBuild/content/eng/sdkArchiveDiffs.targets similarity index 82% rename from src/SourceBuild/content/eng/build.targets rename to src/SourceBuild/content/eng/sdkArchiveDiffs.targets index 008c7c2ad929..c80f6a245ff1 100644 --- a/src/SourceBuild/content/eng/build.targets +++ b/src/SourceBuild/content/eng/sdkArchiveDiffs.targets @@ -1,13 +1,12 @@ + - + DependsOnTargets="DetermineSourceBuiltSdkVersion"> @@ -37,6 +36,12 @@ <_changedFiles Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' != 'Unchanged'" /> + + $(ArtifactsLogDir)\SdkArchiveDiffs\SdkArchiveDiffs.csv + + + + @@ -44,9 +49,7 @@ Importance="High" Condition="'@(_changedFiles->Count())' == '0'" /> - - + diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index 95b45311372f..a196b48cf9a8 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -117,7 +117,8 @@ - + From 36d3cfa39404d03b08d05e8801a61732dd4f20f8 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:46:02 -0800 Subject: [PATCH 32/37] Update src/SourceBuild/content/eng/sdkArchiveDiffs.targets Co-authored-by: Viktor Hofer --- src/SourceBuild/content/eng/sdkArchiveDiffs.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/sdkArchiveDiffs.targets b/src/SourceBuild/content/eng/sdkArchiveDiffs.targets index c80f6a245ff1..79ec32b2e22d 100644 --- a/src/SourceBuild/content/eng/sdkArchiveDiffs.targets +++ b/src/SourceBuild/content/eng/sdkArchiveDiffs.targets @@ -37,7 +37,7 @@ - $(ArtifactsLogDir)\SdkArchiveDiffs\SdkArchiveDiffs.csv + $(ArtifactsLogDir)SdkArchiveDiffs.csv From 2c5bee478848cafce6986fb4130ec7924d50af32 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:57:01 -0800 Subject: [PATCH 33/37] Fix typo --- .../eng/{sdkArchiveDiffs.targets => sdkArchiveDiff.targets} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/SourceBuild/content/eng/{sdkArchiveDiffs.targets => sdkArchiveDiff.targets} (100%) diff --git a/src/SourceBuild/content/eng/sdkArchiveDiffs.targets b/src/SourceBuild/content/eng/sdkArchiveDiff.targets similarity index 100% rename from src/SourceBuild/content/eng/sdkArchiveDiffs.targets rename to src/SourceBuild/content/eng/sdkArchiveDiff.targets From b290188e395cfcf308e3d22f119e72ec46138a72 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:59:14 -0800 Subject: [PATCH 34/37] Use remote file name for downloaded sdk --- .../GetClosestArchive.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs index b78f4c91b8f1..85592191d0b4 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs @@ -82,7 +82,7 @@ public async Task ExecuteAsync() var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; - ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + $".{ArchiveName}-{BuiltVersion}-{BuiltRid}.closest.{ArchiveExtension}"); + ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + "." + Path.GetFileName(packageUriPath)); Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}"); using (var file = File.Create(ClosestOfficialArchivePath)) { From 60c0a7e489992d4df2bf2373055a9d76256c5116 Mon Sep 17 00:00:00 2001 From: Jackson Schuster <36744439+jtschuster@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:01:10 -0800 Subject: [PATCH 35/37] Add args to WriteLinesToFile --- src/SourceBuild/content/eng/sdkArchiveDiff.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/sdkArchiveDiff.targets b/src/SourceBuild/content/eng/sdkArchiveDiff.targets index 79ec32b2e22d..500faa8c937a 100644 --- a/src/SourceBuild/content/eng/sdkArchiveDiff.targets +++ b/src/SourceBuild/content/eng/sdkArchiveDiff.targets @@ -40,7 +40,7 @@ $(ArtifactsLogDir)SdkArchiveDiffs.csv - + Date: Wed, 6 Mar 2024 13:37:06 -0800 Subject: [PATCH 36/37] Use .diff instead of csv for sdk content diffs --- eng/pipelines/templates/jobs/vmr-build.yml | 10 ++++++---- src/SourceBuild/content/eng/sdkArchiveDiff.targets | 13 +++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 16d676186c87..b0dfd4a05a04 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -208,7 +208,7 @@ jobs: exit 1 fi displayName: Setup Previously Source-Built SDK - + - ${{ if eq(parameters.targetOS, 'windows') }}: - script: | call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine ${{ parameters.extraProperties }} @@ -305,7 +305,7 @@ jobs: for envVar in $customEnvVars; do customDockerRunArgs="$customDockerRunArgs -e $envVar" done - + if [[ '${{ parameters.runOnline }}' == 'False' ]]; then customDockerRunArgs="$customDockerRunArgs --network none" fi @@ -342,10 +342,10 @@ jobs: # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. - powershell: | function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) { - Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { + Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { $targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName) New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null - Copy-Item $_.FullName -Destination $targetPath -Force + Copy-Item $_.FullName -Destination $targetPath -Force } } @@ -356,6 +356,7 @@ jobs: CopyWithRelativeFolders "artifacts/" $targetFolder "*.binlog" CopyWithRelativeFolders "artifacts/" $targetFolder "*.log" + CopyWithRelativeFolders "artifacts/" $targetFolder "*.diff" CopyWithRelativeFolders "src/" $targetFolder "*.binlog" CopyWithRelativeFolders "src/" $targetFolder "*.log" CopyWithRelativeFolders "test/" $targetFolder "*.binlog" @@ -382,6 +383,7 @@ jobs: cd "$(sourcesPath)" find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \; if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \; fi diff --git a/src/SourceBuild/content/eng/sdkArchiveDiff.targets b/src/SourceBuild/content/eng/sdkArchiveDiff.targets index 500faa8c937a..f0863f1bc166 100644 --- a/src/SourceBuild/content/eng/sdkArchiveDiff.targets +++ b/src/SourceBuild/content/eng/sdkArchiveDiff.targets @@ -34,13 +34,22 @@ <_changedFiles Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' != 'Unchanged'" /> + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Added'" > + + + + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Removed'" > + - + + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Unchanged'" > + + - $(ArtifactsLogDir)SdkArchiveDiffs.csv + $(ArtifactsLogDir)SdkArchiveContent.diff - + Date: Wed, 6 Mar 2024 21:31:51 -0800 Subject: [PATCH 37/37] Use Item transform instead of batching to output lines to file --- src/SourceBuild/content/eng/sdkArchiveDiff.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/sdkArchiveDiff.targets b/src/SourceBuild/content/eng/sdkArchiveDiff.targets index f0863f1bc166..c3bc8078763a 100644 --- a/src/SourceBuild/content/eng/sdkArchiveDiff.targets +++ b/src/SourceBuild/content/eng/sdkArchiveDiff.targets @@ -49,7 +49,7 @@ $(ArtifactsLogDir)SdkArchiveContent.diff - +