diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 16d676186c87..b0dfd4a05a04 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -208,7 +208,7 @@ jobs: exit 1 fi displayName: Setup Previously Source-Built SDK - + - ${{ if eq(parameters.targetOS, 'windows') }}: - script: | call $(sourcesPath)\build.cmd -ci -cleanWhileBuilding -prepareMachine ${{ parameters.extraProperties }} @@ -305,7 +305,7 @@ jobs: for envVar in $customEnvVars; do customDockerRunArgs="$customDockerRunArgs -e $envVar" done - + if [[ '${{ parameters.runOnline }}' == 'False' ]]; then customDockerRunArgs="$customDockerRunArgs --network none" fi @@ -342,10 +342,10 @@ jobs: # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. - powershell: | function CopyWithRelativeFolders($sourcePath, $targetFolder, $filter) { - Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { + Get-ChildItem -Path $sourcePath -Filter $filter -Recurse | ForEach-Object { $targetPath = Join-Path $targetFolder (Resolve-Path -Relative $_.FullName) New-Item -ItemType Directory -Path (Split-Path -Parent $targetPath) -Force | Out-Null - Copy-Item $_.FullName -Destination $targetPath -Force + Copy-Item $_.FullName -Destination $targetPath -Force } } @@ -356,6 +356,7 @@ jobs: CopyWithRelativeFolders "artifacts/" $targetFolder "*.binlog" CopyWithRelativeFolders "artifacts/" $targetFolder "*.log" + CopyWithRelativeFolders "artifacts/" $targetFolder "*.diff" CopyWithRelativeFolders "src/" $targetFolder "*.binlog" CopyWithRelativeFolders "src/" $targetFolder "*.log" CopyWithRelativeFolders "test/" $targetFolder "*.binlog" @@ -382,6 +383,7 @@ jobs: cd "$(sourcesPath)" find artifacts/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \; find artifacts/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \; + find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \; if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \; fi diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 52bed083548c..216830039f4e 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -203,6 +203,7 @@ $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll')) $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll')) + $([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll')) diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 04f6a25b6f6f..d18fb9a508a7 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -20,5 +20,7 @@ + + diff --git a/src/SourceBuild/content/eng/build.sourcebuild.targets b/src/SourceBuild/content/eng/build.sourcebuild.targets index 00b1439ef7d6..a05905856cf0 100644 --- a/src/SourceBuild/content/eng/build.sourcebuild.targets +++ b/src/SourceBuild/content/eng/build.sourcebuild.targets @@ -35,10 +35,6 @@ - - - - $(ArtifactsAssetsDir)dotnet-symbols-sdk-$(SourceBuiltSdkVersion)-$(TargetRid)$(ArchiveExtension) diff --git a/src/SourceBuild/content/eng/sdkArchiveDiff.targets b/src/SourceBuild/content/eng/sdkArchiveDiff.targets new file mode 100644 index 000000000000..c3bc8078763a --- /dev/null +++ b/src/SourceBuild/content/eng/sdkArchiveDiff.targets @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + <_changedFiles Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' != 'Unchanged'" /> + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Added'" > + + + + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Removed'" > + - + + <_sdkFilesDiff Include="@(_ContentDifferences)" Condition="'%(_contentDifferences.Kind)' == 'Unchanged'" > + + + + + + $(ArtifactsLogDir)SdkArchiveContent.diff + + + + + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index 4d6b4d316651..a196b48cf9a8 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -15,6 +15,7 @@ UnpackTarballs; BuildXPlatTasks; BuildMSBuildSdkResolver; + BuildSdkArchiveDiff; BuildLeakDetection; ExtractToolPackage; GenerateRootFs; @@ -116,6 +117,15 @@ + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs new file mode 100644 index 000000000000..1417c88cad2c --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Formats.Tar; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +public abstract class Archive : IDisposable +{ + public static async Task Create(string path, CancellationToken cancellationToken = default) + { + if (path.EndsWith(".tar.gz")) + return await TarArchive.Create(path, cancellationToken); + else if (path.EndsWith(".zip")) + return ZipFileArchive.Create(path); + else + throw new NotSupportedException("Unsupported archive type"); + } + + public abstract string[] GetFileNames(); + + public abstract bool Contains(string relativePath); + + public abstract void Dispose(); + + public class TarArchive : Archive + { + private string _extractedFolder; + + private TarArchive(string extractedFolder) + { + _extractedFolder = extractedFolder; + } + + public static new async Task Create(string path, CancellationToken cancellationToken = default) + { + var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs)); + using (var gzStream = File.OpenRead(path)) + using (var gzipStream = new GZipStream(gzStream, CompressionMode.Decompress)) + { + await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken); + } + return new TarArchive(tmpFolder.FullName); + } + + public override bool Contains(string relativePath) + { + return File.Exists(Path.Combine(_extractedFolder, relativePath)); + } + + public override string[] GetFileNames() + { + return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray(); + } + + public override void Dispose() + { + if (Directory.Exists(_extractedFolder)) + Directory.Delete(_extractedFolder, true); + } + } + + public class ZipFileArchive : Archive + { + private ZipArchive _archive; + + private ZipFileArchive(ZipArchive archive) + { + _archive = archive; + } + + public static ZipFileArchive Create(string path) + { + return new ZipFileArchive(new ZipArchive(File.OpenRead(path))); + } + + public override bool Contains(string relativePath) + { + return _archive.GetEntry(relativePath) != null; + } + + public override string[] GetFileNames() + { + return _archive.Entries.Select(e => e.FullName).ToArray(); + } + + public override void Dispose() + { + _archive.Dispose(); + } + } + + private static string GetArchiveExtension(string path) + { + if (path.EndsWith(".tar.gz")) + { + return ".tar.gz"; + } + else if (path.EndsWith(".zip")) + { + return ".zip"; + } + else + { + throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip"); + } + } + + public static (string Version, string Rid, string extension) GetInfoFromFileName(string filename, string packageName) + { + var extension = GetArchiveExtension(filename); + var Version = VersionIdentifier.GetVersion(filename); + if (Version is null) + throw new ArgumentException("Invalid archive file name '{filename}': No valid version found in file name."); + // Once we've removed the version, package name, and extension, we should be left with the RID + var Rid = filename + .Replace(extension, "") + .Replace(Version, "") + .Replace(packageName, "") + .Trim('-', '.', '_'); + + // A RID with '.' must have a version number after the first '.' in each part of the RID. For example, alpine.3.10-arm64. + // Otherwise, it's likely an archive of another type of file that we don't handle here, for example, .msi.wixpack.zip. + var ridParts = Rid.Split('-'); + foreach(var item in ridParts.SelectMany(p => p.Split('.').Skip(1))) + { + if (!int.TryParse(item, out _)) + throw new ArgumentException($"Invalid Rid '{Rid}' in archive file name '{filename}'. Expected RID with '.' to be part of a version. This likely means the file is an archive of a different file type."); + } + return (Version, Rid, extension); + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs new file mode 100644 index 000000000000..008fe67f0e16 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Diff.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +static class Diff +{ + public static ITaskItem TaskItemFromDiff((string, DifferenceKind) diff) + { + var item = new TaskItem(diff.Item1); + item.SetMetadata("Kind", Enum.GetName(diff.Item2)); + return item; + } + + public enum DifferenceKind + { + /// + /// Present in the test but not in the baseline + /// + Added, + + /// + /// Present in the baseline but not in the test + /// + Removed, + + /// + /// Present in both the baseline and test + /// + Unchanged + } + + /// + /// Uses the Longest Common Subsequence algorithm (as used in 'git diff') to find the differences between two lists of strings. + /// Returns a list of the joined lists with the differences marked as either added or removed. + /// + public static List<(string, DifferenceKind DifferenceKind)> GetDiffs( + Span baselineSequence, + Span testSequence, + Func equalityComparer, + Func? formatter = null, + CancellationToken cancellationToken = default) + { + // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence + // cancellationToken.ThrowIfCancellationRequested(); + formatter ??= static s => s; + + // Optimization: remove common prefix + int i = 0; + List<(string, DifferenceKind)> prefix = []; + while (i < baselineSequence.Length && i < testSequence.Length && equalityComparer(baselineSequence[i], testSequence[i])) + { + prefix.Add((formatter(baselineSequence[i]), DifferenceKind.Unchanged)); + i++; + } + + baselineSequence = baselineSequence[i..]; + testSequence = testSequence[i..]; + + // Initialize first row and column + int[,] m = new int[baselineSequence.Length + 1, testSequence.Length + 1]; + for (i = 0; i <= baselineSequence.Length; i++) + { + m[i, 0] = i; + } + for (i = 0; i <= testSequence.Length; i++) + { + m[0, i] = i; + } + + // Compute edit distance + for (i = 1; i <= baselineSequence.Length; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + for (int j = 1; j <= testSequence.Length; j++) + { + if (equalityComparer(baselineSequence[i - 1], testSequence[j - 1])) + { + m[i, j] = m[i - 1, j - 1]; + } + else + { + m[i, j] = 1 + Math.Min(m[i - 1, j], m[i, j - 1]); + } + } + } + + // Trace back the edits + int row = baselineSequence.Length; + int col = testSequence.Length; + List<(string, DifferenceKind)> diff = []; + while (row > 0 || col > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + if (row > 0 && col > 0 && equalityComparer(baselineSequence[row - 1], testSequence[col - 1])) + { + diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Unchanged)); + row--; + col--; + } + else if (col > 0 && (row == 0 || m[row, col - 1] <= m[row - 1, col])) + { + diff.Add((formatter(testSequence[col - 1]), DifferenceKind.Added)); + col--; + } + else if (row > 0 && (col == 0 || m[row, col - 1] > m[row - 1, col])) + { + diff.Add((formatter(baselineSequence[row - 1]), DifferenceKind.Removed)); + row--; + } + else + { + throw new UnreachableException(); + } + } + diff.Reverse(); + prefix.AddRange(diff); + return prefix; + } + +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs new file mode 100644 index 000000000000..9fc19a7385c0 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Task = System.Threading.Tasks.Task; + +public class FindArchiveDiffs : Microsoft.Build.Utilities.Task, ICancelableTask +{ + [Required] + public required ITaskItem BaselineArchive { get; init; } + + [Required] + public required ITaskItem TestArchive { get; init; } + + [Output] + public ITaskItem[] ContentDifferences { get; set; } = []; + + private CancellationTokenSource _cancellationTokenSource = new(); + private CancellationToken cancellationToken => _cancellationTokenSource.Token; + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + var baselineTask = Archive.Create(BaselineArchive.ItemSpec); + var testTask = Archive.Create(TestArchive.ItemSpec); + Task.WaitAll([baselineTask, testTask], cancellationToken); + using var baseline = await baselineTask; + using var test = await testTask; + var baselineFiles = baseline.GetFileNames(); + var testFiles = test.GetFileNames(); + ContentDifferences = + Diff.GetDiffs(baselineFiles, testFiles, VersionIdentifier.AreVersionlessEqual, static p => VersionIdentifier.RemoveVersions(p, "{VERSION}"), cancellationToken) + .Select(Diff.TaskItemFromDiff) + .ToArray(); + return true; + } + + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs new file mode 100644 index 000000000000..85592191d0b4 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestArchive.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +public abstract class GetClosestArchive : Microsoft.Build.Utilities.Task, ICancelableTask +{ + [Required] + public required string BuiltArchivePath { get; init; } + + [Output] + public string ClosestOfficialArchivePath { get; set; } = ""; + + private string? _builtVersion; + protected string BuiltVersion + { + get => _builtVersion ?? throw new InvalidOperationException(); + private set => _builtVersion = value; + } + + private string? _builtRid; + protected string BuiltRid + { + get => _builtRid ?? throw new InvalidOperationException(); + private set => _builtRid = value; + } + + private string? _archiveExtension; + protected string ArchiveExtension + { + get => _archiveExtension ?? throw new InvalidOperationException(); + private set => _archiveExtension = value; + } + + /// + /// The name of the package to find the closest official archive for. For example, "dotnet-sdk" or "aspnetcore-runtime". + /// + protected abstract string ArchiveName { get; } + + private CancellationTokenSource _cancellationTokenSource = new(); + protected CancellationToken CancellationToken => _cancellationTokenSource.Token; + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + + /// + /// Get the URL of the latest official archive for the given version string and RID. + /// + public abstract Task GetClosestOfficialArchiveUrl(); + + public abstract Task GetClosestOfficialArchiveVersion(); + + public override bool Execute() + { + return Task.Run(ExecuteAsync).Result; + } + + public async Task ExecuteAsync() + { + CancellationToken.ThrowIfCancellationRequested(); + var filename = Path.GetFileName(BuiltArchivePath); + (BuiltVersion, BuiltRid, ArchiveExtension) = Archive.GetInfoFromFileName(filename, ArchiveName); + Log.LogMessage($"Finding closest official archive for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + + string? downloadUrl = await GetClosestOfficialArchiveUrl(); + if (downloadUrl == null) + { + Log.LogError($"Failed to find a download URL for '{ArchiveName}' version '{BuiltVersion}' RID '{BuiltRid}'"); + return false; + } + + HttpClient client = new HttpClient(); + + Log.LogMessage(MessageImportance.High, $"Downloading {downloadUrl}"); + HttpResponseMessage packageResponse = await client.GetAsync(downloadUrl, CancellationToken); + + var packageUriPath = packageResponse.RequestMessage!.RequestUri!.LocalPath; + + ClosestOfficialArchivePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + "." + Path.GetFileName(packageUriPath)); + Log.LogMessage($"Copying {packageUriPath} to {ClosestOfficialArchivePath}"); + using (var file = File.Create(ClosestOfficialArchivePath)) + { + await packageResponse.Content.CopyToAsync(file, CancellationToken); + } + + return true; + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs new file mode 100644 index 000000000000..bb038516c434 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetClosestOfficialSdk.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Build.Framework; + +public class GetClosestOfficialSdk : GetClosestArchive +{ + protected override string ArchiveName => "dotnet-sdk"; + + HttpClient client = new HttpClient(new HttpClientHandler() { AllowAutoRedirect = false }); + + private string? closestVersion; + private string? closestUrl; + + public override async Task GetClosestOfficialArchiveUrl() + { + // Channel in the form of 9.0.1xx + var channel = BuiltVersion[..5] + "xx"; + var akaMsUrl = $"https://aka.ms/dotnet/{channel}/daily/{ArchiveName}-{BuiltRid}{ArchiveExtension}"; + var redirectResponse = await client.GetAsync(akaMsUrl, CancellationToken); + // aka.ms returns a 301 for valid redirects and a 302 to Bing for invalid URLs + if (redirectResponse.StatusCode != HttpStatusCode.Moved) + { + Log.LogMessage(MessageImportance.High, $"Failed to find package at '{akaMsUrl}': invalid aka.ms URL"); + return null; + } + closestUrl = redirectResponse.Headers.Location!.ToString(); + closestVersion = VersionIdentifier.GetVersion(closestUrl); + return closestUrl; + } + + public override async Task GetClosestOfficialArchiveVersion() + { + if (closestUrl is not null) + { + return closestVersion; + } + _ = await GetClosestOfficialArchiveUrl(); + return closestVersion; + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs new file mode 100644 index 000000000000..bd27213b80b3 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/GetValidArchiveItems.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Build.Framework; + +public class GetValidArchiveItems : Microsoft.Build.Utilities.Task +{ + [Required] + public required ITaskItem[] ArchiveItems { get; init; } + + [Required] + public required string ArchiveName { get; init; } + + [Output] + public ITaskItem[] ValidArchiveItems { get; set; } = []; + + public override bool Execute() + { + List archiveItems = new(); + foreach (var item in ArchiveItems) + { + var filename = Path.GetFileName(item.ItemSpec); + try + { + // Ensure the version and RID info can be parsed from the item + _ = Archive.GetInfoFromFileName(filename, ArchiveName); + archiveItems.Add(item); + } + catch (ArgumentException e) + { + Log.LogMessage($"'{item.ItemSpec}' is not a valid archive name: '{e.Message}'"); + continue; + } + } + switch (archiveItems.Count) + { + case 0: + Log.LogMessage(MessageImportance.High, "No valid archive items found"); + ValidArchiveItems = []; + return false; + case 1: + Log.LogMessage(MessageImportance.High, $"{archiveItems[0]} is the only valid archive item found"); + ValidArchiveItems = archiveItems.ToArray(); + break; + default: + archiveItems.Sort((a, b) => a.ItemSpec.Length - b.ItemSpec.Length); + Log.LogMessage(MessageImportance.High, $"Multiple valid archive items found: '{string.Join("', '", archiveItems)}'"); + ValidArchiveItems = archiveItems.ToArray(); + break; + } + return true; + } + +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj new file mode 100644 index 000000000000..ff3bf2402a01 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj @@ -0,0 +1,12 @@ + + + + $(NetCurrent) + enable + + + + + + + diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs new file mode 100644 index 000000000000..2150a721197f --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/VersionIdentifier.cs @@ -0,0 +1,257 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +// Copied from https://github.com/dotnet/arcade/blob/db45698020f58f88eef75b23b2598a59872918f6/src/Microsoft.DotNet.VersionTools/lib/src/BuildManifest/VersionIdentifier.cs +// Conflicting MSBuild versions and some customizations make it difficult to use the Arcade assembly. +public static class VersionIdentifier +{ + private static readonly HashSet _knownTags = new HashSet + { + "alpha", + "beta", + "preview", + "prerelease", + "servicing", + "rtm", + "rc" + }; + + private static readonly SortedDictionary _sequencesToReplace = + new SortedDictionary + { + { "-.", "." }, + { "..", "." }, + { "--", "-" }, + { "//", "/" }, + { "_.", "." } + }; + + private const string _finalSuffix = "final"; + + private static readonly char[] _delimiters = new char[] { '.', '-', '_' }; + + /// + /// Identify the version of an asset. + /// + /// Asset names can come in two forms: + /// - Blobs that include the full path + /// - Packages that do not include any path elements. + /// + /// There may be multiple different version numbers in a blob path. + /// This method starts at the last segment of the path and works backward to find a version number. + /// + /// Asset name + /// Version, or null if none is found. + public static string? GetVersion(string assetName) + { + string[] pathSegments = assetName.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + string? potentialVersion = null; + for (int i = pathSegments.Length - 1; i >= 0; i--) + { + potentialVersion = GetVersionForSingleSegment(pathSegments[i]); + if (potentialVersion != null) + { + return potentialVersion; + } + } + + return potentialVersion; + } + + /// + /// Identify the version number of an asset segment. + /// + /// Asset segment + /// Version number, or null if none was found + /// + /// Identifying versions is not particularly easy. To constrain the problem, + /// we apply the following assumptions which are generally valid for .NET Core. + /// - We always have major.minor.patch, and it always begins the version string. + /// - The only pre-release or build metadata labels we use begin with the _knownTags shown above. + /// - We use additional numbers in our version numbers after the initial + /// major.minor.patch-prereleaselabel.prereleaseiteration segment, + /// but any non-numeric element will end the version string. + /// - The we use in versions and file names are ., -, and _. + /// + private static string? GetVersionForSingleSegment(string assetPathSegment) + { + + // Find the start of the version number by finding the major.minor.patch. + // Scan the string forward looking for a digit preceded by one of the delimiters, + // then look for a minor.patch, completing the major.minor.patch. Continue to do so until we get + // to something that is NOT major.minor.patch (this is necessary because we sometimes see things like: + // VS.Redist.Common.NetCore.Templates.x86.2.2.3.0.101-servicing-014320.nupkg + // Continue iterating until we find ALL potential versions. Return the one that is the latest in the segment + // This is to deal with files with multiple major.minor.patchs in the file name, for example: + // Microsoft.NET.Workload.Mono.ToolChain.Manifest-6.0.100.Msi.x64.6.0.0-rc.1.21380.2.symbols.nupkg + + int currentIndex = 0; + // Stack of major.minor.patch. + Stack<(int versionNumber, int index)> majorMinorPatchStack = new Stack<(int, int)>(3); + string? majorMinorPatch = null; + int majorMinorPatchIndex = 0; + StringBuilder versionSuffix = new StringBuilder(); + char prevDelimiterCharacter = char.MinValue; + char nextDelimiterCharacter = char.MinValue; + Dictionary majorMinorPatchDictionary = new Dictionary(); + while (true) + { + string nextSegment; + prevDelimiterCharacter = nextDelimiterCharacter; + int nextDelimiterIndex = assetPathSegment.IndexOfAny(_delimiters, currentIndex); + if (nextDelimiterIndex != -1) + { + nextDelimiterCharacter = assetPathSegment[nextDelimiterIndex]; + nextSegment = assetPathSegment.Substring(currentIndex, nextDelimiterIndex - currentIndex); + } + else + { + nextSegment = assetPathSegment.Substring(currentIndex); + } + + // If we have not yet found the major/minor/patch, then there are four cases: + // - There have been no potential major/minor/patch numbers found and the current segment is a number. Push onto the majorMinorPatch stack + // and continue. + // - There has been at least one number found, but less than 3, and the current segment not a number or not preceded by '.'. In this case, + // we should clear out the stack and continue the search. + // - There have been at least 2 numbers found and the current segment is a number and preceded by '.'. Push onto the majorMinorPatch stack and continue + // - There have been at least 3 numbers found and the current segment is not a number or not preceded by '-'. In this case, we can call this the major minor + // patch number and no longer need to continue searching + if (majorMinorPatch == null) + { + bool isNumber = int.TryParse(nextSegment, out int potentialVersionSegment); + if ((majorMinorPatchStack.Count == 0 && isNumber) || + (majorMinorPatchStack.Count > 0 && prevDelimiterCharacter == '.' && isNumber)) + { + majorMinorPatchStack.Push((potentialVersionSegment, currentIndex)); + } + // Check for partial major.minor.patch cases, like: 2.2.bar or 2.2-100.bleh + else if (majorMinorPatchStack.Count > 0 && majorMinorPatchStack.Count < 3 && + (prevDelimiterCharacter != '.' || !isNumber)) + { + majorMinorPatchStack.Clear(); + } + + // Determine whether we are done with major.minor.patch after this update. + if (majorMinorPatchStack.Count >= 3 && (prevDelimiterCharacter != '.' || !isNumber || nextDelimiterIndex == -1)) + { + // Done with major.minor.patch, found. Pop the top 3 elements off the stack. + (int patch, int patchIndex) = majorMinorPatchStack.Pop(); + (int minor, int minorIndex) = majorMinorPatchStack.Pop(); + (int major, int majorIndex) = majorMinorPatchStack.Pop(); + majorMinorPatch = $"{major}.{minor}.{patch}"; + majorMinorPatchIndex = majorIndex; + } + } + + // Don't use else, so that we don't miss segments + // in case we are just deciding that we've finished major minor patch. + if (majorMinorPatch != null) + { + // Now look at the next segment. If it looks like it could be part of a version, append to what we have + // and continue. If it can't, then we're done. + // + // Cases where we should break out and be done: + // - We have an empty pre-release label and the delimiter is not '-'. + // - We have an empty pre-release label and the next segment does not start with a known tag. + // - We have a non-empty pre-release label and the current segment is not a number and also not 'final' + // A corner case of versioning uses .final to represent a non-date suffixed final pre-release version: + // 3.1.0-preview.10.final + if (versionSuffix.Length == 0 && + (prevDelimiterCharacter != '-' || !_knownTags.Any(tag => nextSegment.StartsWith(tag, StringComparison.OrdinalIgnoreCase)))) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, majorMinorPatch); + majorMinorPatch = null; + versionSuffix = new StringBuilder(); + } + else if (versionSuffix.Length != 0 && !int.TryParse(nextSegment, out int potentialVersionSegment) && nextSegment != _finalSuffix) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}"); + majorMinorPatch = null; + versionSuffix = new StringBuilder(); + } + else + { + // Append the delimiter character and then the current segment + versionSuffix.Append(prevDelimiterCharacter); + versionSuffix.Append(nextSegment); + } + } + + if (nextDelimiterIndex != -1) + { + currentIndex = nextDelimiterIndex + 1; + } + else + { + break; + } + } + + if (majorMinorPatch != null) + { + majorMinorPatchDictionary.Add(majorMinorPatchIndex, $"{majorMinorPatch}{versionSuffix.ToString()}"); + } + + if (!majorMinorPatchDictionary.Any()) + { + return null; + } + + int maxKey = majorMinorPatchDictionary.Keys.Max(); + return majorMinorPatchDictionary[maxKey]; + } + + /// + /// Given an asset name, remove all .NET Core version numbers (as defined by GetVersionForSingleSegment) + /// from the string + /// + /// Asset + /// Asset name without versions + public static string RemoveVersions(string assetName, string replacement = "") + { + string[] pathSegments = assetName.Split('/'); + + // Remove the version number from each segment, then join back together and + // remove any useless character sequences. + + for (int i = 0; i < pathSegments.Length; i++) + { + if (!string.IsNullOrEmpty(pathSegments[i])) + { + string? versionForSegment = GetVersionForSingleSegment(pathSegments[i]); + if (versionForSegment != null) + { + pathSegments[i] = pathSegments[i].Replace(versionForSegment, replacement); + } + } + } + + // Continue replacing things until there is nothing left to replace. + string assetWithoutVersions = string.Join("/", pathSegments); + bool anyReplacements = true; + while (anyReplacements) + { + string replacementIterationResult = assetWithoutVersions; + foreach (var sequence in _sequencesToReplace) + { + replacementIterationResult = replacementIterationResult.Replace(sequence.Key, sequence.Value); + } + anyReplacements = replacementIterationResult != assetWithoutVersions; + assetWithoutVersions = replacementIterationResult; + } + + return assetWithoutVersions; + } + + + public static bool AreVersionlessEqual(string assetName1, string assetName2) + { + return RemoveVersions(assetName1) == RemoveVersions(assetName2); + } +}