Add SdkArchiveDiff task to verify the sdk archive has all the expected files

src/SourceBuild/content/Directory.Build.props

@@ -203,6 +203,7 @@
   <PropertyGroup>
     <XPlatSourceBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll'))</XPlatSourceBuildTasksAssembly>
     <LeakDetectionTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll'))</LeakDetectionTasksAssembly>
+    <SdkArchiveDiffTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.dll'))</SdkArchiveDiffTasksAssembly>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(EnablePoison)' == 'true'">

src/SourceBuild/content/build.proj

@@ -20,5 +20,6 @@
   </Target>
 
   <Import Project="$(RepositoryEngineeringDir)build.sourcebuild.targets" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
+  <Import Project="$(RepositoryEngineeringDir)build.targets" />
 
 </Project>

src/SourceBuild/content/eng/build.targets

@@ -0,0 +1,29 @@
+<Project>
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="FindArchiveDiffs" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetClosestOfficialSdk" />
+  <UsingTask AssemblyFile="$(SdkArchiveDiffTasksAssembly)" TaskName="GetSingleTarballItem" />
+
+  <Target Name="ReportSdkArchiveDiffs"
+          AfterTargets="Build"
+          DependsOnTargets="DetermineSourceBuiltSdkVersion" >
+
+      <GetSingleTarballItem SdkTarballItems="@(SdkTarballItem)">
+        <Output TaskParameter="BestSdkTarballItem" PropertyName="_TestSdkArchivePath"/>
+      </GetSingleTarballItem>
+
+      <GetClosestOfficialSdk BuiltSdkPath="$(_TestSdkArchivePath)" Condition="'$(_TestSdkArchivePath)' != ''">
+        <Output TaskParameter="ClosestOfficialSdkPath" PropertyName="ClosestOfficialSdkPath" />
+      </GetClosestOfficialSdk>
+
+      <Message Text="Failed to find closest official SDK archive." Importance="High" Condition="'$(_TestSdkArchivePath)' != '' AND '$(ClosestOfficialSdkPath)' == ''" />
+
+      <FindArchiveDiffs BaselineArchive="$(_TestSdkArchivePath)" TestArchive="$(ClosestOfficialSdkPath)" Condition="'$(_TestSdkArchivePath)' != '' AND '$(ClosestOfficialSdkPath)' != ''">
+        <Output TaskParameter="ContentDifferences" ItemName="ContentDifferences" />
+      </FindArchiveDiffs>
+
+      <Message Text="Difference in sdk archive: %(ContentDifferences.Kind): %(ContentDifferences.Identity)" Importance="High" Condition="'%(ContentDifferences.Identity)' != '' AND '%(ContentDifferences.Kind)' != 'Unchanged'"/>
+      <Delete Files="$(ClosestOfficialSdkPath)" Condition="'$(ClosestOfficialSdkPath)' == ''"/>
+
+  </Target>
+
+</Project>

src/SourceBuild/content/eng/tools/init-build.proj

@@ -15,6 +15,7 @@
       UnpackTarballs;
       BuildXPlatTasks;
       BuildMSBuildSdkResolver;
+      BuildSdkArchiveDiff;
       BuildLeakDetection;
       ExtractToolPackage;
       GenerateRootFs;
@@ -116,6 +117,14 @@
     </Touch>
   </Target>
 
+  <Target Name="BuildSdkArchiveDiff" >
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Restore"
+             Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
+    <MSBuild Projects="tasks\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff\Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff.csproj"
+             Targets="Build" />
+  </Target>
+
   <Target Name="GenerateRootFs"
           Condition="'$(BuildOS)' != 'windows' and '$(CrossBuild)' == 'true' and '$(ROOTFS_DIR)' == ''">
     <PropertyGroup>

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/Archive.cs

@@ -0,0 +1,160 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Formats.Tar;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+public abstract class Archive : IDisposable
+{
+    public static async Task<Archive> Create(string path)
+    {
+        if (path.EndsWith(".tar.gz"))
+            return await TarArchive.Create(path);
+        else if (path.EndsWith(".zip"))
+            return ZipFileArchive.Create(path);
+        else
+            throw new NotSupportedException("Unsupported archive type");
+    }
+
+    public abstract bool Contains(string relativePath);
+
+    public abstract string[] GetFileNames();
+
+    public abstract string[] GetFileLines(string relativePath);
+
+    public abstract Task<byte[]> GetFileBytesAsync(string relativePath);
+
+    public abstract void Dispose();
+
+    public class TarArchive : Archive
+    {
+        private string _extractedFolder;
+
+        private TarArchive(string extractedFolder)
+        {
+            _extractedFolder = extractedFolder;
+        }
+
+        public static async Task<TarArchive> Create(string path, CancellationToken cancellationToken = default)
+        {
+            var tmpFolder = Directory.CreateTempSubdirectory(nameof(FindArchiveDiffs));
+            using (var gzStream = File.OpenRead (path))
+            using (var gzipStream = new GZipStream (gzStream, CompressionMode.Decompress))
+            {
+                await TarFile.ExtractToDirectoryAsync(gzipStream, tmpFolder.FullName, true, cancellationToken);
+            }
+            return new TarArchive(tmpFolder.FullName);
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return File.Exists(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override string[] GetFileNames()
+        {
+            return Directory.GetFiles(_extractedFolder, "*", SearchOption.AllDirectories).Select(f => f.Substring(_extractedFolder.Length + 1)).ToArray();
+        }
+
+        public override string[] GetFileLines(string relativePath)
+        {
+            return File.ReadAllLines(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override Task<byte[]> GetFileBytesAsync(string relativePath)
+        {
+            var filePath = Path.Combine(_extractedFolder, relativePath);
+            if (!File.Exists(filePath))
+                return Task.FromResult<byte[]>([]);
+            return File.ReadAllBytesAsync(Path.Combine(_extractedFolder, relativePath));
+        }
+
+        public override void Dispose()
+        {
+            if (Directory.Exists(_extractedFolder))
+                Directory.Delete(_extractedFolder, true);
+        }
+    }
+
+    public class ZipFileArchive : Archive
+    {
+        private ZipArchive _archive;
+
+        private ZipFileArchive(ZipArchive archive)
+        {
+            _archive = archive;
+        }
+
+        public static new ZipFileArchive Create(string path)
+        {
+            return new ZipFileArchive(new ZipArchive(File.OpenRead(path)));
+        }
+
+        public override bool Contains(string relativePath)
+        {
+            return _archive.GetEntry(relativePath) != null;
+        }
+
+        public override string[] GetFileNames()
+        {
+            return _archive.Entries.Select(e => e.FullName).ToArray();
+        }
+
+        public override string[] GetFileLines(string relativePath)
+        {
+            var entry = _archive.GetEntry(relativePath);
+            if (entry == null)
+                throw new ArgumentException("File not found");
+            return entry.Lines();
+        }
+        public override Task<byte[]> GetFileBytesAsync(string relativePath)
+        {
+            using (var entry = _archive.GetEntry(relativePath)?.Open())
+            {
+                if (entry == null)
+                {
+                    return Task.FromResult<byte[]>([]);
+                }
+                return entry.ReadToEndAsync();
+            }
+        }
+
+        public override void Dispose()
+        {
+            _archive.Dispose();
+        }
+    }
+
+    public static (string Version, string Rid, string extension) GetInfoFromArchivePath(string path)
+    {
+        string extension;
+        if (path.EndsWith(".tar.gz"))
+        {
+            extension = ".tar.gz";
+        }
+        else if (path.EndsWith(".zip"))
+        {
+            extension = ".zip";
+        }
+        else
+        {
+            throw new ArgumentException($"Invalid archive extension '{path}': must end with .tar.gz or .zip");
+        }
+
+        string filename = Path.GetFileName(path)[..^extension.Length];
+        var dashDelimitedParts = filename.Split('-');
+        var (rid, versionString) = dashDelimitedParts switch
+        {
+            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(first) => (third + '-' + fourth, first + '-' + second),
+            ["dotnet", "sdk", var first, var second, var third, var fourth] when PathWithVersions.IsVersionString(third) => (first + '-' + second, third + '-' + fourth),
+            _ => throw new ArgumentException($"Invalid archive file name '{filename}': file name should include full build version and rid in the format dotnet-sdk-<version>-<rid>{extension} or dotnet-sdk-<rid>-<version>{extension}")
+        };
+
+        return (versionString, rid, extension);
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.SdkArchiveDiff/FindArchiveDiffs.cs

@@ -0,0 +1,134 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Task = System.Threading.Tasks.Task;
+
+public class FindArchiveDiffs : Microsoft.Build.Utilities.Task
+{
+    public class ArchiveItem
+    {
+        public required string Path { get; init; }
+    }
+
+    [Required]
+    public required ITaskItem BaselineArchive { get; init; }
+
+    [Required]
+    public required ITaskItem TestArchive { get; init; }
+
+    [Output]
+    public ITaskItem[] ContentDifferences { get; set; } = [];
+
+    public override bool Execute()
+    {
+        return Task.Run(ExecuteAsync).Result;
+    }
+
+    public async Task<bool> ExecuteAsync()
+    {
+        var baselineTask = Archive.Create(BaselineArchive.ItemSpec);
+        var testTask = Archive.Create(TestArchive.ItemSpec);
+        Task.WaitAll(baselineTask, testTask);
+        using var baseline = await baselineTask;
+        using var test = await testTask;
+        var baselineFiles = baseline.GetFileNames();
+        var testFiles = test.GetFileNames();
+        ContentDifferences =
+            GetDiffs(baselineFiles, testFiles, PathWithVersions.Equal, PathWithVersions.GetVersionlessPath)
+            .Select(FromDiff)
+            .ToArray();
+        return true;
+    }
+
+    static ITaskItem FromDiff((string, DifferenceKind) diff)
+    {
+        var item = new TaskItem(diff.Item1);
+        item.SetMetadata("Kind", Enum.GetName(diff.Item2));
+        return item;
+    }
+
+    public enum DifferenceKind
+    {
+        Added,
+        Removed,
+        Unchanged
+    }
+
+    public static List<(string, DifferenceKind DifferenceKind)> GetDiffs(
+        string[] originalPathsWithVersions,
+        string[] modifiedPathsWithVersions,
+        Func<string, string, bool> equalityComparer,
+        Func<string, string>? formatter = null)
+    {
+        formatter ??= static s => s;
+        // Edit distance algorithm: https://en.wikipedia.org/wiki/Longest_common_subsequence
+
+        int[,] dp = new int[originalPathsWithVersions.Length + 1, modifiedPathsWithVersions.Length + 1];
+
+        // Initialize first row and column
+        for (int i = 0; i <= originalPathsWithVersions.Length; i++)
+        {
+            dp[i, 0] = i;
+        }
+        for (int j = 0; j <= modifiedPathsWithVersions.Length; j++)
+        {
+            dp[0, j] = j;
+        }
+
+        // Compute edit distance
+        for (int i = 1; i <= originalPathsWithVersions.Length; i++)
+        {
+            for (int j = 1; j <= modifiedPathsWithVersions.Length; j++)
+            {
+                if (equalityComparer(originalPathsWithVersions[i - 1], modifiedPathsWithVersions[j - 1]))
+                {
+                    dp[i, j] = dp[i - 1, j - 1];
+                }
+                else
+                {
+                    dp[i, j] = 1 + Math.Min(dp[i - 1, j], dp[i, j - 1]);
+                }
+            }
+        }
+
+        // Trace back the edits
+        int row = originalPathsWithVersions.Length;
+        int col = modifiedPathsWithVersions.Length;
+
+        List<(string, DifferenceKind)> formattedDiff = [];
+        while (row > 0 || col > 0)
+        {
+            var baselineItem = originalPathsWithVersions[row - 1];
+            var testItem = modifiedPathsWithVersions[col - 1];
+            if (row > 0 && col > 0 && PathWithVersions.Equal(baselineItem, testItem))
+            {
+                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Unchanged));
+                row--;
+                col--;
+            }
+            else if (col > 0 && (row == 0 || dp[row, col - 1] <= dp[row - 1, col]))
+            {
+                formattedDiff.Add((formatter(modifiedPathsWithVersions[col - 1]), DifferenceKind.Added));
+                col--;
+            }
+            else if (row > 0 && (col == 0 || dp[row, col - 1] > dp[row - 1, col]))
+            {
+                formattedDiff.Add((formatter(originalPathsWithVersions[row - 1]), DifferenceKind.Removed));
+                row--;
+            }
+            else
+            {
+                throw new UnreachableException();
+            }
+        }
+        formattedDiff.Reverse();
+        return formattedDiff;
+    }
+}