From 842ac9b2fec1c10cae480030dd41e1020a7d49b2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 10 Apr 2024 20:49:36 -0700 Subject: [PATCH 1/5] In progress --- .../IRemoteSourceGenerationService.cs | 4 +- ...lutionCompilationState_SourceGenerators.cs | 7 ++- .../RemoteSourceGenerationService.cs | 60 +++++++++++++++++-- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs b/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs index d56c7ff41d3bb..d75ed83198f0b 100644 --- a/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs +++ b/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs @@ -33,10 +33,10 @@ ValueTask> GetContentsAsync( Checksum solutionChecksum, ProjectId projectId, ImmutableArray documentIds, CancellationToken cancellationToken); /// - /// Whether or not the specified has source generators or not. + /// Whether or not the specified analyzer references have source generators or not. /// ValueTask HasGeneratorsAsync( - Checksum solutionChecksum, ProjectId projectId, CancellationToken cancellationToken); + Checksum solutionChecksum, ProjectId projectId, ImmutableArray analyzerReferenceChecksums, CancellationToken cancellationToken); } /// diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs index 915b3adc72a60..672875731e42b 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs @@ -94,6 +94,8 @@ static SourceGeneratorMap ComputeSourceGenerators(ProjectState projectState) public async Task HasSourceGeneratorsAsync(ProjectId projectId, CancellationToken cancellationToken) { var projectState = this.SolutionState.GetRequiredProjectState(projectId); + if (projectState.AnalyzerReferences.Count == 0) + return false; if (!s_hasSourceGeneratorsMap.TryGetValue(projectState, out var lazy)) { @@ -120,10 +122,13 @@ static async Task ComputeHasSourceGeneratorsAsync( // Out of process, call to the remote to figure this out. var projectId = projectState.Id; + var projectStateChecksums = await projectState.GetStateChecksumsAsync(cancellationToken).ConfigureAwait(false); + var analyzerReferences = projectStateChecksums.AnalyzerReferences.Children; + var result = await client.TryInvokeAsync( solution, projectId, - (service, solution, cancellationToken) => service.HasGeneratorsAsync(solution, projectId, cancellationToken), + (service, solution, cancellationToken) => service.HasGeneratorsAsync(solution, projectId, analyzerReferences, cancellationToken), cancellationToken).ConfigureAwait(false); return result.HasValue && result.Value; } diff --git a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs index 6cee072584d86..bd1a9e94f0683 100644 --- a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs +++ b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs @@ -3,9 +3,13 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.CodeAnalysis.Diagnostics; using Microsoft.CodeAnalysis.PooledObjects; using Microsoft.CodeAnalysis.Shared.Extensions; using Microsoft.CodeAnalysis.SourceGeneration; @@ -13,6 +17,8 @@ namespace Microsoft.CodeAnalysis.Remote; +using AnalyzerReferenceMap = ConditionalWeakTable>; + internal sealed partial class RemoteSourceGenerationService(in BrokeredServiceBase.ServiceConstructionArguments arguments) : BrokeredServiceBase(arguments), IRemoteSourceGenerationService { @@ -64,11 +70,57 @@ public ValueTask> GetContentsAsync( }, cancellationToken); } - public ValueTask HasGeneratorsAsync(Checksum solutionChecksum, ProjectId projectId, CancellationToken cancellationToken) + private static readonly ImmutableArray<(string language, AnalyzerReferenceMap analyzerReferenceMap, AnalyzerReferenceMap.CreateValueCallback callback)> s_languageToAnalyzerReferenceMap = + [ + (LanguageNames.CSharp, new(), static analyzerReference => AsyncLazy.Create(cancellationToken => HasSourceGeneratorsAsync(analyzerReference, LanguageNames.CSharp, cancellationToken))), + (LanguageNames.VisualBasic, new(), static analyzerReference => AsyncLazy.Create(cancellationToken => HasSourceGeneratorsAsync(analyzerReference, LanguageNames.VisualBasic, cancellationToken))) + ]; + + private static async Task HasSourceGeneratorsAsync( + AnalyzerReference analyzerReference, string language, CancellationToken cancellationToken) { - return RunServiceAsync(solutionChecksum, async solution => + var generators = analyzerReference.GetGenerators(langauge); + return generators.Any(); + } + + public async ValueTask HasGeneratorsAsync( + Checksum solutionChecksum, + ProjectId projectId, + ImmutableArray analyzerReferenceChecksums, + string language, + CancellationToken cancellationToken) + { + if (analyzerReferenceChecksums.Length == 0) + return false; + + var workspace = GetWorkspace(); + var assetProvider = workspace.CreateAssetProvider(solutionChecksum, WorkspaceManager.SolutionAssetCache, SolutionAssetSource); + + using var _1 = PooledHashSet.GetInstance(out var checksums); + checksums.AddRange(analyzerReferenceChecksums); + + // Fetch the analyzer references specified by the host. Note: this will only serialize this information over + // the first time needed. After that, it will be cached in the WorkspaceManager.SolutionAssetCache on the remote + // side, so it will be a no-op to fetch them in the future. + using var _2 = ArrayBuilder.GetInstance(checksums.Count, out var analyzerReferences); + await assetProvider.GetAssetsAsync>( + projectId, + checksums, + static (_, analyzerReference, analyzerReferences) => analyzerReferences.Add(analyzerReference), + analyzerReferences, + cancellationToken).ConfigureAwait(false); + + var tuple = s_languageToAnalyzerReferenceMap.Single(static (val, language) => val.language == language, language); + var analyzerReferenceMap = tuple.analyzerReferenceMap; + var callback = tuple.callback; + + foreach (var analyzerReference in analyzerReferences) { - return await solution.CompilationState.HasSourceGeneratorsAsync(projectId, cancellationToken).ConfigureAwait(false); - }, cancellationToken); + var hasGeneratorsLazy = analyzerReferenceMap.GetValue(analyzerReference, callback); + if (await hasGeneratorsLazy.GetValueAsync(cancellationToken).ConfigureAwait(false)) + return true; + } + + return false; } } From e8e9c4993a7a576b378237b9691f859aa7534077 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 10 Apr 2024 20:58:51 -0700 Subject: [PATCH 2/5] docs --- .../IRemoteSourceGenerationService.cs | 2 +- .../RemoteSourceGenerationService.cs | 30 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs b/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs index d75ed83198f0b..659de283e8849 100644 --- a/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs +++ b/src/Workspaces/Core/Portable/SourceGeneration/IRemoteSourceGenerationService.cs @@ -36,7 +36,7 @@ ValueTask> GetContentsAsync( /// Whether or not the specified analyzer references have source generators or not. /// ValueTask HasGeneratorsAsync( - Checksum solutionChecksum, ProjectId projectId, ImmutableArray analyzerReferenceChecksums, CancellationToken cancellationToken); + Checksum solutionChecksum, ProjectId projectId, ImmutableArray analyzerReferenceChecksums, string language, CancellationToken cancellationToken); } /// diff --git a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs index bd1a9e94f0683..23245daae557b 100644 --- a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs +++ b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs @@ -17,7 +17,8 @@ namespace Microsoft.CodeAnalysis.Remote; -using AnalyzerReferenceMap = ConditionalWeakTable>; +// Can use AnalyzerReference as a key here as we will will always get back the same instance back for the same checksum. +using AnalyzerReferenceMap = ConditionalWeakTable>; internal sealed partial class RemoteSourceGenerationService(in BrokeredServiceBase.ServiceConstructionArguments arguments) : BrokeredServiceBase(arguments), IRemoteSourceGenerationService @@ -72,15 +73,15 @@ public ValueTask> GetContentsAsync( private static readonly ImmutableArray<(string language, AnalyzerReferenceMap analyzerReferenceMap, AnalyzerReferenceMap.CreateValueCallback callback)> s_languageToAnalyzerReferenceMap = [ - (LanguageNames.CSharp, new(), static analyzerReference => AsyncLazy.Create(cancellationToken => HasSourceGeneratorsAsync(analyzerReference, LanguageNames.CSharp, cancellationToken))), - (LanguageNames.VisualBasic, new(), static analyzerReference => AsyncLazy.Create(cancellationToken => HasSourceGeneratorsAsync(analyzerReference, LanguageNames.VisualBasic, cancellationToken))) + (LanguageNames.CSharp, new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.CSharp)), + (LanguageNames.VisualBasic, new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.VisualBasic)) ]; - private static async Task HasSourceGeneratorsAsync( - AnalyzerReference analyzerReference, string language, CancellationToken cancellationToken) + private static StrongBox HasSourceGenerators( + AnalyzerReference analyzerReference, string language) { - var generators = analyzerReference.GetGenerators(langauge); - return generators.Any(); + var generators = analyzerReference.GetGenerators(language); + return new(generators.Any()); } public async ValueTask HasGeneratorsAsync( @@ -93,6 +94,11 @@ public async ValueTask HasGeneratorsAsync( if (analyzerReferenceChecksums.Length == 0) return false; + // Do not use RunServiceAsync here. We don't want to actually synchronize a solution instance on this remote + // side to service this request. Specifically, solution syncing is expensive, and will pull over a lot of data + // that we don't need (like document contents). All we need to do is synchronize over the analyzer-references + // (which are actually quite small as they are represented as file-paths), and then answer the question based on + // them directly. We can then cache that result for future requests. var workspace = GetWorkspace(); var assetProvider = workspace.CreateAssetProvider(solutionChecksum, WorkspaceManager.SolutionAssetCache, SolutionAssetSource); @@ -102,6 +108,12 @@ public async ValueTask HasGeneratorsAsync( // Fetch the analyzer references specified by the host. Note: this will only serialize this information over // the first time needed. After that, it will be cached in the WorkspaceManager.SolutionAssetCache on the remote // side, so it will be a no-op to fetch them in the future. + // + // From this point on, the host won't call into us for the same project-state (as it caches the data itself). If + // the project state changes, it will just call into us with the checksums for its analyzer references. As + // those will almost always be the same, we'll just fetch the precomputed values on our end, return them, and + // the host will cache it. We'll only actually fetch something new and compute something new when an actual new + // analyzer reference is added. using var _2 = ArrayBuilder.GetInstance(checksums.Count, out var analyzerReferences); await assetProvider.GetAssetsAsync>( projectId, @@ -116,8 +128,8 @@ await assetProvider.GetAssetsAsync Date: Wed, 10 Apr 2024 21:12:04 -0700 Subject: [PATCH 3/5] Cleanup --- ...lutionCompilationState_SourceGenerators.cs | 33 +++++++++++++------ .../RemoteSourceGenerationService.cs | 15 ++++----- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs index 672875731e42b..ab0b6e27b3d8d 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Collections.Frozen; +using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Runtime.CompilerServices; @@ -17,6 +18,11 @@ namespace Microsoft.CodeAnalysis; +// Cache of list of analyzer references to whether or not they have source generators. Keyed based off +// IReadOnlyList so that we can cache the value even as project-states fork based on +// document edits. +using AnalyzerReferenceMap = ConditionalWeakTable, AsyncLazy>; + internal partial class SolutionCompilationState { private sealed record SourceGeneratorMap( @@ -38,7 +44,11 @@ private sealed record SourceGeneratorMap( /// process (if present) and having it make the determination, without the host necessarily loading generators /// itself. /// - private static readonly ConditionalWeakTable> s_hasSourceGeneratorsMap = new(); + private static readonly Dictionary s_languageToAnalyzerReferenceMap = new() + { + { LanguageNames.CSharp, new() }, + {LanguageNames.VisualBasic, new() }, + }; /// /// This method should only be called in a .net core host like our out of process server. @@ -97,20 +107,23 @@ public async Task HasSourceGeneratorsAsync(ProjectId projectId, Cancellati if (projectState.AnalyzerReferences.Count == 0) return false; - if (!s_hasSourceGeneratorsMap.TryGetValue(projectState, out var lazy)) + if (!RemoteSupportedLanguages.IsSupported(projectState.Language)) + return false; + + var analyzerReferenceMap = s_languageToAnalyzerReferenceMap[projectState.Language]; + if (!analyzerReferenceMap.TryGetValue(projectState.AnalyzerReferences, out var lazy)) { // Extracted into local function to prevent allocations in the case where we find a value in the cache. - lazy = GetLazy(projectState); + lazy = GetLazy(analyzerReferenceMap, projectState); } return await lazy.GetValueAsync(cancellationToken).ConfigureAwait(false); - AsyncLazy GetLazy(ProjectState projectState) - => s_hasSourceGeneratorsMap.GetValue( - projectState, - projectState => AsyncLazy.Create( - static (tuple, cancellationToken) => ComputeHasSourceGeneratorsAsync(tuple.@this, tuple.projectState, cancellationToken), - (@this: this, projectState))); + AsyncLazy GetLazy(AnalyzerReferenceMap analyzerReferenceMap, ProjectState projectState) + => analyzerReferenceMap.GetValue( + projectState.AnalyzerReferences, + _ => AsyncLazy.Create( + cancellationToken => ComputeHasSourceGeneratorsAsync(this, projectState, cancellationToken))); static async Task ComputeHasSourceGeneratorsAsync( SolutionCompilationState solution, ProjectState projectState, CancellationToken cancellationToken) @@ -128,7 +141,7 @@ static async Task ComputeHasSourceGeneratorsAsync( var result = await client.TryInvokeAsync( solution, projectId, - (service, solution, cancellationToken) => service.HasGeneratorsAsync(solution, projectId, analyzerReferences, cancellationToken), + (service, solution, cancellationToken) => service.HasGeneratorsAsync(solution, projectId, analyzerReferences, projectState.Language, cancellationToken), cancellationToken).ConfigureAwait(false); return result.HasValue && result.Value; } diff --git a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs index 23245daae557b..5a5a17d0021a4 100644 --- a/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs +++ b/src/Workspaces/Remote/ServiceHub/Services/SourceGeneration/RemoteSourceGenerationService.cs @@ -71,11 +71,11 @@ public ValueTask> GetContentsAsync( }, cancellationToken); } - private static readonly ImmutableArray<(string language, AnalyzerReferenceMap analyzerReferenceMap, AnalyzerReferenceMap.CreateValueCallback callback)> s_languageToAnalyzerReferenceMap = - [ - (LanguageNames.CSharp, new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.CSharp)), - (LanguageNames.VisualBasic, new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.VisualBasic)) - ]; + private static readonly Dictionary s_languageToAnalyzerReferenceMap = new() + { + { LanguageNames.CSharp, (new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.CSharp)) }, + { LanguageNames.VisualBasic, (new(), static analyzerReference => HasSourceGenerators(analyzerReference, LanguageNames.VisualBasic)) }, + }; private static StrongBox HasSourceGenerators( AnalyzerReference analyzerReference, string language) @@ -122,10 +122,7 @@ await assetProvider.GetAssetsAsync val.language == language, language); - var analyzerReferenceMap = tuple.analyzerReferenceMap; - var callback = tuple.callback; - + var (analyzerReferenceMap, callback) = s_languageToAnalyzerReferenceMap[language]; foreach (var analyzerReference in analyzerReferences) { var hasGenerators = analyzerReferenceMap.GetValue(analyzerReference, callback); From df6429d26a3d2997385a960c578c08cccb3f1830 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 10 Apr 2024 21:22:11 -0700 Subject: [PATCH 4/5] formatting --- .../Solution/SolutionCompilationState_SourceGenerators.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs index ab0b6e27b3d8d..4a94680e0bd91 100644 --- a/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs +++ b/src/Workspaces/Core/Portable/Workspace/Solution/SolutionCompilationState_SourceGenerators.cs @@ -47,7 +47,7 @@ private sealed record SourceGeneratorMap( private static readonly Dictionary s_languageToAnalyzerReferenceMap = new() { { LanguageNames.CSharp, new() }, - {LanguageNames.VisualBasic, new() }, + { LanguageNames.VisualBasic, new() }, }; /// From 078c2ed5f1a61e763293b750f546521863788fa2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Thu, 11 Apr 2024 11:14:17 -0700 Subject: [PATCH 5/5] Add docs --- src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs b/src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs index ad5f1ca91a42e..37c6881906c63 100644 --- a/src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs +++ b/src/Workspaces/Remote/ServiceHub/Host/AssetProvider.cs @@ -57,6 +57,10 @@ public override async ValueTask GetAssetsAsync( await this.SynchronizeAssetsAsync(assetPath, checksums, callback, arg, cancellationToken).ConfigureAwait(false); } + /// + /// This is the function called when we are not doing an incremental update, but are instead doing a bulk + /// full sync. + /// public async ValueTask SynchronizeSolutionAssetsAsync(Checksum solutionChecksum, CancellationToken cancellationToken) { var timer = SharedStopwatch.StartNew();