Merge branch 'bmurri/simplify-task-management' into bmurri/multiple-e…

…xecutors
microsoft · Feb 27, 2025 · ab117a5 · ab117a5
2 parents 0566e08 + acf6fd3
commit ab117a5
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -1,3 +1,5 @@
+### Announcement: Deployment [changes](https://github.com/microsoft/ga4gh-tes/wiki/6.0.0-Migration-Guide) in v6.0.0
+
 # GA4GH TES on Azure
 
 This project is an implementation of the [GA4GH Task Execution Service (TES)](https://github.com/ga4gh/task-execution-schemas), that provides distributed batch task execution on Microsoft Azure. The TES API is an effort to define a standardized schema and API for describing batch execution tasks. A task defines a set of input files, a set of Docker containers and commands to run, a set of output files, and some other logging and metadata.

diff --git a/src/TesApi.Web/Events/RunnerEventsProcessor.cs b/src/TesApi.Web/Events/RunnerEventsProcessor.cs
@@ -411,11 +411,19 @@ IAsyncEnumerable<string> GetProcessLogsAsync(Tes.Runner.Events.EventMessage mess
                 var blobNameStartsWith = message.Name switch
                 {
                     Tes.Runner.Events.EventsPublisher.DownloadEndEvent => "download_std",
+
                     Tes.Runner.Events.EventsPublisher.ExecutorEndEvent => $"exec-{ParseExecutorIndex(message.EventData):D3}_std",
                     Tes.Runner.Events.EventsPublisher.UploadEndEvent => "upload_std",
                     _ => string.Empty,
                 };
 
+                if (message.EventData.TryGetValue("executor", out var value) &&
+                    int.TryParse(value.Split('/', 2, StringSplitOptions.TrimEntries)[0], out var executor))
+                {
+                    // Maintain format with Tes.RunnerCLI.Commands.CommandLauncher.LaunchesExecutorCommandAsSubProcessAsync()
+                    blobNameStartsWith += $"{executor:D3}_std";
+                }
+
                 return GetAvailableProcessLogsAsync(blobNameStartsWith, tesTask, cancellationToken).Select(t => t.Uri.AbsoluteUri);
             }
 

diff --git a/src/deploy-tes-on-azure/Deployer.cs b/src/deploy-tes-on-azure/Deployer.cs
@@ -629,12 +629,16 @@ await Task.WhenAll(
                             {
                                 ConsoleEx.WriteLine("Unable to assign 'Storage Blob Data Contributor' for deployment identity to the storage account. If the deployment fails as a result, the storage account must be precreated and the deploying user must have the 'Storage Blob Data Contributor' role for the storage account.", ConsoleColor.Yellow);
                             }
+                            else
+                            {
+                                await Task.Delay(TimeSpan.FromMinutes(5), cts.Token);
+                            }
 
-                            await WritePersonalizedFilesToStorageAccountAsync(storageAccountData);
                             await AssignVmAsContributorToStorageAccountAsync(managedIdentity, storageAccount);
                             await AssignVmAsDataOwnerToStorageAccountAsync(managedIdentity, storageAccount);
                             await AssignManagedIdOperatorToResourceAsync(managedIdentity, resourceGroup);
                             await AssignMIAsNetworkContributorToResourceAsync(managedIdentity, resourceGroup);
+                            await WritePersonalizedFilesToStorageAccountAsync(storageAccountData);
                         }),
                     ]);
 

diff --git a/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj b/src/deploy-tes-on-azure/deploy-tes-on-azure.csproj
@@ -66,4 +66,19 @@
     <ProjectReference Include="..\build-push-acr\build-push-acr.csproj" />
   </ItemGroup>
 
+  <!-- Not needed in NET SDK 9 -->
+  <Target Name="RemoveSingleFileHostsFromDependencies" BeforeTargets="_ComputeCopyToPublishDirectoryItems">
+    <PropertyGroup>
+      <SingleFileHostIntermediatePath Condition=" '$(SingleFileHostIntermediatePath)' == '' ">$(AppHostIntermediatePath)</SingleFileHostIntermediatePath>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <__ItemsToRemove Remove="@(__ItemsToRemove)" />
+      <__ItemsToRemove Condition=" '%(Filename)' == '$(_DotNetSingleFileHostExecutableNameWithoutExtension)' And '%(FullPath)' != '$(SingleFileHostIntermediatePath)' "
+                       Include="@(_SourceItemsToCopyToPublishDirectoryAlways)" />
+      <_SourceItemsToCopyToPublishDirectoryAlways Remove="@(__ItemsToRemove)" />
+      <__ItemsToRemove Remove="@(__ItemsToRemove)" />
+    </ItemGroup>
+  </Target>
+
 </Project>
diff --git a/src/deploy-tes-on-azure/scripts/helm/templates/tes-deployment.yaml b/src/deploy-tes-on-azure/scripts/helm/templates/tes-deployment.yaml
@@ -16,15 +16,15 @@ spec:
       labels:
         io.kompose.service: tes
         azure.workload.identity/use: "true"
-      annotations:
-        container.apparmor.security.beta.kubernetes.io/tes: runtime/default
     spec:
       serviceAccountName: {{ .Values.identity.name }}-sa
       securityContext:
         runAsUser: 1000
         runAsGroup: 1000
         fsGroup: 1000
         supplementalGroups: [1000]
+        appArmorProfile:
+          type: RuntimeDefault
       terminationGracePeriodSeconds: 90
       containers:
         - env: