support background steps

src/Runner.Common/JobServerQueue.cs

@@ -837,6 +837,16 @@ private List<TimelineRecord> MergeTimelineRecords(List<TimelineRecord> timelineR
                             timelineRecord.Variables[variable.Key] = variable.Value.Clone();
                         }
                     }
+
+                    // Merge background step metadata
+                    if (rec.IsBackground)
+                    {
+                        timelineRecord.IsBackground = rec.IsBackground;
+                    }
+                    timelineRecord.StepType = rec.StepType ?? timelineRecord.StepType;
+                    timelineRecord.WaitStepIds = rec.WaitStepIds ?? timelineRecord.WaitStepIds;
+                    timelineRecord.CancelStepId = rec.CancelStepId ?? timelineRecord.CancelStepId;
+                    timelineRecord.ParallelGroupId = rec.ParallelGroupId ?? timelineRecord.ParallelGroupId;
                 }
                 else
                 {

src/Runner.Worker/ActionCommandManager.cs

@@ -178,7 +178,7 @@ private void ValidateStopToken(IExecutionContext context, string stopToken)
                     Message = $"Invoked ::stopCommand:: with token: [{stopToken}]",
                     Type = JobTelemetryType.ActionCommand
                 };
-                context.Global.JobTelemetry.Add(telemetry);
+                lock (context.Global.CollectionLock) { context.Global.JobTelemetry.Add(telemetry); }
             }
 
             if (isTokenInvalid && !allowUnsecureStopCommandTokens)
@@ -326,7 +326,7 @@ public void ProcessCommand(IExecutionContext context, string line, ActionCommand
                     Type = JobTelemetryType.ActionCommand,
                     Message = "DeprecatedCommand: set-output"
                 };
-                context.Global.JobTelemetry.Add(telemetry);
+                lock (context.Global.CollectionLock) { context.Global.JobTelemetry.Add(telemetry); }
             }
 
             if (!command.Properties.TryGetValue(SetOutputCommandProperties.Name, out string outputName) || string.IsNullOrEmpty(outputName))
@@ -372,7 +372,7 @@ public void ProcessCommand(IExecutionContext context, string line, ActionCommand
                     Type = JobTelemetryType.ActionCommand,
                     Message = "DeprecatedCommand: save-state"
                 };
-                context.Global.JobTelemetry.Add(telemetry);
+                lock (context.Global.CollectionLock) { context.Global.JobTelemetry.Add(telemetry); }
             }
 
             if (!command.Properties.TryGetValue(SaveStateCommandProperties.Name, out string stateName) || string.IsNullOrEmpty(stateName))

src/Runner.Worker/ActionManager.cs

@@ -1068,11 +1068,14 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
                                 }
 
                                 executionContext.Debug($"Created symlink from cached directory '{cacheDirectory}' to '{destDirectory}'");
-                                executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                                lock (executionContext.Global.CollectionLock)
                                 {
-                                    Type = JobTelemetryType.General,
-                                    Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache} via symlink"
-                                });
+                                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                                    {
+                                        Type = JobTelemetryType.General,
+                                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache} via symlink"
+                                    });
+                                }
 
                                 Trace.Info("Finished getting action repository.");
                                 return;
@@ -1108,11 +1111,14 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont
                     }
                 }
 
-                executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                lock (executionContext.Global.CollectionLock)
                 {
-                    Type = JobTelemetryType.General,
-                    Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
-                });
+                    executionContext.Global.JobTelemetry.Add(new JobTelemetry()
+                    {
+                        Type = JobTelemetryType.General,
+                        Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}"
+                    });
+                }
 
                 if (!useActionArchiveCache)
                 {

src/Runner.Worker/ActionManifestManagerWrapper.cs

@@ -446,7 +446,7 @@ private void RecordMismatch(IExecutionContext context, string methodName)
             {
                 context.Global.HasActionManifestMismatch = true;
                 var telemetry = new JobTelemetry { Type = JobTelemetryType.General, Message = $"ActionManifestMismatch: {methodName}" };
-                context.Global.JobTelemetry.Add(telemetry);
+                lock (context.Global.CollectionLock) { context.Global.JobTelemetry.Add(telemetry); }
             }
         }
 
@@ -456,7 +456,7 @@ private void RecordComparisonError(IExecutionContext context, string errorDetail
             {
                 context.Global.HasActionManifestMismatch = true;
                 var telemetry = new JobTelemetry { Type = JobTelemetryType.General, Message = $"ActionManifestComparisonError: {errorDetails}" };
-                context.Global.JobTelemetry.Add(telemetry);
+                lock (context.Global.CollectionLock) { context.Global.JobTelemetry.Add(telemetry); }
             }
         }
 

src/Runner.Worker/BackgroundStepContext.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace GitHub.Runner.Worker
+{
+    /// <summary>
+    /// Tracks a background step's execution state.
+    /// </summary>
+    internal sealed class BackgroundStepContext
+    {
+        public string StepId { get; }
+        public IStep Step { get; }
+        public Task ExecutionTask { get; set; }
+        public CancellationTokenSource Cts { get; set; }
+        public GitHub.DistributedTask.WebApi.TaskResult? Result { get; set; }
+        public bool IsCompleted => ExecutionTask?.IsCompleted ?? false;
+        public string ExternalId => Step.ExecutionContext.Id.ToString("N");
+
+        public BackgroundStepContext(string stepId, IStep step)
+        {
+            StepId = stepId;
+            Step = step;
+        }
+    }
+}

src/Runner.Worker/ControlFlowStepData.cs

@@ -0,0 +1,31 @@
+using System;
+
+namespace GitHub.Runner.Worker
+{
+    public enum ControlFlowType
+    {
+        Wait,
+        WaitAll,
+        Cancel,
+    }
+
+    /// <summary>
+    /// Data for control-flow steps (wait, wait-all, cancel).
+    /// Used with JobExtensionRunner instead of dedicated IStep implementations.
+    /// </summary>
+    public sealed class ControlFlowStepData
+    {
+        public ControlFlowType Type { get; set; }
+        public Guid StepId { get; set; }
+        public string StepName { get; set; }
+
+        // Wait: IDs of background steps to wait for
+        public string[] WaitStepIds { get; set; }
+
+        // Cancel: IDs of background steps to cancel
+        public string[] CancelStepIds { get; set; }
+
+        // Parallel group ID for grouping steps in the UI
+        public string ParallelGroupId { get; set; }
+    }
+}

src/Runner.Worker/ExecutionContext.cs

@@ -85,6 +85,12 @@ public interface IExecutionContext : IRunnerService
         IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List<Issue> embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null);
         IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary<string, string> intraActionState = null, string siblingScopeName = null);
 
+        // Background step deferral properties
+        Dictionary<string, string> DeferredOutputs { get; set; }
+        Dictionary<string, string> DeferredEnvironmentVariables { get; set; }
+        List<string> DeferredPrependPath { get; set; }
+        bool DeferOutcomeConclusion { get; set; }
+
         // logging
         long Write(string tag, string message);
         void QueueAttachFile(string type, string name, string filePath);
@@ -100,11 +106,18 @@ public interface IExecutionContext : IRunnerService
         void SetGitHubContext(string name, string value);
         void SetOutput(string name, string value, out string reference);
         void SetTimeout(TimeSpan? timeout);
+
+        // Background step deferral flush methods
+        void FlushDeferredOutputs();
+        void FlushDeferredEnvironment();
+        void FlushDeferredOutcomeConclusion();
+
         void AddIssue(Issue issue, ExecutionContextLogOptions logOptions);
         void Progress(int percentage, string currentOperation = null);
         void UpdateDetailTimelineRecord(TimelineRecord record);
 
         void UpdateTimelineRecordDisplayName(string displayName);
+        void SetBackgroundStepMetadata(bool isBackground = false, string stepType = null, string[] waitStepIds = null, string cancelStepId = null, string parallelGroupId = null);
 
         // matchers
         void Add(OnMatcherChanged handler);
@@ -279,6 +292,12 @@ public JobContext JobContext
 
         public List<string> StepEnvironmentOverrides { get; } = new List<string>();
 
+        // Background step deferral properties
+        public Dictionary<string, string> DeferredOutputs { get; set; }
+        public Dictionary<string, string> DeferredEnvironmentVariables { get; set; }
+        public List<string> DeferredPrependPath { get; set; }
+        public bool DeferOutcomeConclusion { get; set; }
+
         public override void Initialize(IHostContext hostContext)
         {
             base.Initialize(hostContext);
@@ -511,6 +530,12 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
                     Annotations = new List<Annotation>()
                 };
 
+                // Populate background step metadata from timeline record fields
+                stepResult.IsBackground = _record.IsBackground;
+                stepResult.StepType = _record.StepType;
+                stepResult.WaitStepIds = _record.WaitStepIds;
+                stepResult.CancelStepId = _record.CancelStepId;
+
                 _record.Issues?.ForEach(issue =>
                 {
                     var annotation = issue.ToAnnotation();
@@ -536,7 +561,7 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
                         var annotation = issue.ToAnnotation();
                         if (annotation != null)
                         {
-                            Global.JobAnnotations.Add(annotation.Value);
+                            lock (Global.CollectionLock) { Global.JobAnnotations.Add(annotation.Value); }
                             if (annotation.Value.IsInfrastructureIssue && string.IsNullOrEmpty(Global.InfrastructureFailureCategory))
                             {
                                 Global.InfrastructureFailureCategory = issue.Category;
@@ -554,11 +579,22 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation =
 
             _logger.End();
 
-            UpdateGlobalStepsContext();
+            if (!DeferOutcomeConclusion)
+            {
+                UpdateGlobalStepsContext();
+            }
 
             return Result.Value;
         }
 
+        public void FlushDeferredOutcomeConclusion()
+        {
+            if (DeferOutcomeConclusion)
+            {
+                UpdateGlobalStepsContext();
+            }
+        }
+
         public void UpdateGlobalStepsContext()
         {
             // Skip if generated context name. Generated context names start with "__". After 3.2 the server will never send an empty context name.
@@ -634,6 +670,40 @@ public void SetOutput(string name, string value, out string reference)
             Global.StepsContext.SetOutput(ScopeName, ContextName, name, value, out reference);
         }
 
+        public void FlushDeferredOutputs()
+        {
+            if (DeferredOutputs == null || DeferredOutputs.Count == 0)
+            {
+                return;
+            }
+
+            foreach (var kvp in DeferredOutputs)
+            {
+                Global.StepsContext.SetOutput(ScopeName, ContextName, kvp.Key, kvp.Value, out _);
+            }
+        }
+
+        public void FlushDeferredEnvironment()
+        {
+            if (DeferredEnvironmentVariables != null)
+            {
+                foreach (var kvp in DeferredEnvironmentVariables)
+                {
+                    Global.EnvironmentVariables[kvp.Key] = kvp.Value;
+                    SetEnvContext(kvp.Key, kvp.Value);
+                }
+            }
+
+            if (DeferredPrependPath != null)
+            {
+                foreach (var path in DeferredPrependPath)
+                {
+                    Global.PrependPath.RemoveAll(x => string.Equals(x, path, StringComparison.CurrentCulture));
+                    Global.PrependPath.Add(path);
+                }
+            }
+        }
+
         public void SetTimeout(TimeSpan? timeout)
         {
             if (timeout != null)
@@ -807,6 +877,16 @@ public void UpdateTimelineRecordDisplayName(string displayName)
             _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
         }
 
+        public void SetBackgroundStepMetadata(bool isBackground = false, string stepType = null, string[] waitStepIds = null, string cancelStepId = null, string parallelGroupId = null)
+        {
+            _record.IsBackground = isBackground;
+            _record.StepType = stepType;
+            _record.WaitStepIds = waitStepIds;
+            _record.CancelStepId = cancelStepId;
+            _record.ParallelGroupId = parallelGroupId;
+            _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
+        }
+
         public void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token)
         {
             // Validation
@@ -1191,7 +1271,7 @@ public void PublishStepTelemetry()
                     }
 
                     Trace.Info($"Publish step telemetry for current step {StringUtil.ConvertToJson(StepTelemetry)}.");
-                    Global.StepsTelemetry.Add(StepTelemetry);
+                    lock (Global.CollectionLock) { Global.StepsTelemetry.Add(StepTelemetry); }
                     _stepTelemetryPublished = true;
                 }
             }
@@ -1330,7 +1410,10 @@ public void ApplyContinueOnError(TemplateToken continueOnErrorToken)
                 Trace.Info($"Updated step result (continue on error)");
             }
 
-            UpdateGlobalStepsContext();
+            if (!DeferOutcomeConclusion)
+            {
+                UpdateGlobalStepsContext();
+            }
         }
 
         internal IPipelineTemplateEvaluator ToPipelineTemplateEvaluatorInternal(bool allowServiceContainerCommand, ObjectTemplating.ITraceWriter traceWriter = null)

src/Runner.Worker/FileCommandManager.cs

@@ -122,8 +122,18 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container
                     {
                         continue;
                     }
-                    context.Global.PrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture));
-                    context.Global.PrependPath.Add(line);
+                    if (context.DeferredPrependPath != null)
+                    {
+                        // Background step: buffer path additions until wait/wait-all
+                        context.DeferredPrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture));
+                        context.DeferredPrependPath.Add(line);
+                        context.Debug($"Deferred prepend path '{line}' for background step");
+                    }
+                    else
+                    {
+                        context.Global.PrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture));
+                        context.Global.PrependPath.Add(line);
+                    }
                 }
             }
         }
@@ -172,8 +182,17 @@ private static void SetEnvironmentVariable(
             string name,
             string value)
         {
-            context.Global.EnvironmentVariables[name] = value;
-            context.SetEnvContext(name, value);
+            if (context.DeferredEnvironmentVariables != null)
+            {
+                // Background step: buffer env changes until wait/wait-all
+                context.DeferredEnvironmentVariables[name] = value;
+                context.Debug($"Deferred env '{name}' for background step");
+            }
+            else
+            {
+                context.Global.EnvironmentVariables[name] = value;
+                context.SetEnvContext(name, value);
+            }
             context.Debug($"{name}='{value}'");
         }
 
@@ -302,7 +321,16 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container
             var pairs = new EnvFileKeyValuePairs(context, filePath);
             foreach (var pair in pairs)
             {
-                context.SetOutput(pair.Key, pair.Value, out var reference);
+                if (context.DeferredOutputs != null)
+                {
+                    // Background step: buffer outputs until wait/wait-all
+                    context.DeferredOutputs[pair.Key] = pair.Value;
+                    context.Debug($"Deferred output '{pair.Key}' for background step");
+                }
+                else
+                {
+                    context.SetOutput(pair.Key, pair.Value, out var reference);
+                }
                 context.Debug($"Set output {pair.Key} = {pair.Value}");
             }
         }

src/Runner.Worker/GlobalContext.cs

@@ -12,6 +12,9 @@ namespace GitHub.Runner.Worker
 {
     public sealed class GlobalContext
     {
+        // Lock for thread-safe access to shared collections during concurrent background step execution
+        public readonly object CollectionLock = new object();
+
         public ContainerInfo Container { get; set; }
         public List<ServiceEndpoint> Endpoints { get; set; }
         public IDictionary<String, String> EnvironmentVariables { get; set; }