diff --git a/.changes/unreleased/ENHANCEMENTS-441-20240724-130109.yaml b/.changes/unreleased/ENHANCEMENTS-441-20240724-130109.yaml new file mode 100644 index 00000000..8a338e26 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-441-20240724-130109.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: '`AgentPool`: Add the ability to configure scale-up and scale-down autoscaling times separately via the `cooldown.scaleUpSeconds` and `cooldown.scaleDownSeconds` attributes, respectively.' +time: 2024-07-24T13:01:09.985507-04:00 +custom: + PR: "441" diff --git a/.changes/unreleased/NOTES-441-20240725-083719.yaml b/.changes/unreleased/NOTES-441-20240725-083719.yaml new file mode 100644 index 00000000..6ebbe5de --- /dev/null +++ b/.changes/unreleased/NOTES-441-20240725-083719.yaml @@ -0,0 +1,5 @@ +kind: NOTES +body: The `AgentPool` CRD has been changed. Please follow the Helm chart instructions on how to upgrade it. +time: 2024-07-25T08:37:19.168477+02:00 +custom: + PR: "441" diff --git a/api/v1alpha2/agentpool_types.go b/api/v1alpha2/agentpool_types.go index 8e225ea9..708c3eb8 100644 --- a/api/v1alpha2/agentpool_types.go +++ b/api/v1alpha2/agentpool_types.go @@ -73,6 +73,21 @@ type AgentDeploymentAutoscaling struct { //+optional //+kubebuilder:default:=300 CooldownPeriodSeconds *int32 `json:"cooldownPeriodSeconds,omitempty"` + + // CoolDownPeriod configures the period to wait between scaling up and scaling down + //+optional + CooldownPeriod *AgentDeploymentAutoscalingCooldownPeriod `json:"cooldownPeriod,omitempty"` +} + +// AgentDeploymentAutoscalingCooldownPeriod configures the period to wait between scaling up and scaling down +type AgentDeploymentAutoscalingCooldownPeriod struct { + // ScaleUpSeconds is the time to wait before scaling up. + //+optional + ScaleUpSeconds *int32 `json:"scaleUpSeconds,omitempty"` + + // ScaleDownSeconds is the time to wait before scaling down. + //+optional + ScaleDownSeconds *int32 `json:"scaleDownSeconds,omitempty"` } type AgentDeployment struct { @@ -80,7 +95,7 @@ type AgentDeployment struct { Spec *v1.PodSpec `json:"spec,omitempty"` } -// AgentPoolSpec defines the desired stak get ste of AgentPool. +// AgentPoolSpec defines the desired state of AgentPool. type AgentPoolSpec struct { // Agent Pool name. // More information: diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index cb29376c..79fc07c6 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -64,6 +64,11 @@ func (in *AgentDeploymentAutoscaling) DeepCopyInto(out *AgentDeploymentAutoscali *out = new(int32) **out = **in } + if in.CooldownPeriod != nil { + in, out := &in.CooldownPeriod, &out.CooldownPeriod + *out = new(AgentDeploymentAutoscalingCooldownPeriod) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentDeploymentAutoscaling. @@ -76,6 +81,31 @@ func (in *AgentDeploymentAutoscaling) DeepCopy() *AgentDeploymentAutoscaling { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentDeploymentAutoscalingCooldownPeriod) DeepCopyInto(out *AgentDeploymentAutoscalingCooldownPeriod) { + *out = *in + if in.ScaleUpSeconds != nil { + in, out := &in.ScaleUpSeconds, &out.ScaleUpSeconds + *out = new(int32) + **out = **in + } + if in.ScaleDownSeconds != nil { + in, out := &in.ScaleDownSeconds, &out.ScaleDownSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentDeploymentAutoscalingCooldownPeriod. +func (in *AgentDeploymentAutoscalingCooldownPeriod) DeepCopy() *AgentDeploymentAutoscalingCooldownPeriod { + if in == nil { + return nil + } + out := new(AgentDeploymentAutoscalingCooldownPeriod) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentDeploymentAutoscalingStatus) DeepCopyInto(out *AgentDeploymentAutoscalingStatus) { *out = *in diff --git a/charts/terraform-cloud-operator/README.md b/charts/terraform-cloud-operator/README.md index 90f3165e..5a4855fa 100644 --- a/charts/terraform-cloud-operator/README.md +++ b/charts/terraform-cloud-operator/README.md @@ -91,6 +91,14 @@ For a more detailed explanation, please refer to the [FAQ](../../docs/faq.md#gen #### Upgrade recommendations + - `2.5.0` to `2.6.0` + + - The `AgentPool` CRD has been changed: + + ```console + $ kubectl replace -f https://raw.githubusercontent.com/hashicorp/terraform-cloud-operator/v2.6.0/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml + ``` + - `2.3.0` to `2.4.0` - The `Workspace` CRD has been changed: diff --git a/charts/terraform-cloud-operator/README.md.gotmpl b/charts/terraform-cloud-operator/README.md.gotmpl index 30fe8724..2bd2c852 100644 --- a/charts/terraform-cloud-operator/README.md.gotmpl +++ b/charts/terraform-cloud-operator/README.md.gotmpl @@ -91,6 +91,15 @@ For a more detailed explanation, please refer to the [FAQ](../../docs/faq.md#gen #### Upgrade recommendations + - `2.5.0` to `2.6.0` + + - The `AgentPool` CRD has been changed: + + ```console + $ kubectl replace -f https://raw.githubusercontent.com/hashicorp/terraform-cloud-operator/v2.6.0/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml + ``` + + - `2.3.0` to `2.4.0` - The `Workspace` CRD has been changed: diff --git a/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml b/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml index a3cd1224..120c70bd 100644 --- a/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml +++ b/charts/terraform-cloud-operator/crds/app.terraform.io_agentpools.yaml @@ -40,7 +40,7 @@ spec: metadata: type: object spec: - description: AgentPoolSpec defines the desired stak get ste of AgentPool. + description: AgentPoolSpec defines the desired state of AgentPool. properties: agentDeployment: description: Agent deployment settings @@ -7549,6 +7549,21 @@ spec: autoscaling: description: Agent deployment settings properties: + cooldownPeriod: + description: CoolDownPeriod configures the period to wait between + scaling up and scaling down + properties: + scaleDownSeconds: + description: ScaleDownSeconds is the time to wait before scaling + down. + format: int32 + type: integer + scaleUpSeconds: + description: ScaleUpSeconds is the time to wait before scaling + up. + format: int32 + type: integer + type: object cooldownPeriodSeconds: default: 300 description: CooldownPeriodSeconds is the time to wait between diff --git a/config/crd/bases/app.terraform.io_agentpools.yaml b/config/crd/bases/app.terraform.io_agentpools.yaml index a4cefe1d..f245fd0e 100644 --- a/config/crd/bases/app.terraform.io_agentpools.yaml +++ b/config/crd/bases/app.terraform.io_agentpools.yaml @@ -37,7 +37,7 @@ spec: metadata: type: object spec: - description: AgentPoolSpec defines the desired stak get ste of AgentPool. + description: AgentPoolSpec defines the desired state of AgentPool. properties: agentDeployment: description: Agent deployment settings @@ -7546,6 +7546,21 @@ spec: autoscaling: description: Agent deployment settings properties: + cooldownPeriod: + description: CoolDownPeriod configures the period to wait between + scaling up and scaling down + properties: + scaleDownSeconds: + description: ScaleDownSeconds is the time to wait before scaling + down. + format: int32 + type: integer + scaleUpSeconds: + description: ScaleUpSeconds is the time to wait before scaling + up. + format: int32 + type: integer + type: object cooldownPeriodSeconds: default: 300 description: CooldownPeriodSeconds is the time to wait between diff --git a/controllers/agentpool_controller.go b/controllers/agentpool_controller.go index d9cdf084..0aa9cee2 100644 --- a/controllers/agentpool_controller.go +++ b/controllers/agentpool_controller.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "strconv" - "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -103,10 +102,9 @@ func (r *AgentPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( ap.log.Info("Agent Pool Controller", "msg", "successfully reconcilied agent pool") r.Recorder.Eventf(&ap.instance, corev1.EventTypeNormal, "ReconcileAgentPool", "Successfully reconcilied agent pool ID %s", ap.instance.Status.AgentPoolID) - // Re-queue custom resource after the cool down period if applicable. - if t := ap.coolDownSecondsRemaining(); t > 0 { - return requeueAfter(time.Duration(t) * time.Second) - } + // TODO: + // - Add a `metadata` field to the `agentPoolInstance` structure. The `metadata` structure can be used to carry information + // related to an agent pool instance, such as requeue after time. return requeueAfter(AgentPoolSyncPeriod) } diff --git a/controllers/agentpool_controller_autoscaling.go b/controllers/agentpool_controller_autoscaling.go index 5a2f56a9..5ddb33c2 100644 --- a/controllers/agentpool_controller_autoscaling.go +++ b/controllers/agentpool_controller_autoscaling.go @@ -151,16 +151,35 @@ func (r *AgentPoolReconciler) scaleAgentDeployment(ctx context.Context, ap *agen return r.Client.Update(ctx, &deployment) } -// coolDownSecondsRemaining returns the remaining seconds in the Cool Down stage. +// cooldownSecondsRemaining returns the remaining seconds in the Cool Down stage. // A negative value indicates expired Cool Down. -func (a *agentPoolInstance) coolDownSecondsRemaining() int { - if s := a.instance.Status.AgentDeploymentAutoscalingStatus; s != nil && s.LastScalingEvent != nil { - lastScalingEventSeconds := int(time.Since(s.LastScalingEvent.Time).Seconds()) - cooldownPeriodSeconds := int(*a.instance.Spec.AgentDeploymentAutoscaling.CooldownPeriodSeconds) - return cooldownPeriodSeconds - lastScalingEventSeconds +func (a *agentPoolInstance) cooldownSecondsRemaining(currentReplicas, desiredReplicas int32) int { + status := a.instance.Status.AgentDeploymentAutoscalingStatus + if status == nil || status.LastScalingEvent == nil { + return -1 } - return -1 + cooldownPeriodSeconds := int(*a.instance.Spec.AgentDeploymentAutoscaling.CooldownPeriodSeconds) + + cooldownPeriod := a.instance.Spec.AgentDeploymentAutoscaling.CooldownPeriod + if cooldownPeriod != nil { + if v := cooldownPeriod.ScaleDownSeconds; v != nil { + cooldownPeriodSeconds = int(*v) + if currentReplicas > desiredReplicas { + a.log.Info("Reconcile Agent Autoscaling", "msg", fmt.Sprintf("Agents scaling down, using configured scale down period: %v", cooldownPeriodSeconds)) + } + } + + if v := cooldownPeriod.ScaleUpSeconds; v != nil { + if desiredReplicas > currentReplicas { + cooldownPeriodSeconds = int(*v) + a.log.Info("Reconcile Agent Autoscaling", "msg", fmt.Sprintf("Agents scaling up, using configured scale down period: %v", cooldownPeriodSeconds)) + } + } + } + + lastScalingEventSeconds := int(time.Since(status.LastScalingEvent.Time).Seconds()) + return cooldownPeriodSeconds - lastScalingEventSeconds } func (r *AgentPoolReconciler) reconcileAgentAutoscaling(ctx context.Context, ap *agentPoolInstance) error { @@ -170,11 +189,6 @@ func (r *AgentPoolReconciler) reconcileAgentAutoscaling(ctx context.Context, ap ap.log.Info("Reconcile Agent Autoscaling", "msg", "new reconciliation event") - if ap.coolDownSecondsRemaining() > 0 { - ap.log.Info("Reconcile Agent Autoscaling", "msg", "autoscaler is within the cooldown period, skipping") - return nil - } - requiredAgents, err := computeRequiredAgents(ctx, ap) if err != nil { ap.log.Error(err, "Reconcile Agent Autoscaling", "msg", "Failed to get agents needed") @@ -195,6 +209,11 @@ func (r *AgentPoolReconciler) reconcileAgentAutoscaling(ctx context.Context, ap maxReplicas := *ap.instance.Spec.AgentDeploymentAutoscaling.MaxReplicas desiredReplicas := computeDesiredReplicas(requiredAgents, minReplicas, maxReplicas) if desiredReplicas != currentReplicas { + if ap.cooldownSecondsRemaining(currentReplicas, desiredReplicas) > 0 { + ap.log.Info("Reconcile Agent Autoscaling", "msg", "autoscaler is within the cooldown period, skipping") + return nil + } + scalingEvent := fmt.Sprintf("Scaling agent deployment from %v to %v replicas", currentReplicas, desiredReplicas) ap.log.Info("Reconcile Agent Autoscaling", "msg", strings.ToLower(scalingEvent)) r.Recorder.Event(&ap.instance, corev1.EventTypeNormal, "AutoscaleAgentPoolDeployment", scalingEvent) diff --git a/controllers/agentpool_controller_test.go b/controllers/agentpool_controller_test.go index d23ab36e..55b02200 100644 --- a/controllers/agentpool_controller_test.go +++ b/controllers/agentpool_controller_test.go @@ -388,6 +388,35 @@ var _ = Describe("Agent Pool controller", Ordered, func() { Expect(instance.Spec.AgentDeploymentAutoscaling.CooldownPeriodSeconds).To(Equal(pointer.PointerOf(int32(60)))) }) + It("can autoscale agent deployments with seperate scale-up and scale-down periods", func() { + createTestAgentPool(instance) + + Expect(k8sClient.Get(ctx, namespacedName, instance)).Should(Succeed()) + Expect(instance.Spec.AgentDeployment).To(BeNil()) + + instance.Spec.AgentDeployment = &appv1alpha2.AgentDeployment{} + instance.Spec.AgentDeploymentAutoscaling = &appv1alpha2.AgentDeploymentAutoscaling{ + MinReplicas: pointer.PointerOf(int32(3)), + MaxReplicas: pointer.PointerOf(int32(5)), + CooldownPeriod: &appv1alpha2.AgentDeploymentAutoscalingCooldownPeriod{ + ScaleUpSeconds: pointer.PointerOf(int32(30)), + ScaleDownSeconds: pointer.PointerOf(int32(600)), + }, + } + Expect(k8sClient.Update(ctx, instance)).Should(Succeed()) + + Expect(k8sClient.Get(ctx, namespacedName, instance)).Should(Succeed()) + Expect(instance.Spec.AgentDeployment).ToNot(BeNil()) + Expect(instance.Spec.AgentDeployment.Replicas).To(BeNil()) + Expect(instance.Spec.AgentDeployment.Spec).To(BeNil()) + Expect(instance.Spec.AgentDeploymentAutoscaling).ToNot(BeNil()) + Expect(instance.Spec.AgentDeploymentAutoscaling.TargetWorkspaces).To(BeNil()) + Expect(instance.Spec.AgentDeploymentAutoscaling.MinReplicas).To(Equal(pointer.PointerOf(int32(3)))) + Expect(instance.Spec.AgentDeploymentAutoscaling.MaxReplicas).To(Equal(pointer.PointerOf(int32(5)))) + Expect(instance.Spec.AgentDeploymentAutoscaling.CooldownPeriod.ScaleUpSeconds).To(Equal(pointer.PointerOf(int32(30)))) + Expect(instance.Spec.AgentDeploymentAutoscaling.CooldownPeriod.ScaleDownSeconds).To(Equal(pointer.PointerOf(int32(600)))) + }) + It("can autoscale agent deployments by targeting specific workspaces", func() { createTestAgentPool(instance) @@ -491,7 +520,6 @@ func validateAgentPoolTestTokens(ctx context.Context, instance *appv1alpha2.Agen return compareAgentTokens(ct, kt) }).Should(BeTrue()) - } func validateAgentPoolDeployment(ctx context.Context, instance *appv1alpha2.AgentPool) { diff --git a/docs/api-reference.md b/docs/api-reference.md index b292bbe9..212b6d68 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -47,6 +47,22 @@ _Appears in:_ | `minReplicas` _integer_ | MinReplicas is the minimum number of replicas for the Agent deployment. | | `targetWorkspaces` _[TargetWorkspace](#targetworkspace)_ | TargetWorkspaces is a list of HCP Terraform Workspaces which
the agent pool should scale up to meet demand. When this field
is ommited the autoscaler will target all workspaces that are
associated with the AgentPool. | | `cooldownPeriodSeconds` _integer_ | CooldownPeriodSeconds is the time to wait between scaling events. Defaults to 300. | +| `cooldownPeriod` _[AgentDeploymentAutoscalingCooldownPeriod](#agentdeploymentautoscalingcooldownperiod)_ | CoolDownPeriod configures the period to wait between scaling up and scaling down | + + +#### AgentDeploymentAutoscalingCooldownPeriod + + + +AgentDeploymentAutoscalingCooldownPeriod configures the period to wait between scaling up and scaling down + +_Appears in:_ +- [AgentDeploymentAutoscaling](#agentdeploymentautoscaling) + +| Field | Description | +| --- | --- | +| `scaleUpSeconds` _integer_ | ScaleUpSeconds is the time to wait before scaling up. | +| `scaleDownSeconds` _integer_ | ScaleDownSeconds is the time to wait before scaling down. | #### AgentDeploymentAutoscalingStatus @@ -86,7 +102,7 @@ AgentPool is the Schema for the agentpools API. -AgentPoolSpec defines the desired stak get ste of AgentPool. +AgentPoolSpec defines the desired state of AgentPool. _Appears in:_ - [AgentPool](#agentpool)