Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🌱 Add v1beta2 RollingOut condition #11463

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions api/v1beta1/cluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,29 @@ const (
ClusterRemoteConnectionProbeSucceededV1Beta2Reason = "ProbeSucceeded"
)

// Cluster's RollingOut condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// ClusterRollingOutV1Beta2Condition is the summary of `RollingOut` conditions from ControlPlane, MachineDeployments
// and MachinePools.
ClusterRollingOutV1Beta2Condition = RollingOutV1Beta2Condition

// ClusterRollingOutV1Beta2Reason surfaces when at least one of the Cluster's control plane, MachineDeployments,
// or MachinePools are rolling out.
ClusterRollingOutV1Beta2Reason = RollingOutV1Beta2Reason

// ClusterNotRollingOutV1Beta2Reason surfaces when none of the Cluster's control plane, MachineDeployments,
// or MachinePools are rolling out.
ClusterNotRollingOutV1Beta2Reason = NotRollingOutV1Beta2Reason

// ClusterRollingOutUnknownV1Beta2Reason surfaces when one of the Cluster's control plane, MachineDeployments,
// or MachinePools rolling out condition is unknown, and none true.
ClusterRollingOutUnknownV1Beta2Reason = "RollingOutUnknown"

// ClusterRollingOutInternalErrorV1Beta2Reason surfaces unexpected failures when listing machines
// or computing the RollingOut condition.
ClusterRollingOutInternalErrorV1Beta2Reason = InternalErrorV1Beta2Reason
)

// Cluster's ScalingUp condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// ClusterScalingUpV1Beta2Condition is the summary of `ScalingUp` conditions from ControlPlane, MachineDeployments,
Expand Down
15 changes: 15 additions & 0 deletions api/v1beta1/machinedeployment_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,21 @@ const (
MachineDeploymentMachinesUpToDateInternalErrorV1Beta2Reason = InternalErrorV1Beta2Reason
)

// MachineDeployment's RollingOut condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// MachineDeploymentRollingOutV1Beta2Condition is true if there is at least one machine not up-to-date.
MachineDeploymentRollingOutV1Beta2Condition = RollingOutV1Beta2Condition

// MachineDeploymentRollingOutV1Beta2Reason surfaces when there is at least one machine not up-to-date.
MachineDeploymentRollingOutV1Beta2Reason = RollingOutV1Beta2Reason

// MachineDeploymentNotRollingOutV1Beta2Reason surfaces when all the machines are up-to-date.
MachineDeploymentNotRollingOutV1Beta2Reason = NotRollingOutV1Beta2Reason

// MachineDeploymentRollingOutInternalErrorV1Beta2Reason surfaces unexpected failures when listing machines.
MachineDeploymentRollingOutInternalErrorV1Beta2Reason = InternalErrorV1Beta2Reason
)

// MachineDeployment's ScalingUp condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// MachineDeploymentScalingUpV1Beta2Condition is true if actual replicas < desired replicas.
Expand Down
13 changes: 13 additions & 0 deletions api/v1beta1/v1beta2_condition_consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ const (
// the same condition type exists.
MachinesUpToDateV1Beta2Condition = "MachinesUpToDate"

// RollingOutV1Beta2Condition reports if an object is rolling out changes to machines; Cluster API usually
// rolls out changes to machines by replacing not up-to-date machines with new ones.
// Note: This condition type is defined to ensure consistent naming of conditions across objects.
// Please use object specific variants of this condition which provides more details for each context where
// the same condition type exists.
RollingOutV1Beta2Condition = "RollingOut"

// ScalingUpV1Beta2Condition reports if an object is scaling up.
// Note: This condition type is defined to ensure consistent naming of conditions across objects.
// Please use object specific variants of this condition which provides more details for each context where
Expand Down Expand Up @@ -114,6 +121,12 @@ const (
// UpToDateUnknownV1Beta2Reason applies to a condition surfacing object up-tp-date unknown.
UpToDateUnknownV1Beta2Reason = "UpToDateUnknown"

// RollingOutV1Beta2Reason surfaces when an object is rolling out.
RollingOutV1Beta2Reason = "RollingOut"

// NotRollingOutV1Beta2Reason surfaces when an object is not rolling out.
NotRollingOutV1Beta2Reason = "NotRollingOut"

// ScalingUpV1Beta2Reason surfaces when an object is scaling up.
ScalingUpV1Beta2Reason = "ScalingUp"

Expand Down
15 changes: 15 additions & 0 deletions controlplane/kubeadm/api/v1beta1/v1beta2_condition_consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,21 @@ const (
KubeadmControlPlaneMachinesUpToDateInternalErrorV1Beta2Reason = clusterv1.InternalErrorV1Beta2Reason
)

// KubeadmControlPlane's RollingOut condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// KubeadmControlPlaneRollingOutV1Beta2Condition is true if there is at least one machine not up-to-date.
KubeadmControlPlaneRollingOutV1Beta2Condition = clusterv1.RollingOutV1Beta2Condition

// KubeadmControlPlaneRollingOutV1Beta2Reason surfaces when there is at least one machine not up-to-date.
KubeadmControlPlaneRollingOutV1Beta2Reason = clusterv1.RollingOutV1Beta2Reason

// KubeadmControlPlaneNotRollingOutV1Beta2Reason surfaces when all the machines are up-to-date.
KubeadmControlPlaneNotRollingOutV1Beta2Reason = clusterv1.NotRollingOutV1Beta2Reason

// KubeadmControlPlaneRollingOutInternalErrorV1Beta2Reason surfaces unexpected failures when listing machines.
KubeadmControlPlaneRollingOutInternalErrorV1Beta2Reason = clusterv1.InternalErrorV1Beta2Reason
)

// KubeadmControlPlane's ScalingUp condition and corresponding reasons that will be used in v1Beta2 API version.
const (
// KubeadmControlPlaneScalingUpV1Beta2Condition is true if actual replicas < desired replicas.
Expand Down
1 change: 1 addition & 0 deletions controlplane/kubeadm/internal/controllers/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,7 @@ func patchKubeadmControlPlane(ctx context.Context, patchHelper *patch.Helper, kc
controlplanev1.KubeadmControlPlaneControlPlaneComponentsHealthyV1Beta2Condition,
controlplanev1.KubeadmControlPlaneMachinesReadyV1Beta2Condition,
controlplanev1.KubeadmControlPlaneMachinesUpToDateV1Beta2Condition,
controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
controlplanev1.KubeadmControlPlaneScalingUpV1Beta2Condition,
controlplanev1.KubeadmControlPlaneScalingDownV1Beta2Condition,
controlplanev1.KubeadmControlPlaneRemediatingV1Beta2Condition,
Expand Down
58 changes: 58 additions & 0 deletions controlplane/kubeadm/internal/controllers/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ func (r *KubeadmControlPlaneReconciler) updateV1Beta2Status(ctx context.Context,

setReplicas(ctx, controlPlane.KCP, controlPlane.Machines)
setInitializedCondition(ctx, controlPlane.KCP)
setRollingOutCondition(ctx, controlPlane.KCP, controlPlane.Machines)
setScalingUpCondition(ctx, controlPlane.KCP, controlPlane.Machines, controlPlane.InfraMachineTemplateIsNotFound, controlPlane.PreflightCheckResults)
setScalingDownCondition(ctx, controlPlane.KCP, controlPlane.Machines, controlPlane.PreflightCheckResults)
setMachinesReadyCondition(ctx, controlPlane.KCP, controlPlane.Machines)
Expand Down Expand Up @@ -210,6 +211,63 @@ func setInitializedCondition(_ context.Context, kcp *controlplanev1.KubeadmContr
})
}

func setRollingOutCondition(_ context.Context, kcp *controlplanev1.KubeadmControlPlane, machines collections.Machines) {
// Count machines rolling out and collect reasons why a rollout is happening.
// Note: The code below collects all the reasons for which at least a machine is rolling out; under normal circumstances
// all the machines are rolling out for the same reasons, however, in case of changes to KCP
// before a previous changes is not fully rolled out, there could be machines rolling out for
// different reasons.
rollingOutReplicas := 0
rolloutReasons := sets.Set[string]{}
for _, machine := range machines {
upToDateCondition := v1beta2conditions.Get(machine, clusterv1.MachineUpToDateV1Beta2Condition)
if upToDateCondition == nil || upToDateCondition.Status != metav1.ConditionFalse {
continue
}
rollingOutReplicas++
if upToDateCondition.Message != "" {
rolloutReasons.Insert(strings.Split(upToDateCondition.Message, "; ")...)
}
}

if rollingOutReplicas == 0 {
var message string
v1beta2conditions.Set(kcp, metav1.Condition{
Type: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: controlplanev1.KubeadmControlPlaneNotRollingOutV1Beta2Reason,
Message: message,
})
return
}

// Rolling out.
message := fmt.Sprintf("Rolling out %d not up-to-date replicas", rollingOutReplicas)
if rolloutReasons.Len() > 0 {
// Surface rollout reasons ensuring that if there is a version change, it goes first.
reasons := rolloutReasons.UnsortedList()
sort.Slice(reasons, func(i, j int) bool {
if strings.HasPrefix(reasons[i], "Version") && !strings.HasPrefix(reasons[j], "Version") {
return true
}
if !strings.HasPrefix(reasons[i], "Version") && strings.HasPrefix(reasons[j], "Version") {
return false
}
return reasons[i] < reasons[j]
})
for i := range reasons {
reasons[i] = fmt.Sprintf("* %s", reasons[i])
}
message += fmt.Sprintf("\n%s", strings.Join(reasons, "\n"))
}
v1beta2conditions.Set(kcp, metav1.Condition{
Type: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
Status: metav1.ConditionTrue,
Reason: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Reason,
Message: message,
})
}

func setScalingUpCondition(_ context.Context, kcp *controlplanev1.KubeadmControlPlane, machines collections.Machines, infrastructureObjectNotFound bool, preflightChecks internal.PreflightCheckResults) {
if kcp.Spec.Replicas == nil {
v1beta2conditions.Set(kcp, metav1.Condition{
Expand Down
93 changes: 93 additions & 0 deletions controlplane/kubeadm/internal/controllers/status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,99 @@ func Test_setInitializedCondition(t *testing.T) {
}
}

func Test_setRollingOutCondition(t *testing.T) {
upToDateCondition := metav1.Condition{
Type: clusterv1.MachineUpToDateV1Beta2Condition,
Status: metav1.ConditionTrue,
Reason: clusterv1.MachineUpToDateV1Beta2Reason,
}

tests := []struct {
name string
kcp *controlplanev1.KubeadmControlPlane
machines []*clusterv1.Machine
expectCondition metav1.Condition
}{
{
name: "no machines",
kcp: &controlplanev1.KubeadmControlPlane{},
machines: []*clusterv1.Machine{},
expectCondition: metav1.Condition{
Type: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: controlplanev1.KubeadmControlPlaneNotRollingOutV1Beta2Reason,
},
},
{
name: "all machines are up to date",
kcp: &controlplanev1.KubeadmControlPlane{},
machines: []*clusterv1.Machine{
{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{upToDateCondition}}}},
{ObjectMeta: metav1.ObjectMeta{Name: "m2"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{upToDateCondition}}}},
},
expectCondition: metav1.Condition{
Type: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: controlplanev1.KubeadmControlPlaneNotRollingOutV1Beta2Reason,
},
},
{
name: "one up-to-date, two not up-to-date, one reporting up-to-date unknown",
kcp: &controlplanev1.KubeadmControlPlane{},
machines: []*clusterv1.Machine{
{ObjectMeta: metav1.ObjectMeta{Name: "m1"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{upToDateCondition}}}},
{ObjectMeta: metav1.ObjectMeta{Name: "m2"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{
{
Type: clusterv1.MachineUpToDateV1Beta2Condition,
Status: metav1.ConditionUnknown,
Reason: clusterv1.InternalErrorV1Beta2Reason,
},
}}}},
{ObjectMeta: metav1.ObjectMeta{Name: "m3"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{
{
Type: clusterv1.MachineUpToDateV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: clusterv1.MachineNotUpToDateV1Beta2Reason,
Message: "Version v1.25.0, v1.26.0 required",
},
}}}},
{ObjectMeta: metav1.ObjectMeta{Name: "m4"}, Status: clusterv1.MachineStatus{V1Beta2: &clusterv1.MachineV1Beta2Status{Conditions: []metav1.Condition{
{
Type: clusterv1.MachineUpToDateV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: clusterv1.MachineNotUpToDateV1Beta2Reason,
Message: "Failure domain failure-domain1, failure-domain2 required; InfrastructureMachine is not up-to-date",
},
}}}},
},
expectCondition: metav1.Condition{
Type: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition,
Status: metav1.ConditionTrue,
Reason: controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Reason,
Message: "Rolling out 2 not up-to-date replicas\n" +
"* Version v1.25.0, v1.26.0 required\n" +
"* Failure domain failure-domain1, failure-domain2 required\n" +
"* InfrastructureMachine is not up-to-date",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)

var machines collections.Machines
if tt.machines != nil {
machines = collections.FromMachines(tt.machines...)
}
setRollingOutCondition(ctx, tt.kcp, machines)

condition := v1beta2conditions.Get(tt.kcp, controlplanev1.KubeadmControlPlaneRollingOutV1Beta2Condition)
g.Expect(condition).ToNot(BeNil())
g.Expect(*condition).To(v1beta2conditions.MatchCondition(tt.expectCondition, v1beta2conditions.IgnoreLastTransitionTime(true)))
})
}
}

func Test_setScalingUpCondition(t *testing.T) {
tests := []struct {
name string
Expand Down
1 change: 1 addition & 0 deletions internal/controllers/cluster/cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,7 @@ func patchCluster(ctx context.Context, patchHelper *patch.Helper, cluster *clust
clusterv1.ClusterMachinesReadyV1Beta2Condition,
clusterv1.ClusterMachinesUpToDateV1Beta2Condition,
clusterv1.ClusterRemoteConnectionProbeV1Beta2Condition,
clusterv1.ClusterRollingOutV1Beta2Condition,
clusterv1.ClusterScalingUpV1Beta2Condition,
clusterv1.ClusterScalingDownV1Beta2Condition,
clusterv1.ClusterRemediatingV1Beta2Condition,
Expand Down
72 changes: 72 additions & 0 deletions internal/controllers/cluster/cluster_controller_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ func (r *Reconciler) updateStatus(ctx context.Context, s *scope) {
setWorkersAvailableCondition(ctx, s.cluster, expv1.MachinePoolList{}, s.descendants.machineDeployments, s.getDescendantsSucceeded)
setMachinesReadyCondition(ctx, s.cluster, allMachines, s.getDescendantsSucceeded)
setMachinesUpToDateCondition(ctx, s.cluster, allMachines, s.getDescendantsSucceeded)
setRollingOutCondition(ctx, s.cluster, s.controlPlane, expv1.MachinePoolList{}, s.descendants.machineDeployments, s.controlPlaneIsNotFound, s.getDescendantsSucceeded)
setScalingUpCondition(ctx, s.cluster, s.controlPlane, expv1.MachinePoolList{}, s.descendants.machineDeployments, s.descendants.machineSets, s.controlPlaneIsNotFound, s.getDescendantsSucceeded)
setScalingDownCondition(ctx, s.cluster, s.controlPlane, expv1.MachinePoolList{}, s.descendants.machineDeployments, s.descendants.machineSets, s.controlPlaneIsNotFound, s.getDescendantsSucceeded)
setRemediatingCondition(ctx, s.cluster, machinesToBeRemediated, unhealthyMachines, s.getDescendantsSucceeded)
Expand Down Expand Up @@ -760,6 +761,77 @@ func setRemediatingCondition(ctx context.Context, cluster *clusterv1.Cluster, ma
})
}

func setRollingOutCondition(ctx context.Context, cluster *clusterv1.Cluster, controlPlane *unstructured.Unstructured, machinePools expv1.MachinePoolList, machineDeployments clusterv1.MachineDeploymentList, controlPlaneIsNotFound bool, getDescendantsSucceeded bool) {
log := ctrl.LoggerFrom(ctx)

// If there was some unexpected errors in getting control plane or listing descendants (this should never happen), surface it.
if (cluster.Spec.ControlPlaneRef != nil && controlPlane == nil && !controlPlaneIsNotFound) || !getDescendantsSucceeded {
v1beta2conditions.Set(cluster, metav1.Condition{
Type: clusterv1.ClusterRollingOutV1Beta2Condition,
Status: metav1.ConditionUnknown,
Reason: clusterv1.ClusterRollingOutInternalErrorV1Beta2Reason,
Message: "Please check controller logs for errors",
})
return
}

if controlPlane == nil && len(machinePools.Items)+len(machineDeployments.Items) == 0 {
v1beta2conditions.Set(cluster, metav1.Condition{
Type: clusterv1.ClusterRollingOutV1Beta2Condition,
Status: metav1.ConditionFalse,
Reason: clusterv1.ClusterNotRollingOutV1Beta2Reason,
})
return
}

ws := make([]aggregationWrapper, 0, len(machinePools.Items)+len(machineDeployments.Items)+1)
if controlPlane != nil {
// control plane is considered only if it is reporting the condition (the contract does not require conditions to be reported)
// Note: this implies that it won't surface as "Conditions RollingOut not yet reported from ...".
if c, err := v1beta2conditions.UnstructuredGet(controlPlane, clusterv1.RollingOutV1Beta2Condition); err == nil && c != nil {
ws = append(ws, aggregationWrapper{cp: controlPlane})
}
}
for _, mp := range machinePools.Items {
ws = append(ws, aggregationWrapper{mp: &mp})
}
for _, md := range machineDeployments.Items {
ws = append(ws, aggregationWrapper{md: &md})
}

rollingOutCondition, err := v1beta2conditions.NewAggregateCondition(
ws, clusterv1.RollingOutV1Beta2Condition,
v1beta2conditions.TargetConditionType(clusterv1.ClusterRollingOutV1Beta2Condition),
// Instruct aggregate to consider RollingOut condition with negative polarity.
v1beta2conditions.NegativePolarityConditionTypes{clusterv1.RollingOutV1Beta2Condition},
// Using a custom merge strategy to override reasons applied during merge and to ensure merge
// takes into account the fact the RollingOut has negative polarity.
v1beta2conditions.CustomMergeStrategy{
MergeStrategy: v1beta2conditions.DefaultMergeStrategy(
v1beta2conditions.TargetConditionHasPositivePolarity(false),
v1beta2conditions.ComputeReasonFunc(v1beta2conditions.GetDefaultComputeMergeReasonFunc(
clusterv1.ClusterRollingOutV1Beta2Reason,
clusterv1.ClusterRollingOutUnknownV1Beta2Reason,
clusterv1.ClusterNotRollingOutV1Beta2Reason,
)),
v1beta2conditions.GetPriorityFunc(v1beta2conditions.GetDefaultMergePriorityFunc(clusterv1.RollingOutV1Beta2Condition)),
),
},
)
if err != nil {
log.Error(err, "Failed to aggregate ControlPlane, MachinePool, MachineDeployment, MachineSet's RollingOut conditions")
v1beta2conditions.Set(cluster, metav1.Condition{
Type: clusterv1.ClusterRollingOutV1Beta2Condition,
Status: metav1.ConditionUnknown,
Reason: clusterv1.ClusterRollingOutInternalErrorV1Beta2Reason,
Message: "Please check controller logs for errors",
})
return
}

v1beta2conditions.Set(cluster, *rollingOutCondition)
}

func setScalingUpCondition(ctx context.Context, cluster *clusterv1.Cluster, controlPlane *unstructured.Unstructured, machinePools expv1.MachinePoolList, machineDeployments clusterv1.MachineDeploymentList, machineSets clusterv1.MachineSetList, controlPlaneIsNotFound bool, getDescendantsSucceeded bool) {
log := ctrl.LoggerFrom(ctx)

Expand Down
Loading