diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index e0f9fd3d..4c986ecc 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -446,6 +446,16 @@ func (c *Service) ChangeHPAFromTortoiseRecommendation(tortoise *autoscalingv1bet hpa.Spec.MinReplicas = &minToActuallyApply if tortoise.Spec.UpdateMode != autoscalingv1beta3.UpdateModeOff && recordMetrics { // We don't want to record applied* metric when UpdateMode is Off. + netChangeMaxReplicas := float64(hpa.Spec.MaxReplicas - recommendMax) + netChangeMinReplicas := float64(*hpa.Spec.MinReplicas) - float64(recommendMin) + if netChangeMaxReplicas > 0 || netChangeMinReplicas < 0 { + metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) + } + if netChangeMaxReplicas < 0 || netChangeMinReplicas > 0 { + metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) + } + metrics.NetHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMinReplicas) + metrics.NetHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(netChangeMaxReplicas) metrics.AppliedHPAMinReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(*hpa.Spec.MinReplicas)) metrics.AppliedHPAMaxReplicas.WithLabelValues(tortoise.Name, tortoise.Namespace, hpa.Name).Set(float64(hpa.Spec.MaxReplicas)) } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d19b7987..b981c3f5 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -46,6 +46,36 @@ var ( Help: "memory request (byte) that tortoises actually applys", }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) + DecreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "decrease_apply_counter", + Help: "counter for number of resource decreases applied by tortoise", + }, []string{"tortoise_name", "namespace"}) + + IncreaseApplyCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "increase_apply_counter", + Help: "counter for number of resource increases applied by tortoise", + }, []string{"tortoise_name", "namespace"}) + + NetHPAMinReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_minreplicas", + Help: "net hpa minReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) + + NetHPAMaxReplicas = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_hpa_maxreplicas", + Help: "net hpa maxReplicas that tortoises actually applys to hpa", + }, []string{"tortoise_name", "namespace", "hpa_name"}) + + NetCPURequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_cpu_request", + Help: "net cpu request (millicore) that tortoises actually applys", + }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) + + NetMemoryRequest = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "net_memory_request", + Help: "net memory request (byte) that tortoises actually applys", + }, []string{"tortoise_name", "namespace", "container_name", "controller_name", "controller_kind"}) + ProposedHPATargetUtilization = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "proposed_hpa_utilization_target", Help: "recommended hpa utilization target values that tortoises propose", @@ -87,6 +117,12 @@ func init() { AppliedHPAMinReplicas, AppliedCPURequest, AppliedMemoryRequest, + IncreaseApplyCounter, + DecreaseApplyCounter, + NetHPAMaxReplicas, + NetHPAMinReplicas, + NetCPURequest, + NetMemoryRequest, ProposedHPATargetUtilization, ProposedHPAMinReplicas, ProposedHPAMaxReplicas, diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go index db42336f..ef1e0490 100644 --- a/pkg/tortoise/tortoise.go +++ b/pkg/tortoise/tortoise.go @@ -665,6 +665,14 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T oldTortoise := tortoise.DeepCopy() + oldRequestMap := map[string]map[corev1.ResourceName]resource.Quantity{} + for _, r := range tortoise.Status.Conditions.ContainerResourceRequests { + oldRequestMap[r.ContainerName] = map[corev1.ResourceName]resource.Quantity{} + for resourcename, value := range r.Resource { + oldRequestMap[r.ContainerName][resourcename] = value + } + } + newRequests := make([]v1beta3.ContainerResourceRequests, 0, len(tortoise.Status.Recommendations.Vertical.ContainerResourceRecommendation)) for _, r := range tortoise.Status.Recommendations.Vertical.ContainerResourceRecommendation { recommendation := r.RecommendedResource.DeepCopy() @@ -759,12 +767,22 @@ func (c *Service) UpdateResourceRequest(ctx context.Context, tortoise *v1beta3.T for _, r := range tortoise.Status.Conditions.ContainerResourceRequests { // only record metrics once in every reconcile loop. for resourcename, value := range r.Resource { + oldRequest := oldRequestMap[r.ContainerName][resourcename] + netChange := float64(oldRequest.MilliValue() - value.MilliValue()) if resourcename == corev1.ResourceCPU { // We don't want to record applied* metric when UpdateMode is Off. metrics.AppliedCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.MilliValue())) + metrics.NetCPURequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(netChange) } if resourcename == corev1.ResourceMemory { metrics.AppliedMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(value.Value())) + metrics.NetMemoryRequest.WithLabelValues(tortoise.Name, tortoise.Namespace, r.ContainerName, tortoise.Spec.TargetRefs.ScaleTargetRef.Name, tortoise.Spec.TargetRefs.ScaleTargetRef.Kind).Set(float64(netChange)) + } + if netChange > 0 { + metrics.IncreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) + } + if netChange < 0 { + metrics.DecreaseApplyCounter.WithLabelValues(tortoise.Name, tortoise.Namespace).Add(1) } } }