Merge pull request #168 from ciaranRoche/expose-stor-metrics

add exposing max and allocated storage metrics for postgres
integr8ly · Jul 27, 2020 · 63297c7 · 63297c7
2 parents 05b1f60 + 33f447d
commit 63297c7
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 16 deletions.
diff --git a/pkg/providers/aws/provider_postgres.go b/pkg/providers/aws/provider_postgres.go
@@ -974,6 +974,14 @@ func (p *PostgresProvider) exposePostgresMetrics(ctx context.Context, cr *v1alph
 	} else {
 		resources.SetMetric(resources.DefaultPostgresAvailMetricName, genericLabels, 1)
 	}
+
+	// cloud watch only provides us with free storage space, we need to expose more metrics to allow for more accurate alerting
+	// as `predict_linear` is hard to predict on non-linear growth & results in false positives
+	// we should follow the approach AWS take to auto scaling, and alert when free storage space is less than 10%
+	if instance != nil && instance.AllocatedStorage != nil {
+		// convert allocated storage to bytes and expose as a metric
+		resources.SetMetric(resources.DefaultPostgresAllocatedStorageMetricName, genericLabels, float64(*instance.AllocatedStorage*resources.BytesInGibiBytes))
+	}
 }
 
 // set metrics about the postgres instance being deleted

diff --git a/pkg/resources/custom_metrics.go b/pkg/resources/custom_metrics.go
@@ -18,22 +18,25 @@ import (
 )
 
 const (
-	sleepytime                              = 3600
-	DefaultPostgresMaintenanceMetricName    = "cro_postgres_service_maintenance"
-	DefaultPostgresInfoMetricName           = "cro_postgres_info"
-	DefaultPostgresAvailMetricName          = "cro_postgres_available"
-	DefaultPostgresConnectionMetricName     = "cro_postgres_connection"
-	DefaultPostgresStatusMetricName         = "cro_postgres_status_phase"
-	DefaultPostgresDeletionMetricName       = "cro_postgres_deletion_timestamp"
-	DefaultPostgresSnapshotStatusMetricName = "cro_postgres_snapshot_status_phase"
-	DefaultRedisMaintenanceMetricName       = "cro_redis_service_maintenance"
-	DefaultRedisInfoMetricName              = "cro_redis_info"
-	DefaultRedisAvailMetricName             = "cro_redis_available"
-	DefaultRedisConnectionMetricName        = "cro_redis_connection"
-	DefaultRedisStatusMetricName            = "cro_redis_status_phase"
-	DefaultRedisDeletionMetricName          = "cro_redis_deletion_timestamp"
-	DefaultRedisSnapshotStatusMetricName    = "cro_redis_snapshot_status_phase"
-	DefaultBlobStorageStatusMetricName      = "cro_blobstorage_status_phase"
+	sleepytime                                = 3600
+	DefaultPostgresMaintenanceMetricName      = "cro_postgres_service_maintenance"
+	DefaultPostgresInfoMetricName             = "cro_postgres_info"
+	DefaultPostgresAvailMetricName            = "cro_postgres_available"
+	DefaultPostgresConnectionMetricName       = "cro_postgres_connection"
+	DefaultPostgresStatusMetricName           = "cro_postgres_status_phase"
+	DefaultPostgresDeletionMetricName         = "cro_postgres_deletion_timestamp"
+	DefaultPostgresSnapshotStatusMetricName   = "cro_postgres_snapshot_status_phase"
+	DefaultPostgresAllocatedStorageMetricName = "cro_postgres_current_allocated_storage"
+	DefaultRedisMaintenanceMetricName         = "cro_redis_service_maintenance"
+	DefaultRedisInfoMetricName                = "cro_redis_info"
+	DefaultRedisAvailMetricName               = "cro_redis_available"
+	DefaultRedisConnectionMetricName          = "cro_redis_connection"
+	DefaultRedisStatusMetricName              = "cro_redis_status_phase"
+	DefaultRedisDeletionMetricName            = "cro_redis_deletion_timestamp"
+	DefaultRedisSnapshotStatusMetricName      = "cro_redis_snapshot_status_phase"
+	DefaultBlobStorageStatusMetricName        = "cro_blobstorage_status_phase"
+
+	BytesInGibiBytes = 1073741824
 )
 
 var (