From 0f483ae689090cb85936aa973847d10df6e73b2a Mon Sep 17 00:00:00 2001 From: Achyut Madhusudan Date: Wed, 22 Nov 2023 23:04:54 +0530 Subject: [PATCH] Extracted DB and Object Storage Logs to display on DSPA Signed-off-by: Achyut Madhusudan --- controllers/database.go | 25 ++++++------ controllers/dspipeline_controller.go | 15 +++++--- controllers/storage.go | 57 ++++++++++++++++------------ 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/controllers/database.go b/controllers/database.go index b0e6d891f..1fbc84b89 100644 --- a/controllers/database.go +++ b/controllers/database.go @@ -21,10 +21,11 @@ import ( b64 "encoding/base64" "fmt" + "time" + _ "github.com/go-sql-driver/mysql" dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/config" - "time" ) const dbSecret = "mariadb/secret.yaml.tmpl" @@ -37,7 +38,7 @@ var mariadbTemplates = []string{ } // extract to var for mocking in testing -var ConnectAndQueryDatabase = func(host, port, username, password, dbname string, dbConnectionTimeout time.Duration) bool { +var ConnectAndQueryDatabase = func(host, port, username, password, dbname string, dbConnectionTimeout time.Duration) (bool, string) { // Create a context with a timeout of 1 second ctx, cancel := context.WithTimeout(context.Background(), dbConnectionTimeout) defer cancel() @@ -45,22 +46,23 @@ var ConnectAndQueryDatabase = func(host, port, username, password, dbname string connectionString := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", username, password, host, port, dbname) db, err := sql.Open("mysql", connectionString) if err != nil { - return false + return false, err.Error() } defer db.Close() testStatement := "SELECT 1;" _, err = db.QueryContext(ctx, testStatement) - return err == nil + return err == nil, "" } func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication, - params *DSPAParams) bool { + params *DSPAParams) (bool, string) { log := r.Log.WithValues("namespace", dsp.Namespace).WithValues("dspa_name", dsp.Name) if params.DatabaseHealthCheckDisabled(dsp) { - log.V(1).Info("Database health check disabled, assuming database is available and ready.") - return true + infoMessage := "Database health check disabled, assuming database is available and ready." + log.V(1).Info(infoMessage) + return true, infoMessage } log.Info("Performing Database Health Check") @@ -68,8 +70,9 @@ func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1al usingExternalDB := params.UsingExternalDB(dsp) usingMariaDB := !databaseSpecified || dsp.Spec.Database.MariaDB != nil if !usingMariaDB && !usingExternalDB { - log.Info("Could not connect to Database: Unsupported Type") - return false + infoMessage := "Could not connect to Database: Unsupported Type" + log.Info(infoMessage) + return false, infoMessage } decodePass, _ := b64.StdEncoding.DecodeString(params.DBConnection.Password) @@ -77,7 +80,7 @@ func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1al log.V(1).Info(fmt.Sprintf("Database Heath Check connection timeout: %s", dbConnectionTimeout)) - dbHealthCheckPassed := ConnectAndQueryDatabase(params.DBConnection.Host, + dbHealthCheckPassed, message := ConnectAndQueryDatabase(params.DBConnection.Host, params.DBConnection.Port, params.DBConnection.Username, string(decodePass), @@ -88,7 +91,7 @@ func (r *DSPAReconciler) isDatabaseAccessible(ctx context.Context, dsp *dspav1al } else { log.Info("Unable to connect to Database") } - return dbHealthCheckPassed + return dbHealthCheckPassed, message } func (r *DSPAReconciler) ReconcileDatabase(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication, diff --git a/controllers/dspipeline_controller.go b/controllers/dspipeline_controller.go index 14faf6b27..6f7935d5e 100644 --- a/controllers/dspipeline_controller.go +++ b/controllers/dspipeline_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/go-logr/logr" @@ -224,8 +225,8 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } // Get Prereq Status (DB and ObjStore Ready) - dbAvailable := r.isDatabaseAccessible(ctx, dspa, params) - objStoreAvailable := r.isObjectStorageAccessible(ctx, dspa, params) + dbAvailable, dbAvailableMessage := r.isDatabaseAccessible(ctx, dspa, params) + objStoreAvailable, objStoreAvailableMessage := r.isObjectStorageAccessible(ctx, dspa, params) dspaPrereqsReady := dbAvailable && objStoreAvailable if dspaPrereqsReady { @@ -269,7 +270,7 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } - conditions, err := r.GenerateStatus(ctx, dspa, params, dbAvailable, objStoreAvailable) + conditions, err := r.GenerateStatus(ctx, dspa, params, dbAvailable, objStoreAvailable, dbAvailableMessage, objStoreAvailableMessage) if err != nil { log.Info(err.Error()) return ctrl.Result{}, err @@ -411,14 +412,15 @@ func (r *DSPAReconciler) handleReadyCondition(ctx context.Context, dspa *dspav1a } func (r *DSPAReconciler) GenerateStatus(ctx context.Context, dspa *dspav1alpha1.DataSciencePipelinesApplication, - params *DSPAParams, dbAvailableStatus, objStoreAvailableStatus bool) ([]metav1.Condition, error) { + params *DSPAParams, dbAvailableStatus bool, objStoreAvailableStatus bool, dbAvailableMessage string, objStoreAvailableMessage string) ([]metav1.Condition, error) { // Create Database Availability Condition databaseAvailable := r.buildCondition(config.DatabaseAvailable, dspa, config.DatabaseAvailable) if dbAvailableStatus { databaseAvailable.Status = metav1.ConditionTrue databaseAvailable.Message = "Database connectivity successfully verified" } else { - databaseAvailable.Message = "Could not connect to database" + databaseAvailable.Status = metav1.ConditionFalse + databaseAvailable.Message = dbAvailableMessage } // Create Object Storage Availability Condition @@ -427,7 +429,8 @@ func (r *DSPAReconciler) GenerateStatus(ctx context.Context, dspa *dspav1alpha1. objStoreAvailable.Status = metav1.ConditionTrue objStoreAvailable.Message = "Object Store connectivity successfully verified" } else { - objStoreAvailable.Message = "Could not connect to Object Store" + objStoreAvailable.Status = metav1.ConditionFalse + objStoreAvailable.Message = objStoreAvailableMessage } // Create APIServer Readiness Condition diff --git a/controllers/storage.go b/controllers/storage.go index 90606da30..3ba892a2d 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -24,13 +24,14 @@ import ( "fmt" "net/http" + "time" + "github.com/go-logr/logr" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/config" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/util" - "time" ) const storageSecret = "minio/secret.yaml.tmpl" @@ -94,7 +95,7 @@ func getHttpsTransportWithCACert(log logr.Logger, pemCerts []byte) (*http.Transp return transport, nil } -var ConnectAndQueryObjStore = func(ctx context.Context, log logr.Logger, endpoint, bucket string, accesskey, secretkey []byte, secure bool, pemCerts []byte, objStoreConnectionTimeout time.Duration) bool { +var ConnectAndQueryObjStore = func(ctx context.Context, log logr.Logger, endpoint, bucket string, accesskey, secretkey []byte, secure bool, pemCerts []byte, objStoreConnectionTimeout time.Duration) (bool, string) { cred := createCredentialProvidersChain(string(accesskey), string(secretkey)) opts := &minio.Options{ @@ -105,16 +106,18 @@ var ConnectAndQueryObjStore = func(ctx context.Context, log logr.Logger, endpoin if len(pemCerts) != 0 { tr, err := getHttpsTransportWithCACert(log, pemCerts) if err != nil { - log.Error(err, "Encountered error when processing custom ca bundle.") - return false + errorMessage := "Encountered error when processing custom ca bundle." + log.Error(err, errorMessage) + return false, errorMessage } opts.Transport = tr } minioClient, err := minio.New(endpoint, opts) if err != nil { + errorMessage := "Could not connect to object storage endpoint" log.Info(fmt.Sprintf("Could not connect to object storage endpoint: %s", endpoint)) - return false + return false, errorMessage } ctx, cancel := context.WithTimeout(ctx, objStoreConnectionTimeout) @@ -128,60 +131,66 @@ var ConnectAndQueryObjStore = func(ctx context.Context, log logr.Logger, endpoin // In the case that the Error is NoSuchKey (or NoSuchBucket), we can verify that the endpoint worked and the object just doesn't exist case minio.ErrorResponse: if err.Code == "NoSuchKey" || err.Code == "NoSuchBucket" { - return true + return true, err.Error() } } if util.IsX509UnknownAuthorityError(err) { - log.Error(err, "Encountered x509 UnknownAuthorityError when connecting to ObjectStore. "+ - "If using an tls S3 connection with self-signed certs, you may specify a custom CABundle "+ - "to mount on the DSP API Server via the DSPA cr under the spec.cABundle field. If you have already "+ - "provided a CABundle, verify the validity of the provided CABundle.") - return false + errorMessage := "Encountered x509 UnknownAuthorityError when connecting to ObjectStore. " + + "If using an tls S3 connection with self-signed certs, you may specify a custom CABundle " + + "to mount on the DSP API Server via the DSPA cr under the spec.cABundle field. If you have already " + + "provided a CABundle, verify the validity of the provided CABundle." + log.Error(err, errorMessage) + return false, errorMessage } // Every other error means the endpoint in inaccessible, or the credentials provided do not have, at a minimum GetObject, permissions + errorMessage := err.Error() log.Info(fmt.Sprintf("Could not connect to (%s), Error: %s", endpoint, err.Error())) - return false + return false, errorMessage } // Getting here means the health check passed - return true + return true, "" } func (r *DSPAReconciler) isObjectStorageAccessible(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication, - params *DSPAParams) bool { + params *DSPAParams) (bool, string) { log := r.Log.WithValues("namespace", dsp.Namespace).WithValues("dspa_name", dsp.Name) if params.ObjectStorageHealthCheckDisabled(dsp) { - log.V(1).Info("Object Storage health check disabled, assuming object store is available and ready.") - return true + infoMessage := "Object Storage health check disabled, assuming object store is available and ready." + log.V(1).Info(infoMessage) + return true, infoMessage } log.Info("Performing Object Storage Health Check") endpoint, err := joinHostPort(params.ObjectStorageConnection.Host, params.ObjectStorageConnection.Port) if err != nil { - log.Error(err, "Could not determine Object Storage Endpoint") - return false + errorMessage := "Could not determine Object Storage Endpoint" + log.Error(err, errorMessage) + return false, errorMessage } accesskey, err := base64.StdEncoding.DecodeString(params.ObjectStorageConnection.AccessKeyID) if err != nil { - log.Error(err, "Could not decode Object Storage Access Key ID") - return false + errorMessage := "Could not decode Object Storage Access Key ID" + log.Error(err, errorMessage) + return false, errorMessage } secretkey, err := base64.StdEncoding.DecodeString(params.ObjectStorageConnection.SecretAccessKey) if err != nil { - log.Error(err, "Could not decode Object Storage Secret Access Key") - return false + errorMessage := "Could not decode Object Storage Secret Access Key" + log.Error(err, errorMessage) + return false, errorMessage } objStoreConnectionTimeout := config.GetDurationConfigWithDefault(config.ObjStoreConnectionTimeoutConfigName, config.DefaultObjStoreConnectionTimeout) log.V(1).Info(fmt.Sprintf("Object Store connection timeout: %s", objStoreConnectionTimeout)) - verified := ConnectAndQueryObjStore(ctx, log, endpoint, params.ObjectStorageConnection.Bucket, accesskey, secretkey, + verified, message := ConnectAndQueryObjStore(ctx, log, endpoint, params.ObjectStorageConnection.Bucket, accesskey, secretkey, *params.ObjectStorageConnection.Secure, params.APICustomPemCerts, objStoreConnectionTimeout) if verified { @@ -189,7 +198,7 @@ func (r *DSPAReconciler) isObjectStorageAccessible(ctx context.Context, dsp *dsp } else { log.Info("Object Storage Health Check Failed") } - return verified + return verified, message } // ReconcileStorage will set up Storage Connection.