This is prime branch for all GORM TO SQLBOILER migrations. (botlabs-g…

…g#1655)

* all: migrate executed command logging away from gorm to sqlboiler (botlabs-gg#1654)

* common: add schemas for executed command logs

* common: configure and run sqlboiler for executed command log model

* all: migrate executed command logging to sqlboiler

* reminders: migrate away from gorm to sqlboiler (botlabs-gg#1652)

* reminders: add database schemas

* reminders: set up and run sqlboiler

* reminders: migrate to sqlboiler

While at it, lightly refactor and simplify the code as applicable. The most substantial (non-database) refactor is renaming and moving strReminders to DisplayReminders.

To aid in the refactor, also add discordgo.ParseID as a shortcut for strconv.ParseInt(...). This operation is often needed to convert string IDs in database to int64's and having a shorter name helps.

* reminders: auto-delete old entries with null guild_id as necessary

* soundboard: remove stray references to gorm (botlabs-gg#1651)

The soundboard module used to use gorm, but was migrated to sqlboiler some years ago in commit 628ea9a. There are still two (erroneous) references to gorm remaining which were not caught since this section of code ignores errors. This commit changes these to use sqlboiler as well.

* schemamigration: skip executing ALTER COLUMN SET NOT NULL queries when possible (botlabs-gg#1650)

* schemamigration: skip executing ALTER COLUMN SET NOT NULL queries when possible

* Use quantifier * not + in regex for consistency

It really should be +, but the current code works and hey -- when in Rome, do as the Romans.

* youtube: migrate away from gorm to sqlboiler (botlabs-gg#1660)

* youtube: add database schemas

* youtube: set up and run sqlboiler

* youtube: migrate to sqlboiler

* notifications: migrate away from gorm to sqlboiler (botlabs-gg#1659)

* notifications: decouple from configstore

* notifications: add database schemas

* notifications: set up and run sqlboiler

* notifications: migrate to sqlboiler

* moderation: migrate away from gorm to sqlboiler (botlabs-gg#1658)

* moderation: decouple from configstore

* moderation: add database schemas

* moderation: set up and run sqlboiler

* moderation: migrate to sqlboiler

* web: support null.Int64 in forms and validator

We currently use gorilla/schema to parse HTML form values into structs -- this allows us to specify, e,g., `<input ... name=SomeField>` and have `payload.SomeField` set correctly in Go handlers. This approach mostly works nicely, with the major exception of null.Int64 (formerly sql.NullInt64 with gorm) fields, which show up in the moderation plugin, e.g., Config.DefaultMuteDuration.

Particularly, since gorilla/schema has no native support for null.Int64, we have instead instructed gorilla/schema to directly set the Int64 value using <input ... name=DefaultMuteDuration.Int64> and then manually set DefaultMuteDuration.Valid=true in the web handler, without which the changes are not saved to database. This approach is, for obvious reasons, rather brittle. In this commit we therefore register a custom gorilla/schema decoder for null.Int64, obviating the need for both pointing to the Int64 field and manually setting Valid=true. While at it, we also fix the validator so that it checks null.Int64 fields properly.

* web: support validating types.Int64Array

gorm uses pq.Int64Array but sqlboiler uses types.Int64Array, so we need also support the latter now.

* common/configstore: remove package (botlabs-gg#1668)

* all: remove last traces of gorm (botlabs-gg#1670)

* common: connect to postgres via database/sql

...instead of through gorm.

* common: remove gorm log hook

* common: remove gorm SmallModel

* deps: run go mod tidy

---------

Co-authored-by: Joe L <56809242+jo3-l@users.noreply.github.com>

automod/automod_web.go

@@ -506,8 +506,8 @@ func (p *Plugin) handlePostAutomodUpdateRule(w http.ResponseWriter, r *http.Requ
 		}
 	}
 	if anyMute {
-		conf, err := moderation.GetConfig(g.ID)
-		if err != nil || conf.MuteRole == "" {
+		conf, err := moderation.GetCachedConfigOrDefault(g.ID)
+		if err != nil || conf.MuteRole == 0 {
 			tx.Rollback()
 			tmpl.AddAlerts(web.ErrorAlert("No mute role set, please configure one."))
 			return tmpl, nil

commands/commands.go

@@ -49,10 +49,6 @@ func (p *Plugin) PluginInfo() *common.PluginInfo {
 func RegisterPlugin() {
 	plugin := &Plugin{}
 	common.RegisterPlugin(plugin)
-	err := common.GORM.AutoMigrate(&common.LoggedExecutedCommand{}).Error
-	if err != nil {
-		logger.WithError(err).Fatal("Failed migrating logged commands database")
-	}
 
 	common.InitSchemas("commands", DBSchemas...)
 }

commands/yagcommmand.go

@@ -20,7 +20,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/sirupsen/logrus"
+	"github.com/volatiletech/sqlboiler/v4/boil"
 	"github.com/volatiletech/sqlboiler/v4/queries/qm"
+
+	commonmodels "github.com/botlabs-gg/yagpdb/v2/common/models"
 )
 
 type ContextKey int
@@ -233,7 +236,7 @@ func (yc *YAGCommand) Run(data *dcmd.Data) (interface{}, error) {
 	}
 
 	// Set up log entry for later use
-	logEntry := &common.LoggedExecutedCommand{
+	logEntry := &commonmodels.ExecutedCommand{
 		UserID:    discordgo.StrID(data.Author.ID),
 		ChannelID: discordgo.StrID(data.ChannelID),
 
@@ -243,8 +246,7 @@ func (yc *YAGCommand) Run(data *dcmd.Data) (interface{}, error) {
 	}
 
 	if data.GuildData != nil {
-		logEntry.GuildID = discordgo.StrID(data.GuildData.GS.ID)
-
+		logEntry.GuildID.SetValid(discordgo.StrID(data.GuildData.GS.ID))
 	}
 
 	metricsExcecutedCommands.With(prometheus.Labels{"name": "(other)", "trigger_type": triggerType}).Inc()
@@ -284,7 +286,7 @@ func (yc *YAGCommand) Run(data *dcmd.Data) (interface{}, error) {
 	}
 
 	// Create command log entry
-	err := common.GORM.Create(logEntry).Error
+	err := logEntry.InsertG(data.Context(), boil.Infer())
 	if err != nil {
 		logger.WithError(err).Error("Failed creating command execution log")
 	}

common/common.go

@@ -17,8 +17,6 @@ import (
 
 	"github.com/botlabs-gg/yagpdb/v2/common/cacheset"
 	"github.com/botlabs-gg/yagpdb/v2/lib/discordgo"
-	"github.com/jinzhu/gorm"
-	_ "github.com/jinzhu/gorm/dialects/postgres"
 	"github.com/jmoiron/sqlx"
 	"github.com/mediocregopher/radix/v3"
 	"github.com/prometheus/client_golang/prometheus"
@@ -30,7 +28,6 @@ import (
 var (
 	VERSION = "unknown"
 
-	GORM *gorm.DB
 	PQ   *sql.DB
 	SQLX *sqlx.DB
 
@@ -131,6 +128,10 @@ func Init() error {
 
 	logger.Info("Initializing core schema")
 	InitSchemas("core_configs", CoreServerConfDBSchema, localIDsSchema)
+
+	logger.Info("Initializing executed command log schema")
+	InitSchemas("executed_commands", ExecutedCommandDBSchemas...)
+
 	initQueuedSchemas()
 
 	return err
@@ -274,17 +275,15 @@ func connectDB(host, user, pass, dbName string, maxConns int) error {
 		passwordPart = " password='" + pass + "'"
 	}
 
-	db, err := gorm.Open("postgres", fmt.Sprintf("host=%s user=%s dbname=%s sslmode=disable%s", host, user, dbName, passwordPart))
-	GORM = db
-	PQ = db.DB()
+	db, err := sql.Open("postgres", fmt.Sprintf("host=%s user=%s dbname=%s sslmode=disable%s", host, user, dbName, passwordPart))
+	PQ = db
 	SQLX = sqlx.NewDb(PQ, "postgres")
 	boil.SetDB(PQ)
 	if err == nil {
 		PQ.SetMaxOpenConns(maxConns)
 		PQ.SetMaxIdleConns(maxConns)
 		logger.Infof("Set max PG connections to %d", maxConns)
 	}
-	GORM.SetLogger(&GORMLogger{})
 
 	return err
 }

common/executedcommandschema.go

@@ -0,0 +1,39 @@
+package common
+
+var ExecutedCommandDBSchemas = []string{`
+CREATE TABLE IF NOT EXISTS executed_commands (
+	id SERIAL PRIMARY KEY,
+	created_at TIMESTAMP WITH TIME ZONE NOT NULL,
+	updated_at TIMESTAMP WITH TIME ZONE NOT NULL,
+
+	user_id TEXT NOT NULL, -- text not bigint for legacy compatibility
+	channel_id TEXT NOT NULL,
+	guild_id TEXT,
+
+	command TEXT NOT NULL,
+	raw_command TEXT NOT NULL,
+	error TEXT,
+
+	time_stamp TIMESTAMP WITH TIME ZONE NOT NULL,
+	response_time BIGINT NOT NULL
+);
+`, `
+-- Preexisting tables created prior to sqlboiler are missing non-null constraints,
+-- so add them retraoctively.
+
+ALTER TABLE executed_commands ALTER COLUMN created_at SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN updated_at SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN user_id SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN channel_id SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN command SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN raw_command SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN time_stamp SET NOT NULL;
+`, `
+ALTER TABLE executed_commands ALTER COLUMN response_time SET NOT NULL;
+`}

common/loghooks.go

@@ -137,13 +137,6 @@ func DiscordGatewayLogger(shardID int, connectionID int, msgL int, msgf string,
 	}
 }
 
-type GORMLogger struct {
-}
-
-func (g *GORMLogger) Print(v ...interface{}) {
-	logrus.WithField("stck", "...").Error(v...)
-}
-
 type LoggingTransport struct {
 	Inner http.RoundTripper
 }