Skip to content

Commit

Permalink
it now sends an extra alarm after a successful backup if the one befo…
Browse files Browse the repository at this point in the history 
…re was a faillure
  • Loading branch information
@LobbyLobster
LobbyLobster committed Jun 27, 2024
1 parent 6ff7e6b commit 2f2b5d5
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 23 deletions.
38 changes: 33 additions & 5 deletions backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,19 @@ import (
"time"
)

var didItWork bool = true // this variable is for determining if the app should send a notification after a failed backup to inform that it works now
func itWorksNow(message string, worked bool) {
oldOnlyOnError := params.Notify.Webhook.OnlyOnError
if !didItWork && worked && oldOnlyOnError {
if oldOnlyOnError {
params.Notify.Webhook.OnlyOnError = false
}
notify.SendAlarm(message, false)
params.Notify.Webhook.OnlyOnError = oldOnlyOnError
}
didItWork = worked
}

func getDBList() (dbList []string) {
switch params.Database {
case "postgresql":
Expand Down Expand Up @@ -90,13 +103,13 @@ func Backup() {
}
return
}

for _, db := range params.Databases {
dst := strings.TrimSuffix(params.BackupDestination, "/") + "/" + db
if params.BackupAsTables && db != "mysql" {
dumpPaths, names, err := dumpDBWithTables(db, dst)
if err != nil {
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
} else {
logger.Info("Successfully backed up database:" + db + " with its tables separately, at " + params.BackupDestination + "/" + db)
notify.SendAlarm("Successfully backed up "+db+" at "+params.BackupDestination+"/"+db, false)
Expand All @@ -110,6 +123,7 @@ func Backup() {
filePath, name, err := dumpDB(db, dst)
if err != nil {
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
} else {
logger.Info("Successfully backed up database:" + db + " at " + filePath)
notify.SendAlarm("Successfully backed up "+db+" at "+filePath, false)
Expand Down Expand Up @@ -160,6 +174,7 @@ func uploadWhileDumping(db string) {
err := dumpAndUpload(db, pipeWriters)
if err != nil {
notify.SendAlarm("Problem during backing up "+db+" - Error: "+err.Error(), true)
itWorksNow("", false)
}
for _, writer := range pipeWriters {
writer.Close()
Expand All @@ -169,24 +184,37 @@ func uploadWhileDumping(db string) {
if uploadErr != nil {
logger.Error(strconv.Itoa(i+1) + ") " + db + " - " + "Couldn't upload to S3: " + uploaders[i].instance.Endpoint + " - Error: " + err.Error())
notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Couldn't upload to S3: "+uploaders[i].instance.Endpoint+" - Error: "+err.Error(), true)
itWorksNow("", false)
} else {
logger.Info(strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint)
notify.SendAlarm(strconv.Itoa(i+1)+") "+db+" - "+"Successfully uploaded to S3: "+uploaders[i].instance.Endpoint, false)
message := strconv.Itoa(i+1) + ") " + db + " - " + "Successfully uploaded to S3: " + uploaders[i].instance.Endpoint
notify.SendAlarm(message, false)
itWorksNow(message, true)
}
}
}

func upload(name, db, filePath string) {
var err error
switch params.BackupType.Type {
case "s3", "minio":
uploadToS3(filePath, name, db)
err = uploadToS3(filePath, name, db)
if err != nil {
itWorksNow("", false)
}
case "sftp":
for _, target := range params.BackupType.Info[0].Targets {
SendSFTP(filePath, name, db, target)
err = SendSFTP(filePath, name, db, target)
if err != nil {
itWorksNow("", false)
}
}
case "rsync":
for _, target := range params.BackupType.Info[0].Targets {
SendRsync(filePath, name, target)
err = SendRsync(filePath, name, target)
if err != nil {
itWorksNow("", false)
}
}
}
}
18 changes: 14 additions & 4 deletions backup/rsync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"strings"
)

func SendRsync(srcPath, dstPath string, target config.Target) {
func SendRsync(srcPath, dstPath string, target config.Target) error {
var dst string

if target.Path != "" {
Expand All @@ -18,7 +18,11 @@ func SendRsync(srcPath, dstPath string, target config.Target) {
}

err := sendRsync(srcPath, dst, target)
if err == nil && params.Rotation.Enabled {
if err != nil {
return err
}

if params.Rotation.Enabled {
extension := strings.Split(dstPath, ".")
shouldRotate, dstPath := rotate(extension[0])
for i := 1; i < len(extension); i++ {
Expand All @@ -28,9 +32,13 @@ func SendRsync(srcPath, dstPath string, target config.Target) {
dstPath = target.Path + "/" + dstPath
}
if shouldRotate {
_ = sendRsync(srcPath, dstPath, target)
err = sendRsync(srcPath, dstPath, target)
if err != nil {
return err
}
}
}
return nil
}

func sendRsync(srcPath, dstPath string, target config.Target) error {
Expand Down Expand Up @@ -65,7 +73,9 @@ func sendRsync(srcPath, dstPath string, target config.Target) error {
}

logger.Info("Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath)
notify.SendAlarm("Successfully uploaded "+srcPath+" to "+target.Host+":"+dstPath, false)
message := "Successfully uploaded " + srcPath + " to " + target.Host + ":" + dstPath
notify.SendAlarm(message, false)
itWorksNow(message, true)

return nil
}
4 changes: 3 additions & 1 deletion backup/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ func uploadFileToS3(src, dst, db string, reader io.Reader, s3Instance *uploaderS
return err
}
logger.Info("Successfully uploaded " + src + " to S3\nBucket: " + bucketName + " path: " + dst)
notify.SendAlarm("Successfully uploaded "+src+" to S3\nBucket: "+bucketName+" path: "+dst, false)
message := "Successfully uploaded " + src + " to S3\nBucket: " + bucketName + " path: " + dst
notify.SendAlarm(message, false)
itWorksNow(message, true)
if params.Rotation.Enabled {
if db == "mysql" {
db = db + "_users"
Expand Down
35 changes: 22 additions & 13 deletions backup/sftp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ import (
"golang.org/x/crypto/ssh/agent"
)

func SendSFTP(srcPath, dstPath, db string, target config.Target) {
func SendSFTP(srcPath, dstPath, db string, target config.Target) error {
dstPath = target.Path + "/" + nameWithPath(dstPath)
logger.Info("SFTP transfer started.\n Source: " + srcPath + " - Destination: " + target.Host + ":" + dstPath)
sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
if err != nil {
logger.Error("Couldn't get environment variable SSH_AUTH_SOCK - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't get environment variable SSH_AUTH_SOCK - Error: "+err.Error(), true)
return
return err
}

sockAgent := agent.NewClient(sock)
Expand All @@ -28,7 +28,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) {
if err != nil {
logger.Error("Couldn't get signers for ssh keys - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't get signers for ssh keys - Error: "+err.Error(), true)
return
return err
}
auths := []ssh.AuthMethod{ssh.PublicKeys(signers...)}

Expand All @@ -51,7 +51,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) {
if err != nil {
logger.Error("Couldn't create an SFTP client - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create an SFTP client - Error: "+err.Error(), true)
return
return err
}
defer func() {
err = sftpCli.Close()
Expand All @@ -65,7 +65,7 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) {
if err != nil {
logger.Error("Couldn't open source file " + srcPath + " for copying - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't open source file "+srcPath+" for copying - Error: "+err.Error(), true)
return
return err
}
defer func() {
err = src.Close()
Expand All @@ -75,7 +75,10 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) {
}
}()

sendOverSFTP(srcPath, dstPath, src, target, sftpCli)
err = sendOverSFTP(srcPath, dstPath, src, target, sftpCli)
if err != nil {
return err
}

if params.Rotation.Enabled {
shouldRotate, newDst := rotate(db)
Expand All @@ -85,13 +88,16 @@ func SendSFTP(srcPath, dstPath, db string, target config.Target) {
newDst = newDst + "." + extension[i]
}
newDst = target.Path + "/" + newDst
sendOverSFTP(srcPath, newDst, src, target, sftpCli)
err = sendOverSFTP(srcPath, newDst, src, target, sftpCli)
if err != nil {
return err
}
}
}

return nil
}

func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, sftpCli *sftp.Client) {
func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, sftpCli *sftp.Client) error {
fullPath := strings.Split(dstPath, "/")
newPath := "/"
for i := 0; i < len(fullPath)-1; i++ {
Expand All @@ -101,13 +107,13 @@ func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, s
if err != nil {
logger.Error("Couldn't create folders " + newPath + " - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create folders "+newPath+" - Error: "+err.Error(), true)
return
return err
}
dst, err := sftpCli.Create(dstPath)
if err != nil {
logger.Error("Couldn't create file " + dstPath + " - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't create file "+dstPath+" - Error: "+err.Error(), true)
return
return err
}
defer func() {
err = dst.Close()
Expand All @@ -121,8 +127,11 @@ func sendOverSFTP(srcPath, dstPath string, src *os.File, target config.Target, s
if _, err := dst.ReadFrom(src); err != nil {
logger.Error("Couldn't read from file " + srcPath + " to write at " + dstPath + " - Error: " + err.Error())
notify.SendAlarm("Couldn't upload backup "+srcPath+" to "+target.Host+":"+dstPath+"\nCouldn't read from file "+srcPath+" to write at "+dstPath+" - Error: "+err.Error(), true)
return
return err
}
logger.Info("Successfully copied " + srcPath + " to " + target.Host + ":" + dstPath)
notify.SendAlarm("Successfully copied "+srcPath+" to "+target.Host+":"+dstPath, false)
message := "Successfully copied " + srcPath + " to " + target.Host + ":" + dstPath
notify.SendAlarm(message, false)
itWorksNow(message, true)
return nil
}

0 comments on commit 2f2b5d5

Please sign in to comment.