Merge pull request #42 from teknologi-umum/chore/retry-processor

chore(backend): retry processor

backend/monitor_processor.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"math"
+	"time"
 
 	"github.com/rs/zerolog/log"
 )
@@ -26,16 +28,32 @@ func (m *Processor) ProcessResponse(response Response) {
 		uniqueId = uniqueId[:255]
 	}
 
-	// TODO: Retry write if it fails
-	// Write the response to the historical writer
-	err := m.historicalWriter.Write(context.Background(), MonitorHistorical{
-		MonitorID: uniqueId,
-		Status:    status,
-		Latency:   response.RequestDuration,
-		Timestamp: response.Timestamp,
-	})
-	if err != nil {
-		log.Error().Err(err).Msg("failed to write historical data")
+	attemptRemaining := 3
+	attemptedEntries := 0
+	for attemptRemaining > 0 {
+		err := m.historicalWriter.Write(context.Background(), MonitorHistorical{
+			MonitorID: uniqueId,
+			Status:    status,
+			Latency:   response.RequestDuration,
+			Timestamp: response.Timestamp,
+		})
+		if err != nil {
+			attemptedEntries++
+			if attemptRemaining == 0 {
+				log.Error().Err(err).Msgf("failed to write historical data. Attempt %d failed\n", attemptedEntries)
+				return
+			}
+
+			delay := time.Second * time.Duration(math.Pow(2, math.Abs(float64(attemptedEntries))))
+			log.Error().Err(err).Msgf("failed to write historical data. Attempt %d failed. Retrying in %v...\n", attemptedEntries, delay)
+
+			time.Sleep(delay)
+
+			attemptRemaining -= 1
+			continue
+		}
+
+		break
 	}
 
 	go func() {