Add retry mechanism to CNI interface discovery on kube-proxy handler

We noticed that even though RA pod starts running only after the node is
ready, sometimes the kube-proxy handler fails to discover the CNI interface a
fter the node is rebooted.

This PR adds a retry to CNI discovery.

Fixes: #3120

Signed-off-by: Yossi Boaron <yboaron@redhat.com>

pkg/routeagent_driver/handlers/kubeproxy/kp_iptables.go

@@ -21,6 +21,7 @@ package kubeproxy
 import (
 	"net"
 	"os"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/submariner-io/admiral/pkg/log"
@@ -31,6 +32,8 @@ import (
 	"github.com/submariner-io/submariner/pkg/netlink"
 	cniapi "github.com/submariner-io/submariner/pkg/routeagent_driver/cni"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/utils/set"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 )
@@ -91,8 +94,16 @@ func (kp *SyncHandler) GetNetworkPlugins() []string {
 	return networkPlugins
 }
 
+var discoverCNIRetryConfig = wait.Backoff{
+	Cap:      1 * time.Minute,
+	Duration: 4 * time.Second,
+	Factor:   1.2,
+	Steps:    12,
+}
+
 func (kp *SyncHandler) Init() error {
 	var err error
+	var cniIface *cniapi.Interface
 
 	kp.hostname, err = os.Hostname()
 	if err != nil {
@@ -104,7 +115,17 @@ func (kp *SyncHandler) Init() error {
 		return errors.Wrapf(err, "Unable to find the default interface on host: %s", kp.hostname)
 	}
 
-	cniIface, err := cniapi.Discover(kp.localClusterCidr[0])
+	err = retry.OnError(discoverCNIRetryConfig, func(err error) bool {
+		logger.Infof("Waiting for CNI interface discovery: %s", err)
+		return true
+	}, func() error {
+		cniIface, err = cniapi.Discover(kp.localClusterCidr[0])
+		if err != nil {
+			return errors.Wrapf(err, "Error discovering the CNI interface")
+		}
+
+		return nil
+	})
 	if err == nil {
 		// Configure CNI Specific changes
 		kp.cniIface = cniIface