Merge pull request #6 from CommercialTribe/develop

Remove labels and retry redis-cli connection

.dockerignore

@@ -0,0 +1,2 @@
+*
+!sidecar.sh

.travis.yml

@@ -24,7 +24,7 @@ deploy:
   - provider: script
     script: ./build-and-push.sh
     on:
-      branch: master
+      all_branches: true
   - provider: script
     script: ./deploy.sh gke_commercial-tribe-staging_us-central1-a_staging develop
     on:

Dockerfile

@@ -1,10 +1,10 @@
 FROM redis:3.2
 MAINTAINER Jason Waldrip <jwaldrip@commercialtribe.com>
 
-ADD https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/kubectl /usr/local/bin/kubectl
+ADD https://storage.googleapis.com/kubernetes-release/release/v1.6.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl
 RUN chmod +x /usr/local/bin/kubectl
 
 WORKDIR /app
-ADD sidecar.sh /app/sidecar.sh
+ADD . /app
 RUN chmod +x /app/sidecar.sh
 CMD /app/sidecar.sh

build-and-push.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-image="commercialtribe/redis-sentinel-sidecar"
+image="commercialtribe/redis-sentinel-sidecar:v20170407.1"
 
 login(){
   docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD

k8s/redis-node-statefulset.yaml

@@ -15,7 +15,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
         # Redis
-        - name: redis
+        - name: redis-node
           image: redis:3.2
           command:
             - redis-server
@@ -40,7 +40,7 @@ spec:
             <<: *healthcheck
 
         # Sentinel
-        - name: sentinel
+        - name: redis-sentinel
           image: redis:3.2
           command: [ "bash", "-c", "touch sentinel.conf && redis-sentinel sentinel.conf" ]
           ports:
@@ -57,8 +57,8 @@ spec:
             <<: *healthcheck
 
         # Sidecar
-        - name: sidecar
-          image: commercialtribe/redis-sentinel-sidecar:latest
+        - name: redis-sidecar
+          image: commercialtribe/redis-sentinel-sidecar:v20170407.1
           imagePullPolicy: Always
           env:
             - name: POD_NAMESPACE

sidecar.sh

@@ -5,7 +5,7 @@ set -e
 # Set vars
 ip=${POD_IP-`hostname -i`}                                  # ip address of pod
 redis_port=${NODE_PORT_NUMBER-6379}                         # redis port
-sentinel_port=${SENTINEL_PORT_NUMBER-26379}                    # sentinel port
+sentinel_port=${SENTINEL_PORT_NUMBER-26379}                 # sentinel port
 group_name="$POD_NAMESPACE-$(hostname | sed 's/-[0-9]$//')" # master group name
 quorum="${SENTINEL_QUORUM-2}"                               # quorum needed
 
@@ -17,12 +17,30 @@ parallel_syncs=${PARALEL_SYNCS-1}
 # Get all the kubernetes pods
 labels=`echo $(cat /etc/pod-info/labels) | tr -d '"' | tr " " ","`
 
+try_step_interval=${TRY_STEP_INTERVAL-"1"}
+max_tries=${MAX_TRIES-"3"}
+retry() {
+  local tries=0
+  until $@ ; do
+    status=$?
+    tries=$(($tries + 1))
+    if [ $tries -gt $max_tries ] ; then
+      >&2 echo "Failed to run \`$@\` after $max_tries tries..."
+      return $status
+    fi
+    sleepsec=$(($tries * $try_step_interval))
+    >&2 echo "Failed: \`$@\`, retyring in $sleepsec seconds..."
+    sleep $sleepsec
+  done
+  return $?
+}
+
 cli(){
-  redis-cli -p $redis_port $@
+  retry redis-cli -p $redis_port $@
 }
 
 sentinel-cli(){
-  redis-cli -p $sentinel_port $@
+  retry redis-cli -p $sentinel_port $@
 }
 
 ping() {
@@ -74,8 +92,9 @@ hosts(){
 }
 
 boot(){
+  set-role-label "none" # set roll label to nothing
   sleep $(($failover_timeout / 1000))
-  ping-both || exit 1
+  ping-both
   master=$(active-master)
   if [[ -n "$master" ]] ; then
     become-slave-of $master
@@ -87,13 +106,17 @@ boot(){
   touch booted
 }
 
+set-role-label(){
+  kubectl label --overwrite pods `hostname` role=$1
+}
+
 monitor-label(){
   last_role=none
   while true ; do
     ping-both || exit 1
     current_role=`role`
     if [[ "$last_role" != "$current_role" ]] ; then
-      kubectl label --overwrite pods `hostname` role=$current_role
+      set-role-label $current_role
       last_role=$current_role
     fi
     sleep 1