feed-ingress-deployment-nlb.yml

# Example deployment for launching feed-ingress, the nginx ingress controller.
#
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: feed-ingress
  namespace: kube-system
  labels:
    app: feed-ingress
spec:
  replicas: 3
  selector:
    matchLabels:
      app: feed-ingress
  template:
    metadata:
      labels:
        app: feed-ingress
    spec:

      # Listen directly on host interface, required so NLBs can contact nginx.
      hostNetwork: true

      # Time to wait for requests to gracefully terminate when updating the deployment.
      # Needs to be at least as long as the -drain-delay setting below, with a buffer for other teardown.
      # Recommend to set this to 2x -drain-delay.
      terminationGracePeriodSeconds: 60

      # Always restart on death.
      restartPolicy: Always

      containers:
      - image: skycirrus/feed-ingress:v2.0.0
        name: feed-ingress

        resources:
          requests:
            cpu: "1"
            memory: 300Mi
          limits:
            memory: 300Mi

        ports:
        - hostPort: 8080
          containerPort: 8080
          name: ingress
          protocol: TCP
        - hostPort: 8081
          containerPort: 8081
          name: ingress-health
          protocol: TCP
        # Health port of the controller.
        - containerPort: 12082
          name: health
          protocol: TCP

        args:
        - nlb

        # Ingress nginx port that NLBs will direct traffic towards.
        - --ingress-port=8080

        # Health port on nginx, used by NLBs to determine health.
        - --ingress-health-port=8081

        # Default security whitelist for ingress. Can be overridden with the sky.uk/allow annotation.
        - --ingress-allow=10.0.0.0/8

        # Health port for the controller to respond on.
        - --health-port=12082

        # Log level of nginx. Recommended to leave at error, or set to crit if too much spam.
        - --nginx-loglevel=error

        # How often to reload nginx if needed. Setting too low can cause 504s from the NLB in the case of heavy
        # ingress updates.
        - --nginx-update-period=5m

        # Use pushgateway for prometheus metrics. Optional - metrics available at /metrics on health port.
        - --pushgateway=mypushgateway.com
        - --pushgateway-interval=20
        - --pushgateway-label=k8s_cluster=dev
        - --pushgateway-label=environment=dev
        - --pushgateway-label=version=v1.0.2

        # Set status to unhealthy if fewer than this number of matching NLBs are found
        - --nlb-expected-number=2

        # Attach to the NLBs with label sky.uk/KubernetesClusterFrontend set to this value.
        - --nlb-frontend-tag-value=dev

        # Attach to the NLBs with label sky.uk/KubernetesClusterIngressClass set to this value
        # and adopt ingress resources with a matching kubernetes.io/ingress.class value
        - --ingress-class=main

        # Only consider ingresses in namespaces with this label. Optional.
        - --ingress-controller-namespace-selector=app=myapp

        # NLB drain time - time to wait while NLB drains requests from feed when stopping. Should be
        # at least as long as the NLBs drain timeout.
        - --drain-delay=30s

        # Each worker uses a full cpu, so scale up vertically on a box by increasing this value.
        - --nginx-workers=1

        # Connections*workers needs to be less than available ephemeral ports. Linux default is 60999-32768=28231.
        - --nginx-worker-connections=20000

        # Needs to be greater than any frontend idle timeout.
        - --nginx-keepalive-seconds=304

        # CIDRs of the NLBs to trust X-Forwarded-For, for determining client IP allow/deny.
        - --nginx-trusted-frontends=10.0.0.0/8

        # Max number of idle connections to a backend.
        - --nginx-backend-keepalive-count=1024

        # Default max time for a request to a backend. Can be overridden by an annotation on the ingress resource.
        - --nginx-default-backend-timeout-seconds=5

        # Needs to be greater than 64 to support very large domain names.
        - --nginx-server-names-hash-bucket-size=128

        # Access logs turned on - add or remove the "-access-log" flag to turn them on/off.
        - --access-log
        - --access-log-dir=/var/log/nginx

        # Add custom headers to the access logs.
        - --nginx-log-headers=X-Amzn-Trace-Id

        # Controller health determines readiness. This has no effect on ingress traffic from NLBs.
        readinessProbe:
          httpGet:
            path: /health
            port: 12082
            scheme: HTTP
          initialDelaySeconds: 1
          timeoutSeconds: 1
          periodSeconds: 1
          failureThreshold: 1

        # Only consider liveness of ingress itself, favouring uptime over controller health.
        livenessProbe:
          httpGet:
            path: /health
            port: 8081
            scheme: HTTP
          initialDelaySeconds: 30
          timeoutSeconds: 1
          periodSeconds: 10
          failureThreshold: 3

        # Access logs volume.
        volumeMounts:
        - name: nginx-log
          mountPath: /var/log/nginx

      volumes:
      - name: nginx-log
        emptyDir: {}